From 2cbfb03eb60b40921e477b415495d005f633f28f Mon Sep 17 00:00:00 2001 From: Nathan Rashleigh Date: Sun, 30 Jun 2024 17:57:54 +1000 Subject: [PATCH] changes for prod container --- .dockerignore | 20 ++++++++++++----- .gitignore | 2 ++ archiver/Dockerfile | 22 ++++--------------- archiver/build/index.js | 8 +++++++ archiver/index.ts | 12 +++++++---- archiver/package.json | 3 +-- archiver/src/memes.ts | 45 +++++++++++++++++++++++++++++++++------ archiver/src/scraper.ts | 2 +- client/.dockerignore | 1 + client/Dockerfile | 10 +++++++-- client/dev.Dockerfile | 7 ++++++ client/src/App.js | 12 +++++++++-- docker-compose.build.yaml | 7 ++++++ docker-compose.yaml | 30 +++++++++++++++----------- 14 files changed, 128 insertions(+), 53 deletions(-) create mode 100644 client/.dockerignore create mode 100644 client/dev.Dockerfile create mode 100644 docker-compose.build.yaml diff --git a/.dockerignore b/.dockerignore index d2820a5..a0fa393 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,16 @@ -data/ -node_modules/ - - .env -vision-credentials.json \ No newline at end of file +*.env +.env* + +.venv/ +vision-credentials.json +__pycache__/ + +node_modules + +# drop the workspace file cause it leaks info about personal machines +*.sublime-workspace + +data/ +meili_data/ +data.bak/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 756047f..a0fa393 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .env +*.env +.env* .venv/ vision-credentials.json diff --git a/archiver/Dockerfile b/archiver/Dockerfile index 7ecd093..9a73990 100644 --- a/archiver/Dockerfile +++ b/archiver/Dockerfile @@ -1,21 +1,7 @@ -FROM node:20 as builder -WORKDIR /app - -COPY . ./ -RUN npm i -RUN npm run build - -# execution FROM node:20 -WORKDIR /app -COPY --from=builder /app/build /app - -# packages are not bundled in the build because sharp uses native modules -COPY package.json /app/package.json -RUN npm i --include prod - -# prisma setup -COPY ./prisma /app/prisma +WORKDIR /usr/src/memestream +COPY . /usr/src/memestream +RUN npm i RUN npx prisma generate -CMD node index.js +CMD npm start diff --git a/archiver/build/index.js b/archiver/build/index.js index e01b7ae..5953f1b 100644 --- a/archiver/build/index.js +++ b/archiver/build/index.js @@ -1,5 +1,7 @@ // index.ts import "dotenv/config"; +import { exec as execCb } from "child_process"; +import { promisify } from "util"; import pino from "pino"; import * as Minio from "minio"; import Vision from "@google-cloud/vision"; @@ -302,6 +304,7 @@ async function memeService(baseLog, prisma, meili) { BigInt.prototype.toJSON = function bigIntStringify() { return this.toString(); }; +var exec = promisify(execCb); async function main() { const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default; const log = pino({ base: {} }); @@ -331,6 +334,11 @@ async function main() { 1095060717117374687n // aella memes ]; + async function migrateDb() { + const output = await exec("prisma migrate deploy"); + log.info(output); + } + await migrateDb(); await Promise.all(CHANNEL_IDS.map((id) => scraper2.processChannel(id))); } main().then(() => true).catch((e) => console.error(e)); diff --git a/archiver/index.ts b/archiver/index.ts index 9575fa5..6289dfe 100644 --- a/archiver/index.ts +++ b/archiver/index.ts @@ -23,17 +23,19 @@ BigInt.prototype.toJSON = function bigIntStringify() { const exec = promisify(execCb) async function main() { - const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default - const log = pino({ base: {} }); + const VISION_CREDENTIALS = JSON.parse(process.env.VISION_CREDENTIALS) const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS }) const prisma = new PrismaClient() const minio = new Minio.Client({ - endPoint: process.env.STORAGE_URL, + endPoint: process.env.STORAGE_ENDPOINT, accessKey: process.env.STORAGE_ACCESS_KEY, secretKey: process.env.STORAGE_SECRET_KEY, + useSSL: false, + port: 9000, }) + const meili = new MeiliSearch({ host: process.env.MEILI_URL, apiKey: process.env.MEILI_MASTER_KEY, @@ -48,7 +50,7 @@ async function main() { 367936467592413194n, // tpm screenshots 1028511952277864501n, // tpm sludge 725835129326665768n, // machinegirl memes - 1095060717117374687n, // aella memes + // 1095060717117374687n, // aella memes ] async function migrateDb() { @@ -58,6 +60,8 @@ async function main() { await migrateDb() + log.info('ready') + await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id))) } diff --git a/archiver/package.json b/archiver/package.json index 20056f3..e98442e 100644 --- a/archiver/package.json +++ b/archiver/package.json @@ -5,8 +5,7 @@ "main": "index.ts", "type": "module", "scripts": { - "start": "esrun index.ts | pino-pretty", - "start:bundle": "node index.js", + "start": "esrun index.ts", "dev": "esrun --watch index.ts | pino-pretty", "build": "esbuild index.ts --bundle --format=esm --outdir=build --platform=node --target=node22.1 --packages=external", "migrate:make": "prisma migrate dev --name", diff --git a/archiver/src/memes.ts b/archiver/src/memes.ts index 67ff338..f5ef0ba 100644 --- a/archiver/src/memes.ts +++ b/archiver/src/memes.ts @@ -6,11 +6,23 @@ import _ from 'lodash' import * as attachments from './attachments' +// where to post the imgs to const STORAGE_BUCKET = process.env.STORAGE_BUCKET +// the host the client will fetch the img from +const STORAGE_HOST = process.env.STORAGE_HOST + +// name for the key the client uses +const MEILI_SEARCH_KEY_UUID = process.env.MEILI_SEARCH_KEY_UUID +const MEILI_SEARCH_KEY_NAME = process.env.MEILI_SEARCH_KEY_NAME +const MEILI_INDEX = process.env.MEILI_INDEX || 'memes' function memeUrl(a: Attachment) { - const STORAGE_URL = process.env.STORAGE_URL - const u = new URL(`${STORAGE_BUCKET}/${attachments.objectName(a)}`, `https://${STORAGE_URL}`) + const STORAGE_ENDPOINT = process.env.STORAGE_ENDPOINT + + // thisis broken + // needs to be actual url + + const u = new URL(`${STORAGE_BUCKET}/${attachments.objectName(a)}`, `${STORAGE_HOST}`) return u.href } @@ -39,18 +51,37 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient, return m } + async function ensureMeiliSearchKey() { + const keys = await meili.getKeys() + const searchKey = keys.results.find(k => k.name === MEILI_SEARCH_KEY_NAME) + if (searchKey) { + log.info(`Public search key=${searchKey.key}`) + return searchKey + } + const created = meili.createKey({ + name: MEILI_SEARCH_KEY_NAME, + actions: ['search', 'documents.get', 'indexes.get', 'stats.get', 'version'], + indexes: [MEILI_INDEX], + // never expire + expiresAt: null, + uid: MEILI_SEARCH_KEY_UUID, + description: 'Memestream - Public Search Key' + }) + return created + } + async function ensureMeiliIndex() { - const task = await meili.createIndex('memes', { primaryKey: 'id' }) + const task = await meili.createIndex(MEILI_INDEX, { primaryKey: 'id' }) const result = await meili.waitForTask(task.taskUid) log.debug(result); - const index = await meili.index('memes') + const index = await meili.index(MEILI_INDEX) await index.updateSortableAttributes(['postedAt']) return index; } // BLOWS IT UP async function resetMeili() { - const delTask = await meili.deleteIndex('memes') + const delTask = await meili.deleteIndex(MEILI_INDEX) const delResult = await meili.waitForTask(delTask.taskUid) const index = await ensureMeiliIndex() const memes = (await prisma.meme.findMany()) @@ -64,16 +95,18 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient, } async function persist(m: Meme) { - const index = meili.index('memes') + const index = meili.index(MEILI_INDEX) const doc = ({ ...m, postedAt: m.postedAt.valueOf() }) return index.updateDocuments([doc]) } await ensureMeiliIndex(); + await ensureMeiliSearchKey() // BAM // await resetMeili(); + return { ensureMeiliIndex, resetMeili, diff --git a/archiver/src/scraper.ts b/archiver/src/scraper.ts index 624eacb..c5f6646 100644 --- a/archiver/src/scraper.ts +++ b/archiver/src/scraper.ts @@ -22,7 +22,7 @@ function convertSnowflakeToDate(snowflake) { return new Date(Number(milliseconds) + DISCORD_EPOCH) } -export default async function scraper( +export default async function discordScraper( discordAuth, vision: ImageAnnotatorClient, minio: Minio.Client, diff --git a/client/.dockerignore b/client/.dockerignore new file mode 100644 index 0000000..40b878d --- /dev/null +++ b/client/.dockerignore @@ -0,0 +1 @@ +node_modules/ \ No newline at end of file diff --git a/client/Dockerfile b/client/Dockerfile index 94934fb..6349db4 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -1,6 +1,12 @@ # build step FROM node:20 AS build-env -WORKDIR /app +WORKDIR /usr/src/memestream-client + +# super annoying +# ARG REACT_APP_MEILI_ENDPOINT +# ARG REACT_APP_MEILI_KEY +# ENV REACT_APP_MEILI_ENDPOINT $REACT_APP_MEILI_ENDPOINT +# ENV REACT_APP_MEILI_KEY $REACT_APP_MEILI_KEY COPY . ./ RUN npm i @@ -9,5 +15,5 @@ RUN npm run build # static server FROM nginx WORKDIR /srv/app -COPY --from=build-env /app/build /srv/app +COPY --from=build-env /usr/src/memestream-client/build /srv/app COPY ./nginx.conf /etc/nginx/nginx.conf diff --git a/client/dev.Dockerfile b/client/dev.Dockerfile new file mode 100644 index 0000000..bdbe6d6 --- /dev/null +++ b/client/dev.Dockerfile @@ -0,0 +1,7 @@ +# build step +FROM node:20 AS build-env +WORKDIR /usr/src/memestream-client + +COPY . ./ +RUN npm i +CMD npm start diff --git a/client/src/App.js b/client/src/App.js index b1d080b..c66bca8 100644 --- a/client/src/App.js +++ b/client/src/App.js @@ -11,9 +11,17 @@ import "./App.css"; import { instantMeiliSearch } from "@meilisearch/instant-meilisearch"; import { convertSnowflakeToDate } from "./util"; +const MEILI_ENDPOINT = process.env.REACT_APP_MEILI_ENDPOINT || + "https://search.memestream.strix.systems" + +const MEILI_KEY = process.env.REACT_APP_MEILI_KEY || + '040586799aee83b633a4935d8c0e045e9fde9e78cff0dfe542ee10583272d40d' + +console.log({ MEILI_ENDPOINT, MEILI_KEY }) + const { searchClient } = instantMeiliSearch( - "https://search.memestream.strix.systems", - "ef006ab414a12b49eb797e0637219d5d6bcf3c84b8195c9fac49f2629713705a", + MEILI_ENDPOINT, + MEILI_KEY, { finitePagination: false, placeholderSearch: true, diff --git a/docker-compose.build.yaml b/docker-compose.build.yaml new file mode 100644 index 0000000..aa51865 --- /dev/null +++ b/docker-compose.build.yaml @@ -0,0 +1,7 @@ +services: + client: + image: registry.strix.systems/memestream-client + build: + context: ./client + env_file: + - .env.client.prod diff --git a/docker-compose.yaml b/docker-compose.yaml index c66c3a1..e4ae212 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,10 +3,8 @@ services: pg: image: postgres:latest - environment: - POSTGRES_USER: '${POSTGRES_USER}' - POSTGRES_DB: '${POSTGRES_DB}' - POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}' + env_file: + - .env.pg ports: - '5432:5432' volumes: @@ -14,12 +12,12 @@ services: meili: image: getmeili/meilisearch:v1.7 + env_file: + - .env.meili ports: - '7700:7700' volumes: - './data/meili:/meili_data' - environment: - MEILI_MASTER_KEY: '${MEILI_MASTER_KEY}' minio: image: quay.io/minio/minio @@ -29,17 +27,23 @@ services: command: ['minio', 'server', '/data', '--console-address', ':9001'] volumes: - './data/minio:/data' - environment: - MINIO_ROOT_USER: '${MINIO_ROOT_USER}' - MINIO_ROOT_PASSWORD: '${MINIO_ROOT_PASSWORD}' archiver: + image: registry.strix.systems/memestream-archiver + command: npm run dev build: ./archiver + env_file: + - .env.archiver volumes: - - './.env:/app/.env' - - './archiver/vision-credentials.json:/app/vision-credentials.json' + - './archiver:/usr/src/memestream' client: - build: ./client + build: + context: ./client + dockerfile: ./dev.Dockerfile + env_file: + - .env.client.dev ports: - - 8080:8080 + - 3000:3000 + volumes: + - './client:/usr/src/memestream-client'