diff --git a/.gitignore b/.gitignore index b936a6f..6389603 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ node_modules/ data/ db/ -.env \ No newline at end of file +.env + +*sql.gz \ No newline at end of file diff --git a/api/.env b/api/.env deleted file mode 100644 index d406536..0000000 --- a/api/.env +++ /dev/null @@ -1,7 +0,0 @@ -# Environment variables declared in this file are automatically made available to Prisma. -# See the documentation for more detail: https://pris.ly/d/prisma-schema#accessing-environment-variables-from-the-schema - -# Prisma supports the native connection string format for PostgreSQL, MySQL, SQLite, SQL Server, MongoDB and CockroachDB. -# See the documentation for all the connection string options: https://pris.ly/d/connection-strings - -DATABASE_URL="postgresql://mcrates:digdeep@0.0.0.0:45432/mcrates?schema=public" \ No newline at end of file diff --git a/api/api.Dockerfile b/api/api.Dockerfile index 3541195..d958b82 100644 --- a/api/api.Dockerfile +++ b/api/api.Dockerfile @@ -1,12 +1,12 @@ FROM node:20 WORKDIR /opt/crates/api -COPY . ./ # puppeteer deps RUN apt-get update # don't think about it RUN apt-get install -y ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 lsb-release wget xdg-utils +COPY . ./ RUN npm i # not sure why doesn't do it itself RUN node node_modules/puppeteer/install.mjs diff --git a/api/bugs.txt b/api/bugs.txt new file mode 100755 index 0000000..fbfbc4a --- /dev/null +++ b/api/bugs.txt @@ -0,0 +1 @@ +crates-api-7b8d754768-r4f5q 19:40:38.635Z ERROR mcrates: sync error ││ crates-api-7b8d754768-r4f5q TypeError: Cannot read properties of undefined (reading 'status') ││ crates-api-7b8d754768-r4f5q at populateListings (/opt/crates/api/src/events.ts:42:28) ││ crates-api-7b8d754768-r4f5q at updateWantListings (/opt/crates/api/src/events.ts:57:30) ││ crates-api-7b8d754768-r4f5q at EventEmitter.sync (/opt/crates/api/src/events.ts:83:13) ││ crates-api-7b8d754768-r4f5q 19:40:40.707Z INFO mcrates: (event=Sync, wants=192) ││ crates-api-7b8d754768-r4f5q 19:42:14.756Z ERROR mcrates: Navigation timeout of 30000 ms exceeded ││ crates-api-7b8d754768-r4f5q TimeoutError: Navigation timeout of 30000 ms exceeded ││ crates-api-7b8d754768-r4f5q at new Deferred (/opt/crates/api/node_modules/puppeteer-core/src/util/Deferred.ts:73:2 ││ crates-api-7b8d754768-r4f5q at Function.create (/opt/crates/api/node_modules/puppeteer-core/src/util/Deferred.ts:2 ││ crates-api-7b8d754768-r4f5q at new LifecycleWatcher (/opt/crates/api/node_modules/puppeteer-core/src/cdp/Lifecycle ││ crates-api-7b8d754768-r4f5q at CdpFrame.goto (/opt/crates/api/node_modules/puppeteer-core/src/cdp/Frame.ts:159:21) ││ crates-api-7b8d754768-r4f5q at CdpFrame. (/opt/crates/api/node_modules/puppeteer-core/src/util/decorato ││ crates-api-7b8d754768-r4f5q at CdpPage.goto (/opt/crates/api/node_modules/puppeteer-core/src/api/Page.ts:1649:35) ││ crates-api-7b8d754768-r4f5q at Object.listings (/opt/crates/api/src/discogs.ts:80:32) ││ crates-api-7b8d754768-r4f5q at populateListings (/opt/crates/api/src/events.ts:37:30) ││ crates-api-7b8d754768-r4f5q at updateWantListings (/opt/crates/api/src/events.ts:57:30) ││ crates-api-7b8d754768-r4f5q at EventEmitter.sync (/opt/crates/api/src/events.ts:83:13) ││ crates-api-7b8d754768-r4f5q 19:42:14.989Z ERROR mcrates: sync error ││ crates-api-7b8d754768-r4f5q TypeError: Cannot read properties of undefined (reading 'status') ││ crates-api-7b8d754768-r4f5q at populateListings (/opt/crates/api/src/events.ts:42:28) ││ crates-api-7b8d754768-r4f5q at updateWantListings (/opt/crates/api/src/events.ts:57:30) ││ crates-api-7b8d754768-r4f5q at EventEmitter.sync (/opt/crates/api/src/events.ts:83:13) ││ crates-api-7b8d754768-r4f5q 19:42:25.047Z INFO mcrates: (event=Sync, wants=202) ││ crates-api-7b8d754768-r4f5q 19:44:01.988Z ERROR mcrates: Timed out after waiting 30000ms ││ crates-api-7b8d754768-r4f5q TimeoutError: Timed out after waiting 30000ms ││ crates-api-7b8d754768-r4f5q at /opt/crates/api/node_modules/puppeteer-core/src/common/util.ts:302:17 ││ crates-api-7b8d754768-r4f5q at /opt/crates/api/node_modules/puppeteer-core/lib/cjs/third_party/rxjs/rxjs.js:3986:3 ││ crates-api-7b8d754768-r4f5q at OperatorSubscriber2._this._next (/opt/crates/api/node_modules/puppeteer-core/lib/cj ││ crates-api-7b8d754768-r4f5q at OperatorSubscriber2.Subscriber2.next (/opt/crates/api/node_modules/puppeteer-core/l ││ crates-api-7b8d754768-r4f5q at AsyncAction2. (/opt/crates/api/node_modules/puppeteer-core/lib/cjs/third ││ crates-api-7b8d754768-r4f5q at AsyncAction2._execute (/opt/crates/api/node_modules/puppeteer-core/lib/cjs/third_pa ││ crates-api-7b8d754768-r4f5q at AsyncAction2.execute (/opt/crates/api/node_modules/puppeteer-core/lib/cjs/third_par ││ crates-api-7b8d754768-r4f5q at AsyncScheduler2.flush (/opt/crates/api/node_modules/puppeteer-core/lib/cjs/third_pa ││ crates-api-7b8d754768-r4f5q at listOnTimeout (node:internal/timers:581:17) ││ crates-api-7b8d754768-r4f5q at processTimers (node:internal/timers:519:7) ││ crates-api-7b8d754768-r4f5q 19:44:04.209Z ERROR mcrates: sync error ││ crates-api-7b8d754768-r4f5q TypeError: Cannot read properties of undefined (reading 'status') ││ crates-api-7b8d754768-r4f5q at populateListings (/opt/crates/api/src/events.ts:42:28) ││ crates-api-7b8d754768-r4f5q at updateWantListings (/opt/crates/api/src/events.ts:57:30) ││ crates-api-7b8d754768-r4f5q at EventEmitter.sync (/opt/crates/api/src/events.ts:83:13) ││ crates-api-7b8d754768-r4f5q 19:44:22.261Z INFO mcrates: (event=Sync, wants=216) ││ crates-api-7b8d754768-r4f5q ProtocolError: Page.setLifecycleEventsEnabled timed out. Increase the 'protocolTimeout' settin ││ crates-api-7b8d754768-r4f5q at Callback. (/opt/crates/api/node_modules/puppeteer-core/sr ││ crates-api-7b8d754768-r4f5q at new Callback (/opt/crates/api/node_modules/puppeteer-core/src/common/CallbackRegistry.t ││ crates-api-7b8d754768-r4f5q at CallbackRegistry.create (/opt/crates/api/node_modules/puppeteer-core/src/common/Callbac ││ crates-api-7b8d754768-r4f5q at Connection._rawSend (/opt/crates/api/node_modules/puppeteer-core/src/cdp/Connection.ts: ││ crates-api-7b8d754768-r4f5q at CdpCDPSession.send (/opt/crates/api/node_modules/puppeteer-core/src/cdp/CDPSession.ts:9 ││ crates-api-7b8d754768-r4f5q at FrameManager.initialize (/opt/crates/api/node_modules/puppeteer-core/src/cdp/FrameManag ││ crates-api-7b8d754768-r4f5q at FrameManager.onAttachedToTarget (/opt/crates/api/node_modules/puppeteer-core/src/cdp/Fr ││ crates-api-7b8d754768-r4f5q at #onAttachedToTarget (/opt/crates/api/node_modules/puppeteer-core/src/cdp/Page.ts:330:24 ││ crates-api-7b8d754768-r4f5q at /opt/crates/api/node_modules/puppeteer-core/lib/cjs/third_party/mitt/mitt.js:62:7 ││ crates-api-7b8d754768-r4f5q at Array.map () ││ crates-api-7b8d754768-r4f5q [ERROR] 19:45:26 ProtocolError: Page.setLifecycleEventsEnabled timed out. Increase the 'protoc \ No newline at end of file diff --git a/api/index.ts b/api/index.ts index 098cd32..dc0309c 100644 --- a/api/index.ts +++ b/api/index.ts @@ -33,7 +33,7 @@ async function start() { await notify.listen(); await discord.connect(); - events.emit(Events.Sync); + // events.emit(Events.Sync); api.listen(); } diff --git a/api/package-lock.json b/api/package-lock.json old mode 100644 new mode 100755 diff --git a/api/package.json b/api/package.json index 888e520..aea13e1 100644 --- a/api/package.json +++ b/api/package.json @@ -6,7 +6,7 @@ "scripts": { "test": "mocha", "test:watch": "mocha --watch", - "migrate": "knex migrate:latest", + "migrate": "prisma migrate status", "migrate:make": "prisma migrate dev --name", "start": "ts-node-dev --respawn --transpile-only index.ts | bunyan -o short" }, diff --git a/api/src/events.ts b/api/src/events.ts index 5a2059a..97ab61e 100644 --- a/api/src/events.ts +++ b/api/src/events.ts @@ -84,14 +84,14 @@ export default function createEvents(log: Log, pg: Pg, discogs, offers, discord) if (wants.length === 0) { log.info('all listings up to date'); - await setTimeout() + await setTimeout(10000) + // forever + events.emit(Events.Sync); } } catch (e) { log.error(e, 'sync error') } - // forever - events.emit(Events.Sync); }); // events.on(Events.ListingsUpdated, async function listingsUpdated() { diff --git a/client/client.dev.Dockerfile b/client/client.dev.Dockerfile new file mode 100755 index 0000000..4382245 --- /dev/null +++ b/client/client.dev.Dockerfile @@ -0,0 +1,8 @@ +# build step +FROM node:16 AS build-env +WORKDIR /opt/crates/client + +COPY . ./ +RUN npm i + +CMD ["npm", "start"] \ No newline at end of file diff --git a/client/src/app.tsx b/client/src/app.tsx index ebd4a87..bb6370a 100644 --- a/client/src/app.tsx +++ b/client/src/app.tsx @@ -18,6 +18,7 @@ import Info from './routes/info'; if (window.location.host === 'matescrates.au') LogRocket.init('yh0dy3/mcrates'); + const Mcrates = () => { const dispatch = useDispatch(); diff --git a/docker-compose.MCRATES.yaml b/docker-compose.MCRATES.yaml deleted file mode 100644 index 1c29e2c..0000000 --- a/docker-compose.MCRATES.yaml +++ /dev/null @@ -1,38 +0,0 @@ -version: "3.3" - -services: - pg: - image: postgres:latest - environment: - POSTGRES_USER: '${POSTGRES_USER}' - POSTGRES_DB: '${POSTGRES_DB}' - POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}' - - # allow prisma client to push changes from outside docker etc - # ufw blocks this from being publicly accessible - ports: - - '45432:5432' - volumes: - - "./db:/var/lib/postgresql/data" - - api: - image: node:16 - working_dir: /opt/mcrates/api - entrypoint: npm start - environment: - DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@pg:5432/${POSTGRES_DB}?schema=public' - DISCORD_TOKEN: '${DISCORD_TOKEN}' - DISCOGS_TOKEN: '${DISCOGS_TOKEN}' - DISCORD_ENABLED: 'true' - volumes: - - "./api:/opt/mcrates/api" - - nginx: - image: nginx:latest - volumes: - - './nginx/matescrates.au.nginx.conf:/etc/nginx/conf.d/default.conf' - - './client/build:/srv/matescrates' - - '/etc/letsencrypt:/etc/letsencrypt' - ports: - - '80:80' - - '443:443' diff --git a/docker-compose.TEST.yaml b/docker-compose.TEST.yaml index 3405354..f4e45f8 100644 --- a/docker-compose.TEST.yaml +++ b/docker-compose.TEST.yaml @@ -4,18 +4,18 @@ services: pg: image: postgres:latest environment: - POSTGRES_USER: 'mcrates' + POSTGRES_USER: 'crates' POSTGRES_DB: 'test' POSTGRES_PASSWORD: 'digdeep' api: image: node:16 - working_dir: /opt/mcrates/api + working_dir: /opt/crates/api entrypoint: npm run test:watch depends_on: - pg environment: - DATABASE_URL: "postgresql://mcrates:digdeep@pg:5432/test?schema=public" + DATABASE_URL: "postgresql://crates:digdeep@pg:5432/test?schema=public" volumes: - - "./api:/opt/mcrates/api" + - "./api:/opt/crates/api" diff --git a/docker-compose.yaml b/docker-compose.yaml index 5a5d8e0..dfa0466 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,42 +1,57 @@ version: "3.3" services: - pg: - image: postgres:latest - environment: - POSTGRES_USER: '${POSTGRES_USER}' - POSTGRES_DB: '${POSTGRES_DB}' - POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}' - ports: - - '45432:5432' - volumes: - - "./db:/var/lib/postgresql/data" + # pg: + # image: postgres:latest + # environment: + # POSTGRES_USER: '${POSTGRES_USER}' + # POSTGRES_DB: '${POSTGRES_DB}' + # POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}' + # ports: + # - '45432:5432' + # volumes: + # - "./db:/var/lib/postgresql/data" api: - image: node:20 + image: crates/api build: context: ./api dockerfile: ./api.Dockerfile - working_dir: /opt/mcrates/api - depends_on: - - 'pg' - entrypoint: npm start + working_dir: /opt/crates/api + # depends_on: + # - 'pg' + command: npm start environment: - DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@pg:5432/${POSTGRES_DB}?schema=public' + DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?schema=public' DISCORD_TOKEN: '${DISCORD_TOKEN}' DISCOGS_TOKEN: '${DISCOGS_TOKEN}' + CHOKIDAR_USEPOLLING: 1 volumes: - - "./api:/opt/mcrates/api" + - "./api:/opt/crates/api" + - "/opt/crates/api/node_modules" + extra_hosts: + - "pg:host-gateway" + + migrate: + restart: on-failure:3 + extends: api + command: npm run migrate status client: image: node:16 - working_dir: /opt/mcrates/client + working_dir: /opt/crates/client + build: + context: ./client + dockerfile: ./client.dev.Dockerfile entrypoint: npm start ports: - '8080:8080' # fwd the hmr websocket + environment: + CHOKIDAR_USEPOLLING: 1 volumes: - - "./client:/opt/mcrates/client" - - "./api:/opt/mcrates/api" + - "./client:/opt/crates/client" + - "./api:/opt/crates/api" + - "/opt/crates/client/node_modules" nginx: image: nginx:latest