update for ntwl.xyz

.gitignore

@@ -1,3 +1,7 @@
 node_modules/
 data/
 db/
+
+.env
+
+*sql.gz

TODO.md

@@ -1,15 +1,30 @@
 # TODO
-push offers
 
-add last synced time to want
+- accept new offers that are under highest accepted price
+
+env
+    discogs api key
+    discord api key
+
+discord bot
+    notify on sign up
+    want added
+    offer accepted
+
+email ppl who signed up
+    say we have updates
+
+main list filters
+    order by
+    offers only
+
+sample offer with tooltips
 
 go over and reorganise event logic
+    should be connection.service ?
 
 contact options on landing page
 
 skip unofficial release
 
 status stepper
-
-queue up wantlist adds to gracefully handle rate limiting
-

api/.dockerignore

@@ -0,0 +1 @@
+node_modules/

api/.env

@@ -1,7 +0,0 @@
-# Environment variables declared in this file are automatically made available to Prisma.
-# See the documentation for more detail: https://pris.ly/d/prisma-schema#accessing-environment-variables-from-the-schema
-
-# Prisma supports the native connection string format for PostgreSQL, MySQL, SQLite, SQL Server, MongoDB and CockroachDB.
-# See the documentation for all the connection string options: https://pris.ly/d/connection-strings
-
-DATABASE_URL="postgresql://mcrates:digdeep@0.0.0.0:45432/mcrates?schema=public"

api/api.Dockerfile

@@ -0,0 +1,14 @@
+FROM node:20
+WORKDIR /opt/crates/api
+
+# puppeteer deps
+RUN apt-get update
+# don't think about it
+RUN apt-get install -y ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 lsb-release wget xdg-utils
+
+COPY . ./
+RUN npm i
+# not sure why doesn't do it itself
+RUN node node_modules/puppeteer/install.mjs
+
+CMD ["npm", "start"]

api/bugs.txt

@@ -0,0 +1 @@
+crates-api-7b8d754768-r4f5q 19:40:38.635Z ERROR mcrates: sync error                                                        ││ crates-api-7b8d754768-r4f5q     TypeError: Cannot read properties of undefined (reading 'status')                          ││ crates-api-7b8d754768-r4f5q         at populateListings (/opt/crates/api/src/events.ts:42:28)                              ││ crates-api-7b8d754768-r4f5q         at updateWantListings (/opt/crates/api/src/events.ts:57:30)                            ││ crates-api-7b8d754768-r4f5q         at EventEmitter.sync (/opt/crates/api/src/events.ts:83:13)                             ││ crates-api-7b8d754768-r4f5q 19:40:40.707Z  INFO mcrates:  (event=Sync, wants=192)                                          ││ crates-api-7b8d754768-r4f5q 19:42:14.756Z ERROR mcrates: Navigation timeout of 30000 ms exceeded                           ││ crates-api-7b8d754768-r4f5q     TimeoutError: Navigation timeout of 30000 ms exceeded                                      ││ crates-api-7b8d754768-r4f5q         at new Deferred (/opt/crates/api/node_modules/puppeteer-core/src/util/Deferred.ts:73:2 ││ crates-api-7b8d754768-r4f5q         at Function.create (/opt/crates/api/node_modules/puppeteer-core/src/util/Deferred.ts:2 ││ crates-api-7b8d754768-r4f5q         at new LifecycleWatcher (/opt/crates/api/node_modules/puppeteer-core/src/cdp/Lifecycle ││ crates-api-7b8d754768-r4f5q         at CdpFrame.goto (/opt/crates/api/node_modules/puppeteer-core/src/cdp/Frame.ts:159:21) ││ crates-api-7b8d754768-r4f5q         at CdpFrame.<anonymous> (/opt/crates/api/node_modules/puppeteer-core/src/util/decorato ││ crates-api-7b8d754768-r4f5q         at CdpPage.goto (/opt/crates/api/node_modules/puppeteer-core/src/api/Page.ts:1649:35)  ││ crates-api-7b8d754768-r4f5q         at Object.listings (/opt/crates/api/src/discogs.ts:80:32)                              ││ crates-api-7b8d754768-r4f5q         at populateListings (/opt/crates/api/src/events.ts:37:30)                              ││ crates-api-7b8d754768-r4f5q         at updateWantListings (/opt/crates/api/src/events.ts:57:30)                            ││ crates-api-7b8d754768-r4f5q         at EventEmitter.sync (/opt/crates/api/src/events.ts:83:13)                             ││ crates-api-7b8d754768-r4f5q 19:42:14.989Z ERROR mcrates: sync error                                                        ││ crates-api-7b8d754768-r4f5q     TypeError: Cannot read properties of undefined (reading 'status')                          ││ crates-api-7b8d754768-r4f5q         at populateListings (/opt/crates/api/src/events.ts:42:28)                              ││ crates-api-7b8d754768-r4f5q         at updateWantListings (/opt/crates/api/src/events.ts:57:30)                            ││ crates-api-7b8d754768-r4f5q         at EventEmitter.sync (/opt/crates/api/src/events.ts:83:13)                             ││ crates-api-7b8d754768-r4f5q 19:42:25.047Z  INFO mcrates:  (event=Sync, wants=202)                                          ││ crates-api-7b8d754768-r4f5q 19:44:01.988Z ERROR mcrates: Timed out after waiting 30000ms                                   ││ crates-api-7b8d754768-r4f5q     TimeoutError: Timed out after waiting 30000ms                                              ││ crates-api-7b8d754768-r4f5q         at /opt/crates/api/node_modules/puppeteer-core/src/common/util.ts:302:17               ││ crates-api-7b8d754768-r4f5q         at /opt/crates/api/node_modules/puppeteer-core/lib/cjs/third_party/rxjs/rxjs.js:3986:3 ││ crates-api-7b8d754768-r4f5q         at OperatorSubscriber2._this._next (/opt/crates/api/node_modules/puppeteer-core/lib/cj ││ crates-api-7b8d754768-r4f5q         at OperatorSubscriber2.Subscriber2.next (/opt/crates/api/node_modules/puppeteer-core/l ││ crates-api-7b8d754768-r4f5q         at AsyncAction2.<anonymous> (/opt/crates/api/node_modules/puppeteer-core/lib/cjs/third ││ crates-api-7b8d754768-r4f5q         at AsyncAction2._execute (/opt/crates/api/node_modules/puppeteer-core/lib/cjs/third_pa ││ crates-api-7b8d754768-r4f5q         at AsyncAction2.execute (/opt/crates/api/node_modules/puppeteer-core/lib/cjs/third_par ││ crates-api-7b8d754768-r4f5q         at AsyncScheduler2.flush (/opt/crates/api/node_modules/puppeteer-core/lib/cjs/third_pa ││ crates-api-7b8d754768-r4f5q         at listOnTimeout (node:internal/timers:581:17)                                         ││ crates-api-7b8d754768-r4f5q         at processTimers (node:internal/timers:519:7)                                          ││ crates-api-7b8d754768-r4f5q 19:44:04.209Z ERROR mcrates: sync error                                                        ││ crates-api-7b8d754768-r4f5q     TypeError: Cannot read properties of undefined (reading 'status')                          ││ crates-api-7b8d754768-r4f5q         at populateListings (/oLine truncated

api/index.ts

@@ -24,12 +24,16 @@ async function start() {
         prisma,
         pg,
         discogs,
+        discord,
         api,
         events,
         notify,
-    } = createServices(log);
+    } = await createServices(log);
 
     await notify.listen();
+    await discord.connect();
+
+    // events.emit(Events.Sync);
 
     api.listen();
 }

api/package.json

@@ -1,12 +1,12 @@
 {
-  "name": "mates-crates",
+  "name": "crates",
   "version": "1.0.0",
   "description": "",
   "main": "index.js",
   "scripts": {
     "test": "mocha",
     "test:watch": "mocha --watch",
-    "migrate": "knex migrate:latest",
+    "migrate": "prisma migrate status",
     "migrate:make": "prisma migrate dev --name",
     "start": "ts-node-dev --respawn --transpile-only index.ts | bunyan -o short"
   },
@@ -21,7 +21,6 @@
     "@types/mocha": "^9.1.1",
     "axios": "^0.27.2",
     "mocha": "^10.0.0",
-    "node-dev": "^7.4.3",
     "prisma": "^4.5.0",
     "ts-node-dev": "^2.0.0",
     "typescript": "^4.7.4"
@@ -34,6 +33,8 @@
     "bluebird": "^3.7.2",
     "bunyan": "^1.8.15",
     "cheerio": "^1.0.0-rc.12",
+    "discord.js": "^14.7.1",
+    "esrun": "^3.2.26",
     "koa": "^2.13.4",
     "koa-body": "^5.0.0",
     "koa-passport": "^5.0.0",
@@ -42,7 +43,10 @@
     "koa-ws": "^1.0.1",
     "lodash": "^4.17.21",
     "passport-local": "^1.0.0",
-    "pg-listen": "^1.7.0"
+    "pg-listen": "^1.7.0",
+    "puppeteer": "^23.3.0",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
   },
   "overrides": {
     "koa-passport": {

api/prisma/schema.prisma

@@ -26,7 +26,7 @@ model Listing {
     offers     Offer[]
 
     createdAt  DateTime @default(now())
-    updatedAt  DateTime @updatedAt
+    updatedAt  DateTime @default(now())
 
     @@index([seller])
     @@map("listings")
@@ -38,7 +38,7 @@ model User {
     password   String
 
     createdAt  DateTime   @default(now())
-    updatedAt  DateTime   @updatedAt
+    updatedAt  DateTime   @default(now())
 
     wants      Want[]
     offers     Offer[]
@@ -55,7 +55,7 @@ model Want {
     user       User       @relation(fields: [userId], references: [id], onDelete: Cascade, onUpdate: Cascade)
 
     createdAt  DateTime   @default(now())
-    updatedAt  DateTime   @updatedAt
+    updatedAt  DateTime   @default(now())
 
     listings   Listing[]
     offers     Offer[]

api/src/api.test.ts

@@ -48,12 +48,4 @@ describe('api', function apiTests() {
             assert(res.headers['set-cookie']);
         })
     });
-
-    describe('offers', function offerTests() {
-        it('create offers', async function createOffersTest() {
-            const { data: bundles } = await api.get('/bundles');
-            const listingIds = bundles[0].items.map(i => i.id);
-            const { data } = await api.post('/offers', { listingIds, shipping: 50.0, baseShipping: 15.0 });
-        });
-    });
 });

api/src/api.ts

@@ -29,6 +29,11 @@ export default function createApi(log: Log, pg: Pg, discogs: Discogs, offers: Of
 
     const app = koaWebsocket(new koa());
 
+    // @ts-ignore - ts don't know wtf it talking about
+    BigInt.prototype.toJSON = function bigIntStringify() {
+        return this.toString();
+    };
+
     app.use(async (ctx, next) => {
         try {
             await next();
@@ -46,6 +51,7 @@ export default function createApi(log: Log, pg: Pg, discogs: Discogs, offers: Of
     function listen() {
         const server = app.listen(PORT);
         log.info({ PORT }, 'listening');
+        return server;
     }
 
     app.use(cors({ credentials: true }));
@@ -54,7 +60,7 @@ export default function createApi(log: Log, pg: Pg, discogs: Discogs, offers: Of
     app.keys = [SECRET];
     app.use(session(app));
 
-    const { authRouter, passport } = routerAuth(log, pg);
+    const { authRouter, passport } = routerAuth(log, pg, events);
     app.use(passport.initialize());
     app.use(passport.session())
     app.use(authRouter.routes());
@@ -66,9 +72,10 @@ export default function createApi(log: Log, pg: Pg, discogs: Discogs, offers: Of
     router.post('/want/discogs', async function wantListPost(ctx, next) {
         ctx.assert(ctx.isAuthenticated(), 401, 'log in required');
         const { username } = ctx.request.body;
+        ctx.assert(username, 'no username supplied');
         const user = await pg.userFindId(ctx.state.user.id);
         const wants = await discogs.wantListSync(username, user);
-        events.emit(Events.WantsAdded, [wants]);
+        events.emit(Events.WantsAdded, wants);
         ctx.response.body = wants;
     });
 

api/src/discogs.ts

@@ -1,23 +1,21 @@
-import fs from 'fs/promises';
-
 import axios from 'axios';
-import cheerio from 'cheerio';
-import Bluebird from 'bluebird';
+import pptr from 'puppeteer-extra'
+import StealthPlugin from 'puppeteer-extra-plugin-stealth'
 
 import type Log from 'bunyan';
-import type { PrismaClient, Want, User, Listing } from '@prisma/client';
+import type { Want, User } from '@prisma/client';
 
-import { logAxiosError, wantDetails } from './utils';
+import { wantDetails } from './utils';
 import { Pg } from './pg';
 
-// margh
-// vanderbabe
-// jmcdermid1
-
-export default function createDiscogs(log: Log, pg: Pg) {
-    const TOKEN = 'Discogs token=wRTXuYZGOPsIQgKRGKCAkUkidGtvpSfQgPWkYIpD';
+export default async function createDiscogs(log: Log, pg: Pg) {
+    const TOKEN = process.env.DISCOGS_TOKEN;
     const apiBase = 'https://api.discogs.com'
-    const discogsBase = 'https://www.discogs.com'
+
+    pptr.use(StealthPlugin())
+    const browser = await pptr.launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'] })
+
+    log.info('browser created')
 
     const api = axios.create({
         headers: {
@@ -27,24 +25,27 @@ export default function createDiscogs(log: Log, pg: Pg) {
         },
     });
 
-    async function handleApiError(e) {
-        logAxiosError(e);
-        if (e.response.status === 429) {
-            throw new Error('rate limited');
-        }
-        return null;
+    function onSuccess(response) {
+        return response.data;
     }
 
-    async function wantListSync(username: string, user: User): Promise<Want[]> {
-        const { data: { wants, pagination }} = await api.get(`${apiBase}/users/${username}/wants`);
-        const releases = await Promise.all(wants.map(want => release(want.id)));
+    function onError(error) {
+        throw error;
+    }
+    api.interceptors.response.use(onSuccess, onError);
 
+    async function wantListSync(username: string, user: User): Promise<Want[]> {
+        const res = await api.get(`${apiBase}/users/${username}/wants`);
+        const { wants, pagination } = res;
+
+        const releases = await Promise.all(wants.map(want => release(want.id)));
         log.info({ wants: wants.length, user: username, userId: user.id }, 'fetched discogs wantlist');
 
         // TODO filter releases to be vinyl format only
 
         // convert want object to actual release and ignore duplicate errors
         const results = await Promise.allSettled(releases.map(release => pg.wantCreate(release, user)));
+        log.info(results);
         // @ts-ignore
         const additions = results.filter(r => r.status === 'fulfilled').map(r => r.value);
 
@@ -54,7 +55,7 @@ export default function createDiscogs(log: Log, pg: Pg) {
     }
 
     async function release(id: DiscogsRelease): Promise<any> {
-        const { data } = await api.get(`${apiBase}/releases/${id}`);
+        const data = await api.get(`${apiBase}/releases/${id}`);
         return data;
     }
 
@@ -63,44 +64,59 @@ export default function createDiscogs(log: Log, pg: Pg) {
         url.searchParams.append('query', query);
         url.searchParams.append('type', 'release');
         url.searchParams.append('format', 'vinyl');
-        const { data: { results } } = await api.get(url.href);
+        const res = await api.get(url.href);
+        const { results } = res;
         return results;
     }
 
     async function inventory(user: string) {
         const res = await api.get(`${apiBase}/users/${user}/inventory`);
-        return res.data.listings;
+        return res.listings;
     }
 
     async function listings(want: Want) {
-        const res = await api.get(`https://www.discogs.com/sell/release/${want.release}?limit=250`);
-        const $ = cheerio.load(res.data);
-        const ids = $('.item_description_title[href]')
-            .map(function getListingId() {
-                const id = Number.parseInt(this.attribs.href.replace('/sell/item/', ''), 10);
-                return id;
-            })
-            .get();
+        const page = await browser.newPage()
+        const url = `https://www.discogs.com/sell/release/${want.release}?limit=250`
+        const res = await page.goto(url)
+        log.debug({ status: res.status() }, 'status')
+        if (!res.ok()) throw new Error(res.statusText())
+        const els = await page.$$('.item_description_title[href]')
+        const hrefs = await Promise.all(els.map(e => e.evaluate(e => e.href)))
+        const ids = hrefs
+            .map(h => new URL(h))
+            .map(p => p.pathname.replace('/sell/item/', ''))
+            .map(p => Number.parseInt(p, 10))
+
+        await page.close()
+
+        if (ids.length === 0) log.warn({ res, url, ...wantDetails(want) }, "Want had zero listings")
 
         const unlisted = await pg.listingRmUnlisted(want, ids);
         if (unlisted) log.info({ unlisted: unlisted, ...wantDetails(want) }, 'missing listings removed');
 
         // preferentially take listings from sellers if we hit req limits
-        return Bluebird.mapSeries(ids, async id => listing(id, want));
+
+        // sometimes we get dodgy listings from discogs that throw perm errors
+        // we just want to ignore them
+        // const results = await Promise.allSettled(ids.map(id => listing(id, want)));
+
+        const results = []
+        for await (const id of ids) {
+            const r = await listing(id, want)
+            results.push(r)
+        }
+        return results;
+
     }
 
     async function listing(id, want) {
         const listing = await pg.listingFind(id);
         if (listing) return listing;
 
-        try {
-            log.debug('fetching listing', id);
-            const res = await api.get(`${apiBase}/marketplace/listings/${id}`);
-            log.debug(res.data);
-            return pg.listingUpsert(res.data, want);
-        } catch (e) {
-            return handleApiError(e);
-        }
+        log.debug('fetching listing', id);
+        const discogsListing = await api.get(`${apiBase}/marketplace/listings/${id}`);
+        log.debug(discogsListing);
+        return pg.listingUpsert(discogsListing, want);
     }
 
     return {

api/src/discord.ts

@@ -0,0 +1,36 @@
+import type Log from 'bunyan';
+
+import { Client, GatewayIntentBits } from 'discord.js';
+
+export default function createDiscord(log: Log) {
+
+    const client = new Client({ intents: [GatewayIntentBits.Guilds] });
+
+    const TOKEN = process.env.DISCORD_TOKEN;
+
+    async function connect() {
+        return new Promise((resolve, reject) => {
+            client.once('ready', () => {
+                log.info({ bot: client.user.tag }, 'discord client logged in');
+                resolve(client);
+            });
+            client.login(TOKEN);
+        });
+    }
+
+    function message(msg) {
+        if (!process.env.DISCORD_ENABLED) return false;
+        log.info({ msg }, 'sending discord message');
+        const channel = client.channels.cache.get('1072490127324090418');
+        // @ts-ignore - wrong
+        const str = JSON.stringify(msg, null, 4);
+        channel.send(`\`\`\`${str}\`\`\``);
+    }
+
+    return {
+        connect,
+        message,
+    };
+}
+
+export type Discord = ReturnType<typeof createDiscord>;

api/src/events.ts

@@ -17,81 +17,112 @@ export enum Events {
     ListingsUpdated = 'ListingsUpdated',
     OffersUpdated = 'OffersUpdated',
     WantsUpdated = 'WantsUpdated',
+
+    DiscordMessage = 'DiscordMessage',
 }
 
-export default function createEvents(log: Log, pg: Pg, discogs, offers) {
+export default function createEvents(log: Log, pg: Pg, discogs, offers, discord) {
     const events = new EventEmitter({ captureRejections: true });
     const connections = new Map<UUID, Rpc>();
 
-    async function populateListings(want: Want) {
-        try {
-            const listings = await discogs.listings(want);
-            return listings;
-        } catch (e) {
-            if (e.message === 'rate limited') {
-                log.warn('rate limited, trying again in 60s');
-                await setTimeout(60 * 1000);
-                return populateListings(want);
-            }
-            throw e;
-        }
-    }
-
     events.on('error', function eventError(e) {
         log.error('event error', e);
     });
 
+    const MAX_ATTEMPTS = 3;
+    const SYNC_EVERY_MS = 60 * 1000
+    const RATE_LIMIT_WAIT_MS = (attempts) => attempts * 70 * 1000
+    async function populateListings(want: Want, attempts = 1) {
+        try {
+            const listings = await discogs.listings(want);
+            await pg.wantSynced(want);
+            return listings;
+        } catch (e) {
+            log.error(e)
+            if (e.response.status === 429) {
+                log.warn({ RATE_LIMIT_WAIT_MS: RATE_LIMIT_WAIT_MS(attempts), ...e.response.headers }, 'rate limited, trying again');
+
+                await setTimeout(RATE_LIMIT_WAIT_MS(attempts));
+                if (attempts > MAX_ATTEMPTS) {
+                    log.error(wantDetails(want), 'failed to populate listings')
+                }
+                return populateListings(want, attempts++);
+            }
+            throw e;
+        }
+    }
+
     async function updateWantListings(wants: Want[]) {
         for (const want of wants) {
             const listings = await populateListings(want);
             log.info({ listings: listings.length, ...wantDetails(want) }, 'listings found');
-            events.emit(Events.ListingsUpdated);
+            await offers.generate();
+            await notifyUser(want.userId);
         }
 
         log.info({ wants: wants.length }, 'finished syncing');
     }
 
+    async function notifyUser(id: UUID) {
+        const rpc = connections.get(id);
+        if (!rpc) return null;
+        rpc.pushUserWants();
+        rpc.pushUserOffers();
+    };
+
     events.on(Events.WantsAdded, async function wantsAdded(wants: Want[]) {
         log.info({ event: Events.WantsAdded, wants: wants.map(w => wantDetails(w)) }, 'starting listings population');
         await updateWantListings(wants);
         log.info({ event: Events.WantsAdded, wants: wants.map(w => wantDetails(w)) }, 'finished populating listings');
-        events.emit(Events.ListingsUpdated);
     });
 
     events.on(Events.Sync, async function sync() {
-        const wants = await pg.wantsToSync();
-        log.info({ event: Events.Sync, wants: wants.length });
-        await updateWantListings(wants);
-        events.emit(Events.ListingsUpdated);
-    });
+        try {
+            const wants = await pg.wantsToSync();
+            log.info({ event: Events.Sync, wants: wants.length });
+            await updateWantListings(wants);
 
-    events.on(Events.ListingsUpdated, async function listingsUpdated() {
-        const updatedOffers = await offers.generate()
-        log.debug({ offers: updatedOffers.length }, 'offers up to date');
-
-        for (const [id, rpc] of connections) {
-            rpc.pushUserOffers();
+            if (wants.length === 0) {
+                log.info('all listings up to date');
+                await setTimeout(10000)
+                // forever
+                events.emit(Events.Sync);
+            }
+        } catch (e) {
+            log.error(e, 'sync error')
         }
+
     });
 
+    // events.on(Events.ListingsUpdated, async function listingsUpdated() {
+    //     const updatedOffers = await offers.generate()
+    //     log.debug({ offers: updatedOffers.length }, 'offers up to date');
+    // });
+
     events.on(Events.Connection, async function onConnection({ rpc, id }) {
         log.debug('new rpc connection', { id, rpc });
         connections.set(id, rpc);
     })
 
-    events.on(Events.OffersUpdated, async function onOffersUpdated(offer: Offer) {
-        const rpc = connections.get(offer.userId);
-        if (!rpc) return null;
-        log.debug({ event: Events.OffersUpdated, rpc, offer });
-        return rpc.pushUserOffers();
-    });
+    events.on(Events.DiscordMessage, async function onDiscordMessage(msg) {
+        discord.message(msg);
+    })
 
-    events.on(Events.WantsUpdated, async function onWantsUpdated(want: Want) {
-        const rpc = connections.get(want.userId);
-        if (!rpc) return null;
-        log.debug({ event: Events.WantsUpdated, rpc, want });
-        return rpc.pushUserWants();
-    });
+    // emitted by listen/notify
+    // disabled for now
+    // events.on(Events.OffersUpdated, async function onOffersUpdated(offer: Offer) {
+    //     const rpc = connections.get(offer.userId);
+    //     if (!rpc) return null;
+    //     log.debug({ event: Events.OffersUpdated, rpc, offer });
+    //     return rpc.pushUserOffers();
+    // });
+
+    // events.on(Events.WantsUpdated, async function onWantsUpdated(want: Want) {
+    //     const rpc = connections.get(want.userId);
+    //     if (!rpc) return null;
+    //     log.debug({ event: Events.WantsUpdated, rpc, want });
+    //     return rpc.pushUserWants();
+    // });
 
     return events;
 }

api/src/offers.ts

@@ -63,19 +63,20 @@ export default function createOffers(log: Log, pg: Pg) {
 
         // check the user actually has this offer
         const offer = await pg.userOfferFind(ctx.state.user, id);
-        return pg.offerStatusSet(offer.id, Status.Accepted);
+        return pg.offerStatusSet([offer.id], Status.Accepted);
     }
 
-    async function statusSet(user: User, id: UUID, status: Status) {
-        log.info({ status, id }, 'setting offer status');
+    async function statusSet(user: User, ids: UUID[], status: Status) {
+        log.info({ status, ids }, 'setting offer status');
 
-        const offer = await pg.offerFindId(id);
-        const update = await pg.offerStatusSet(offer.id, status);
+        // const offer = await pg.offerFindId(ids);
+        const update = await pg.offerStatusSet(ids, status);
 
-        if (status === Status.Complete) {
-            log.info({ id, status }, 'completed offer, removing want...');
-            await pg.wantRm(offer.wantId);
-        }
+        // forget about this for now
+        // if (status === Status.Complete) {
+        //     log.info({ id, status }, 'completed offer, removing want...');
+        //     await pg.wantRm(offer.wantId);
+        // }
 
         return update;
     }

api/src/pg.ts

@@ -5,6 +5,7 @@ import * as Bluebird from 'bluebird';
 import _ from 'lodash';
 
 import { Status } from './offers';
+import { wantDetails } from './utils';
 
 export interface UserCreateParams {
     email: string,
@@ -21,11 +22,12 @@ export default function createPg(log: Log, prisma: PrismaClient) {
     }
 
     async function userWants(user: User) {
-        return prisma.want.findMany({
+        const wants = await prisma.want.findMany({
             where: { userId: user.id },
             include: { _count: { select: { offers: true }} },
-            orderBy: { offers: { _count: 'desc' } }
-        });
+        })
+
+        return _.sortBy(wants, w => `${w.data.artists_sort} - ${w.data.title}`);
     }
 
     async function userOffers(user: User) {
@@ -111,7 +113,12 @@ export default function createPg(log: Log, prisma: PrismaClient) {
     }
 
     async function wantsToSync() {
-        return prisma.want.findMany({});
+        return prisma.want.findMany({
+            orderBy: { updatedAt: 'asc' },
+            where: {
+                updatedAt: { lte: new Date(Date.now() - 1000 * 60 * 15) }
+            }
+        });
     }
 
     async function wantRm(id: UUID) {
@@ -132,6 +139,13 @@ export default function createPg(log: Log, prisma: PrismaClient) {
         });
     }
 
+    async function wantSynced(want: Want) {
+        return prisma.want.update({
+            where: { id: want.id },
+            data: { updatedAt: new Date() },
+        });
+    }
+
     async function listingRmUnlisted(want: Want, currentListingIds: DiscogsListing[]) {
         return prisma.$transaction(async (prisma) => {
             log.debug({ currentListingIds });
@@ -152,7 +166,7 @@ export default function createPg(log: Log, prisma: PrismaClient) {
                 }
             });
 
-            if (offersRemoved > 0) log.info({ offersRemoved }, 'pending offers removed');
+            if (offersRemoved > 0) log.info({ offersRemoved, ...wantDetails(want) }, 'pending offers removed');
 
             // now we have to filter listingIds with confirmed offers
             // and get their listingIds to filter out
@@ -179,7 +193,7 @@ export default function createPg(log: Log, prisma: PrismaClient) {
                 },
             });
 
-            log.info({ count }, 'second stage open offers removed');
+            if (count) log.info({ count }, 'second stage open offers removed');
 
             return count;
         });
@@ -275,9 +289,9 @@ export default function createPg(log: Log, prisma: PrismaClient) {
         );
     }
 
-    async function offerStatusSet(id: UUID, status: Status) {
-        return prisma.offer.update({
-            where: { id: id },
+    async function offerStatusSet(ids: UUID[], status: Status) {
+        return prisma.offer.updateMany({
+            where: { id: { in: ids } },
             data: { status },
         });
     }
@@ -304,6 +318,7 @@ export default function createPg(log: Log, prisma: PrismaClient) {
         wantCreate,
         wantList,
         wantsToSync,
+        wantSynced,
         wantRm,
 
         listingUpsert,

api/src/routers/router.auth.ts

@@ -4,8 +4,9 @@ import { Strategy as LocalStrategy } from 'passport-local';
 import Router from '@koa/router'
 
 import type { Pg } from './../pg';
+import { Events } from './../events';
 
-export default function authRouter(log, pg: Pg) {
+export default function authRouter(log, pg: Pg, events) {
     const passport = new KoaPassport();
 
     async function verifyPassword(user, password) {
@@ -69,6 +70,7 @@ export default function authRouter(log, pg: Pg) {
             id: user.id,
             email: user.email,
         };
+        events.emit(Events.DiscordMessage, { msg: 'user register', email: user.email });
     });
 
     return { authRouter, passport };

api/src/rpc.ts

@@ -75,7 +75,7 @@ export type RpcMessage =
 
 
 type OfferStatusSetParams = {
-    id: UUID,
+    ids: UUID[],
     status: Status,
 };
 
@@ -109,13 +109,15 @@ export default function createRpc(log: Log, pg: Pg, ws, discogs: Discogs, offers
     }
 
     async function offerStatusSet(params: OfferStatusSetParams): Promise<OffersMessage> {
-        const { id, status } = params;
-        assert(id, 'no id supplied');
+        const { ids, status } = params;
+        assert(ids, 'no ids supplied');
         assert(Status[status], 'unknown status');
 
-        const offer = await offers.statusSet(user, id, status);
+        const offer = await offers.statusSet(user, ids, status);
         const userOffers = await offers.userOffers(user);
 
+        events.emit(Events.DiscordMessage, { msg: 'user offer event', user: user.email, offer })
+
         return [RpcMessageType.Offers, userOffers]
     }
 

api/src/services.ts

@@ -1,25 +1,28 @@
 import { PrismaClient } from '@prisma/client'
 
-import createDiscogs from './discogs';
-import createOffers from './offers';
-import createLog from './log';
 import createApi from './api';
+import createDiscogs from './discogs';
 import createEvents from './events';
-import createPg from './pg';
+import createLog from './log';
 import createNotify from './notify';
+import createOffers from './offers';
+import createPg from './pg';
+import createDiscord from './discord';
 
-export default function createServices(log) {
+export default async function createServices(log) {
     const prisma = new PrismaClient();
+    const discord = createDiscord(log);
     const pg = createPg(log, prisma);
-    const discogs = createDiscogs(log, pg);
+    const discogs = await createDiscogs(log, pg);
     const offers = createOffers(log, pg);
-    const events = createEvents(log, pg, discogs, offers);
+    const events = createEvents(log, pg, discogs, offers, discord);
     const notify = createNotify(log, pg, events);
     const api = createApi(log, pg, discogs, offers, events);
 
     return {
         api,
         discogs,
+        discord,
         events,
         notify,
         pg,

api/src/utils.ts

@@ -17,6 +17,8 @@ export function logAxiosError(e) {
 export function wantDetails(want: Want) {
     const data = want.data as any;
 
+    if (!data) console.warn(want);
+
     return {
         title: data.title,
         release: data.id,

client/.eslintrc.js

@@ -8,23 +8,27 @@ module.exports = {
         "build/"
     ],
     rules: {
+        'arrow-body-style': [0],
+
+        'max-len': ['error', 120],
+
+        'no-console': [0],
+        'no-multi-spaces': [0],
         // prevents stupid complaints a la
         // (req) {
         //   req.something = x;
         // }
         'no-param-reassign': [2, { props: false }],
-        'no-multi-spaces': [0],
-        'max-len': ['error', 120],
-        'import/no-extraneous-dependencies': [0],
-        'prefer-arrow-callback': [0],
-        'arrow-body-style': [0],
         'no-return-assign': 0,
-        'no-console': [0],
+
+        'prefer-arrow-callback': [0],
+
         // i like loops
         'no-plusplus': [0],
-
         'no-await-in-loop': [0],
+
         indent: ['error', 4],
+
         'keyword-spacing': ['error'],
         'key-spacing': ['error'],
     },

client/client.Dockerfile

@@ -0,0 +1,12 @@
+# build step
+FROM node:16 AS build-env
+WORKDIR /opt/crates/client
+
+COPY . ./
+RUN npm run build
+
+# static server
+FROM nginx
+WORKDIR /srv/app
+COPY --from=build-env /opt/crates/client/build /srv/app
+COPY ./client.nginx.conf /etc/nginx/nginx.conf

client/client.dev.Dockerfile

@@ -0,0 +1,8 @@
+# build step
+FROM node:16 AS build-env
+WORKDIR /opt/crates/client
+
+COPY . ./
+RUN npm i
+
+CMD ["npm", "start"]

client/client.nginx.conf

@@ -0,0 +1,49 @@
+events {
+    worker_connections 1024;
+}
+
+http {
+    include mime.types;
+    sendfile on;
+
+    server {
+        listen 8080;
+        listen [::]:8080;
+
+        autoindex off;
+
+        root /srv/app;
+
+        gzip on;
+        gzip_disable "msie6";
+
+        gzip_vary on;
+        gzip_proxied any;
+        gzip_comp_level 6;
+        gzip_buffers 16 8k;
+        gzip_http_version 1.1;
+        gzip_min_length 256;
+        gzip_types
+            application/atom+xml
+            application/geo+json
+            application/javascript
+            application/x-javascript
+            application/json
+            application/ld+json
+            application/manifest+json
+            application/rdf+xml
+            application/rss+xml
+            application/xhtml+xml
+            application/xml
+            font/eot
+            font/otf
+            font/ttf
+            image/svg+xml
+            text/css
+            text/javascript
+            text/plain
+            text/xml;
+
+        error_page 404 /;
+    }
+}

client/package-lock.json

@@ -22,6 +22,7 @@
             },
             "devDependencies": {
                 "@babel/plugin-transform-react-jsx-source": "^7.18.6",
+                "@types/lodash": "^4.17.10",
                 "@typescript-eslint/eslint-plugin": "^5.30.6",
                 "@typescript-eslint/parser": "^5.30.6",
                 "eslint": "^8.20.0",
@@ -3009,6 +3010,13 @@
             "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==",
             "dev": true
         },
+        "node_modules/@types/lodash": {
+            "version": "4.17.10",
+            "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.10.tgz",
+            "integrity": "sha512-YpS0zzoduEhuOWjAotS6A5AVCva7X4lVlYLF0FYHAY9sdraBfnatttHItlWeZdGhuEkf+OzMNg2ZYAx8t+52uQ==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/@types/mime": {
             "version": "3.0.0",
             "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.0.tgz",
@@ -23437,6 +23445,12 @@
             "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==",
             "dev": true
         },
+        "@types/lodash": {
+            "version": "4.17.10",
+            "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.10.tgz",
+            "integrity": "sha512-YpS0zzoduEhuOWjAotS6A5AVCva7X4lVlYLF0FYHAY9sdraBfnatttHItlWeZdGhuEkf+OzMNg2ZYAx8t+52uQ==",
+            "dev": true
+        },
         "@types/mime": {
             "version": "3.0.0",
             "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.0.tgz",

client/package.json

@@ -6,7 +6,7 @@
     "scripts": {
         "build": "preact build --no-prerender",
         "start": "preact --no-clear watch",
-        "lint": "eslint src"
+        "lint": "eslint --fix src"
     },
     "dependencies": {
         "@reduxjs/toolkit": "^1.8.5",
@@ -22,6 +22,7 @@
     },
     "devDependencies": {
         "@babel/plugin-transform-react-jsx-source": "^7.18.6",
+        "@types/lodash": "^4.17.10",
         "@typescript-eslint/eslint-plugin": "^5.30.6",
         "@typescript-eslint/parser": "^5.30.6",
         "eslint": "^8.20.0",

client/src/app.tsx

@@ -16,7 +16,15 @@ import Search from './routes/search';
 import List from './routes/list';
 import Info from './routes/info';
 
-if (window.location.host === 'matescrates.au') LogRocket.init('yh0dy3/mcrates');
+if (window.location.host === 'crates.ntwl.xyz') LogRocket.init('yh0dy3/mcrates');
+
+
+// ██     ██ ███████      █████  ██████  ███████     ███████  ██████      ██████   █████   ██████ ██   ██
+// ██     ██ ██          ██   ██ ██   ██ ██          ██      ██    ██     ██   ██ ██   ██ ██      ██  ██
+// ██  █  ██ █████       ███████ ██████  █████       ███████ ██    ██     ██████  ███████ ██      █████
+// ██ ███ ██ ██          ██   ██ ██   ██ ██               ██ ██    ██     ██   ██ ██   ██ ██      ██  ██
+//  ███ ███  ███████     ██   ██ ██   ██ ███████     ███████  ██████      ██████  ██   ██  ██████ ██   ██
+
 
 const Mcrates = () => {
     const dispatch = useDispatch();
@@ -46,7 +54,7 @@ const Mcrates = () => {
                 <Route path="/register" component={Login} mode="register" />
                 <Route path="/search" component={Search} />
                 <Route path="/list" component={List} />
-                <Route path="/info" component={Info} />
+                {/*<Route path="/info" component={Info} />*/}
 
                 {/* ACP */}
                 <Route path="/acp" component={Acp} />

client/src/components/nav.tsx

@@ -25,7 +25,7 @@ const Nav = () => {
         }
         route(DEFAULT_ROUTE)
 
-        if (window.location.host === 'matescrates.au') {
+        if (window.location.host === 'crates.ntwl.xyz') {
             LogRocket.identify(user.id, {
                 name: user.email,
                 email: user.email,
@@ -72,7 +72,7 @@ const Nav = () => {
     return (
         <header>
             {user ? nav() : anonNav()}
-            {showAdmin(user) ? adminNav() : <div></div>}
+            {showAdmin(user) ? adminNav() : <div />}
             {user ? userNav() : null}
         </header>
     );

client/src/components/offer.group.tsx

@@ -1,6 +1,7 @@
 import { h } from 'preact';
 import { ExternalLink, Check } from 'react-feather';
-import { useState } from 'preact/hooks';
+import { useState, useCallback } from 'preact/hooks';
+import _ from 'lodash'
 
 import { useSelector, useDispatch } from 'react-redux'
 import { State } from './../store';
@@ -58,6 +59,58 @@ function OfferPriceBtn(offer) {
     )
 }
 
+function OfferConditionSlider([condition, offers], i, groupId, rangeMin, rangeMax) {
+    const step = 'any'
+    const id = `${groupId}-${i}`
+    const markersId = `${id}-markers`
+    const steps = offers.map(o => o.total)
+    const initialValue = _.last(offers.filter(o => o.status === 'Accepted'))?.total || 0
+
+    const ws = useSelector((state: State) => state.ws);
+    const [acceptedPrice, setAcceptedPrice] = useState(initialValue)
+
+    const sendOffers = useCallback(_.debounce(function send(v) {
+        const accepted = offers.filter(o => Number(o.total) <= v).map(o => o.id)
+        const pending = offers.filter(o => Number(o.total) > v).map(o => o.id)
+        ws.sendOfferStatusSet(accepted, 'Accepted')
+        ws.sendOfferStatusSet(pending, 'Pending')
+    }, 1000), [])
+
+    function onChange(target) {
+        const highest = () => {
+            return _.findLast(steps, s => target.value >= Number(s)) || 0
+        }
+
+        const v = highest()
+        target.value = v
+        setAcceptedPrice(v)
+        sendOffers(v)
+    }
+
+    const acceptedPriceLabel = acceptedPrice ? `$${acceptedPrice}` : '-'
+
+    return (
+        <div key={id}>
+            <label for={id}>{condition}</label><br />
+            <label for={id}>{acceptedPriceLabel}</label><br />
+            <input
+                type="range"
+                id={id}
+                name={id}
+                list={markersId}
+                step={step}
+                min={rangeMin}
+                max={rangeMax}
+                value={acceptedPrice}
+                onChange={e => onChange((e.target as HTMLInputElement)) } />
+
+            <datalist id={markersId}>
+                {offers.map(o => <option value={o.total}></option>)}
+            </datalist>
+        </div>
+    )
+}
+
 
 export default function OfferGroup(group, i) {
     const ws = useSelector((state: State) => state.ws);
@@ -66,19 +119,32 @@ export default function OfferGroup(group, i) {
     const descriptions = group.details.formats.map(f => f.descriptions.join(' ')).join(' ')
     const detailText = `${format} ${descriptions}`;
 
+    const conditions = Object.entries(_.groupBy(group.offers, 'condition'))
+    const rangeMin = 0
+    const rangeMax = _.max(group.offers.map(o => Number(o.total)))
+
+    const SHOW_BUTTONS = false
+    const buttons = SHOW_BUTTONS ? (
+        <div class="lines">
+            {group.offers.map(o => OfferPriceBtn(o))}
+        </div>
+    ) : null
+
+
     return (
         <div class="offer" key={i} id={group.details.id}>
             <section>
-                <h3>{group.details.artists_sort} - {group.details.title} <a href={group.details.uri} target="_blank"><ExternalLink /></a></h3>
+                <h3>{group.details.artists_sort} - {group.details.title} <a href={group.details.uri} target="_blank" rel="noreferrer"><ExternalLink /></a></h3>
                 <div class='alt-text details'>
                     {group.details.year} - {detailText}
                 </div>
                 <div class='alt-text details'>Status: {group.status}</div>
             </section>
             <img src={group.details.thumb} />
-            <div class="lines">
-                {group.offers.map(o => OfferPriceBtn(o))}
+            <div>
+                {conditions.map((c, i) => OfferConditionSlider(c, i, group.details.id, rangeMin, rangeMax))}
             </div>
+            { buttons }
         </div>
     )
 }

client/src/components/offer.tsx

@@ -29,7 +29,7 @@ export default function Offer(offer) {
     return (
         <div class="offer">
             <section>
-                <h3>{details.artists_sort} - {details.title} <a href={details.uri} target="_blank"><ExternalLink /></a></h3>
+                <h3>{details.artists_sort} - {details.title} <a href={details.uri} target="_blank" rel="noreferrer"><ExternalLink /></a></h3>
                 <div class='alt-text' >{details.year}</div>
                 <div class='alt-text'>{details.formats[0].descriptions.join(' ')}</div>
                 <div class='number'>mcrates price: ${offer.total}</div>

client/src/components/want.tsx

@@ -27,7 +27,7 @@ function Want(want) {
     return (
         <article key={r.id}>
             <div>
-                <h3>{r.artists_sort} - {r.title} <a href={r.uri} target="_blank"><ExternalLink /></a></h3>
+                <h3>{r.artists_sort} - {r.title} <a href={r.uri} target="_blank" rel="noreferrer"><ExternalLink /></a></h3>
             </div>
             <img src={r.thumb} />
             <div class='details'>

client/src/http.ts

@@ -15,8 +15,8 @@ declare module 'axios' {
 }
 
 const api = axios.create({
-  baseURL: '/api',
-  withCredentials: true
+    baseURL: '/api',
+    withCredentials: true
 });
 
 function onSuccess(response) {

client/src/routes/acp.tsx

@@ -34,11 +34,11 @@ function Item({ item }) {
 
     return (
         <div key={item.id} class={bold}>
-            <a href={item.listing.data.uri} target="_blank" >
+            <a href={item.listing.data.uri} target="_blank" rel="noreferrer" >
                 <h4>{item.listing.data.release.description}</h4>
             </a>
             <h5>{item.user.email}</h5>
-            <h5 class='alt-text'>{item.total}</h5>
+            <h5 class='alt-text'>{item.total} - {item.condition}</h5>
             <h5>{item.status}</h5>
             <label>Set Status:</label>
             <select onChange={onChange}>
@@ -53,7 +53,7 @@ function SellerGroup(group) {
 
     return (
         <div class="bundle">
-           <section>
+            <section>
                 <h2>{group.username} - {group.seller}</h2>
                 <h3>{ready} / {group.offers.length} offers</h3>
                 <h4>Ships: {group.mentionsAustralia && 'AU'} {group.mentionsWorldwide && 'WORLD'}</h4>
@@ -75,7 +75,7 @@ function Home() {
     }
 
     return (
-        <div>
+        <div class='acp'>
             <button onClick={sync}> sync listings </button>
             <button onClick={sync}> generate offers </button>
             <hr />

client/src/routes/info.tsx

@@ -18,6 +18,7 @@ const Info = () => {
                     <h2>How It Works</h2>
                     <p> First, add some records to your <span class='alt'>List</span> on the <a href='/search'>Search Page</a>. </p>
                     <p> Mates Crates will then make <span class='alt'>Offers</span> to you for releases you add to your <a href='/list'>List</a>. </p>
+                    <p> (this can take a few minutes time!) </p>
                     <p> Select all <span class='alt'>Offers</span> you are interested in, the more you accept the more likely the item will be <span class='alt-text'>Confirmed</span>.</p>
                     <p> Check back regularly for updated <span class='alt'>Offers</span>, the global record marketplace is a living thing. </p>
                 </section>

client/src/routes/login.tsx

@@ -93,7 +93,7 @@ const Login = ({ mode }) => {
                     <input type="password" name="password" id="password" />
                 </div>
                 <div class="item">
-                    <input type="submit" value="login"/>
+                    <input type="submit" value="login" />
                 </div>
             </form>
             <button onClick={gotoRegister}> create account </button>
@@ -107,9 +107,9 @@ function LoginPage({ mode }) {
             <Login mode={mode} />
             <Swirl />
             <div class='words'>
-                <p>Mates Crates is a record importing service.</p>
-                <p>We bundle orders together from thousands of online stores to reduce the cost of shipping vinyl.</p>
-                <p>Records can then be picked up from Preston.</p>
+                <p>record importing service</p>
+                <p>we bundle orders together from thousands of online stores to reduce the cost of shipping vinyl</p>
+                <p>then we get together and play them</p>
             </div>
         </div>
     )

client/src/routes/main.tsx

@@ -8,7 +8,9 @@ import * as toast from './../toast';
 import OfferGroup from './../components/offer.group';
 
 const GroupNav = (want) => {
-    const href = want._count.offers ? `#${want.release}` : null;
+    if (!want._count.offers) return null
+    const href = `#${want.release}`;
+
     return (
         <div class=''>
             <a href={href}>{want.data.artists_sort} - {want.data.title}</a>

client/src/routes/search.tsx

@@ -4,6 +4,7 @@ import { ExternalLink, Check } from 'react-feather';
 import { useSelector } from 'react-redux'
 
 import { State } from './../store';
+import { info, error as toastError } from './../toast';
 
 import http from '../http';
 
@@ -24,7 +25,7 @@ function Result(r, wants, ws) {
         <article key={r.id}>
             <button onClick={onClick} disabled={wanted} class={classes} >{btnText}</button>
             <div>
-                <h3>{r.title} <a href={`https://discogs.com${r.uri}`} target="_blank"><ExternalLink /></a></h3>
+                <h3>{r.title} <a href={`https://discogs.com${r.uri}`} target="_blank" rel="noreferrer"><ExternalLink /></a></h3>
                 <h5 class='alt-text' >{r.year}</h5>
                 <h5 class='alt-text'>{r.country}</h5>
                 <h5 class='alt-text'>{r.format.join(' ')}</h5>
@@ -36,10 +37,8 @@ function Result(r, wants, ws) {
 
 const User = () => {
     const [query, setQuery] = useState('')
-    const [title, setTitle] = useState('');
     const [username, setUsername] = useState('');
     const [results, setResults] = useState([]);
-    const [infoText, setInfoText] = useState('');
 
     const wants = useSelector((state: State) => state.wants);
     const ws = useSelector((state: State) => state.ws);
@@ -47,23 +46,33 @@ const User = () => {
     async function fetchResults() {
         const [error, results] = await http.postWantSearch(query);
 
-        if (results.length === 0) {
+        if (error) {
             setResults([]);
-            return setInfoText('No results');
+            toastError('Error', error);
+        }
+
+        if (results.length === 0) {
+            setResults([]);
+            return info('No Results', 'No results returned from search');
         }
 
-        setInfoText('');
         setResults(results);
     }
 
     async function submitWantlist() {
         const [error, wants] = await http.postDiscogs(username);
-        if (wants.length === 0) {
+
+        if (error) {
             setResults([]);
-            return setInfoText('No new additions');
+            toastError('Error', error);
         }
 
-        setInfoText(`Added ${wants.length} items to your list.`);
+        if (wants.length === 0) {
+            setResults([]);
+            return info('Success', 'No new additions');
+        }
+
+        info('Success', `Added ${wants.length} items to your list.`);
     }
 
     const searchRef = useRef<HTMLInputElement>();
@@ -87,10 +96,6 @@ const User = () => {
         ? <button onClick={() => resultsRef.current.scrollTo(0, 0)}> top </button>
         : null;
 
-    const infoEl = infoText
-        ? <article><div></div><div>{infoText}</div></article>
-        : null;
-
     return (
         <div class='search'>
             <div>
@@ -111,7 +116,6 @@ const User = () => {
                 <p>Data provided by Discogs.</p>
             </div>
             <div class='results' ref={resultsRef}>
-                {infoEl}
                 {results.map(r => Result(r, wants, ws))}
                 {topEl}
             </div>

client/src/store.tsx

@@ -2,58 +2,58 @@ import { createStore, combineReducers } from 'redux'
 import { configureStore, createSlice } from '@reduxjs/toolkit'
 
 export const userState = createSlice({
-  name: 'user',
-  initialState: null,
-  reducers: {
-    set: (state, action) => action.payload
-  },
+    name: 'user',
+    initialState: null,
+    reducers: {
+        set: (state, action) => action.payload
+    },
 });
 
 export const offersState = createSlice({
-  name: 'offers',
-  initialState: null,
-  reducers: {
-    set: (state, action) => action.payload
-  },
+    name: 'offers',
+    initialState: null,
+    reducers: {
+        set: (state, action) => action.payload
+    },
 });
 
 export const acpOffersState = createSlice({
-  name: 'acpOffers',
-  initialState: [],
-  reducers: {
-    set: (state, action) => action.payload
-  },
+    name: 'acpOffers',
+    initialState: [],
+    reducers: {
+        set: (state, action) => action.payload
+    },
 });
 
 export const wantsState = createSlice({
-  name: 'wants',
-  initialState: null,
-  reducers: {
-    set: (state, action) => action.payload
-  },
+    name: 'wants',
+    initialState: null,
+    reducers: {
+        set: (state, action) => action.payload
+    },
 });
 
 export const wsState = createSlice({
-  name: 'ws',
-  initialState: null,
-  reducers: {
-    set: (state, action) => action.payload
-  },
+    name: 'ws',
+    initialState: null,
+    reducers: {
+        set: (state, action) => action.payload
+    },
 });
 
 const reducer = combineReducers({
-  offers: offersState.reducer,
-  wants: wantsState.reducer,
-  ws: wsState.reducer,
-  user: userState.reducer,
+    offers: offersState.reducer,
+    wants: wantsState.reducer,
+    ws: wsState.reducer,
+    user: userState.reducer,
 
-  acpOffers: acpOffersState.reducer,
+    acpOffers: acpOffersState.reducer,
 });
 
 const store = createStore(
-  reducer,
-  // @ts-ignore -- devtools window exists on dev
-  window.__REDUX_DEVTOOLS_EXTENSION__ && window.__REDUX_DEVTOOLS_EXTENSION__()
+    reducer,
+    // @ts-ignore -- devtools window exists on dev
+    window.__REDUX_DEVTOOLS_EXTENSION__ && window.__REDUX_DEVTOOLS_EXTENSION__()
 );
 
 export default store;

client/src/style/index.less

@@ -75,6 +75,7 @@ html {
 
     --color-blue: #475977;
     --color-green: #476d4a;
+    --color-red: #ae4442;
 
     --color-accepted: var(--color-blue);
     --color-confirmed: var(--color-green);

client/src/style/styles.less

@@ -22,6 +22,8 @@ body {
 .landing {
     display: grid;
 
+    overflow: scroll;
+
     grid-template-columns: 1fr 2fr;
     grid-gap: calc(var(--spacing-double) * 2);
     grid-template-areas:
@@ -49,6 +51,8 @@ body {
 .info {
     display: grid;
 
+    overflow: hidden;
+
     grid-template-areas:
         "words swirl"
         "words contact";
@@ -63,10 +67,12 @@ body {
     .contact {
         grid-area: contact;
         text-align: center;
+        overflow: scroll;
     }
 
     .words {
         grid-area: words;
+        overflow: scroll;
     }
 
     .swirl {
@@ -236,29 +242,33 @@ main {
     }
 }
 
-.bundles {
-    flex-flow: row;
+.acp {
+    overflow: scroll;
 
-    .bundle {
-        margin-bottom: var(--spacing-double);
+    .bundles {
+        flex-flow: row;
 
-        section {
-            margin-bottom: var(--spacing);
-        }
+        .bundle {
+            margin-bottom: var(--spacing-double);
 
-        .items {
-            display: grid;
-            grid-template-columns: repeat(5, 1fr);
-
-            div {
-                margin-right: var(--spacing);
+            section {
                 margin-bottom: var(--spacing);
+            }
 
+            .items {
                 display: grid;
+                grid-template-columns: repeat(5, 1fr);
 
-                &.bold {
-                    font-weight: bold;
-                    background: var(--background-2);
+                div {
+                    margin-right: var(--spacing);
+                    margin-bottom: var(--spacing);
+
+                    display: grid;
+
+                    &.bold {
+                        font-weight: bold;
+                        background: var(--background-2);
+                    }
                 }
             }
         }
@@ -340,6 +350,8 @@ main {
     margin-bottom: var(--spacing);
 }
 
+input[type='range']{ accent-color: var(--color-accepted); }
+
 @media (max-width:900px)  {
     .landing {
         grid-template-columns: 1fr;

client/src/template.html

@@ -2,7 +2,7 @@
 <html lang="en">
   <head>
     <meta charset="utf-8">
-    <title>matescrates.au - record import service</title>
+    <title>crates - record import service</title>
     <meta name="viewport" content="width=device-width,initial-scale=1">
     <meta name="mobile-web-app-capable" content="yes">
     <meta name="apple-mobile-web-app-capable" content="yes">

client/src/toast.tsx

@@ -13,3 +13,18 @@ export function info(title, message) {
         theme: 'dark',
     });
 }
+
+export function error(title, message) {
+    toast.error({
+        title,
+        message: message || '',
+        position: 'bottomCenter',
+        color: '#ae4442',
+        titleColor: '#F4F1D0',
+        messageColor: '#F4F1D0',
+        iconColor: '#F4F1D0',
+        icon: '',
+        theme: 'dark',
+    });
+}
+

client/src/ws.ts

@@ -1,4 +1,4 @@
-import debounce from 'lodash/debounce';
+import throttle from 'lodash/throttle';
 
 import { wantsState, userState, offersState, acpOffersState } from './store'
 import * as toast from './toast';
@@ -29,8 +29,8 @@ export default function createSocket(dispatch) {
         // setTimeout(sendPing, 50);
     }
 
-    function sendOfferStatusSet(id, status) {
-        send(['OfferStatusSet', { id, status }]);
+    function sendOfferStatusSet(ids, status) {
+        send(['OfferStatusSet', { ids, status }]);
     }
 
     function sendWantRm(id) {
@@ -77,7 +77,13 @@ export default function createSocket(dispatch) {
         dispatch(wantsState.actions.set(wants));
     }
 
+    const offersNotification = throttle((offers) => {
+        if (!offers.length) return false;
+        toast.info('Offers Updated', 'New offers available.')
+    }, 60 * 1000);
+
     function onOffers(offers) {
+        offersNotification(offers);
         dispatch(offersState.actions.set(offers));
     }
 

docker-compose.K8S.yaml

@@ -0,0 +1,14 @@
+version: "3.3"
+
+services:
+  client:
+    image: registry.ntwl.xyz/crates-client
+    build:
+      context: ./client
+      dockerfile: ./client.Dockerfile
+
+  api:
+    image: registry.ntwl.xyz/crates-api
+    build:
+      context: ./api
+      dockerfile: ./api.Dockerfile

docker-compose.MCRATES.yaml

@@ -1,35 +0,0 @@
-version: "3.3"
-
-services:
-  pg:
-    image: postgres:latest
-    environment:
-      POSTGRES_USER: 'mcrates'
-      POSTGRES_DB: 'mcrates'
-      POSTGRES_PASSWORD: 'digdeep'
-
-    # allow prisma client to push changes from outside docker etc
-    # ufw blocks this from being publicly accessible
-    ports:
-      - '45432:5432'
-    volumes:
-      - "./db:/var/lib/postgresql/data"
-
-  api:
-    image: node:16
-    working_dir: /opt/mcrates/api
-    entrypoint: npm start
-    environment:
-      DATABASE_URL: 'postgresql://mcrates:digdeep@pg:5432/mcrates?schema=public'
-    volumes:
-      - "./api:/opt/mcrates/api"
-
-  nginx:
-    image: nginx:latest
-    volumes:
-      - './nginx/matescrates.au.nginx.conf:/etc/nginx/conf.d/default.conf'
-      - './client/build:/srv/matescrates'
-      - '/etc/letsencrypt:/etc/letsencrypt'
-    ports:
-      - '80:80'
-      - '443:443'

docker-compose.TEST.yaml

@@ -4,18 +4,18 @@ services:
   pg:
     image: postgres:latest
     environment:
-      POSTGRES_USER: 'mcrates'
+      POSTGRES_USER: 'crates'
       POSTGRES_DB: 'test'
       POSTGRES_PASSWORD: 'digdeep'
 
   api:
     image: node:16
-    working_dir: /opt/mcrates/api
+    working_dir: /opt/crates/api
     entrypoint: npm run test:watch
     depends_on:
       - pg
     environment:
-      DATABASE_URL: "postgresql://mcrates:digdeep@pg:5432/test?schema=public"
+      DATABASE_URL: "postgresql://crates:digdeep@pg:5432/test?schema=public"
     volumes:
-      - "./api:/opt/mcrates/api"
+      - "./api:/opt/crates/api"
 

docker-compose.yaml

@@ -4,34 +4,54 @@ services:
   pg:
     image: postgres:latest
     environment:
-      POSTGRES_USER: 'mcrates'
-      POSTGRES_DB: 'mcrates'
-      POSTGRES_PASSWORD: 'digdeep'
+      POSTGRES_USER: '${POSTGRES_USER}'
+      POSTGRES_DB: '${POSTGRES_DB}'
+      POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
+    volumes:
+      # so ez
+      # insane
+      - "./pg:/docker-entrypoint-initdb.d"
     ports:
       - '45432:5432'
-    volumes:
-      - "./db:/var/lib/postgresql/data"
 
   api:
-    image: node:16
-    working_dir: /opt/mcrates/api
+    image: crates/api
+    build:
+      context: ./api
+      dockerfile: ./api.Dockerfile
+    working_dir: /opt/crates/api
     depends_on:
       - 'pg'
-    entrypoint: npm start
+    command: npm start
     environment:
-      DATABASE_URL: 'postgresql://mcrates:digdeep@pg:5432/mcrates?schema=public'
+      DATABASE_URL: 'postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?schema=public'
+      DISCORD_TOKEN: '${DISCORD_TOKEN}'
+      DISCOGS_TOKEN: '${DISCOGS_TOKEN}'
+      CHOKIDAR_USEPOLLING: 1
     volumes:
-      - "./api:/opt/mcrates/api"
+      - "./api:/opt/crates/api"
+      - "/opt/crates/api/node_modules"
+
+  migrate:
+    restart: on-failure:3
+    extends: api
+    command: npm run migrate status
 
   client:
     image: node:16
-    working_dir: /opt/mcrates/client
+    working_dir: /opt/crates/client
+    build:
+      context: ./client
+      dockerfile: ./client.dev.Dockerfile
     entrypoint: npm start
     ports:
       - '8080:8080' # fwd the hmr websocket
+    environment:
+      CHOKIDAR_USEPOLLING: 1
     volumes:
-      - "./client:/opt/mcrates/client"
-      - "./api:/opt/mcrates/api"
+      - "./client:/opt/crates/client"
+      - "./api:/opt/crates/api"
+      - "/opt/crates/client/node_modules"
 
   nginx:
     image: nginx:latest

nginx/localhost.nginx.conf

@@ -1,8 +1,8 @@
-upstream mcrates_api {
+upstream crates_api {
     server api:41337;
 }
 
-upstream mcrates_client {
+upstream crates_client {
     server client:8080;
 }
 
@@ -13,12 +13,12 @@ map $http_upgrade $connection_upgrade {
 
 server {
     location / {
-        proxy_pass http://mcrates_client;
+        proxy_pass http://crates_client;
         proxy_read_timeout 600s;
     }
 
     location /api/ws {
-        proxy_pass http://mcrates_api;
+        proxy_pass http://crates_api;
         proxy_http_version 1.1;
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection $connection_upgrade;
@@ -28,7 +28,7 @@ server {
 
     location /api/ {
         # trailing slash means api will receive url WITHOUT /api
-        proxy_pass http://mcrates_api/;
+        proxy_pass http://crates_api/;
         proxy_read_timeout 600s;
     }
 }