we're so back

api/.dockerignore

@@ -0,0 +1 @@
+node_modules/

api/api.Dockerfile

@@ -1,4 +1,13 @@
-FROM node:16
+FROM node:20
 WORKDIR /opt/crates/api
 COPY . ./
+
+# puppeteer deps
+RUN apt-get update
+RUN apt-get install -y ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 lsb-release wget xdg-utils
+
+RUN npm i
+# not sure why doesn't do it itself
+RUN node node_modules/puppeteer/install.mjs
+
 CMD ["npm", "start"]

api/index.ts

@@ -28,7 +28,7 @@ async function start() {
         api,
         events,
         notify,
-    } = createServices(log);
+    } = await createServices(log);
 
     await notify.listen();
     await discord.connect();

api/package.json

@@ -34,6 +34,7 @@
     "bunyan": "^1.8.15",
     "cheerio": "^1.0.0-rc.12",
     "discord.js": "^14.7.1",
+    "esrun": "^3.2.26",
     "koa": "^2.13.4",
     "koa-body": "^5.0.0",
     "koa-passport": "^5.0.0",
@@ -42,7 +43,10 @@
     "koa-ws": "^1.0.1",
     "lodash": "^4.17.21",
     "passport-local": "^1.0.0",
-    "pg-listen": "^1.7.0"
+    "pg-listen": "^1.7.0",
+    "puppeteer": "^23.3.0",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
   },
   "overrides": {
     "koa-passport": {

api/src/discogs.ts

@@ -1,33 +1,21 @@
-import fs from 'fs/promises';
-
 import axios from 'axios';
-import cheerio from 'cheerio';
+import pptr from 'puppeteer-extra'
-import Bluebird from 'bluebird';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth'
 
 import type Log from 'bunyan';
-import type { PrismaClient, Want, User, Listing } from '@prisma/client';
+import type { Want, User } from '@prisma/client';
 
-import { logAxiosError, wantDetails } from './utils';
+import { wantDetails } from './utils';
 import { Pg } from './pg';
 
-type ApiResponse = [string | null, any | null];
+export default async function createDiscogs(log: Log, pg: Pg) {
-
-declare module 'axios' {
-  export interface AxiosInstance {
-    request(config: AxiosRequestConfig): Promise<ApiResponse>;
-    get(url: string, config?: AxiosRequestConfig): Promise<ApiResponse>;
-    delete(url: string, config?: AxiosRequestConfig): Promise<ApiResponse>;
-    head(url: string, config?: AxiosRequestConfig): Promise<ApiResponse>;
-    post(url: string, data?: any, config?: AxiosRequestConfig): Promise<ApiResponse>;
-    put(url: string, data?: any, config?: AxiosRequestConfig): Promise<ApiResponse>;
-    patch(url: string, data?: any, config?: AxiosRequestConfig): Promise<ApiResponse>;
-  }
-}
-
-export default function createDiscogs(log: Log, pg: Pg) {
     const TOKEN = process.env.DISCOGS_TOKEN;
     const apiBase = 'https://api.discogs.com'
-    const discogsBase = 'https://www.discogs.com'
+
+    pptr.use(StealthPlugin())
+    const browser = await pptr.launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'] })
+
+    log.info('browser created')
 
     const api = axios.create({
         headers: {
@@ -38,22 +26,16 @@ export default function createDiscogs(log: Log, pg: Pg) {
     });
 
     function onSuccess(response) {
-        return [null, response.data];
+        return response.data;
     }
 
     function onError(error) {
-        if (error.response.status === 429) {
+        throw error;
-            return [new Error('rate limited, try again in 1 minute'), null];
-        }
-        if (error.response) return [error.response.data, null];
-        if (error.request) return ['Network Error', null];
-        return ['Unknown Error', null];
     }
     api.interceptors.response.use(onSuccess, onError);
 
     async function wantListSync(username: string, user: User): Promise<Want[]> {
-        const [err, res] = await api.get(`${apiBase}/users/${username}/wants`);
+        const res = await api.get(`${apiBase}/users/${username}/wants`);
-        if (err) throw err;
         const { wants, pagination } = res;
 
         const releases = await Promise.all(wants.map(want => release(want.id)));
@@ -73,8 +55,7 @@ export default function createDiscogs(log: Log, pg: Pg) {
     }
 
     async function release(id: DiscogsRelease): Promise<any> {
-        const [err, data] = await api.get(`${apiBase}/releases/${id}`);
+        const data = await api.get(`${apiBase}/releases/${id}`);
-        if (err) throw err;
         return data;
     }
 
@@ -83,29 +64,41 @@ export default function createDiscogs(log: Log, pg: Pg) {
         url.searchParams.append('query', query);
         url.searchParams.append('type', 'release');
         url.searchParams.append('format', 'vinyl');
-        const [err, res] = await api.get(url.href);
+        const res = await api.get(url.href);
-        if (err) throw err;
         const { results } = res;
         return results;
     }
 
     async function inventory(user: string) {
-        const [err, res] = await api.get(`${apiBase}/users/${user}/inventory`);
+        const res = await api.get(`${apiBase}/users/${user}/inventory`);
-        if (err) throw err;
         return res.listings;
     }
 
     async function listings(want: Want) {
-        const [err, res] = await api.get(`https://www.discogs.com/sell/release/${want.release}?limit=250`);
+        const page = await browser.newPage()
-        if (err) throw err;
+        const url = `https://www.discogs.com/sell/release/${want.release}?limit=250`
 
-        const $ = cheerio.load(res);
+        log.info({ url }, 'fetching')
-        const ids = $('.item_description_title[href]')
+
-            .map(function getListingId() {
+        const res = await page.goto(url)
-                const id = Number.parseInt(this.attribs.href.replace('/sell/item/', ''), 10);
+
-                return id;
+        log.info(res)
-            })
+
-            .get();
+        ----------------
+
+
+        not catching the rate limiting here i thnk
+
+        if (res.status() === 429) throw new Error("rate limited")
+
+        const els = await page.$$('.item_description_title[href]')
+        const hrefs = await Promise.all(els.map(e => e.evaluate(e => e.href)))
+        const ids = hrefs
+            .map(h => new URL(h))
+            .map(p => p.pathname.replace('/sell/item/', ''))
+            .map(p => Number.parseInt(p, 10))
+
+        await page.close()
 
         const unlisted = await pg.listingRmUnlisted(want, ids);
         if (unlisted) log.info({ unlisted: unlisted, ...wantDetails(want) }, 'missing listings removed');
@@ -133,8 +126,7 @@ export default function createDiscogs(log: Log, pg: Pg) {
         if (listing) return listing;
 
         log.debug('fetching listing', id);
-        const [err, discogsListing] = await api.get(`${apiBase}/marketplace/listings/${id}`);
+        const discogsListing = await api.get(`${apiBase}/marketplace/listings/${id}`);
-        if (err) throw err;
         log.debug(discogsListing);
         return pg.listingUpsert(discogsListing, want);
     }

api/src/events.ts

@@ -36,7 +36,8 @@ export default function createEvents(log: Log, pg: Pg, discogs, offers, discord)
             await pg.wantSynced(want);
             return listings;
         } catch (e) {
-            if (e.message.includes('rate limited')) {
+            log.error(e)
+            if (e.status === 429) {
                 log.warn('rate limited, trying again in 60s');
                 await setTimeout(60 * 1000);
                 if (attempts > MAX_ATTEMPTS) {

api/src/services.ts

@@ -9,11 +9,11 @@ import createOffers from './offers';
 import createPg from './pg';
 import createDiscord from './discord';
 
-export default function createServices(log) {
+export default async function createServices(log) {
     const prisma = new PrismaClient();
     const discord = createDiscord(log);
     const pg = createPg(log, prisma);
-    const discogs = createDiscogs(log, pg);
+    const discogs = await createDiscogs(log, pg);
     const offers = createOffers(log, pg);
     const events = createEvents(log, pg, discogs, offers, discord);
     const notify = createNotify(log, pg, events);

docker-compose.yaml

@@ -13,7 +13,10 @@ services:
       - "./db:/var/lib/postgresql/data"
 
   api:
-    image: node:16
+    image: node:20
+    build:
+      context: ./api
+      dockerfile: ./api.Dockerfile
     working_dir: /opt/mcrates/api
     depends_on:
       - 'pg'