fixes for cloudflare

api/api.Dockerfile

@@ -4,6 +4,7 @@ COPY . ./
 
 # puppeteer deps
 RUN apt-get update
+# don't think about it
 RUN apt-get install -y ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 lsb-release wget xdg-utils
 
 RUN npm i

api/src/discogs.ts

@@ -77,12 +77,9 @@ export default async function createDiscogs(log: Log, pg: Pg) {
     async function listings(want: Want) {
         const page = await browser.newPage()
         const url = `https://www.discogs.com/sell/release/${want.release}?limit=250`
-
-        log.info({ url }, 'fetching')
         const res = await page.goto(url)
         log.debug({ status: res.status() }, 'status')
         if (!res.ok()) throw new Error(res.statusText())
-
         const els = await page.$$('.item_description_title[href]')
         const hrefs = await Promise.all(els.map(e => e.evaluate(e => e.href)))
         const ids = hrefs
@@ -92,6 +89,8 @@ export default async function createDiscogs(log: Log, pg: Pg) {
 
         await page.close()
 
+        if (ids.length === 0) log.warn({ res, url, ...wantDetails(want) }, "Want had zero listings")
+
         const unlisted = await pg.listingRmUnlisted(want, ids);
         if (unlisted) log.info({ unlisted: unlisted, ...wantDetails(want) }, 'missing listings removed');
 
@@ -99,18 +98,14 @@ export default async function createDiscogs(log: Log, pg: Pg) {
 
         // sometimes we get dodgy listings from discogs that throw perm errors
         // we just want to ignore them
-        const results = await Promise.allSettled(ids.map(id => listing(id, want)));
-        // @ts-ignore
-        const validListings = results
-            .map(r => {
-                if (r.status === 'fulfilled') return r.value;
-                if (r.reason.response.status === 429) throw r.reason;
-                log.warn('listing fetch error', r)
-                return null
-            })
-            .filter(v => v);
+        // const results = await Promise.allSettled(ids.map(id => listing(id, want)));
 
-        return validListings;
+        const results = []
+        for await (const id of ids) {
+            const r = await listing(id, want)
+            results.push(r)
+        }
+        return results;
 
     }
 

api/src/events.ts

@@ -29,8 +29,10 @@ export default function createEvents(log: Log, pg: Pg, discogs, offers, discord)
         log.error('event error', e);
     });
 
-    const MAX_ATTEMPTS = 2;
-    async function populateListings(want: Want, attempts = 0) {
+    const MAX_ATTEMPTS = 3;
+    const SYNC_EVERY_MS = 60 * 1000
+    const RATE_LIMIT_WAIT_MS = (attempts) => attempts * 70 * 1000
+    async function populateListings(want: Want, attempts = 1) {
         try {
             const listings = await discogs.listings(want);
             await pg.wantSynced(want);
@@ -38,8 +40,9 @@ export default function createEvents(log: Log, pg: Pg, discogs, offers, discord)
         } catch (e) {
             log.error(e)
             if (e.response.status === 429) {
-                log.warn('rate limited, trying again in 60s');
-                await setTimeout(60 * 1000);
+                log.warn({ RATE_LIMIT_WAIT_MS: RATE_LIMIT_WAIT_MS(attempts), ...e.response.headers }, 'rate limited, trying again');
+
+                await setTimeout(RATE_LIMIT_WAIT_MS(attempts));
                 if (attempts > MAX_ATTEMPTS) {
                     log.error(wantDetails(want), 'failed to populate listings')
                 }
@@ -81,7 +84,7 @@ export default function createEvents(log: Log, pg: Pg, discogs, offers, discord)
 
             if (wants.length === 0) {
                 log.info('all listings up to date');
-                await setTimeout(60 * 1000)
+                await setTimeout()
             }
         } catch (e) {
             log.error(e, 'sync error')

client/src/routes/main.tsx

@@ -8,7 +8,9 @@ import * as toast from './../toast';
 import OfferGroup from './../components/offer.group';
 
 const GroupNav = (want) => {
-    const href = want._count.offers ? `#${want.release}` : null;
+    if (!want._count.offers) return null
+    const href = `#${want.release}`;
+
     return (
         <div class=''>
             <a href={href}>{want.data.artists_sort} - {want.data.title}</a>