diff --git a/api/api.Dockerfile b/api/api.Dockerfile index ee0a924..3541195 100644 --- a/api/api.Dockerfile +++ b/api/api.Dockerfile @@ -4,6 +4,7 @@ COPY . ./ # puppeteer deps RUN apt-get update +# don't think about it RUN apt-get install -y ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 lsb-release wget xdg-utils RUN npm i diff --git a/api/src/discogs.ts b/api/src/discogs.ts index a57f657..71c38d4 100644 --- a/api/src/discogs.ts +++ b/api/src/discogs.ts @@ -77,12 +77,9 @@ export default async function createDiscogs(log: Log, pg: Pg) { async function listings(want: Want) { const page = await browser.newPage() const url = `https://www.discogs.com/sell/release/${want.release}?limit=250` - - log.info({ url }, 'fetching') const res = await page.goto(url) log.debug({ status: res.status() }, 'status') if (!res.ok()) throw new Error(res.statusText()) - const els = await page.$$('.item_description_title[href]') const hrefs = await Promise.all(els.map(e => e.evaluate(e => e.href))) const ids = hrefs @@ -92,6 +89,8 @@ export default async function createDiscogs(log: Log, pg: Pg) { await page.close() + if (ids.length === 0) log.warn({ res, url, ...wantDetails(want) }, "Want had zero listings") + const unlisted = await pg.listingRmUnlisted(want, ids); if (unlisted) log.info({ unlisted: unlisted, ...wantDetails(want) }, 'missing listings removed'); @@ -99,18 +98,14 @@ export default async function createDiscogs(log: Log, pg: Pg) { // sometimes we get dodgy listings from discogs that throw perm errors // we just want to ignore them - const results = await Promise.allSettled(ids.map(id => listing(id, want))); - // @ts-ignore - const validListings = results - .map(r => { - if (r.status === 'fulfilled') return r.value; - if (r.reason.response.status === 429) throw r.reason; - log.warn('listing fetch error', r) - return null - }) - .filter(v => v); + // const results = await Promise.allSettled(ids.map(id => listing(id, want))); - return validListings; + const results = [] + for await (const id of ids) { + const r = await listing(id, want) + results.push(r) + } + return results; } diff --git a/api/src/events.ts b/api/src/events.ts index d1fefae..5a2059a 100644 --- a/api/src/events.ts +++ b/api/src/events.ts @@ -29,8 +29,10 @@ export default function createEvents(log: Log, pg: Pg, discogs, offers, discord) log.error('event error', e); }); - const MAX_ATTEMPTS = 2; - async function populateListings(want: Want, attempts = 0) { + const MAX_ATTEMPTS = 3; + const SYNC_EVERY_MS = 60 * 1000 + const RATE_LIMIT_WAIT_MS = (attempts) => attempts * 70 * 1000 + async function populateListings(want: Want, attempts = 1) { try { const listings = await discogs.listings(want); await pg.wantSynced(want); @@ -38,8 +40,9 @@ export default function createEvents(log: Log, pg: Pg, discogs, offers, discord) } catch (e) { log.error(e) if (e.response.status === 429) { - log.warn('rate limited, trying again in 60s'); - await setTimeout(60 * 1000); + log.warn({ RATE_LIMIT_WAIT_MS: RATE_LIMIT_WAIT_MS(attempts), ...e.response.headers }, 'rate limited, trying again'); + + await setTimeout(RATE_LIMIT_WAIT_MS(attempts)); if (attempts > MAX_ATTEMPTS) { log.error(wantDetails(want), 'failed to populate listings') } @@ -81,7 +84,7 @@ export default function createEvents(log: Log, pg: Pg, discogs, offers, discord) if (wants.length === 0) { log.info('all listings up to date'); - await setTimeout(60 * 1000) + await setTimeout() } } catch (e) { log.error(e, 'sync error') diff --git a/client/src/routes/main.tsx b/client/src/routes/main.tsx index 4749dad..0e406c7 100644 --- a/client/src/routes/main.tsx +++ b/client/src/routes/main.tsx @@ -8,7 +8,9 @@ import * as toast from './../toast'; import OfferGroup from './../components/offer.group'; const GroupNav = (want) => { - const href = want._count.offers ? `#${want.release}` : null; + if (!want._count.offers) return null + const href = `#${want.release}`; + return (