From dde964617983683280412e85a73fcbfdf48af1f7 Mon Sep 17 00:00:00 2001 From: ntr Date: Tue, 26 Jul 2022 00:42:47 +1000 Subject: [PATCH] full blown pg --- .gitignore | 3 +- docker-compose.yaml | 2 + index.ts | 116 ++++++++++++++++++-------- migrations/20220724114647_listings.js | 11 --- migrations/20220725073815_users.js | 17 ++++ migrations/20220725080947_wants.js | 27 ++++++ migrations/20220725083125_listings.js | 23 +++++ sellers.ts | 102 ---------------------- src/discogs.ts | 65 +++++++++------ src/mcrates.d.ts | 21 +++++ src/pg.ts | 96 ++++++++++++++++++++- src/utils.ts | 13 +++ tsconfig.json | 2 +- 13 files changed, 323 insertions(+), 175 deletions(-) delete mode 100644 migrations/20220724114647_listings.js create mode 100644 migrations/20220725073815_users.js create mode 100644 migrations/20220725080947_wants.js create mode 100644 migrations/20220725083125_listings.js delete mode 100644 sellers.ts create mode 100644 src/mcrates.d.ts create mode 100644 src/utils.ts diff --git a/.gitignore b/.gitignore index ff227fe..ca06f06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules/ -data/ \ No newline at end of file +data/ +db/ \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 22c6f82..9e73aca 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,3 +9,5 @@ services: POSTGRES_PASSWORD: 'dig deep' ports: - '45432:5432' + volumes: + - "./db:/var/lib/postgresql/data" diff --git a/index.ts b/index.ts index 8032bb6..794c4ce 100644 --- a/index.ts +++ b/index.ts @@ -3,41 +3,91 @@ import _ from 'lodash'; import Promise from 'bluebird'; import discogsService from './src/discogs'; +import pgService from './src/pg'; +import { logAxiosError } from './src/utils'; async function start() { - const discogs = discogsService(); - const wants = JSON.parse(await fs.readFile('./data/margh.wantlist.json', { encoding: 'utf8' })) - console.log('wants', wants.length); + const pg = pgService(); + const discogs = discogsService(pg); - return Promise.map(wants, async w => { - try { - const l = await discogs.listings(w.id) - return l; - } catch (e) { - // console.error(e); - return null; - } - }) - .then(listings => _ - .chain(listings) - .filter(l => l) - .flatten() - .map(l => ({ - id: l.id, - description: l.release.description, - seller_id: l.seller.id, - seller_name: l.seller.username, - })) - .groupBy(l => l.seller_id) - .sortBy(s => s.length) - .reverse() - .head() - .value()); + const usernames = [ + 'margh', + 'jmcdermid1', + 'vanderbabe', + ]; + + async function fetchListings(w) { + return w.map(discogs.listings) + } + + async function populateUserListings(user) { + console.log(user); + const wants = await discogs.wantlist(user); + console.log(user.discogs, 'has', wants.length, 'wants'); + + for (const want of wants) { + const listings = await discogs.listings(want); + const populated = listings.filter(l => l); + console.log(want.id, 'has', populated.length, '/', listings.length, 'listings'); + } + + // const listings = await Promise + // .all(releases.map(async want => { + // const listings = (await getListings(want)).filter(l => l); + + // const insertions = await Promise.all(listings.map(async l => { + // const listing = await pg.listingUpsert(l, want); + // console.log(listing.id, listing.data.release.description, want.user, listing.seller); + // return listing; + // })); + + // console.log(insertions.length, want.id); + // })); + + // return listings; + } + + return Promise.resolve(usernames) + .map(u => discogs.user(u)) + .map(populateUserListings); + + // return Promise.map(releases, async w => { + // try { + // const l = await discogs.listings(w.id) + // return l; + // } catch (e) { + // // console.error(e); + // return null; + // } + // }) + // .then(listings => _ + // .chain(listings) + // .filter(l => l) + // .flatten() + // // .map(l => ({ + // // id: l.id, + // // description: l.release.description, + // // seller_id: l.seller.id, + // // seller_name: l.seller.username, + // // })) + // // .groupBy(l => l.seller_id) + // .value() + // ); } -start() - .then(r => console.log(r)) - .catch(e => { - console.error(e); - process.exit(1); - }); +function run() { + start() + .then(r => { + console.log(r); + setTimeout(run, 60 * 1000); + }) + .catch(e => { + console.error(e); + if (e.message === 'rate limited') { + return setTimeout(run, 60 * 1000); + } + process.exit(1); + }); +} + +run(); diff --git a/migrations/20220724114647_listings.js b/migrations/20220724114647_listings.js deleted file mode 100644 index 6c8a30c..0000000 --- a/migrations/20220724114647_listings.js +++ /dev/null @@ -1,11 +0,0 @@ -exports.up = async knex => { - await knex.schema.createTable('listings', table => { - table.integer('id').primary(); - table.timestamps(true, true); - table.jsonb('data'); - }); - - return true; -}; - -exports.down = async () => {}; \ No newline at end of file diff --git a/migrations/20220725073815_users.js b/migrations/20220725073815_users.js new file mode 100644 index 0000000..11d54dd --- /dev/null +++ b/migrations/20220725073815_users.js @@ -0,0 +1,17 @@ +exports.up = async knex => { + await knex.schema.createTable('users', table => { + table.uuid('id') + .primary() + .defaultTo(knex.raw('gen_random_uuid()')); + + table.string('discogs') + .unique(); + + table.jsonb('data') + .notNullable(); + + table.timestamps(true, true); + }); +}; + +exports.down = async () => true; diff --git a/migrations/20220725080947_wants.js b/migrations/20220725080947_wants.js new file mode 100644 index 0000000..73adae2 --- /dev/null +++ b/migrations/20220725080947_wants.js @@ -0,0 +1,27 @@ +exports.up = async knex => { + await knex.schema.createTable('wants', table => { + table.uuid('id') + .primary() + .defaultTo(knex.raw('gen_random_uuid()')); + + table.integer('release') + .notNullable() + .index(); + + table.uuid('user') + table.foreign('user') + .references('id') + .inTable('users') + .onDelete('CASCADE'); + + table.jsonb('data') + .notNullable(); + + // user can only want a release once + table.unique(['user', 'release']); + + table.timestamps(true, true); + }); +}; + +exports.down = async () => true; diff --git a/migrations/20220725083125_listings.js b/migrations/20220725083125_listings.js new file mode 100644 index 0000000..20ff3c6 --- /dev/null +++ b/migrations/20220725083125_listings.js @@ -0,0 +1,23 @@ +exports.up = async knex => { + await knex.schema.createTable('listings', table => { + table.integer('id') + .primary(); + + table.uuid('want') + table.foreign('want') + .references('id') + .inTable('wants') + .onDelete('CASCADE'); + + table.integer('seller') + .notNullable() + .index(); + + table.jsonb('data') + .notNullable(); + + table.timestamps(true, true); + }); +}; + +exports.down = async () => true; diff --git a/sellers.ts b/sellers.ts deleted file mode 100644 index c5d72e1..0000000 --- a/sellers.ts +++ /dev/null @@ -1,102 +0,0 @@ -export default [ - '430AM_Studio', - 'EtherWood', - 'spacehall.de', - '3345vinylrecords', - 'AsquithJimbo', - 'gianni70', - 'mutt_discogs', - 'vinylline-rec.', - 'alain.v2', - 'DNP_Shop', - 'facerecords', - 'Filter_Musikk', - 'Impressions_sonores', - 'lofi-records', - 'MEGA-MIX', - 'q-logic', - 'RadialStore', - 'Record-cb', - 'tobow', - 'Wolvie-Records', - 'www.hhv.de', - '1-1', - 'alexkiwiwi', - 'All-Day', - 'camaro4504', - 'caviar_lounge', - 'decks.de', - 'Don.Giovanni', - 'DWTX', - 'EXPM', - 'Freakyrecords', - 'hipcity', - 'japanspin', - 'JPsVinyl', - 'KHVinylStore', - 'KillaCutz', - 'love-vinyl-records', - 'm50', - 'marygowild', - 'mondo_diggers_club', - 'mvc.be', - 'OCCIDENTAL', - 'oopu', - 'Palace-Vinyl', - 'purplewax23', - 'rss.jk', - 'rupertmuiry', - 'Space_is_the_place', - 'supermegarecords', - 'tonvermittlung.de', - 'Vinyl_Dealer_666', - 'vinylminded', - 'Vinylotheque', - 'Werkstraat.Shop', - 'xtcr', - 'zionsgate', - '_records_', - '-Beathoven-', - '-N-R-G-', - '12_Inch_Music', - '12-Inch-Love', - '13loco', - '2Cows_Records', - '44ToursRecords', - '4rond', - '7Lj', - '81Renshaw', - 'adriano_pls', - 'AFRICABRASIL', - 'afrika1974', - 'Afternooner', - 'AG303', - 'al..x', - 'alcovarecords', - 'AlexDahlmann', - 'Alexoj', - 'alfalfap', - 'alleytunesrecords', - 'altarofmadness', - 'Amarcord', - 'ambulant', - 'andrega', - 'andson', - 'andytonks', - 'anton.kolo', - 'antonydaly', - 'APSMUSIC', - 'Art-is-an', - 'Astral_Equilibrium', - 'Atrabilogie', - 'AuroraBorealis87', - 'avez', - 'babacar88', - 'Babatron', - 'Bambatta_studio', - 'bansheesteve', - 'BarcelonaCityRecord', - 'bargainvinyl1', - 'Basement_Sales', - 'beatdigger', -]; diff --git a/src/discogs.ts b/src/discogs.ts index d0af348..10f3e9a 100644 --- a/src/discogs.ts +++ b/src/discogs.ts @@ -3,18 +3,10 @@ import fs from 'fs/promises'; import axios from 'axios'; import cheerio from 'cheerio'; -import sellers from './../sellers'; +import { logAxiosError } from './utils'; +import { Pg } from './pg'; -export async function exists(path) { - try { - await fs.stat(path); - return true; - } catch (e) { - return false; - } -} - -export default function discogs() { +export default function discogs(pg: Pg) { const TOKEN = 'wRTXuYZGOPsIQgKRGKCAkUkidGtvpSfQgPWkYIpD'; const apiBase = 'https://api.discogs.com' const discogsBase = 'https://www.discogs.com' @@ -25,9 +17,28 @@ export default function discogs() { }, }); - async function wantlist(user: string) { - const res = await api.get(`${apiBase}/users/${user}/wants`); - return res.data.wants; + function handleApiError(e) { + logAxiosError(e); + console.log('aaaaa', e.response.status); + if (e.response.status === 429) { + console.log('bbbbbbb') + throw new Error('rate limited'); + } + } + + async function user(discogs: string) { + const user = await pg.userGetName(discogs); + if (user) return user; + console.log('fetching user', discogs); + const res = await api.get(`${apiBase}/users/${discogs}`); + return pg.userUpsert(res.data); + } + + async function wantlist(user: User): Promise { + const stored = await pg.userWants(user); + if (stored.length) return stored; + const { data: { wants, pagination }} = await api.get(`${apiBase}/users/${user.data.username}/wants`); + return Promise.all(wants.map(r => pg.wantUpsert(r, user))) } async function inventory(user: string) { @@ -35,8 +46,8 @@ export default function discogs() { return res.data.listings; } - async function listings(release: number) { - const res = await api.get(`https://www.discogs.com/sell/release/${release}`); + async function listings(want: Want): Promise { + const res = await api.get(`https://www.discogs.com/sell/release/${want.release}`); const $ = cheerio.load(res.data); const ids = $('.item_description_title[href]') .map(function getListingId() { @@ -44,22 +55,24 @@ export default function discogs() { return id; }) .get(); - return Promise.all(ids.map(id => listing(id))); + return Promise.all(ids.map(id => listing(id, want))); } - async function listing(id) { - const path = `./data/listings/${id}.json`; - if (await exists(path)) { - // console.log('cached', id); - return JSON.parse(await fs.readFile(path, { encoding: 'utf8' })); + async function listing(id, want) { + const listing = await pg.listingGet(id); + if (listing) return listing; + + try { + const res = await api.get(`${apiBase}/marketplace/listings/${id}`); + return pg.listingUpsert(res.data, want); + } catch (e) { + handleApiError(e); + return null; } - console.log('fetching', id); - const res = await api.get(`${apiBase}/marketplace/listings/${id}`); - await fs.writeFile(path, JSON.stringify(res.data, null, 4)); - return res.data; } return { + user, wantlist, inventory, listings, diff --git a/src/mcrates.d.ts b/src/mcrates.d.ts new file mode 100644 index 0000000..245b455 --- /dev/null +++ b/src/mcrates.d.ts @@ -0,0 +1,21 @@ +type UUID = string; + +type User = { + id: UUID, + discogs: string, + data: any, +} + +type Listing = { + id: number, + want: UUID, + seller: number, + data: any, +} + +type Want = { + id: UUID, + release: number, + user: UUID, + data: any, +} diff --git a/src/pg.ts b/src/pg.ts index b35921c..8b2b41d 100644 --- a/src/pg.ts +++ b/src/pg.ts @@ -1,4 +1,98 @@ -export default function createPg() { +import Knex from 'knex'; + +import config from '../knexfile'; + +export interface Pg { + userGetId: (id: UUID) => Promise, + userGetName: (name: string) => Promise, + userUpsert: (user: User) => Promise, + userWants: (user: User) => Promise, + + wantUpsert: (want: Want, user: User) => Promise, + listingUpsert: (listing: Listing, want: Want) => Promise, + listingGet: (id: number) => Promise, +} + +export default function createPg(): Pg { + const pg = Knex(config); + + async function userGetId(id): Promise { + return pg('users') + .where({ id }) + .first(); + } + + async function userGetName(name): Promise { + return pg('users') + .where({ discogs: name }) + .first(); + } + + async function userWants(user: User): Promise { + return pg('wants') + .where({ user: user.id }); + } + + async function userUpsert(user): Promise { + const [row] = await pg('users') + .insert({ + discogs: user.username, + data: user, + }) + .onConflict('id') + .merge() + .onConflict('discogs') + .merge() + .returning('*'); + + return row; + } + + async function wantUpsert(want, user): Promise { + const [row] = await pg('wants') + .insert({ + data: want, + release: want.id, + user: user.id, + }) + .onConflict('id') + .merge() + .onConflict(['user', 'release']) + .merge() + .returning('*'); + + return row; + } + + async function listingUpsert(listing, want): Promise { + const [row] = await pg('listings') + .insert({ + id: listing.id, + want: want.id, + seller: listing.seller.id, + data: listing, + }) + .onConflict('id') + .merge() + .returning('*'); + + return row; + } + + async function listingGet(id): Promise { + return pg('listings') + .where({ id }) + .first(); + } + return { + userGetId, + userGetName, + userUpsert, + userWants, + + wantUpsert, + listingUpsert, + listingGet, }; } \ No newline at end of file diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 0000000..4ef9082 --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,13 @@ +export function logAxiosError(e) { + console.warn({ + status: e.response.status, + statusText: e.response.statusText, + url: e.config.url, + data: e.response.data, + rateLimiting: { + limit: e.response.headers['x-discogs-ratelimit'], + remaining: e.response.headers['x-discogs-ratelimit-remaining'], + used: e.response.headers['x-discogs-ratelimit-used'], + }, + }); +} diff --git a/tsconfig.json b/tsconfig.json index bd71b59..ca7fcef 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -32,7 +32,7 @@ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ - // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + "types": ["./index.d.ts"], /* Specify type package names to be included without being referenced in a source file. */ // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ // "resolveJsonModule": true, /* Enable importing .json files. */