full blown pg

.gitignore

@@ -1,2 +1,3 @@
 node_modules/
-data/
+data/
+db/

docker-compose.yaml

@@ -9,3 +9,5 @@ services:
       POSTGRES_PASSWORD: 'dig deep'
     ports:
       - '45432:5432'
+    volumes:
+      - "./db:/var/lib/postgresql/data"

index.ts

@@ -3,41 +3,91 @@ import _ from 'lodash';
 import Promise from 'bluebird';
 
 import discogsService from './src/discogs';
+import pgService from './src/pg';
+import { logAxiosError } from './src/utils';
 
 async function start() {
-    const discogs = discogsService();
+    const pg = pgService();
-    const wants = JSON.parse(await fs.readFile('./data/margh.wantlist.json', { encoding: 'utf8' }))
+    const discogs = discogsService(pg);
-    console.log('wants', wants.length);
 
-    return Promise.map(wants, async w => {
+    const usernames = [
-            try {
+        'margh',
-                const l = await discogs.listings(w.id)
+        'jmcdermid1',
-                return l;
+        'vanderbabe',
-            } catch (e) {
+    ];
-                // console.error(e);
+
-                return null;
+    async function fetchListings(w) {
-            }
+        return w.map(discogs.listings)
-        })
+    }
-        .then(listings => _
+
-        .chain(listings)
+    async function populateUserListings(user) {
-        .filter(l => l)
+        console.log(user);
-        .flatten()
+        const wants = await discogs.wantlist(user);
-        .map(l => ({
+        console.log(user.discogs, 'has', wants.length, 'wants');
-            id: l.id,
+
-            description: l.release.description,
+        for (const want of wants) {
-            seller_id: l.seller.id,
+            const listings = await discogs.listings(want);
-            seller_name: l.seller.username,
+            const populated = listings.filter(l => l);
-        }))
+            console.log(want.id, 'has', populated.length, '/', listings.length, 'listings');
-        .groupBy(l => l.seller_id)
+        }
-        .sortBy(s => s.length)
+
-        .reverse()
+        // const listings = await Promise
-        .head()
+        //     .all(releases.map(async want => {
-        .value());
+        //         const listings = (await getListings(want)).filter(l => l);
+
+        //         const insertions = await Promise.all(listings.map(async l => {
+        //             const listing = await pg.listingUpsert(l, want);
+        //             console.log(listing.id, listing.data.release.description, want.user, listing.seller);
+        //             return listing;
+        //         }));
+
+        //         console.log(insertions.length, want.id);
+        //     }));
+
+        // return listings;
+    }
+
+    return Promise.resolve(usernames)
+        .map(u => discogs.user(u))
+        .map(populateUserListings);
+
+    // return Promise.map(releases, async w => {
+    //         try {
+    //             const l = await discogs.listings(w.id)
+    //             return l;
+    //         } catch (e) {
+    //             // console.error(e);
+    //             return null;
+    //         }
+    //     })
+    //     .then(listings => _
+    //     .chain(listings)
+    //     .filter(l => l)
+    //     .flatten()
+    //     // .map(l => ({
+    //     //     id: l.id,
+    //     //     description: l.release.description,
+    //     //     seller_id: l.seller.id,
+    //     //     seller_name: l.seller.username,
+    //     // }))
+    //     // .groupBy(l => l.seller_id)
+    //     .value()
+    // );
 }
 
-start()
+function run() {
-    .then(r => console.log(r))
+    start()
-    .catch(e => {
+        .then(r => {
-        console.error(e);
+            console.log(r);
-        process.exit(1);
+            setTimeout(run, 60 * 1000);
-    });
+        })
+        .catch(e => {
+            console.error(e);
+            if (e.message === 'rate limited') {
+                return setTimeout(run, 60 * 1000);
+            }
+            process.exit(1);
+        });
+}
+
+run();

migrations/20220724114647_listings.js

@@ -1,11 +0,0 @@
-exports.up = async knex => {
-    await knex.schema.createTable('listings', table => {
-        table.integer('id').primary();
-        table.timestamps(true, true);
-        table.jsonb('data');
-    });
-
-    return true;
-};
-
-exports.down = async () => {};

migrations/20220725073815_users.js

@@ -0,0 +1,17 @@
+exports.up = async knex => {
+    await knex.schema.createTable('users', table => {
+        table.uuid('id')
+            .primary()
+            .defaultTo(knex.raw('gen_random_uuid()'));
+
+        table.string('discogs')
+            .unique();
+
+        table.jsonb('data')
+            .notNullable();
+
+        table.timestamps(true, true);
+    });
+};
+
+exports.down = async () => true;

migrations/20220725080947_wants.js

@@ -0,0 +1,27 @@
+exports.up = async knex => {
+    await knex.schema.createTable('wants', table => {
+        table.uuid('id')
+            .primary()
+            .defaultTo(knex.raw('gen_random_uuid()'));
+
+        table.integer('release')
+            .notNullable()
+            .index();
+
+        table.uuid('user')
+        table.foreign('user')
+            .references('id')
+            .inTable('users')
+            .onDelete('CASCADE');
+
+        table.jsonb('data')
+            .notNullable();
+
+        // user can only want a release once
+        table.unique(['user', 'release']);
+
+        table.timestamps(true, true);
+    });
+};
+
+exports.down = async () => true;

migrations/20220725083125_listings.js

@@ -0,0 +1,23 @@
+exports.up = async knex => {
+    await knex.schema.createTable('listings', table => {
+        table.integer('id')
+            .primary();
+
+        table.uuid('want')
+        table.foreign('want')
+            .references('id')
+            .inTable('wants')
+            .onDelete('CASCADE');
+
+        table.integer('seller')
+            .notNullable()
+            .index();
+
+        table.jsonb('data')
+            .notNullable();
+
+        table.timestamps(true, true);
+    });
+};
+
+exports.down = async () => true;

sellers.ts

@@ -1,102 +0,0 @@
-export default [
-    '430AM_Studio',
-    'EtherWood',
-    'spacehall.de',
-    '3345vinylrecords',
-    'AsquithJimbo',
-    'gianni70',
-    'mutt_discogs',
-    'vinylline-rec.',
-    'alain.v2',
-    'DNP_Shop',
-    'facerecords',
-    'Filter_Musikk',
-    'Impressions_sonores',
-    'lofi-records',
-    'MEGA-MIX',
-    'q-logic',
-    'RadialStore',
-    'Record-cb',
-    'tobow',
-    'Wolvie-Records',
-    'www.hhv.de',
-    '1-1',
-    'alexkiwiwi',
-    'All-Day',
-    'camaro4504',
-    'caviar_lounge',
-    'decks.de',
-    'Don.Giovanni',
-    'DWTX',
-    'EXPM',
-    'Freakyrecords',
-    'hipcity',
-    'japanspin',
-    'JPsVinyl',
-    'KHVinylStore',
-    'KillaCutz',
-    'love-vinyl-records',
-    'm50',
-    'marygowild',
-    'mondo_diggers_club',
-    'mvc.be',
-    'OCCIDENTAL',
-    'oopu',
-    'Palace-Vinyl',
-    'purplewax23',
-    'rss.jk',
-    'rupertmuiry',
-    'Space_is_the_place',
-    'supermegarecords',
-    'tonvermittlung.de',
-    'Vinyl_Dealer_666',
-    'vinylminded',
-    'Vinylotheque',
-    'Werkstraat.Shop',
-    'xtcr',
-    'zionsgate',
-    '_records_',
-    '-Beathoven-',
-    '-N-R-G-',
-    '12_Inch_Music',
-    '12-Inch-Love',
-    '13loco',
-    '2Cows_Records',
-    '44ToursRecords',
-    '4rond',
-    '7Lj',
-    '81Renshaw',
-    'adriano_pls',
-    'AFRICABRASIL',
-    'afrika1974',
-    'Afternooner',
-    'AG303',
-    'al..x',
-    'alcovarecords',
-    'AlexDahlmann',
-    'Alexoj',
-    'alfalfap',
-    'alleytunesrecords',
-    'altarofmadness',
-    'Amarcord',
-    'ambulant',
-    'andrega',
-    'andson',
-    'andytonks',
-    'anton.kolo',
-    'antonydaly',
-    'APSMUSIC',
-    'Art-is-an',
-    'Astral_Equilibrium',
-    'Atrabilogie',
-    'AuroraBorealis87',
-    'avez',
-    'babacar88',
-    'Babatron',
-    'Bambatta_studio',
-    'bansheesteve',
-    'BarcelonaCityRecord',
-    'bargainvinyl1',
-    'Basement_Sales',
-    'beatdigger',
-];

src/discogs.ts

@@ -3,18 +3,10 @@ import fs from 'fs/promises';
 import axios from 'axios';
 import cheerio from 'cheerio';
 
-import sellers from './../sellers';
+import { logAxiosError } from './utils';
+import { Pg } from './pg';
 
-export async function exists(path) {
+export default function discogs(pg: Pg) {
-    try {
-        await fs.stat(path);
-        return true;
-    } catch (e) {
-        return false;
-    }
-}
-
-export default function discogs() {
     const TOKEN = 'wRTXuYZGOPsIQgKRGKCAkUkidGtvpSfQgPWkYIpD';
     const apiBase = 'https://api.discogs.com'
     const discogsBase = 'https://www.discogs.com'
@@ -25,9 +17,28 @@ export default function discogs() {
         },
     });
 
-    async function wantlist(user: string) {
+    function handleApiError(e) {
-        const res = await api.get(`${apiBase}/users/${user}/wants`);
+        logAxiosError(e);
-        return res.data.wants;
+        console.log('aaaaa', e.response.status);
+        if (e.response.status === 429) {
+            console.log('bbbbbbb')
+            throw new Error('rate limited');
+        }
+    }
+
+    async function user(discogs: string) {
+        const user = await pg.userGetName(discogs);
+        if (user) return user;
+        console.log('fetching user', discogs);
+        const res = await api.get(`${apiBase}/users/${discogs}`);
+        return pg.userUpsert(res.data);
+    }
+
+    async function wantlist(user: User): Promise<Want[]> {
+        const stored = await pg.userWants(user);
+        if (stored.length) return stored;
+        const { data: { wants, pagination }} = await api.get(`${apiBase}/users/${user.data.username}/wants`);
+        return Promise.all(wants.map(r => pg.wantUpsert(r, user)))
     }
 
     async function inventory(user: string) {
@@ -35,8 +46,8 @@ export default function discogs() {
         return res.data.listings;
     }
 
-    async function listings(release: number) {
+    async function listings(want: Want): Promise<Listing[]> {
-        const res = await api.get(`https://www.discogs.com/sell/release/${release}`);
+        const res = await api.get(`https://www.discogs.com/sell/release/${want.release}`);
         const $ = cheerio.load(res.data);
         const ids = $('.item_description_title[href]')
             .map(function getListingId() {
@@ -44,22 +55,24 @@ export default function discogs() {
                 return id;
             })
             .get();
-        return Promise.all(ids.map(id => listing(id)));
+        return Promise.all(ids.map(id => listing(id, want)));
     }
 
-    async function listing(id) {
+    async function listing(id, want) {
-        const path = `./data/listings/${id}.json`;
+        const listing = await pg.listingGet(id);
-        if (await exists(path)) {
+        if (listing) return listing;
-            // console.log('cached', id);
+
-            return JSON.parse(await fs.readFile(path, { encoding: 'utf8' }));
+        try {
+            const res = await api.get(`${apiBase}/marketplace/listings/${id}`);
+            return pg.listingUpsert(res.data, want);
+        } catch (e) {
+            handleApiError(e);
+            return null;
         }
-        console.log('fetching', id);
-        const res = await api.get(`${apiBase}/marketplace/listings/${id}`);
-        await fs.writeFile(path, JSON.stringify(res.data, null, 4));
-        return res.data;
     }
 
     return {
+        user,
         wantlist,
         inventory,
         listings,

src/mcrates.d.ts

@@ -0,0 +1,21 @@
+type UUID = string;
+
+type User = {
+    id: UUID,
+    discogs: string,
+    data: any,
+}
+
+type Listing = {
+    id: number,
+    want: UUID,
+    seller: number,
+    data: any,
+}
+
+type Want = {
+    id: UUID,
+    release: number,
+    user: UUID,
+    data: any,
+}

src/pg.ts

@@ -1,4 +1,98 @@
-export default function createPg() {
+import Knex from 'knex';
+
+import config from '../knexfile';
+
+export interface Pg {
+    userGetId: (id: UUID) => Promise<User>,
+    userGetName: (name: string) => Promise<User>,
+    userUpsert: (user: User) => Promise<User>,
+    userWants: (user: User) => Promise<Want[]>,
+
+    wantUpsert: (want: Want, user: User) => Promise<Want>,
+    listingUpsert: (listing: Listing, want: Want) => Promise<Listing>,
+    listingGet: (id: number) => Promise<Listing>,
+}
+
+export default function createPg(): Pg {
+    const pg = Knex(config);
+
+    async function userGetId(id): Promise<User> {
+        return pg('users')
+            .where({ id })
+            .first();
+    }
+
+    async function userGetName(name): Promise<User> {
+        return pg('users')
+            .where({ discogs: name })
+            .first();
+    }
+
+    async function userWants(user: User): Promise<Want[]> {
+        return pg('wants')
+            .where({ user: user.id });
+    }
+
+    async function userUpsert(user): Promise<User> {
+        const [row] = await pg('users')
+            .insert({
+                discogs: user.username,
+                data: user,
+            })
+            .onConflict('id')
+            .merge()
+            .onConflict('discogs')
+            .merge()
+            .returning('*');
+
+        return row;
+    }
+
+    async function wantUpsert(want, user): Promise<Want> {
+        const [row] = await pg('wants')
+            .insert({
+                data: want,
+                release: want.id,
+                user: user.id,
+            })
+            .onConflict('id')
+            .merge()
+            .onConflict(['user', 'release'])
+            .merge()
+            .returning('*');
+
+        return row;
+    }
+
+    async function listingUpsert(listing, want): Promise<Listing> {
+        const [row] = await pg('listings')
+            .insert({
+                id: listing.id,
+                want: want.id,
+                seller: listing.seller.id,
+                data: listing,
+            })
+            .onConflict('id')
+            .merge()
+            .returning('*');
+
+        return row;
+    }
+
+    async function listingGet(id): Promise<Listing> {
+        return pg('listings')
+            .where({ id })
+            .first();
+    }
+
     return {
+        userGetId,
+        userGetName,
+        userUpsert,
+        userWants,
+
+        wantUpsert,
+        listingUpsert,
+        listingGet,
     };
 }

src/utils.ts

@@ -0,0 +1,13 @@
+export function logAxiosError(e) {
+    console.warn({
+        status: e.response.status,
+        statusText: e.response.statusText,
+        url: e.config.url,
+        data: e.response.data,
+        rateLimiting: {
+            limit: e.response.headers['x-discogs-ratelimit'],
+            remaining: e.response.headers['x-discogs-ratelimit-remaining'],
+            used: e.response.headers['x-discogs-ratelimit-used'],
+        },
+    });
+}

tsconfig.json

@@ -32,7 +32,7 @@
     // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
     // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
     // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
-    // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
+    "types": ["./index.d.ts"],                                      /* Specify type package names to be included without being referenced in a source file. */
     // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
     // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
     // "resolveJsonModule": true,                        /* Enable importing .json files. */