ntr/crates
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

full blown pg

Browse Source
This commit is contained in:
ntr committed 2022-07-26 00:42:47 +10:00
1 parent 245616acd4
commit dde9646179
13 files changed
+317 -169
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+1
View File
@@ -1,2 +1,3 @@
node_modules/
data/
db/
docker-compose.yaml
+2
View File
@@ -9,3 +9,5 @@ services:
POSTGRES_PASSWORD: 'dig deep'
ports:
- '45432:5432'
volumes:
- "./db:/var/lib/postgresql/data"
index.ts
+78 -28
View File
@@ -3,41 +3,91 @@ import _ from 'lodash';
import Promise from 'bluebird';
import discogsService from './src/discogs';
import pgService from './src/pg';
import { logAxiosError } from './src/utils';
async function start() {
const discogs = discogsService();
const wants = JSON.parse(await fs.readFile('./data/margh.wantlist.json', { encoding: 'utf8' }))
console.log('wants', wants.length);
const pg = pgService();
const discogs = discogsService(pg);
return Promise.map(wants, async w => {
try {
const l = await discogs.listings(w.id)
return l;
} catch (e) {
// console.error(e);
return null;
}
})
.then(listings => _
.chain(listings)
.filter(l => l)
.flatten()
.map(l => ({
id: l.id,
description: l.release.description,
seller_id: l.seller.id,
seller_name: l.seller.username,
}))
.groupBy(l => l.seller_id)
.sortBy(s => s.length)
.reverse()
.head()
.value());
const usernames = [
'margh',
'jmcdermid1',
'vanderbabe',
];
async function fetchListings(w) {
return w.map(discogs.listings)
}
async function populateUserListings(user) {
console.log(user);
const wants = await discogs.wantlist(user);
console.log(user.discogs, 'has', wants.length, 'wants');
for (const want of wants) {
const listings = await discogs.listings(want);
const populated = listings.filter(l => l);
console.log(want.id, 'has', populated.length, '/', listings.length, 'listings');
}
// const listings = await Promise
// .all(releases.map(async want => {
// const listings = (await getListings(want)).filter(l => l);
// const insertions = await Promise.all(listings.map(async l => {
// const listing = await pg.listingUpsert(l, want);
// console.log(listing.id, listing.data.release.description, want.user, listing.seller);
// return listing;
// }));
// console.log(insertions.length, want.id);
// }));
// return listings;
}
return Promise.resolve(usernames)
.map(u => discogs.user(u))
.map(populateUserListings);
// return Promise.map(releases, async w => {
// try {
// const l = await discogs.listings(w.id)
// return l;
// } catch (e) {
// // console.error(e);
// return null;
// }
// })
// .then(listings => _
// .chain(listings)
// .filter(l => l)
// .flatten()
// // .map(l => ({
// // id: l.id,
// // description: l.release.description,
// // seller_id: l.seller.id,
// // seller_name: l.seller.username,
// // }))
// // .groupBy(l => l.seller_id)
// .value()
// );
}
function run() {
start()
.then(r => console.log(r))
.then(r => {
console.log(r);
setTimeout(run, 60 * 1000);
})
.catch(e => {
console.error(e);
if (e.message === 'rate limited') {
return setTimeout(run, 60 * 1000);
}
process.exit(1);
});
}
run();
migrations/20220724114647_listings.js
-11
View File
@@ -1,11 +0,0 @@
exports.up = async knex => {
await knex.schema.createTable('listings', table => {
table.integer('id').primary();
table.timestamps(true, true);
table.jsonb('data');
});
return true;
};
exports.down = async () => {};
migrations/20220725073815_users.js
+17
View File
@@ -0,0 +1,17 @@
exports.up = async knex => {
await knex.schema.createTable('users', table => {
table.uuid('id')
.primary()
.defaultTo(knex.raw('gen_random_uuid()'));
table.string('discogs')
.unique();
table.jsonb('data')
.notNullable();
table.timestamps(true, true);
});
};
exports.down = async () => true;
migrations/20220725080947_wants.js
+27
View File
@@ -0,0 +1,27 @@
exports.up = async knex => {
await knex.schema.createTable('wants', table => {
table.uuid('id')
.primary()
.defaultTo(knex.raw('gen_random_uuid()'));
table.integer('release')
.notNullable()
.index();
table.uuid('user')
table.foreign('user')
.references('id')
.inTable('users')
.onDelete('CASCADE');
table.jsonb('data')
.notNullable();
// user can only want a release once
table.unique(['user', 'release']);
table.timestamps(true, true);
});
};
exports.down = async () => true;
migrations/20220725083125_listings.js
+23
View File
@@ -0,0 +1,23 @@
exports.up = async knex => {
await knex.schema.createTable('listings', table => {
table.integer('id')
.primary();
table.uuid('want')
table.foreign('want')
.references('id')
.inTable('wants')
.onDelete('CASCADE');
table.integer('seller')
.notNullable()
.index();
table.jsonb('data')
.notNullable();
table.timestamps(true, true);
});
};
exports.down = async () => true;
sellers.ts
-102
View File
@@ -1,102 +0,0 @@
export default [
'430AM_Studio',
'EtherWood',
'spacehall.de',
'3345vinylrecords',
'AsquithJimbo',
'gianni70',
'mutt_discogs',
'vinylline-rec.',
'alain.v2',
'DNP_Shop',
'facerecords',
'Filter_Musikk',
'Impressions_sonores',
'lofi-records',
'MEGA-MIX',
'q-logic',
'RadialStore',
'Record-cb',
'tobow',
'Wolvie-Records',
'www.hhv.de',
'1-1',
'alexkiwiwi',
'All-Day',
'camaro4504',
'caviar_lounge',
'decks.de',
'Don.Giovanni',
'DWTX',
'EXPM',
'Freakyrecords',
'hipcity',
'japanspin',
'JPsVinyl',
'KHVinylStore',
'KillaCutz',
'love-vinyl-records',
'm50',
'marygowild',
'mondo_diggers_club',
'mvc.be',
'OCCIDENTAL',
'oopu',
'Palace-Vinyl',
'purplewax23',
'rss.jk',
'rupertmuiry',
'Space_is_the_place',
'supermegarecords',
'tonvermittlung.de',
'Vinyl_Dealer_666',
'vinylminded',
'Vinylotheque',
'Werkstraat.Shop',
'xtcr',
'zionsgate',
'_records_',
'-Beathoven-',
'-N-R-G-',
'12_Inch_Music',
'12-Inch-Love',
'13loco',
'2Cows_Records',
'44ToursRecords',
'4rond',
'7Lj',
'81Renshaw',
'adriano_pls',
'AFRICABRASIL',
'afrika1974',
'Afternooner',
'AG303',
'al..x',
'alcovarecords',
'AlexDahlmann',
'Alexoj',
'alfalfap',
'alleytunesrecords',
'altarofmadness',
'Amarcord',
'ambulant',
'andrega',
'andson',
'andytonks',
'anton.kolo',
'antonydaly',
'APSMUSIC',
'Art-is-an',
'Astral_Equilibrium',
'Atrabilogie',
'AuroraBorealis87',
'avez',
'babacar88',
'Babatron',
'Bambatta_studio',
'bansheesteve',
'BarcelonaCityRecord',
'bargainvinyl1',
'Basement_Sales',
'beatdigger',
];
src/discogs.ts
+39 -26
View File
@@ -3,18 +3,10 @@ import fs from 'fs/promises';
import axios from 'axios';
import cheerio from 'cheerio';
import sellers from './../sellers';
import { logAxiosError } from './utils';
import { Pg } from './pg';
export async function exists(path) {
try {
await fs.stat(path);
return true;
} catch (e) {
return false;
}
}
export default function discogs() {
export default function discogs(pg: Pg) {
const TOKEN = 'wRTXuYZGOPsIQgKRGKCAkUkidGtvpSfQgPWkYIpD';
const apiBase = 'https://api.discogs.com'
const discogsBase = 'https://www.discogs.com'
@@ -25,9 +17,28 @@ export default function discogs() {
},
});
async function wantlist(user: string) {
const res = await api.get(`${apiBase}/users/${user}/wants`);
return res.data.wants;
function handleApiError(e) {
logAxiosError(e);
console.log('aaaaa', e.response.status);
if (e.response.status === 429) {
console.log('bbbbbbb')
throw new Error('rate limited');
}
}
async function user(discogs: string) {
const user = await pg.userGetName(discogs);
if (user) return user;
console.log('fetching user', discogs);
const res = await api.get(`${apiBase}/users/${discogs}`);
return pg.userUpsert(res.data);
}
async function wantlist(user: User): Promise<Want[]> {
const stored = await pg.userWants(user);
if (stored.length) return stored;
const { data: { wants, pagination }} = await api.get(`${apiBase}/users/${user.data.username}/wants`);
return Promise.all(wants.map(r => pg.wantUpsert(r, user)))
}
async function inventory(user: string) {
@@ -35,8 +46,8 @@ export default function discogs() {
return res.data.listings;
}
async function listings(release: number) {
const res = await api.get(`https://www.discogs.com/sell/release/${release}`);
async function listings(want: Want): Promise<Listing[]> {
const res = await api.get(`https://www.discogs.com/sell/release/${want.release}`);
const $ = cheerio.load(res.data);
const ids = $('.item_description_title[href]')
.map(function getListingId() {
@@ -44,22 +55,24 @@ export default function discogs() {
return id;
})
.get();
return Promise.all(ids.map(id => listing(id)));
return Promise.all(ids.map(id => listing(id, want)));
}
async function listing(id) {
const path = `./data/listings/${id}.json`;
if (await exists(path)) {
// console.log('cached', id);
return JSON.parse(await fs.readFile(path, { encoding: 'utf8' }));
}
console.log('fetching', id);
async function listing(id, want) {
const listing = await pg.listingGet(id);
if (listing) return listing;
try {
const res = await api.get(`${apiBase}/marketplace/listings/${id}`);
await fs.writeFile(path, JSON.stringify(res.data, null, 4));
return res.data;
return pg.listingUpsert(res.data, want);
} catch (e) {
handleApiError(e);
return null;
}
}
return {
user,
wantlist,
inventory,
listings,
src/mcrates.d.ts
Vendored
+21
View File
@@ -0,0 +1,21 @@
type UUID = string;
type User = {
id: UUID,
discogs: string,
data: any,
}
type Listing = {
id: number,
want: UUID,
seller: number,
data: any,
}
type Want = {
id: UUID,
release: number,
user: UUID,
data: any,
}
src/pg.ts
+95 -1
View File
@@ -1,4 +1,98 @@
export default function createPg() {
import Knex from 'knex';
import config from '../knexfile';
export interface Pg {
userGetId: (id: UUID) => Promise<User>,
userGetName: (name: string) => Promise<User>,
userUpsert: (user: User) => Promise<User>,
userWants: (user: User) => Promise<Want[]>,
wantUpsert: (want: Want, user: User) => Promise<Want>,
listingUpsert: (listing: Listing, want: Want) => Promise<Listing>,
listingGet: (id: number) => Promise<Listing>,
}
export default function createPg(): Pg {
const pg = Knex(config);
async function userGetId(id): Promise<User> {
return pg('users')
.where({ id })
.first();
}
async function userGetName(name): Promise<User> {
return pg('users')
.where({ discogs: name })
.first();
}
async function userWants(user: User): Promise<Want[]> {
return pg('wants')
.where({ user: user.id });
}
async function userUpsert(user): Promise<User> {
const [row] = await pg('users')
.insert({
discogs: user.username,
data: user,
})
.onConflict('id')
.merge()
.onConflict('discogs')
.merge()
.returning('*');
return row;
}
async function wantUpsert(want, user): Promise<Want> {
const [row] = await pg('wants')
.insert({
data: want,
release: want.id,
user: user.id,
})
.onConflict('id')
.merge()
.onConflict(['user', 'release'])
.merge()
.returning('*');
return row;
}
async function listingUpsert(listing, want): Promise<Listing> {
const [row] = await pg('listings')
.insert({
id: listing.id,
want: want.id,
seller: listing.seller.id,
data: listing,
})
.onConflict('id')
.merge()
.returning('*');
return row;
}
async function listingGet(id): Promise<Listing> {
return pg('listings')
.where({ id })
.first();
}
return {
userGetId,
userGetName,
userUpsert,
userWants,
wantUpsert,
listingUpsert,
listingGet,
};
}
src/utils.ts
+13
View File
@@ -0,0 +1,13 @@
export function logAxiosError(e) {
console.warn({
status: e.response.status,
statusText: e.response.statusText,
url: e.config.url,
data: e.response.data,
rateLimiting: {
limit: e.response.headers['x-discogs-ratelimit'],
remaining: e.response.headers['x-discogs-ratelimit-remaining'],
used: e.response.headers['x-discogs-ratelimit-used'],
},
});
}
tsconfig.json
+1 -1
View File
@@ -32,7 +32,7 @@
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
// "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */
// "types": [], /* Specify type package names to be included without being referenced in a source file. */
"types": ["./index.d.ts"], /* Specify type package names to be included without being referenced in a source file. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
// "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */
// "resolveJsonModule": true, /* Enable importing .json files. */