client init

server/.env

@@ -0,0 +1,7 @@
+# Environment variables declared in this file are automatically made available to Prisma.
+# See the documentation for more detail: https://pris.ly/d/prisma-schema#accessing-environment-variables-from-the-schema
+
+# Prisma supports the native connection string format for PostgreSQL, MySQL, SQLite, SQL Server, MongoDB and CockroachDB.
+# See the documentation for all the connection string options: https://pris.ly/d/connection-strings
+
+DATABASE_URL="postgresql://mcrates:digdeep@0.0.0.0:45432/mcrates?schema=public"

server/README.md

@@ -0,0 +1,6 @@
+listings
+    doc
+    release
+
+wants
+    

server/docker-compose.yaml

@@ -0,0 +1,13 @@
+version: "3.3"
+
+services:
+  pg:
+    image: postgres:latest
+    environment:
+      POSTGRES_USER: 'mcrates'
+      POSTGRES_DB: 'mcrates'
+      POSTGRES_PASSWORD: 'digdeep'
+    ports:
+      - '45432:5432'
+    volumes:
+      - "./db:/var/lib/postgresql/data"

server/global.d.ts

@@ -0,0 +1,14 @@
+// global type aliases and whatnot
+
+type UUID = string;
+
+type DiscogsName = string;
+type DiscogsListing = number;
+type DiscogsSeller = number;
+
+type BundleCandidate = {
+    seller: DiscogsSeller,
+    count: number,
+    listings: DiscogsListing[],
+}
+

server/index.ts

@@ -0,0 +1,99 @@
+import fs from 'fs/promises';
+import _ from 'lodash';
+import Bunyan from 'bunyan';
+import { PrismaClient } from '@prisma/client'
+import Promise from "bluebird";
+
+import discogsService from './src/discogs';
+import pgService from './src/pg';
+import { logAxiosError } from './src/utils';
+import createLog from './src/log';
+
+async function start(log) {
+    const prisma = new PrismaClient();
+    const pg = pgService(log, prisma);
+    const discogs = discogsService(log, pg);
+
+    async function getBundles() {
+        const bundles = await pg.bundleCandidates();
+
+        bundles.forEach(b => {
+            const data = b.map(l => l.data) as any[];
+            const wants = b.map(l => l.wantId);
+            const total = data.reduce((acc, l) => acc + l.price.value, 0);
+            const items = data.map(l => ({
+                id: l.id,
+                description: l.release.description,
+                price: l.price.value,
+                condition: l.condition,
+                uri: l.uri,
+            }));
+
+            const username = data[0]?.seller.username
+            const seller = data[0]?.seller.id
+            const shipsFrom = data[0]?.ships_from;
+            const currency = data[0]?.price.currency;
+            const shipping = data[0]?.seller.shipping;
+            const mentionsAustralia = shipping.toLowerCase().includes('australia');
+            const mentionsWorldwide = shipping.toLowerCase().includes('world');
+            log.info({ count: b.length, currency, total, items, username, seller, shipsFrom, mentionsAustralia, mentionsWorldwide });
+        });
+
+        // return bundles;
+    }
+
+
+    const usernames = [
+        'vanderbabe',
+        'margh',
+        'jmcdermid1',
+    ];
+
+    async function populateUserListings(user) {
+        log.info(user);
+        const wants = await discogs.wantlist(user);
+
+        // const latest = await discogs.wantlistLatest(user);
+        // if (wants.length !== latest.length)
+        log.info({ username: user.discogs, wantCount: wants.length });
+
+        return Promise.mapSeries(wants, async want => {
+            const listings = await discogs.listings(want);
+            const populated = listings.filter(l => l);
+            log.info({ id: want.id, release: want.release, populated: populated.length, total: listings.length });
+            return populated;
+        });
+    }
+
+    await Promise.mapSeries(usernames, async u => {
+        const user = await discogs.user(u)
+        return populateUserListings(user);
+    });
+
+    return getBundles();
+}
+
+async function run() {
+    const log = createLog();
+
+    process.on('SIGTERM', () => {
+      log.debug('SIGTERM received, exiting');
+      process.exit(0);
+    });
+
+    try {
+        const r = await start(log)
+        // log.info('done syncing', { totalListings: r.flat().length });
+        log.info(r);
+        setTimeout(run, 60 * 1000);
+    } catch (e) {
+        if (e.message === 'rate limited') {
+            log.info('rate limited, trying again in 60s');
+            return setTimeout(run, 60 * 1000);
+        }
+        log.error(e);
+        process.exit(1);
+    }
+}
+
+run();

server/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "mates-crates",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "migrate": "knex migrate:latest",
+    "migrate:make": "prisma migrate dev --name",
+    "start": "ts-node-dev --respawn --transpile-only index.ts | bunyan -o short"
+  },
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@types/bluebird": "^3.5.36",
+    "node-dev": "^7.4.3",
+    "prisma": "^4.1.0",
+    "ts-node-dev": "^2.0.0",
+    "typescript": "^4.7.4"
+  },
+  "dependencies": {
+    "@prisma/client": "^4.1.0",
+    "axios": "^0.27.2",
+    "bluebird": "^3.7.2",
+    "bunyan": "^1.8.15",
+    "express": "^4.18.1",
+    "lodash": "^4.17.21"
+  }
+}

server/prisma/schema.prisma

@@ -0,0 +1,51 @@
+generator client {
+    provider = "prisma-client-js"
+    previewFeatures = ["interactiveTransactions"]
+}
+
+datasource db {
+    provider = "postgresql"
+    url      = env("DATABASE_URL")
+}
+
+model Listing {
+    id         Int      @id
+    seller     Int
+    data       Json
+
+    wantId     String   @db.Uuid
+    want       Want     @relation(fields: [wantId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+
+    createdAt  DateTime @default(now())
+    updatedAt  DateTime @updatedAt
+
+    @@index([seller])
+}
+
+model User {
+    id         String     @id       @default(dbgenerated("gen_random_uuid()")) @db.Uuid
+    discogs    String?    @unique
+    data       Json
+
+    createdAt  DateTime   @default(now())
+    updatedAt  DateTime   @updatedAt
+
+    wants      Want[]
+}
+
+model Want {
+    id         String     @id       @default(dbgenerated("gen_random_uuid()")) @db.Uuid
+    release    Int?
+    data       Json
+
+    userId     String     @db.Uuid
+    user       User       @relation(fields: [userId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+
+    createdAt  DateTime   @default(now())
+    updatedAt  DateTime   @updatedAt
+
+    listings   Listing[]
+
+    @@unique([userId, release])
+    @@index([release])
+}

server/src/discogs.ts

@@ -0,0 +1,95 @@
+import fs from 'fs/promises';
+
+import axios from 'axios';
+import cheerio from 'cheerio';
+import Bluebird from 'bluebird';
+
+import type Log from 'bunyan';
+import type { PrismaClient, Want, User, Listing } from '@prisma/client';
+
+import { logAxiosError } from './utils';
+import { Pg } from './pg';
+
+export default function discogs(log: Log, pg: Pg) {
+    const TOKEN = 'wRTXuYZGOPsIQgKRGKCAkUkidGtvpSfQgPWkYIpD';
+    const apiBase = 'https://api.discogs.com'
+    const discogsBase = 'https://www.discogs.com'
+
+    const api = axios.create({
+        headers: {
+            'Authorization': TOKEN,
+            // discogs only returns the first page without a desktop u/a
+            'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0',
+        },
+    });
+
+    async function handleApiError(e) {
+        logAxiosError(e);
+        if (e.response.status === 429) {
+            throw new Error('rate limited');
+        }
+        return null;
+    }
+
+    async function user(discogs: string) {
+        const user = await pg.userFindName(discogs);
+        if (user) return user;
+        log.debug('fetching user', discogs);
+        const res = await api.get(`${apiBase}/users/${discogs}`);
+        return pg.userUpsert(res.data);
+    }
+
+    async function wantlist(user: User): Promise<Want[]> {
+        const stored = await pg.userWants(user);
+        if (stored.length) return stored;
+        return wantlistLatest(user);
+    }
+
+    async function wantlistLatest(user: User): Promise<Want[]> {
+        const { username } = user.data as { username: string };
+        const { data: { wants, pagination }} = await api.get(`${apiBase}/users/${username}/wants`);
+        return Bluebird.mapSeries(wants, async r => pg.wantUpsert(r, user));
+    }
+
+    async function inventory(user: string) {
+        const res = await api.get(`${apiBase}/users/${user}/inventory`);
+        return res.data.listings;
+    }
+
+    async function listings(want: Want) {
+        const res = await api.get(`https://www.discogs.com/sell/release/${want.release}?limit=250`);
+        const $ = cheerio.load(res.data);
+        const ids = $('.item_description_title[href]')
+            .map(function getListingId() {
+                const id = Number.parseInt(this.attribs.href.replace('/sell/item/', ''), 10);
+                return id;
+            })
+            .get();
+
+        const unlisted = await pg.listingRmUnlisted(want, ids);
+        if (unlisted) log.info({ wantId: want.id, unlisted: unlisted }, 'missing listings removed');
+        return Bluebird.mapSeries(ids, async id => listing(id, want));
+    }
+
+    async function listing(id, want) {
+        const listing = await pg.listingFind(id);
+        if (listing) return listing;
+
+        try {
+            log.debug('fetching listing', id);
+            const res = await api.get(`${apiBase}/marketplace/listings/${id}`);
+            return pg.listingUpsert(res.data, want);
+        } catch (e) {
+            return handleApiError(e);
+        }
+    }
+
+    return {
+        user,
+        wantlist,
+        wantlistLatest,
+        inventory,
+        listings,
+        listing,
+    };
+}

server/src/log.ts

@@ -0,0 +1,10 @@
+import bunyan from 'bunyan';
+
+export default function createLog() {
+    const log = bunyan.createLogger({
+      name: 'mcrates',
+      stream: process.stdout
+    });
+
+    return log;
+}

server/src/pg.ts

@@ -0,0 +1,138 @@
+import type Log from 'bunyan';
+import type { PrismaClient, Want, User, Listing, Prisma } from '@prisma/client';
+import * as Bluebird from 'bluebird';
+
+export interface Pg {
+    userFindId:         (id: UUID) => Promise<User>,
+    userFindName:       (name: DiscogsName) => Promise<User>,
+    userUpsert:         (user: User) => Promise<User>,
+    userWants:          (user: User) => Promise<Want[]>,
+
+    wantUpsert:         (want: Want, user: User) => Promise<Want>,
+    listingUpsert:      (listing: Listing, want: Want) => Promise<Listing>,
+    listingFind:        (id: DiscogsListing) => Promise<Listing>,
+    listingRmUnlisted:  (want: Want, ids: DiscogsListing[]) => Promise<number>,
+
+    bundleCandidates:   () => Promise<Listing[][]>,
+}
+
+export default function createPg(log: Log, prisma: PrismaClient): Pg {
+    async function userFindId(id) {
+        return prisma.user.findUnique({ where: { id }});
+    }
+
+    async function userFindName(name: DiscogsName) {
+        return prisma.user.findUnique({ where: { discogs: name }});
+    }
+
+    async function userWants(user: User) {
+        return prisma.want.findMany({ where: { userId: user.id }});
+    }
+
+    async function userUpsert(user) {
+        const data = {
+            discogs: user.username,
+            data: user as Prisma.JsonObject,
+        };
+
+        return prisma.user.upsert({
+          where: {
+            discogs: user.username,
+          },
+          update: data,
+          create: data,
+        });
+    }
+
+    async function wantUpsert(dcWant, user) {
+        const values: { userId: UUID, release: DiscogsListing, data: any } = {
+            data: dcWant,
+            release: dcWant.id,
+            userId: user.id,
+        };
+
+        return prisma.want.upsert({
+            // get these from the generated types
+            // ./node_modules/.prisma/client/index.d.ts
+            where: { userId_release: {
+                userId: values.userId,
+                release: values.release || null,
+            }},
+            update: values,
+            create: values,
+        });
+    }
+
+    async function listingRmUnlisted(want: Want, ids: DiscogsListing[]) {
+        const { count } = await prisma.listing.deleteMany({
+            where: {
+                wantId: want.id,
+                id: {
+                    notIn: ids,
+                }
+            }
+        });
+
+        return count;
+    }
+
+    async function listingUpsert(listing, want) {
+        const values = {
+            id: listing.id,
+            wantId: want.id,
+            seller: listing.seller.id,
+            data: listing,
+        };
+
+        return prisma.listing.upsert({
+            where: {
+                id: listing.id
+            },
+            update: values,
+            create: values,
+        });
+    }
+
+    async function listingFind(id: DiscogsListing) {
+        return prisma.listing.findUnique({
+            where: {
+                id,
+            }
+        });
+    }
+
+    async function bundleCandidates(): Promise<Listing[][]> {
+        const candidates = await prisma.$queryRaw`
+            select
+                seller, count(distinct("wantId")), array_agg(id) as listings
+            from
+                "Listing"
+            group by
+                seller
+            order by
+                count desc
+            limit 5;
+        ` as BundleCandidate[];
+
+        return Promise.all(candidates.map(async c => prisma.listing.findMany({
+            where: {
+                id: { in: c.listings }
+            },
+            distinct: ['wantId'],
+        })));
+    }
+
+    return {
+        userFindId,
+        userFindName,
+        userUpsert,
+        userWants,
+
+        wantUpsert,
+        listingUpsert,
+        listingFind,
+        listingRmUnlisted,
+
+        bundleCandidates,
+    };
+}

server/src/utils.ts

@@ -0,0 +1,13 @@
+export function logAxiosError(e) {
+    console.warn({
+        status: e.response.status,
+        statusText: e.response.statusText,
+        url: e.config.url,
+        data: e.response.data,
+        rateLimiting: {
+            limit: e.response.headers['x-discogs-ratelimit'],
+            remaining: e.response.headers['x-discogs-ratelimit-remaining'],
+            used: e.response.headers['x-discogs-ratelimit-used'],
+        },
+    });
+}

server/tsconfig.json

@@ -0,0 +1,105 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig to read more about this file */
+
+    /* Projects */
+    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
+    // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
+    // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
+    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
+    // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
+    // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
+
+    /* Language and Environment */
+    "target": "es2020",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
+    // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+    // "jsx": "preserve",                                /* Specify what JSX code is generated. */
+    // "experimentalDecorators": true,                   /* Enable experimental support for TC39 stage 2 draft decorators. */
+    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
+    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
+    // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
+    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
+    // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
+    // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
+    // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
+    // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
+
+    /* Modules */
+    "module": "commonjs",                                /* Specify what module code is generated. */
+    // "rootDir": "./",                                  /* Specify the root folder within your source files. */
+    "moduleResolution": "node",                       /* Specify how TypeScript looks up a file from a given module specifier. */
+    // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
+    // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
+    // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
+    // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
+    "types": [
+      "./index.d.ts"
+    ],                                      /* Specify type package names to be included without being referenced in a source file. */
+    // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
+    // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
+    // "resolveJsonModule": true,                        /* Enable importing .json files. */
+    // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
+
+    /* JavaScript Support */
+    // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
+    // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
+    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
+
+    /* Emit */
+    // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
+    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
+    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
+    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
+    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
+    // "outDir": "./",                                   /* Specify an output folder for all emitted files. */
+    // "removeComments": true,                           /* Disable emitting comments. */
+    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
+    // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
+    // "importsNotUsedAsValues": "remove",               /* Specify emit/checking behavior for imports that are only used for types. */
+    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
+    // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
+    // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
+    // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
+    // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
+    // "newLine": "crlf",                                /* Set the newline character for emitting files. */
+    // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
+    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
+    // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
+    // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
+    // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
+    // "preserveValueImports": true,                     /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
+
+    /* Interop Constraints */
+    // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
+    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
+    "esModuleInterop": true,                             /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
+    // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
+    "forceConsistentCasingInFileNames": true,            /* Ensure that casing is correct in imports. */
+
+    /* Type Checking */
+    "strict": false,                                      /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
+    // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
+    // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
+    // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
+    // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
+    // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
+    // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
+    // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
+    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
+    // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
+    // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
+    // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
+    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
+    // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
+    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
+    // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
+    // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
+
+    /* Completeness */
+    // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
+    "skipLibCheck": true                                 /* Skip type checking all .d.ts files. */
+  }
+}