posted at for sorting

.gitignore

@@ -2,6 +2,7 @@
 
 .venv/
 gimmeme-d052de2b9472.json
+gimmeme-db7225b3cf31.json
 __pycache__/
 
 node_modules

drain.ts

@@ -1,31 +0,0 @@
-import base58 from "bs58";
-import "dotenv/config"
-import * as helpers  from "@solana-developers/helpers";
-import * as web3 from "@solana/web3.js";
-
-const emoKey = helpers.getKeypairFromEnvironment("EMO");
-const emoAddr = new web3.PublicKey(emoKey.publicKey);
-
-const ntrKey = helpers.getKeypairFromEnvironment("NTR");
-const ntrAddr = new web3.PublicKey(ntrKey.publicKey);
-
-const payer = emoKey
-const connection = new web3.Connection(web3.clusterApiUrl("devnet"));
-
-const balance = await connection.getBalance(emoAddr);
-
-const tx = new web3.Transaction();
-const TX_COST = 5000;
-
-const instr = web3.SystemProgram.transfer({
-    fromPubkey: emoAddr,
-    toPubkey: ntrAddr,
-    lamports: balance - TX_COST,
-});
-
-tx.add(instr);
-
-const sig = await web3.sendAndConfirmTransaction(connection, tx, [emoKey]);
-
-console.log(sig);
-

index.ts

@@ -1,40 +1,56 @@
 import "dotenv/config"
 
+import fs from 'fs/promises'
+
+import pino from 'pino'
+
 import * as Minio from 'minio'
 import Vision from '@google-cloud/vision'
 import { PrismaClient } from '@prisma/client'
+import { MeiliSearch } from 'meilisearch'
 
 import discordScraper from './src/scraper'
+import testAttachment from "./src/test.attachment"
+import attachmentService from "./src/attachments"
+import memeService from "./src/memes"
+
+// @ts-ignore - ts don't know wtf it talking about
+BigInt.prototype.toJSON = function bigIntStringify() {
+    return this.toString();
+};
 
 const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default
-const STORAGE_BUCKET = process.env.STORAGE_BUCKET
 
+const log = pino({ base: {} });
+
+const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
+const prisma = new PrismaClient()
 const minio = new Minio.Client({
     endPoint: process.env.STORAGE_URL,
     accessKey: process.env.STORAGE_ACCESS_KEY,
     secretKey: process.env.STORAGE_SECRET_KEY,
 })
-const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
+const meili = new MeiliSearch({
-const prisma = new PrismaClient()
+  host: process.env.MEILI_URL,
+  apiKey: process.env.MEILI_MASTER_KEY,
+})
 
-async function storageBucketExists() {
+const memes = await memeService(log, prisma, meili)
-    const exists = await minio.bucketExists(STORAGE_BUCKET)
+const attachments = await attachmentService(log, minio, prisma, memes)
-    if (exists) return true
+const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments)
-    await minio.makeBucket(STORAGE_BUCKET)
-}
 
-console.log(await storageBucketExists())
-
-// BIGINTS
-// must be strings
 const CHANNEL_IDS = [
-    '367936426962190336',  // TPM shitposts
+    367936426962190336n,  // TPM shitposts
-    '1095060717117374687', // aella memes
+    367936467592413194n,  // tpm screenshots
+    1028511952277864501n,  // tpm sludge
+    725835129326665768n, // machinegirl memes
+    1095060717117374687n, // aella memes
 ]
 
-const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio)
+async function main() {
+    await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id)))
+}
+
+await main()
 
-const msgs = await Promise.all(CHANNEL_IDS.map(c => scraper.getChannelAttachments(c)))
-console.log(msgs)
-const downloads = await Promise.all(msgs.flat().map(scraper.downloadAttachment))
 

memestream.sublime-project

@@ -2,10 +2,7 @@
 	"folders":
 	[
         {
-            "path": ".",
+			"path": "."
-        },
-        {
-			"path": "/home/ntr/memestream"
 		},
     ],
     "build_systems": [
@@ -13,5 +10,9 @@
             "name": "Memestream - npm start",
             "cmd": ["npm", "start"],
         },
-    ],
+        {
+            "name": "Memestream - npm test",
+            "cmd": ["npm", "test"],
+        }
+    ]
 }

package-lock.json

@@ -11,14 +11,20 @@
       "dependencies": {
         "@google-cloud/vision": "^4.2.0",
         "@prisma/client": "^5.12.1",
+        "@theinternetfolks/snowflake": "^1.3.0",
         "axios": "^1.6.8",
         "dotenv": "^16.4.1",
+        "lodash": "^4.17.21",
+        "meilisearch": "^0.38.0",
         "mime-types": "^2.1.35",
         "minio": "^7.1.3",
+        "pino": "^8.20.0",
+        "pino-pretty": "^11.0.0",
         "sharp": "^0.33.3",
         "typescript": "^5.3.3"
       },
       "devDependencies": {
+        "@types/lodash": "^4.17.0",
         "esrun": "^3.2.26",
         "prisma": "^5.12.1"
       }
@@ -996,6 +1002,11 @@
       "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
       "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="
     },
+    "node_modules/@theinternetfolks/snowflake": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@theinternetfolks/snowflake/-/snowflake-1.3.0.tgz",
+      "integrity": "sha512-/5fjyE+JIa95Lm1uDJ9fx52W7/ZHBVvKo5GbpR+swpchbriLzVgFAIUOVwLsscnUTBixyQ+JsEh5pJQM/zKNzQ=="
+    },
     "node_modules/@tootallnate/once": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
@@ -1009,6 +1020,12 @@
       "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz",
       "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg=="
     },
+    "node_modules/@types/lodash": {
+      "version": "4.17.0",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz",
+      "integrity": "sha512-t7dhREVv6dbNj0q17X12j7yDG4bD/DHYX7o5/DbDxobP0HnGPgpRz2Ej77aL7TZT3DSw13fqUTj8J4mMnqa7WA==",
+      "dev": true
+    },
     "node_modules/@types/long": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
@@ -1111,6 +1128,14 @@
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
       "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
     },
+    "node_modules/atomic-sleep": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz",
+      "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==",
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
     "node_modules/available-typed-arrays": {
       "version": "1.0.7",
       "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz",
@@ -1209,6 +1234,29 @@
       "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz",
       "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg=="
     },
+    "node_modules/buffer": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
+      "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "dependencies": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.2.1"
+      }
+    },
     "node_modules/buffer-crc32": {
       "version": "0.2.13",
       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
@@ -1317,6 +1365,11 @@
         "simple-swizzle": "^0.2.2"
       }
     },
+    "node_modules/colorette": {
+      "version": "2.0.20",
+      "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz",
+      "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="
+    },
     "node_modules/combined-stream": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -1328,6 +1381,22 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/cross-fetch": {
+      "version": "3.1.8",
+      "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz",
+      "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==",
+      "dependencies": {
+        "node-fetch": "^2.6.12"
+      }
+    },
+    "node_modules/dateformat": {
+      "version": "4.6.3",
+      "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz",
+      "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==",
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -1516,11 +1585,37 @@
         "node": ">=6"
       }
     },
+    "node_modules/events": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
+      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
+      "engines": {
+        "node": ">=0.8.x"
+      }
+    },
     "node_modules/extend": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
       "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
     },
+    "node_modules/fast-copy": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.2.tgz",
+      "integrity": "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ=="
+    },
+    "node_modules/fast-redact": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.5.0.tgz",
+      "integrity": "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/fast-safe-stringify": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",
+      "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA=="
+    },
     "node_modules/fast-xml-parser": {
       "version": "4.3.6",
       "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.6.tgz",
@@ -1808,6 +1903,11 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/help-me": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz",
+      "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg=="
+    },
     "node_modules/http-proxy-agent": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
@@ -1844,6 +1944,25 @@
         "node": ">= 14"
       }
     },
+    "node_modules/ieee754": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
     "node_modules/inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
@@ -1985,6 +2104,14 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/joycon": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz",
+      "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==",
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/json-bigint": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz",
@@ -2043,6 +2170,14 @@
         "node": ">=10"
       }
     },
+    "node_modules/meilisearch": {
+      "version": "0.38.0",
+      "resolved": "https://registry.npmjs.org/meilisearch/-/meilisearch-0.38.0.tgz",
+      "integrity": "sha512-bHaq8nYxSKw9/Qslq1Zes5g9tHgFkxy/I9o8942wv2PqlNOT0CzptIkh/x98N52GikoSZOXSQkgt6oMjtf5uZw==",
+      "dependencies": {
+        "cross-fetch": "^3.1.6"
+      }
+    },
     "node_modules/mime-db": {
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
@@ -2062,6 +2197,14 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/minimist": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
+      "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/minio": {
       "version": "7.1.3",
       "resolved": "https://registry.npmjs.org/minio/-/minio-7.1.3.tgz",
@@ -2127,6 +2270,14 @@
         "node": ">= 6"
       }
     },
+    "node_modules/on-exit-leak-free": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz",
+      "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==",
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
     "node_modules/once": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
@@ -2147,6 +2298,95 @@
         "url": "https://github.com/sponsors/jonschlinkert"
       }
     },
+    "node_modules/pino": {
+      "version": "8.20.0",
+      "resolved": "https://registry.npmjs.org/pino/-/pino-8.20.0.tgz",
+      "integrity": "sha512-uhIfMj5TVp+WynVASaVEJFTncTUe4dHBq6CWplu/vBgvGHhvBvQfxz+vcOrnnBQdORH3izaGEurLfNlq3YxdFQ==",
+      "dependencies": {
+        "atomic-sleep": "^1.0.0",
+        "fast-redact": "^3.1.1",
+        "on-exit-leak-free": "^2.1.0",
+        "pino-abstract-transport": "^1.1.0",
+        "pino-std-serializers": "^6.0.0",
+        "process-warning": "^3.0.0",
+        "quick-format-unescaped": "^4.0.3",
+        "real-require": "^0.2.0",
+        "safe-stable-stringify": "^2.3.1",
+        "sonic-boom": "^3.7.0",
+        "thread-stream": "^2.0.0"
+      },
+      "bin": {
+        "pino": "bin.js"
+      }
+    },
+    "node_modules/pino-abstract-transport": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.1.0.tgz",
+      "integrity": "sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==",
+      "dependencies": {
+        "readable-stream": "^4.0.0",
+        "split2": "^4.0.0"
+      }
+    },
+    "node_modules/pino-abstract-transport/node_modules/readable-stream": {
+      "version": "4.5.2",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz",
+      "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==",
+      "dependencies": {
+        "abort-controller": "^3.0.0",
+        "buffer": "^6.0.3",
+        "events": "^3.3.0",
+        "process": "^0.11.10",
+        "string_decoder": "^1.3.0"
+      },
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      }
+    },
+    "node_modules/pino-pretty": {
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-11.0.0.tgz",
+      "integrity": "sha512-YFJZqw59mHIY72wBnBs7XhLGG6qpJMa4pEQTRgEPEbjIYbng2LXEZZF1DoyDg9CfejEy8uZCyzpcBXXG0oOCwQ==",
+      "dependencies": {
+        "colorette": "^2.0.7",
+        "dateformat": "^4.6.3",
+        "fast-copy": "^3.0.0",
+        "fast-safe-stringify": "^2.1.1",
+        "help-me": "^5.0.0",
+        "joycon": "^3.1.1",
+        "minimist": "^1.2.6",
+        "on-exit-leak-free": "^2.1.0",
+        "pino-abstract-transport": "^1.0.0",
+        "pump": "^3.0.0",
+        "readable-stream": "^4.0.0",
+        "secure-json-parse": "^2.4.0",
+        "sonic-boom": "^3.0.0",
+        "strip-json-comments": "^3.1.1"
+      },
+      "bin": {
+        "pino-pretty": "bin.js"
+      }
+    },
+    "node_modules/pino-pretty/node_modules/readable-stream": {
+      "version": "4.5.2",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz",
+      "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==",
+      "dependencies": {
+        "abort-controller": "^3.0.0",
+        "buffer": "^6.0.3",
+        "events": "^3.3.0",
+        "process": "^0.11.10",
+        "string_decoder": "^1.3.0"
+      },
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      }
+    },
+    "node_modules/pino-std-serializers": {
+      "version": "6.2.2",
+      "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz",
+      "integrity": "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA=="
+    },
     "node_modules/possible-typed-array-names": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz",
@@ -2171,6 +2411,19 @@
         "node": ">=16.13"
       }
     },
+    "node_modules/process": {
+      "version": "0.11.10",
+      "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
+      "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
+      "engines": {
+        "node": ">= 0.6.0"
+      }
+    },
+    "node_modules/process-warning": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz",
+      "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ=="
+    },
     "node_modules/proto3-json-serializer": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.1.tgz",
@@ -2210,6 +2463,15 @@
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
       "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
     },
+    "node_modules/pump": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
+      "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+      "dependencies": {
+        "end-of-stream": "^1.1.0",
+        "once": "^1.3.1"
+      }
+    },
     "node_modules/query-string": {
       "version": "7.1.3",
       "resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.3.tgz",
@@ -2227,6 +2489,11 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/quick-format-unescaped": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz",
+      "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg=="
+    },
     "node_modules/readable-stream": {
       "version": "3.6.2",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
@@ -2252,6 +2519,14 @@
         "node": ">=8.10.0"
       }
     },
+    "node_modules/real-require": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz",
+      "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==",
+      "engines": {
+        "node": ">= 12.13.0"
+      }
+    },
     "node_modules/require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -2292,11 +2567,24 @@
         }
       ]
     },
+    "node_modules/safe-stable-stringify": {
+      "version": "2.4.3",
+      "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz",
+      "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==",
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/sax": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz",
       "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA=="
     },
+    "node_modules/secure-json-parse": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz",
+      "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="
+    },
     "node_modules/semver": {
       "version": "7.6.0",
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz",
@@ -2374,6 +2662,14 @@
         "is-arrayish": "^0.3.1"
       }
     },
+    "node_modules/sonic-boom": {
+      "version": "3.8.1",
+      "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.8.1.tgz",
+      "integrity": "sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg==",
+      "dependencies": {
+        "atomic-sleep": "^1.0.0"
+      }
+    },
     "node_modules/split-on-first": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz",
@@ -2382,6 +2678,14 @@
         "node": ">=6"
       }
     },
+    "node_modules/split2": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
+      "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
+      "engines": {
+        "node": ">= 10.x"
+      }
+    },
     "node_modules/stream-events": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz",
@@ -2435,6 +2739,17 @@
         "node": ">=8"
       }
     },
+    "node_modules/strip-json-comments": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
+      "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/strnum": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz",
@@ -2483,6 +2798,14 @@
         "node": ">= 6"
       }
     },
+    "node_modules/thread-stream": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.4.1.tgz",
+      "integrity": "sha512-d/Ex2iWd1whipbT681JmTINKw0ZwOUBZm7+Gjs64DHuX34mmw8vJL2bFAaNacaW72zYiTJxSHi5abUuOi5nsfg==",
+      "dependencies": {
+        "real-require": "^0.2.0"
+      }
+    },
     "node_modules/through2": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz",
@@ -3143,6 +3466,11 @@
       "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
       "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="
     },
+    "@theinternetfolks/snowflake": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@theinternetfolks/snowflake/-/snowflake-1.3.0.tgz",
+      "integrity": "sha512-/5fjyE+JIa95Lm1uDJ9fx52W7/ZHBVvKo5GbpR+swpchbriLzVgFAIUOVwLsscnUTBixyQ+JsEh5pJQM/zKNzQ=="
+    },
     "@tootallnate/once": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
@@ -3153,6 +3481,12 @@
       "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz",
       "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg=="
     },
+    "@types/lodash": {
+      "version": "4.17.0",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz",
+      "integrity": "sha512-t7dhREVv6dbNj0q17X12j7yDG4bD/DHYX7o5/DbDxobP0HnGPgpRz2Ej77aL7TZT3DSw13fqUTj8J4mMnqa7WA==",
+      "dev": true
+    },
     "@types/long": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
@@ -3237,6 +3571,11 @@
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
       "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
     },
+    "atomic-sleep": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz",
+      "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="
+    },
     "available-typed-arrays": {
       "version": "1.0.7",
       "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz",
@@ -3305,6 +3644,15 @@
       "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz",
       "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg=="
     },
+    "buffer": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
+      "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
+      "requires": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.2.1"
+      }
+    },
     "buffer-crc32": {
       "version": "0.2.13",
       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
@@ -3384,6 +3732,11 @@
         "simple-swizzle": "^0.2.2"
       }
     },
+    "colorette": {
+      "version": "2.0.20",
+      "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz",
+      "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="
+    },
     "combined-stream": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -3392,6 +3745,19 @@
         "delayed-stream": "~1.0.0"
       }
     },
+    "cross-fetch": {
+      "version": "3.1.8",
+      "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz",
+      "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==",
+      "requires": {
+        "node-fetch": "^2.6.12"
+      }
+    },
+    "dateformat": {
+      "version": "4.6.3",
+      "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz",
+      "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="
+    },
     "debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -3526,11 +3892,31 @@
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
       "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="
     },
+    "events": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
+      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="
+    },
     "extend": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
       "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
     },
+    "fast-copy": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.2.tgz",
+      "integrity": "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ=="
+    },
+    "fast-redact": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.5.0.tgz",
+      "integrity": "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A=="
+    },
+    "fast-safe-stringify": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",
+      "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA=="
+    },
     "fast-xml-parser": {
       "version": "4.3.6",
       "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.6.tgz",
@@ -3718,6 +4104,11 @@
         "function-bind": "^1.1.2"
       }
     },
+    "help-me": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz",
+      "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg=="
+    },
     "http-proxy-agent": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
@@ -3747,6 +4138,11 @@
         "debug": "4"
       }
     },
+    "ieee754": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
+    },
     "inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
@@ -3837,6 +4233,11 @@
         "which-typed-array": "^1.1.14"
       }
     },
+    "joycon": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz",
+      "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="
+    },
     "json-bigint": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz",
@@ -3892,6 +4293,14 @@
         "yallist": "^4.0.0"
       }
     },
+    "meilisearch": {
+      "version": "0.38.0",
+      "resolved": "https://registry.npmjs.org/meilisearch/-/meilisearch-0.38.0.tgz",
+      "integrity": "sha512-bHaq8nYxSKw9/Qslq1Zes5g9tHgFkxy/I9o8942wv2PqlNOT0CzptIkh/x98N52GikoSZOXSQkgt6oMjtf5uZw==",
+      "requires": {
+        "cross-fetch": "^3.1.6"
+      }
+    },
     "mime-db": {
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
@@ -3905,6 +4314,11 @@
         "mime-db": "1.52.0"
       }
     },
+    "minimist": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
+      "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="
+    },
     "minio": {
       "version": "7.1.3",
       "resolved": "https://registry.npmjs.org/minio/-/minio-7.1.3.tgz",
@@ -3950,6 +4364,11 @@
       "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
       "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw=="
     },
+    "on-exit-leak-free": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz",
+      "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA=="
+    },
     "once": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
@@ -3964,6 +4383,87 @@
       "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
       "dev": true
     },
+    "pino": {
+      "version": "8.20.0",
+      "resolved": "https://registry.npmjs.org/pino/-/pino-8.20.0.tgz",
+      "integrity": "sha512-uhIfMj5TVp+WynVASaVEJFTncTUe4dHBq6CWplu/vBgvGHhvBvQfxz+vcOrnnBQdORH3izaGEurLfNlq3YxdFQ==",
+      "requires": {
+        "atomic-sleep": "^1.0.0",
+        "fast-redact": "^3.1.1",
+        "on-exit-leak-free": "^2.1.0",
+        "pino-abstract-transport": "^1.1.0",
+        "pino-std-serializers": "^6.0.0",
+        "process-warning": "^3.0.0",
+        "quick-format-unescaped": "^4.0.3",
+        "real-require": "^0.2.0",
+        "safe-stable-stringify": "^2.3.1",
+        "sonic-boom": "^3.7.0",
+        "thread-stream": "^2.0.0"
+      }
+    },
+    "pino-abstract-transport": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.1.0.tgz",
+      "integrity": "sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==",
+      "requires": {
+        "readable-stream": "^4.0.0",
+        "split2": "^4.0.0"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "4.5.2",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz",
+          "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==",
+          "requires": {
+            "abort-controller": "^3.0.0",
+            "buffer": "^6.0.3",
+            "events": "^3.3.0",
+            "process": "^0.11.10",
+            "string_decoder": "^1.3.0"
+          }
+        }
+      }
+    },
+    "pino-pretty": {
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-11.0.0.tgz",
+      "integrity": "sha512-YFJZqw59mHIY72wBnBs7XhLGG6qpJMa4pEQTRgEPEbjIYbng2LXEZZF1DoyDg9CfejEy8uZCyzpcBXXG0oOCwQ==",
+      "requires": {
+        "colorette": "^2.0.7",
+        "dateformat": "^4.6.3",
+        "fast-copy": "^3.0.0",
+        "fast-safe-stringify": "^2.1.1",
+        "help-me": "^5.0.0",
+        "joycon": "^3.1.1",
+        "minimist": "^1.2.6",
+        "on-exit-leak-free": "^2.1.0",
+        "pino-abstract-transport": "^1.0.0",
+        "pump": "^3.0.0",
+        "readable-stream": "^4.0.0",
+        "secure-json-parse": "^2.4.0",
+        "sonic-boom": "^3.0.0",
+        "strip-json-comments": "^3.1.1"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "4.5.2",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz",
+          "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==",
+          "requires": {
+            "abort-controller": "^3.0.0",
+            "buffer": "^6.0.3",
+            "events": "^3.3.0",
+            "process": "^0.11.10",
+            "string_decoder": "^1.3.0"
+          }
+        }
+      }
+    },
+    "pino-std-serializers": {
+      "version": "6.2.2",
+      "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz",
+      "integrity": "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA=="
+    },
     "possible-typed-array-names": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz",
@@ -3978,6 +4478,16 @@
         "@prisma/engines": "5.12.1"
       }
     },
+    "process": {
+      "version": "0.11.10",
+      "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
+      "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A=="
+    },
+    "process-warning": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz",
+      "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ=="
+    },
     "proto3-json-serializer": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.1.tgz",
@@ -4010,6 +4520,15 @@
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
       "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
     },
+    "pump": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
+      "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+      "requires": {
+        "end-of-stream": "^1.1.0",
+        "once": "^1.3.1"
+      }
+    },
     "query-string": {
       "version": "7.1.3",
       "resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.3.tgz",
@@ -4021,6 +4540,11 @@
         "strict-uri-encode": "^2.0.0"
       }
     },
+    "quick-format-unescaped": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz",
+      "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg=="
+    },
     "readable-stream": {
       "version": "3.6.2",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
@@ -4040,6 +4564,11 @@
         "picomatch": "^2.2.1"
       }
     },
+    "real-require": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz",
+      "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg=="
+    },
     "require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -4060,11 +4589,21 @@
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
       "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
     },
+    "safe-stable-stringify": {
+      "version": "2.4.3",
+      "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz",
+      "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g=="
+    },
     "sax": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz",
       "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA=="
     },
+    "secure-json-parse": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz",
+      "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="
+    },
     "semver": {
       "version": "7.6.0",
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz",
@@ -4123,11 +4662,24 @@
         "is-arrayish": "^0.3.1"
       }
     },
+    "sonic-boom": {
+      "version": "3.8.1",
+      "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.8.1.tgz",
+      "integrity": "sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg==",
+      "requires": {
+        "atomic-sleep": "^1.0.0"
+      }
+    },
     "split-on-first": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz",
       "integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw=="
     },
+    "split2": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
+      "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="
+    },
     "stream-events": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz",
@@ -4172,6 +4724,11 @@
         "ansi-regex": "^5.0.1"
       }
     },
+    "strip-json-comments": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
+      "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="
+    },
     "strnum": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz",
@@ -4213,6 +4770,14 @@
         }
       }
     },
+    "thread-stream": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.4.1.tgz",
+      "integrity": "sha512-d/Ex2iWd1whipbT681JmTINKw0ZwOUBZm7+Gjs64DHuX34mmw8vJL2bFAaNacaW72zYiTJxSHi5abUuOi5nsfg==",
+      "requires": {
+        "real-require": "^0.2.0"
+      }
+    },
     "through2": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz",

package.json

@@ -5,23 +5,30 @@
   "main": "index.ts",
   "type": "module",
   "scripts": {
-    "start": "esrun --watch index.ts",
+    "start": "esrun index.ts | pino-pretty",
+    "dev": "esrun --watch index.ts | pino-pretty",
     "migrate:make": "prisma migrate dev --name",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "esrun src/scraper.test.ts | pino-pretty"
   },
   "author": "",
   "license": "ISC",
   "dependencies": {
     "@google-cloud/vision": "^4.2.0",
     "@prisma/client": "^5.12.1",
+    "@theinternetfolks/snowflake": "^1.3.0",
     "axios": "^1.6.8",
     "dotenv": "^16.4.1",
+    "lodash": "^4.17.21",
+    "meilisearch": "^0.38.0",
     "mime-types": "^2.1.35",
     "minio": "^7.1.3",
+    "pino": "^8.20.0",
+    "pino-pretty": "^11.0.0",
     "sharp": "^0.33.3",
     "typescript": "^5.3.3"
   },
   "devDependencies": {
+    "@types/lodash": "^4.17.0",
     "esrun": "^3.2.26",
     "prisma": "^5.12.1"
   }

prisma/migrations/20240410010107_init/migration.sql

@@ -0,0 +1,38 @@
+-- CreateTable
+CREATE TABLE "attachments" (
+    "id" BIGINT NOT NULL,
+    "msgId" BIGINT NOT NULL,
+    "channelId" BIGINT NOT NULL,
+    "memeId" UUID NOT NULL,
+    "url" TEXT NOT NULL,
+    "contentType" TEXT NOT NULL,
+    "message" JSONB NOT NULL,
+    "guild" JSONB NOT NULL,
+    "channel" JSONB NOT NULL,
+    "vision" JSONB NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "attachments_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "memes" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "url" TEXT NOT NULL,
+    "text" TEXT NOT NULL,
+    "tags" JSONB NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "memes_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "attachments_memeId_key" ON "attachments"("memeId");
+
+-- CreateIndex
+CREATE INDEX "attachments_channelId_idx" ON "attachments"("channelId");
+
+-- AddForeignKey
+ALTER TABLE "attachments" ADD CONSTRAINT "attachments_memeId_fkey" FOREIGN KEY ("memeId") REFERENCES "memes"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

prisma/migrations/20240410015301_memes_attachment_id/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[attachmentId]` on the table `memes` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- AlterTable
+ALTER TABLE "memes" ADD COLUMN     "attachmentId" BIGINT;
+
+-- CreateIndex
+CREATE UNIQUE INDEX "memes_attachmentId_key" ON "memes"("attachmentId");

prisma/migrations/20240410091812_add_channelhistory/migration.sql

@@ -0,0 +1,9 @@
+-- CreateTable
+CREATE TABLE "channelhistory" (
+    "id" BIGINT NOT NULL,
+    "earliestMessageId" BIGINT NOT NULL,
+    "latestMessageId" BIGINT NOT NULL,
+    "backfillComplete" BOOLEAN NOT NULL DEFAULT false,
+
+    CONSTRAINT "channelhistory_pkey" PRIMARY KEY ("id")
+);

prisma/migrations/20240410093508_drop_backfillcomplete/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `backfillComplete` on the `channelhistory` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "channelhistory" DROP COLUMN "backfillComplete";

prisma/migrations/20240414022802_add_timestamp_and_postedat/migration.sql

@@ -0,0 +1,25 @@
+/*
+  Warnings:
+
+  - Added the required column `postedAt` to the `attachments` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `timestamp` to the `memes` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "attachments"
+  ADD COLUMN     "postedAt"
+  TIMESTAMP(3);
+
+UPDATE "attachments"
+  set "postedAt" = TIMESTAMP 'epoch' + ((id >> 22) + 1420070400000) * INTERVAL '1 millisecond';
+
+-- AlterTable
+ALTER TABLE "memes"
+  ADD COLUMN     "postedAt"
+  TIMESTAMP(3);
+
+UPDATE "memes"
+  set "postedAt" = TIMESTAMP 'epoch' + (("attachmentId" >> 22) + 1420070400000) * INTERVAL '1 millisecond';
+
+ALTER TABLE "attachments" ALTER COLUMN "postedAt" SET NOT NULL;
+ALTER TABLE "memes" ALTER COLUMN "postedAt" SET NOT NULL;

prisma/migrations/migration_lock.toml

@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (i.e. Git)
+provider = "postgresql"

prisma/schema.prisma

@@ -26,7 +26,8 @@ model Attachment {
     vision    Json
 
     createdAt  DateTime @default(now())
-    updatedAt  DateTime @updatedAt
+    updatedAt  DateTime @default(now())
+    postedAt   DateTime
 
     @@index([channelId])
     @@map("attachments")
@@ -38,12 +39,28 @@ model Meme {
     text       String
     tags       Json
 
+    // need an timestamp for giving to meili to sort by
+    // i think a usize would actually work but can't
+    // convince the meili-js lib to serialize it properly to json
+    // this one actually has to be .valueOf()'d before we send it
+    // cause js numbers are bigger than postgres int4
+    postedAt   DateTime
     createdAt  DateTime @default(now())
-    updatedAt  DateTime @updatedAt
+    updatedAt  DateTime @default(now())
 
+    // required to prevent the same attachment being uploaded multiple times
+    // nullable because maybe will have memes that don't come from a discord attachment
+    attachmentId BigInt? @unique
     attachment Attachment?
 
     @@map("memes")
 }
 
+model ChannelHistory {
+    id                  BigInt    @id
+    earliestMessageId   BigInt
+    latestMessageId     BigInt
 
+    // we do this because you have to always wrap table names in "" if they have capitals
+    @@map("channelhistory")
+}

search.html

@@ -1,40 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@meilisearch/instant-meilisearch/templates/basic_search.css" />
-  </head>
-  <body>
-      <div id="app" class="wrapper"></div>
-  </body>
-  <script src="https://unpkg.com/react@18/umd/react.development.js" crossorigin></script>
-  <script src="https://unpkg.com/react-dom@18/umd/react-dom.development.js" crossorigin></script>
-  <script src="https://cdn.jsdelivr.net/npm/react-instantsearch-dom@6.7.0/dist/umd/ReactInstantSearchDOM.js"></script>
-  <script src="https://cdn.jsdelivr.net/npm/@meilisearch/instant-meilisearch/dist/instant-meilisearch.umd.min.js"></script>
-  <script>
-    const { InstantSearch, SearchBox, Hits, Highlight, Configure } = ReactInstantSearchDOM;
-    const searchClient = instantMeiliSearch(
-      "http://localhost:7700",
-      '33e714590199031d076954182c55a47b09ae7546e3dbf6f35862eb99cc950e1e'
-    );
-    const App = () => (
-      React.createElement(InstantSearch, {
-        indexName: "memes",
-        searchClient: searchClient
-      }, [
-        React.createElement(SearchBox, { key: 1 }),
-        React.createElement(Hits, { hitComponent: Hit, key: 2 }),
-        React.createElement(Configure, { hitsPerPage: 10 })]
-      )
-    );
-
-    const Hit = (props) => (
-      React.createElement(Highlight, {
-        attribute: "id",
-        hit: props.hit
-      })
-    );
-    ReactDOM.createRoot(document.querySelector('#app')).render(App());
-    console.log(searchClient)
-  </script>
-</html>

src/attachments.ts

@@ -0,0 +1,108 @@
+import sharp from 'sharp'
+import axios from 'axios'
+import mime from 'mime-types'
+
+import type { Logger } from 'pino'
+
+import type Minio from 'minio'
+import type { PrismaClient, Attachment, ChannelHistory } from '@prisma/client'
+
+import type { MemeService } from './memes'
+
+const STORAGE_BUCKET = process.env.STORAGE_BUCKET
+
+export const IMAGE_TYPES = [
+    'image/png',
+    'image/jpeg',
+    'image/webp',
+    'image/gif',
+]
+
+export const VIDEO_TYPES = [
+    'video/mp4',
+    'video/quicktime'
+]
+
+export function isImage(a: Attachment) {
+    return IMAGE_TYPES.includes(a.contentType)
+}
+
+export function objectName(a: Attachment) {
+    if (isImage(a))
+        return `${a.memeId}.webp`
+    return `${a.memeId}.${mime.extension(a.contentType)}`
+}
+
+
+export default async function attachmentService(log: Logger, minio: Minio.Client, prisma: PrismaClient, memes: MemeService) {
+    async function ensureStorageBucketExists() {
+        const exists = await minio.bucketExists(STORAGE_BUCKET)
+        if (exists) return true
+        await minio.makeBucket(STORAGE_BUCKET)
+    }
+
+    async function storeAttachment(a: Attachment) {
+        const download = await axios.get(a.url, { responseType: 'stream' })
+
+        const stream = () => {
+            if (isImage(a)) {
+                const converter = sharp().webp()
+                download.data.pipe(converter)
+                return converter
+            }
+            return download.data
+        }
+
+        return minio.putObject(STORAGE_BUCKET, objectName(a), stream())
+    }
+
+    async function deleteAttachment(a: Attachment) {
+        return minio.removeObject(STORAGE_BUCKET, objectName(a))
+    }
+
+    async function process(a: Attachment, history: Partial<ChannelHistory>) {
+        try {
+            const results = await prisma.$transaction(async prisma => {
+                const meme = memes.fromAttachment(a)
+                const dbMeme = await prisma.meme.upsert({ create: meme, update: meme, where: { id: meme.id } })
+                const dbA = await prisma.attachment.upsert({ create: a, update: a, where: { id: a.id } })
+                log.debug({ msg: 'pending addition to db', dbMeme })
+
+                const upload = await storeAttachment(a)
+                log.debug({ msg: 'successfully uploaded', upload })
+
+                await memes.persist(dbMeme)
+
+                log.info(dbMeme, 'new meme')
+                return { meme: dbMeme, upload, attachment: dbA }
+            }, { timeout: 60000 })
+        } catch (e) {
+
+            // this happens pretty regularly it seems
+            // when ppl reply to an attachment it is included in their message
+            if (e.meta?.modelName !== 'Meme' && e.meta?.target[0] !== 'attachmentId') {
+                log.error(e, 'failed to process attachment')
+                log.error(a, 'failed to process attachment')
+            }
+
+            // deletes the upload if we got that far
+            // (probably not)
+            await deleteAttachment(a)
+        }
+
+        // we assume here that a failure in processing was caused by a duplicate
+        // so we update pg to say we've processed it
+        const newHistory = await prisma.channelHistory.update({ data: history, where: { id: history.id }})
+        log.debug({ newHistory }, 'updated channel history')
+        return true
+    }
+
+    await ensureStorageBucketExists()
+
+    return {
+        process,
+    }
+}
+
+
+export type AttachmentService = Awaited<ReturnType<typeof attachmentService>>

src/memes.ts

@@ -0,0 +1,84 @@
+import type { PrismaClient, Attachment, Meme } from '@prisma/client';
+import type { Meilisearch } from 'meilisearch';
+import type { Logger } from 'pino';
+
+import _ from 'lodash'
+
+import * as attachments from './attachments'
+
+const STORAGE_BUCKET = process.env.STORAGE_BUCKET
+
+function memeUrl(a: Attachment) {
+    const STORAGE_URL = process.env.STORAGE_URL
+    const u = new URL(`${STORAGE_BUCKET}/${attachments.objectName(a)}`, `https://${STORAGE_URL}`)
+    return u.href
+}
+
+export default async function memeService(baseLog: Logger, prisma: PrismaClient, meili: Meilisearch) {
+    const log = baseLog.child({ service: 'meme' })
+
+    function fromAttachment(a: Attachment) {
+        const text = a.vision.fullTextAnnotation?.text || ''
+        const tags = [
+            a.guild.name,
+            a.channel.name,
+            a.message.author.username,
+        ]
+
+        const m: Meme = {
+            id: a.memeId,
+            url: memeUrl(a),
+            text,
+            tags: tags,
+            attachmentId: a.id,
+            postedAt: a.postedAt,
+            createdAt: undefined,
+            updatedAt: undefined
+        }
+
+        return m
+    }
+
+    async function ensureMeiliIndex() {
+        const task = await meili.createIndex('memes', { primaryKey: 'id' })
+        const result = await meili.waitForTask(task.taskUid)
+        log.debug(result);
+        const index = await meili.index('memes')
+        await index.updateSortableAttributes(['postedAt'])
+        return index;
+    }
+
+    // BLOWS IT UP
+    async function resetMeili() {
+        const delTask = await meili.deleteIndex('memes')
+        const delResult = await meili.waitForTask(delTask.taskUid)
+        const index = await ensureMeiliIndex()
+        const memes = (await prisma.meme.findMany())
+            .map(m => ({ ...m, postedAt: m.postedAt.valueOf() }));
+
+        for await (const batch of _.chunk(memes, 1000)) {
+            const task = await index.updateDocuments(batch)
+            const res = await meili.waitForTask(task.taskUid)
+            log.info({ res }, 'added meme batch')
+        }
+    }
+
+    async function persist(m: Meme) {
+        const index = meili.index('memes')
+        return index.updateDocuments([m])
+    }
+
+    await ensureMeiliIndex();
+
+    // BAM
+    // await resetMeili();
+
+    return {
+        ensureMeiliIndex,
+        resetMeili,
+        fromAttachment,
+        persist,
+    }
+}
+
+export type MemeService = Awaited<ReturnType<typeof memeService>>

src/scraper.ts

@@ -1,35 +1,53 @@
-import path from 'path'
-
 import axios from 'axios'
-import mime from 'mime-types'
+
+import _ from 'lodash';
 
 import type { PrismaClient, Attachment } from '@prisma/client';
 import type { ImageAnnotatorClient } from '@google-cloud/vision';
+import type { Logger } from 'pino';
 import type Minio from 'minio'
 
 import sharp from 'sharp';
 
-const IMAGE_TYPES = [
+import type { AttachmentService } from './attachments'
-    'image/png',
+import { IMAGE_TYPES, VIDEO_TYPES } from './attachments'
-    'image/jpeg',
-    'image/webp',
-    'image/gif',
-]
 
-const VIDEO_TYPES = [
+// TODO
-    'video/mp4',
+// also fetch embeds and their descriptions
-    'video/quicktime'
-]
 
-const STORAGE_BUCKET = process.env.STORAGE_BUCKET
+function convertSnowflakeToDate(snowflake) {
+    const DISCORD_EPOCH = 1420070400000
+    const milliseconds = global.BigInt(snowflake) >> 22n
+    return new Date(Number(milliseconds) + DISCORD_EPOCH)
+}
 
-export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio: Minio.Client) {
+export default async function scraper(
+    discordAuth,
+    vision: ImageAnnotatorClient,
+    minio: Minio.Client,
+    prisma: PrismaClient,
+    baseLog: Logger,
+    attachments: AttachmentService
+) {
     const headers = {
         'Authorization': discordAuth,
     }
 
     async function fetchChannelMessages(channelId) {
-        const url = `https://discord.com/api/v9/channels/${channelId}/messages`
+        const url = `https://discord.com/api/v9/channels/${channelId}/messages&limit=100`
+        const res = await axios.get(url, { headers })
+        return res.data
+    }
+
+    // returns msgs in desc id order
+    async function fetchChannelMessagesAfter(channelId, lastMessageId) {
+        const url = `https://discord.com/api/v9/channels/${channelId}/messages?after=${lastMessageId}&limit=100`
+        const res = await axios.get(url, { headers })
+        return res.data
+    }
+
+    async function fetchChannelMessagesBefore(channelId, earliestMessageId) {
+        const url = `https://discord.com/api/v9/channels/${channelId}/messages?before=${earliestMessageId}&limit=100`
         const res = await axios.get(url, { headers })
         return res.data
     }
@@ -37,8 +55,8 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
     async function fetchGuildDetails(guildId) {
         const url = `https://discord.com/api/v9/guilds/${guildId}`
         const res = await axios.get(url, { headers })
-        const keys_to_remove = ['emojis', 'stickers', 'roles']
+        const keys_to_remove = ['emojis', 'stickers', 'roles', 'features']
-        keys_to_remove.forEach(k => delete res[k])
+        keys_to_remove.forEach(k => delete res.data[k])
         return res.data
     }
 
@@ -65,131 +83,114 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
         return res[0] || {}
     }
 
-    async function downloadAttachment(a: Attachment) {
+    async function fetchMissingVision() {
-        const download = await axios.get(a.url, { responseType: 'stream' })
+        const attachments = await prisma.attachment
-        const isImage = IMAGE_TYPES.includes(a.contentType)
+            .findMany({ where: { vision: {}, contentType: { in: IMAGE_TYPES } }})
+        baseLog.info({attachments: attachments.length }, 'missing vision attachments')
+    }
 
-        const objectName = () => {
+    async function processChannel(channelId) {
-            if (isImage)
+        const channel = await fetchChannel(channelId)
-                return `${a.memeId}.webp`
+        const guild = await fetchGuildDetails(channel.guild_id)
+        const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId })
 
-            return `${a.memeId}.${mime.extension(a.contentType)}`
+        async function filterAttachments(messages): Promise<Attachment[]> {
+            const acceptedContentTypes = [
+                ...IMAGE_TYPES,
+                // ...VIDEO_TYPES,
+            ]
+
+            const validAttachments = messages
+                .filter(m => m.attachments)
+                // bundle message and attachment together
+                .flatMap(m => m.attachments.map(a => [a, m]))
+                .filter(([a, m]) => acceptedContentTypes.includes(a.content_type))
+
+            const newAttachments = validAttachments.filter(([a, m]) => ![].includes(a.id))
+            log.info({ newAttachments: newAttachments.length, validAttachments: validAttachments.length })
+
+            return Promise.all(newAttachments
+                .map(async ([a, m]) => {
+                    const attachment: Attachment = {
+                        id: BigInt(a.id),
+                        msgId: BigInt(m.id),
+                        channelId: BigInt(m.channel_id),
+                        url: a.url,
+                        contentType: a.content_type,
+
+                        // create this ahead of time to associate with our copy of the media
+                        memeId: crypto.randomUUID(),
+
+                        // @ts-ignore it's fine
+                        vision: {},
+                        guild,
+                        channel,
+                        message: m,
+                        postedAt: convertSnowflakeToDate(a.id),
+                        createdAt: undefined,
+                        updatedAt: undefined
+                    }
+                    return { ...attachment, vision: await detectText(attachment) }
+                }))
         }
 
-        const stream = async () => {
+        async function* fetchNew(channelId, log, latestMessageId): AsyncIterable<Attachment> {
-            if (isImage) {
+            // reverse the messages because we want to yield them in chronological order
-                const converter = sharp().webp()
+            // the api call returns them most recent first
-                download.data.pipe(converter)
+            const messages = (await fetchChannelMessagesAfter(channelId, latestMessageId)).reverse()
-                return converter
+            if (messages.length === 0) {
+                log.info('no new messages')
+                return
             }
-            return download.data
+
+            log.info({ messages: messages.length, latestMessageId }, 'fetched new messages')
+            yield* await filterAttachments(messages)
+            const cursor = _.last(messages).id
+            yield* fetchNew(channelId, log, cursor)
         }
 
-        return minio.putObject(STORAGE_BUCKET, objectName(), await stream())
+        async function* fetchOld(channelId, log, earliestMessageId): AsyncIterable<Attachment> {
-    }
+            const messages = await fetchChannelMessagesBefore(channelId, earliestMessageId)
+            if (messages.length === 0) {
+                log.info('no new messages')
+                return
+            }
 
-    function filterAttachments(messages, guild, channel) {
+            log.info({ messages: messages.length, earliestMessageId }, 'fetched older messages')
-        const acceptedContentTypes = [
+            yield* await filterAttachments(messages)
-            ...IMAGE_TYPES,
+            const cursor = _.last(messages).id
-            ...VIDEO_TYPES,
+            yield* fetchOld(channelId, log, cursor)
-        ]
+        }
 
-        return Promise.all(messages
+        const ensureHistory = async () => {
-            .filter(m => m.attachments)
+            const stored = await prisma.channelHistory.findFirst({ where: { id: channelId }})
-            // bundle message and attachment together
+            if (stored) return stored
-            .map(m => m.attachments.map(a => [a, m]))
-            .flat()
-            .filter(([a, m]) => acceptedContentTypes.includes(a.content_type))
-            .map(async ([a, m]) => {
-                const attachment: Attachment = {
-                    id: BigInt(a.id),
-                    msgId: BigInt(m.id),
-                    channelId: BigInt(m.channel_id),
-                    url: a.url,
-                    contentType: a.content_type,
-                    message: m,
 
-                    // create this ahead of time to associate with our copy of the media
+            const lastMessageId = channel.last_message_id
-                    memeId: crypto.randomUUID(),
+            return await prisma.channelHistory.create({ data:
-
+                { latestMessageId: lastMessageId, earliestMessageId: lastMessageId, id: channelId }
-                    // @ts-ignore it's fine
-                    vision: {},
-                    guild,
-                    channel,
-
-                    createdAt: undefined,
-                    updatedAt: undefined
-                }
-
-                const visionData = await detectText(attachment)
-                return { ...attachment, vision: visionData }
-            }))
-    }
-
-    async function getChannelAttachments(channelId) {
-        try {
-            const channel = await fetchChannel(channelId)
-            const guild = await fetchGuildDetails(channel.guild_id)
-            const messages = await fetchChannelMessages(channelId)
-
-            return filterAttachments(messages, guild, channel);
-
-            // return { messages, channel, guild }
-        } catch (e) {
-            console.error('scrape error', {
-                err: e.toJSON(),
-                channelId,
             })
         }
+
+        const history = await ensureHistory()
+
+        // we can do this better if we yield a list of attachments instead of one at a time
+        // will have to only update the history if they all succeed, probably better anyway
+        for await (const attachment of fetchNew(channelId, log, history.latestMessageId)) {
+            log.debug({ msg: 'scraped new attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) })
+            const newHistory = { id: history.id, latestMessageId: attachment.msgId }
+            await attachments.process(attachment, newHistory)
+        }
+
+        // assuming history of earliestId doesn't change...
+        for await (const attachment of fetchOld(channelId, log, history.earliestMessageId)) {
+            log.debug({ msg: 'scraped old attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) })
+            const newHistory = { id: history.id, earliestMessageId: attachment.msgId }
+            await attachments.process(attachment, newHistory)
+        }
     }
 
     return {
-        getChannelAttachments,
+        processChannel,
-        downloadAttachment,
     }
 }
-
-
-    // async def download_attachment(attachment: Attachment):
-    //     async with http_session.get(attachment.url) as response:
-    //         if response.status == 200:
-    //             (filename, ext) = attachment.file_details()
-    //             object_name = attachment.object_name()
-
-    //             print(f"uploading object_name={object_name} filename={filename} ext={ext}")
-    //             res_bytes = await response.read()
-
-    //             # make a webp conversion for obvious image types
-    //             if attachment.convert_to_webp():
-    //                 # have to read the whole img before converting to webp
-    //                 image = Image.open(BytesIO(res_bytes))
-    //                 webp_output = BytesIO()
-    //                 image.save(webp_output, format='WebP')
-    //                 webp_output.seek(0)
-    //                 return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=webp_output)
-
-    //             return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=res_bytes)
-
-    // def create_attachments(message, channel, guild):
-    //     urls = [(parse_url(attachment['url']), attachment['id']) for attachment in message['attachments']]
-
-    //     def generate_attachment(url: Url, attachmentId: str):
-    //         if not url.path:
-    //             return None
-
-    //         (filename, ext) = os.path.splitext(os.path.basename(url.path))
-    //         extensions = ['jpg', 'png', 'jpeg', 'webp', 'mov', 'gif', 'mp4']
-    //         if ext.lower()[1:] not in extensions:
-    //             return None
-
-    //         return Attachment(
-    //             id=uuid4(),
-    //             attachmentId=int(attachmentId),
-    //             msgId=int(message['id']),
-    //             message=message,
-    //             guild=guild,
-    //             url=url.url,
-    //             channel=channel
-    //         )
-
-    //     return [generate_attachment(*url) for url in urls]

src/test.attachment.ts

@@ -0,0 +1,121 @@
+export default {
+  id: 1227338263132635217n,
+  msgId: 1227338263010873456n,
+  channelId: 367936426962190336n,
+  url: "https://cdn.discordapp.com/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&",
+  contentType: "image/jpeg",
+  message: {
+    id: "1227338263010873456",
+    type: 0,
+    content: "",
+    channel_id: "367936426962190336",
+    author: {
+      id: "505353587174408193",
+      username: "tgcharlet",
+      avatar: "e800ffab640eb8684020054301668f79",
+      discriminator: "0",
+      public_flags: 0,
+      flags: 0,
+      banner: null,
+      accent_color: null,
+      global_name: "Chehrlet",
+      avatar_decoration_data: null,
+      banner_color: null,
+      clan: null
+    },
+    attachments: [
+      {
+        id: "1227338263132635217",
+        filename: "WOW.jpg",
+        size: 55508,
+        url: "https://cdn.discordapp.com/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&",
+        proxy_url: "https://media.discordapp.net/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&",
+        width: 1080,
+        height: 1022,
+        content_type: "image/jpeg",
+        content_scan_version: 1,
+        placeholder: "YPcFD4LU/YlNhpxWh4iLiJhn+iRpT8AH",
+        placeholder_version: 1
+      }
+    ]
+  },
+  memeId: "7456180d-fa94-4465-bcb6-ea80524c67e8",
+  vision: {
+    textAnnotations: [],
+    localizedObjectAnnotations: [],
+    safeSearchAnnotation: null,
+    imagePropertiesAnnotation: null,
+    error: null,
+    cropHintsAnnotation: null,
+    fullTextAnnotation: {
+      text: "damn thats crazy"
+    },
+    webDetection: null,
+    productSearchResults: null,
+    context: null
+  },
+  guild: {
+    id: "239638846688460802",
+    name: "The Philosopher's Meme",
+    icon: "bda0d70d3bb2ec4df2dee343029d990d",
+    description: "An applied philosophical community of practice for creatives, intellectuals, researchers, and developers. https://tpm.ac",
+    home_header: null,
+    splash: "0d80f531b01ab5abc8d153386f2af3e6",
+    discovery_splash: "b35170de857b5eb58b1065131afa0826",
+    banner: null,
+    owner_id: "103361901899038720",
+    application_id: null,
+    region: "us-south",
+    afk_channel_id: null,
+    afk_timeout: 300,
+    system_channel_id: "934578834534182953",
+    system_channel_flags: 0,
+    widget_enabled: true,
+    widget_channel_id: "301188489490202626",
+    verification_level: 3,
+    default_message_notifications: 1,
+    mfa_level: 1,
+    explicit_content_filter: 2,
+    max_presences: null,
+    max_members: 500000,
+    max_stage_video_channel_users: 50,
+    max_video_channel_users: 25,
+    vanity_url_code: null,
+    premium_tier: 1,
+    premium_subscription_count: 3,
+    preferred_locale: "en-US",
+    rules_channel_id: "301188489490202626",
+    safety_alerts_channel_id: "837815188970537031",
+    public_updates_channel_id: "837815188970537031",
+    hub_type: null,
+    premium_progress_bar_enabled: true,
+    latest_onboarding_question_id: "1136794072313507860",
+    nsfw: false,
+    nsfw_level: 0,
+    incidents_data: null,
+    inventory_settings: null,
+    embed_enabled: true,
+    embed_channel_id: "301188489490202626"
+  },
+  channel: {
+    id: "367936426962190336",
+    type: 0,
+    last_message_id: "1227404638450483340",
+    flags: 0,
+    last_pin_timestamp: "2022-07-12T12:05:47+00:00",
+    guild_id: "239638846688460802",
+    name: "shitposts_memes",
+    parent_id: "1023698856191471667",
+    rate_limit_per_user: 0,
+    topic: "memes, shitposts; memeing, shitposting",
+    position: 38,
+    nsfw: false,
+    icon_emoji: {
+      id: null,
+      name: "🤪"
+    },
+    theme_color: null
+  },
+  createdAt: new Date(),
+  updatedAt: new Date(),
+}

transaction.ts

@@ -1,39 +0,0 @@
-import base58 from "bs58";
-import "dotenv/config"
-import * as helpers  from "@solana-developers/helpers";
-import * as web3 from "@solana/web3.js";
-
-const emoKey = helpers.getKeypairFromEnvironment("EMO");
-const emoAddr = new web3.PublicKey(emoKey.publicKey);
-
-const payer = emoKey
-const connection = new web3.Connection(web3.clusterApiUrl("devnet"));
-
-const PING_PROGRAM_ADDRESS = new web3.PublicKey('ChT1B39WKLS8qUrkLvFDXMhEJ4F1XZzwUNHUt4AU9aVa')
-const PING_PROGRAM_DATA_ADDRESS =  new web3.PublicKey('Ah9K7dQ8EHaZqcAsgBW8w37yN2eAy3koFmUn4x3CJtod')
-
-const transaction = new web3.Transaction()
-const programId = new web3.PublicKey(PING_PROGRAM_ADDRESS)
-const pingProgramDataId = new web3.PublicKey(PING_PROGRAM_DATA_ADDRESS)
-
-
-// const instruction = new web3.TransactionInstruction({
-//   keys: [
-//     {
-//       pubkey: pingProgramDataId,
-//       isSigner: false,
-//       isWritable: true
-//     },
-//   ],
-//   programId
-// });
-
-// transaction.add(instruction)
-
-// const signature = await web3.sendAndConfirmTransaction(
-//   connection,
-//   transaction,
-//   [payer]
-// )
-
-// console.log(`✅ Transaction completed! Signature is ${signature}`)