diff --git a/drain.ts b/drain.ts deleted file mode 100644 index d0f2172..0000000 --- a/drain.ts +++ /dev/null @@ -1,31 +0,0 @@ -import base58 from "bs58"; -import "dotenv/config" -import * as helpers from "@solana-developers/helpers"; -import * as web3 from "@solana/web3.js"; - -const emoKey = helpers.getKeypairFromEnvironment("EMO"); -const emoAddr = new web3.PublicKey(emoKey.publicKey); - -const ntrKey = helpers.getKeypairFromEnvironment("NTR"); -const ntrAddr = new web3.PublicKey(ntrKey.publicKey); - -const payer = emoKey -const connection = new web3.Connection(web3.clusterApiUrl("devnet")); - -const balance = await connection.getBalance(emoAddr); - -const tx = new web3.Transaction(); -const TX_COST = 5000; - -const instr = web3.SystemProgram.transfer({ - fromPubkey: emoAddr, - toPubkey: ntrAddr, - lamports: balance - TX_COST, -}); - -tx.add(instr); - -const sig = await web3.sendAndConfirmTransaction(connection, tx, [emoKey]); - -console.log(sig); - diff --git a/index.ts b/index.ts index b1530ce..b8ef5b9 100644 --- a/index.ts +++ b/index.ts @@ -2,6 +2,8 @@ import "dotenv/config" import fs from 'fs/promises' +import pino from 'pino' + import * as Minio from 'minio' import Vision from '@google-cloud/vision' import { PrismaClient } from '@prisma/client' @@ -19,6 +21,8 @@ BigInt.prototype.toJSON = function bigIntStringify() { const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default const STORAGE_BUCKET = process.env.STORAGE_BUCKET +const log = pino({ base: {} }); + const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS }) const prisma = new PrismaClient() const minio = new Minio.Client({ @@ -26,13 +30,13 @@ const minio = new Minio.Client({ accessKey: process.env.STORAGE_ACCESS_KEY, secretKey: process.env.STORAGE_SECRET_KEY, }) -const client = new MeiliSearch({ +const meili = new MeiliSearch({ host: process.env.MEILI_URL, apiKey: process.env.MEILI_MASTER_KEY, }) -const attachments = attachmentService(minio, prisma) -const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio) +const attachments = attachmentService(log, minio, prisma) +const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log) async function ensureStorageBucketExists() { const exists = await minio.bucketExists(STORAGE_BUCKET) @@ -40,18 +44,37 @@ async function ensureStorageBucketExists() { await minio.makeBucket(STORAGE_BUCKET) } +async function ensureMeiliIndex() { + const delTask = await meili.deleteIndex('memes') + const delResult = await meili.waitForTask(delTask.taskUid) + const task = await meili.createIndex('memes', { primaryKey: 'id' }) + const result = await meili.waitForTask(task.taskUid) + log.info(result); +} + async function processChannels(channelIds) { await ensureStorageBucketExists() const msgs = await Promise.all(channelIds.map(c => scraper.getChannelAttachments(c))) const results = await Promise.allSettled(msgs.flat().map(a => attachments.process(a))) - console.log(results); + log.info(results.map(r => r.status)); return results } + +async function updateMeili() { + await ensureMeiliIndex(); + const index = meili.index('memes') + const memes = await prisma.meme.findMany(); + const task = await index.updateDocuments(memes) + const res = await meili.waitForTask(task.taskUid) + log.info(res) +} + const CHANNEL_IDS = [ 367936426962190336n, // TPM shitposts 1095060717117374687n, // aella memes ] await processChannels(CHANNEL_IDS) +await updateMeili() diff --git a/package-lock.json b/package-lock.json index 20f6d68..771f098 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,8 @@ "meilisearch": "^0.38.0", "mime-types": "^2.1.35", "minio": "^7.1.3", + "pino": "^8.20.0", + "pino-pretty": "^11.0.0", "sharp": "^0.33.3", "typescript": "^5.3.3" }, @@ -1112,6 +1114,14 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/available-typed-arrays": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", @@ -1210,6 +1220,29 @@ "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz", "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==" }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "node_modules/buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -1318,6 +1351,11 @@ "simple-swizzle": "^0.2.2" } }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==" + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -1337,6 +1375,14 @@ "node-fetch": "^2.6.12" } }, + "node_modules/dateformat": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", + "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==", + "engines": { + "node": "*" + } + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -1525,11 +1571,37 @@ "node": ">=6" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, + "node_modules/fast-copy": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.2.tgz", + "integrity": "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ==" + }, + "node_modules/fast-redact": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.5.0.tgz", + "integrity": "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" + }, "node_modules/fast-xml-parser": { "version": "4.3.6", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.6.tgz", @@ -1817,6 +1889,11 @@ "node": ">= 0.4" } }, + "node_modules/help-me": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz", + "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==" + }, "node_modules/http-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", @@ -1853,6 +1930,25 @@ "node": ">= 14" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -1994,6 +2090,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/joycon": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", + "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", + "engines": { + "node": ">=10" + } + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -2079,6 +2183,14 @@ "node": ">= 0.6" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/minio": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/minio/-/minio-7.1.3.tgz", @@ -2144,6 +2256,14 @@ "node": ">= 6" } }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -2164,6 +2284,95 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pino": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-8.20.0.tgz", + "integrity": "sha512-uhIfMj5TVp+WynVASaVEJFTncTUe4dHBq6CWplu/vBgvGHhvBvQfxz+vcOrnnBQdORH3izaGEurLfNlq3YxdFQ==", + "dependencies": { + "atomic-sleep": "^1.0.0", + "fast-redact": "^3.1.1", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^1.1.0", + "pino-std-serializers": "^6.0.0", + "process-warning": "^3.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^3.7.0", + "thread-stream": "^2.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.1.0.tgz", + "integrity": "sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==", + "dependencies": { + "readable-stream": "^4.0.0", + "split2": "^4.0.0" + } + }, + "node_modules/pino-abstract-transport/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/pino-pretty": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-11.0.0.tgz", + "integrity": "sha512-YFJZqw59mHIY72wBnBs7XhLGG6qpJMa4pEQTRgEPEbjIYbng2LXEZZF1DoyDg9CfejEy8uZCyzpcBXXG0oOCwQ==", + "dependencies": { + "colorette": "^2.0.7", + "dateformat": "^4.6.3", + "fast-copy": "^3.0.0", + "fast-safe-stringify": "^2.1.1", + "help-me": "^5.0.0", + "joycon": "^3.1.1", + "minimist": "^1.2.6", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^1.0.0", + "pump": "^3.0.0", + "readable-stream": "^4.0.0", + "secure-json-parse": "^2.4.0", + "sonic-boom": "^3.0.0", + "strip-json-comments": "^3.1.1" + }, + "bin": { + "pino-pretty": "bin.js" + } + }, + "node_modules/pino-pretty/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz", + "integrity": "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==" + }, "node_modules/possible-typed-array-names": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz", @@ -2188,6 +2397,19 @@ "node": ">=16.13" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-warning": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz", + "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==" + }, "node_modules/proto3-json-serializer": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.1.tgz", @@ -2227,6 +2449,15 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/query-string": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.3.tgz", @@ -2244,6 +2475,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" + }, "node_modules/readable-stream": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", @@ -2269,6 +2505,14 @@ "node": ">=8.10.0" } }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "engines": { + "node": ">= 12.13.0" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -2309,11 +2553,24 @@ } ] }, + "node_modules/safe-stable-stringify": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", + "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==", + "engines": { + "node": ">=10" + } + }, "node_modules/sax": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz", "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==" }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==" + }, "node_modules/semver": { "version": "7.6.0", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", @@ -2391,6 +2648,14 @@ "is-arrayish": "^0.3.1" } }, + "node_modules/sonic-boom": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.8.1.tgz", + "integrity": "sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg==", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, "node_modules/split-on-first": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", @@ -2399,6 +2664,14 @@ "node": ">=6" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/stream-events": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", @@ -2452,6 +2725,17 @@ "node": ">=8" } }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/strnum": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", @@ -2500,6 +2784,14 @@ "node": ">= 6" } }, + "node_modules/thread-stream": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.4.1.tgz", + "integrity": "sha512-d/Ex2iWd1whipbT681JmTINKw0ZwOUBZm7+Gjs64DHuX34mmw8vJL2bFAaNacaW72zYiTJxSHi5abUuOi5nsfg==", + "dependencies": { + "real-require": "^0.2.0" + } + }, "node_modules/through2": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", @@ -3254,6 +3546,11 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, + "atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==" + }, "available-typed-arrays": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", @@ -3322,6 +3619,15 @@ "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz", "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==" }, + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -3401,6 +3707,11 @@ "simple-swizzle": "^0.2.2" } }, + "colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==" + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -3417,6 +3728,11 @@ "node-fetch": "^2.6.12" } }, + "dateformat": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", + "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==" + }, "debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -3551,11 +3867,31 @@ "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" }, + "events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" + }, "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, + "fast-copy": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.2.tgz", + "integrity": "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ==" + }, + "fast-redact": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.5.0.tgz", + "integrity": "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==" + }, + "fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" + }, "fast-xml-parser": { "version": "4.3.6", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.6.tgz", @@ -3743,6 +4079,11 @@ "function-bind": "^1.1.2" } }, + "help-me": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz", + "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==" + }, "http-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", @@ -3772,6 +4113,11 @@ "debug": "4" } }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -3862,6 +4208,11 @@ "which-typed-array": "^1.1.14" } }, + "joycon": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", + "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==" + }, "json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -3938,6 +4289,11 @@ "mime-db": "1.52.0" } }, + "minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" + }, "minio": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/minio/-/minio-7.1.3.tgz", @@ -3983,6 +4339,11 @@ "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==" }, + "on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==" + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -3997,6 +4358,87 @@ "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true }, + "pino": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-8.20.0.tgz", + "integrity": "sha512-uhIfMj5TVp+WynVASaVEJFTncTUe4dHBq6CWplu/vBgvGHhvBvQfxz+vcOrnnBQdORH3izaGEurLfNlq3YxdFQ==", + "requires": { + "atomic-sleep": "^1.0.0", + "fast-redact": "^3.1.1", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^1.1.0", + "pino-std-serializers": "^6.0.0", + "process-warning": "^3.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^3.7.0", + "thread-stream": "^2.0.0" + } + }, + "pino-abstract-transport": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.1.0.tgz", + "integrity": "sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==", + "requires": { + "readable-stream": "^4.0.0", + "split2": "^4.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "requires": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + } + } + } + }, + "pino-pretty": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-11.0.0.tgz", + "integrity": "sha512-YFJZqw59mHIY72wBnBs7XhLGG6qpJMa4pEQTRgEPEbjIYbng2LXEZZF1DoyDg9CfejEy8uZCyzpcBXXG0oOCwQ==", + "requires": { + "colorette": "^2.0.7", + "dateformat": "^4.6.3", + "fast-copy": "^3.0.0", + "fast-safe-stringify": "^2.1.1", + "help-me": "^5.0.0", + "joycon": "^3.1.1", + "minimist": "^1.2.6", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^1.0.0", + "pump": "^3.0.0", + "readable-stream": "^4.0.0", + "secure-json-parse": "^2.4.0", + "sonic-boom": "^3.0.0", + "strip-json-comments": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "requires": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + } + } + } + }, + "pino-std-serializers": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz", + "integrity": "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==" + }, "possible-typed-array-names": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz", @@ -4011,6 +4453,16 @@ "@prisma/engines": "5.12.1" } }, + "process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==" + }, + "process-warning": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz", + "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==" + }, "proto3-json-serializer": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.1.tgz", @@ -4043,6 +4495,15 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "query-string": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.3.tgz", @@ -4054,6 +4515,11 @@ "strict-uri-encode": "^2.0.0" } }, + "quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" + }, "readable-stream": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", @@ -4073,6 +4539,11 @@ "picomatch": "^2.2.1" } }, + "real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==" + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -4093,11 +4564,21 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, + "safe-stable-stringify": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", + "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==" + }, "sax": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz", "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==" }, + "secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==" + }, "semver": { "version": "7.6.0", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", @@ -4156,11 +4637,24 @@ "is-arrayish": "^0.3.1" } }, + "sonic-boom": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.8.1.tgz", + "integrity": "sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg==", + "requires": { + "atomic-sleep": "^1.0.0" + } + }, "split-on-first": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", "integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==" }, + "split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==" + }, "stream-events": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", @@ -4205,6 +4699,11 @@ "ansi-regex": "^5.0.1" } }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==" + }, "strnum": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", @@ -4246,6 +4745,14 @@ } } }, + "thread-stream": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.4.1.tgz", + "integrity": "sha512-d/Ex2iWd1whipbT681JmTINKw0ZwOUBZm7+Gjs64DHuX34mmw8vJL2bFAaNacaW72zYiTJxSHi5abUuOi5nsfg==", + "requires": { + "real-require": "^0.2.0" + } + }, "through2": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", diff --git a/package.json b/package.json index 3f4755f..19f7a9f 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,8 @@ "main": "index.ts", "type": "module", "scripts": { - "start": "esrun index.ts", - "dev": "esrun --watch index.ts", + "start": "esrun index.ts | pino-pretty", + "dev": "esrun --watch index.ts | | pino-pretty", "migrate:make": "prisma migrate dev --name", "test": "echo \"Error: no test specified\" && exit 1" }, @@ -20,6 +20,8 @@ "meilisearch": "^0.38.0", "mime-types": "^2.1.35", "minio": "^7.1.3", + "pino": "^8.20.0", + "pino-pretty": "^11.0.0", "sharp": "^0.33.3", "typescript": "^5.3.3" }, diff --git a/src/attachments.ts b/src/attachments.ts index dd55744..941b556 100644 --- a/src/attachments.ts +++ b/src/attachments.ts @@ -2,6 +2,8 @@ import sharp from 'sharp'; import axios from 'axios' import mime from 'mime-types' +import type { Logger } from 'pino'; + import type Minio from 'minio' import type { PrismaClient, Attachment } from '@prisma/client'; @@ -31,7 +33,7 @@ export function objectName(a: Attachment) { return `${a.memeId}.${mime.extension(a.contentType)}` } -export default function attachmentService(minio: Minio.Client, prisma: PrismaClient) { +export default function attachmentService(log: Logger, minio: Minio.Client, prisma: PrismaClient) { async function storeAttachment(a: Attachment) { const download = await axios.get(a.url, { responseType: 'stream' }) @@ -56,10 +58,10 @@ export default function attachmentService(minio: Minio.Client, prisma: PrismaCli const upload = await storeAttachment(a) return prisma.$transaction(async prisma => { const dbMeme = await prisma.meme.upsert({ create: meme, update: meme, where: { id: meme.id } }) - const dbA = prisma.attachment.upsert({ create: a, update: a, where: { id: a.id } }) - console.log('pending addition to db', dbMeme) + const dbA = await prisma.attachment.upsert({ create: a, update: a, where: { id: a.id } }) + log.info({ msg: 'pending addition to db', dbMeme }) const upload = await storeAttachment(a) - console.log('successfully uploaded', upload) + log.info({ msg: 'successfully uploaded', upload }) return { meme: dbMeme, upload, attachment: dbA } }) } diff --git a/src/memes.ts b/src/memes.ts index b5ff226..c36d98d 100644 --- a/src/memes.ts +++ b/src/memes.ts @@ -2,10 +2,11 @@ import type { PrismaClient, Attachment, Meme } from '@prisma/client'; import * as attachments from './attachments' +const STORAGE_BUCKET = process.env.STORAGE_BUCKET + function memeUrl(a: Attachment) { const STORAGE_URL = process.env.STORAGE_URL - const path = attachments.objectName(a) - const u = new URL(path, `https://${STORAGE_URL}`) + const u = new URL(`${STORAGE_BUCKET}/${attachments.objectName(a)}`, `https://${STORAGE_URL}`) return u.href } diff --git a/src/scraper.ts b/src/scraper.ts index 9e45ab8..207395b 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -2,18 +2,14 @@ import axios from 'axios' import type { PrismaClient, Attachment } from '@prisma/client'; import type { ImageAnnotatorClient } from '@google-cloud/vision'; +import type { Logger } from 'pino'; import type Minio from 'minio' import sharp from 'sharp'; import * as attachments from './attachments' -export default function scraper( -discordAuth, -vision: ImageAnnotatorClient, -minio: Minio.Client, -prisma: PrismaClient, -) { +export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio: Minio.Client, prisma: PrismaClient, baseLog: Logger) { const headers = { 'Authorization': discordAuth, } @@ -55,7 +51,7 @@ prisma: PrismaClient, return res[0] || {} } - function filterAttachments(messages, guild, channel) { + async function filterAttachments(log, messages, guild, channel) { const acceptedContentTypes = [ ...attachments.IMAGE_TYPES, ...attachments.VIDEO_TYPES, @@ -69,10 +65,15 @@ prisma: PrismaClient, .filter(([a, m]) => acceptedContentTypes.includes(a.content_type)) const attachmentIds = validAttachments.map(([a, m]) => a.id) + const alreadyProcessedIds = (await prisma.attachment + .findMany({ select: { id: true }, where: { id: { in: attachmentIds }}}) + ).map(r => r.id.toString()) // attachment not a true attachment yet, id still string - const alreadyProcessedIds = prisma.attachment.find + const newAttachments = validAttachments.filter(([a, m]) => !alreadyProcessedIds.includes(a.id)) - return Promise.all(validAttachments + log.info({ newAttachments: newAttachments.length, validAttachments: validAttachments.length }) + + return Promise.all(newAttachments .map(async ([a, m]) => { const attachment: Attachment = { id: BigInt(a.id), @@ -105,14 +106,13 @@ prisma: PrismaClient, const guild = await fetchGuildDetails(channel.guild_id) const messages = await fetchChannelMessages(channelId) - return filterAttachments(messages, guild, channel); + const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId }) + + return filterAttachments(log, messages, guild, channel); // return { messages, channel, guild } } catch (e) { - console.error('scrape error', { - err: e.toJSON(), - channelId, - }) + baseLog.error({ err: e.toJSON(), channelId, }) } } diff --git a/transaction.ts b/transaction.ts deleted file mode 100644 index 263c989..0000000 --- a/transaction.ts +++ /dev/null @@ -1,39 +0,0 @@ -import base58 from "bs58"; -import "dotenv/config" -import * as helpers from "@solana-developers/helpers"; -import * as web3 from "@solana/web3.js"; - -const emoKey = helpers.getKeypairFromEnvironment("EMO"); -const emoAddr = new web3.PublicKey(emoKey.publicKey); - -const payer = emoKey -const connection = new web3.Connection(web3.clusterApiUrl("devnet")); - -const PING_PROGRAM_ADDRESS = new web3.PublicKey('ChT1B39WKLS8qUrkLvFDXMhEJ4F1XZzwUNHUt4AU9aVa') -const PING_PROGRAM_DATA_ADDRESS = new web3.PublicKey('Ah9K7dQ8EHaZqcAsgBW8w37yN2eAy3koFmUn4x3CJtod') - -const transaction = new web3.Transaction() -const programId = new web3.PublicKey(PING_PROGRAM_ADDRESS) -const pingProgramDataId = new web3.PublicKey(PING_PROGRAM_DATA_ADDRESS) - - -// const instruction = new web3.TransactionInstruction({ -// keys: [ -// { -// pubkey: pingProgramDataId, -// isSigner: false, -// isWritable: true -// }, -// ], -// programId -// }); - -// transaction.add(instruction) - -// const signature = await web3.sendAndConfirmTransaction( -// connection, -// transaction, -// [payer] -// ) - -// console.log(`✅ Transaction completed! Signature is ${signature}`) \ No newline at end of file