ITS NODE BABY

src/scraper.ts

@@ -0,0 +1,90 @@
+import axios from 'axios';
+
+export default function scraper(discordAuth) {
+    const headers = {
+        'Authorization': discordAuth,
+    }
+
+    async function fetchChannelMessages(channelId) {
+        const url = `https://discord.com/api/v9/channels/${channelId}/messages`
+        const res = await axios.get(url, { headers })
+        return res.data
+    }
+
+    async function fetchGuildDetails(guildId) {
+        const url = `https://discord.com/api/v9/guilds/${guildId}`
+        const res = await axios.get(url, { headers })
+        const keys_to_remove = ['emojis', 'stickers', 'roles']
+        keys_to_remove.forEach(k => delete res[k])
+        return res.data
+    }
+
+    async function fetchChannel(channelId) {
+        const url = `https://discord.com/api/v9/channels/${channelId}`
+        const res = await axios.get(url, { headers })
+        return res.data
+    }
+
+    async function getChannelAttachments(channelId) {
+        try {
+            const channel = await fetchChannel(channelId)
+            const guild = await fetchGuildDetails(channel.guild_id)
+            const messages = await fetchChannelMessages(channelId)
+            return { messages, channel, guild }
+        } catch (e) {
+            console.error('scrape error', {
+                err: e.toJSON(),
+                channelId,
+            })
+        }
+    }
+
+    return {
+        getChannelAttachments,
+    }
+}
+
+
+    // async def download_attachment(attachment: Attachment):
+    //     async with http_session.get(attachment.url) as response:
+    //         if response.status == 200:
+    //             (filename, ext) = attachment.file_details()
+    //             object_name = attachment.object_name()
+
+    //             print(f"uploading object_name={object_name} filename={filename} ext={ext}")
+    //             res_bytes = await response.read()
+
+    //             # make a webp conversion for obvious image types
+    //             if attachment.convert_to_webp():
+    //                 # have to read the whole img before converting to webp
+    //                 image = Image.open(BytesIO(res_bytes))
+    //                 webp_output = BytesIO()
+    //                 image.save(webp_output, format='WebP')
+    //                 webp_output.seek(0)
+    //                 return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=webp_output)
+
+    //             return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=res_bytes)
+
+    // def create_attachments(message, channel, guild):
+    //     urls = [(parse_url(attachment['url']), attachment['id']) for attachment in message['attachments']]
+
+    //     def generate_attachment(url: Url, attachmentId: str):
+    //         if not url.path:
+    //             return None
+
+    //         (filename, ext) = os.path.splitext(os.path.basename(url.path))
+    //         extensions = ['jpg', 'png', 'jpeg', 'webp', 'mov', 'gif', 'mp4']
+    //         if ext.lower()[1:] not in extensions:
+    //             return None
+
+    //         return Attachment(
+    //             id=uuid4(),
+    //             attachmentId=int(attachmentId),
+    //             msgId=int(message['id']),
+    //             message=message,
+    //             guild=guild,
+    //             url=url.url,
+    //             channel=channel
+    //         )
+
+    //     return [generate_attachment(*url) for url in urls]