downloading channels

gimmeme.py

@@ -0,0 +1,71 @@
+import os
+import asyncio
+import aiohttp
+
+from dotenv import dotenv_values
+
+config = dotenv_values(".env")
+
+CHANNEL_IDS = [
+    367936426962190336, # TPM shitposts
+    1095060717117374687, # aella memes
+]
+
+headers = {
+    'Authorization': config['DISCORD_AUTH'],
+}
+
+async def save_img(session: aiohttp.ClientSession, url, guild_name, channel_name):
+    output_dir = f'./data/{guild_name}/{channel_name}'
+    filename = os.path.basename(url).split('?')[0]
+
+    os.makedirs(output_dir, exist_ok=True);
+
+    async with session.get(url) as response:
+        if response.status == 200:
+            print(filename)
+            with open(f'{output_dir}/{filename}', 'wb') as f:
+                f.write(await response.read())
+
+async def fetch_channel_img_urls(session: aiohttp.ClientSession, channel_id):
+    url = f"https://discord.com/api/v9/channels/{channel_id}/messages"
+    async with session.get(url, headers=headers) as res:
+        json = await res.json()
+        url_lists = map(lambda msg: list(
+            map(lambda attachment: attachment['url'], msg['attachments'])
+        ), json)
+        return [item for group in url_lists for item in group]
+
+async def fetch_guild_details(session: aiohttp.ClientSession, guild_id):
+    url = f"https://discord.com/api/v9/guilds/{guild_id}"
+    async with session.get(url, headers=headers) as res:
+        return await res.json()
+
+
+async def fetch_channel(session: aiohttp.ClientSession, channel_id):
+    url = f"https://discord.com/api/v9/channels/{channel_id}"
+    async with session.get(url, headers=headers) as res:
+        return await res.json()
+
+
+async def download_channel_imgs(session, channel_id):
+    channel = await fetch_channel(session, channel_id)
+    guild = await fetch_guild_details(session, channel['guild_id'])
+    print(f"Fetching messages for channel guild={guild['name']} name={channel['name']} id={channel['id']}")
+    urls = await fetch_channel_img_urls(session, channel_id)
+
+    tasks = [save_img(
+        session,
+        url,
+        channel_name=channel['name'],
+        guild_name=guild['name'],
+    ) for url in urls]
+    await asyncio.gather(*tasks)
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        channels = [download_channel_imgs(session, channel_id) for channel_id in CHANNEL_IDS]
+        await asyncio.gather(*channels)
+
+asyncio.run(main())