// index.ts
import "dotenv/config";
import { exec as execCb } from "child_process";
import { promisify } from "util";
import pino from "pino";
import * as Minio from "minio";
import Vision from "@google-cloud/vision";
import { PrismaClient } from "@prisma/client";
import { MeiliSearch } from "meilisearch";

// src/scraper.ts
import { setTimeout } from "timers/promises";
import axios2 from "axios";
import _2 from "lodash";

// src/attachments.ts
import sharp from "sharp";
import axios from "axios";
import mime from "mime-types";
import _ from "lodash";
var STORAGE_BUCKET = process.env.STORAGE_BUCKET;
var IMAGE_TYPES = [
  "image/png",
  "image/jpeg",
  "image/webp",
  "image/gif"
];
function isImage(a) {
  return IMAGE_TYPES.includes(a.contentType);
}
function objectName(a) {
  if (isImage(a))
    return `${a.memeId}.webp`;
  return `${a.memeId}.${mime.extension(a.contentType)}`;
}
async function attachmentService(log, minio, prisma, memes) {
  async function ensureStorageBucketExists() {
    const exists = await minio.bucketExists(STORAGE_BUCKET);
    if (exists)
      return true;
    await minio.makeBucket(STORAGE_BUCKET);
  }
  async function storeAttachment(a) {
    const download = await axios.get(a.url, { responseType: "stream" });
    const stream = () => {
      if (isImage(a)) {
        const converter = sharp().webp();
        download.data.pipe(converter);
        return converter;
      }
      return download.data;
    };
    return minio.putObject(
      STORAGE_BUCKET,
      objectName(a),
      stream(),
      null,
      // hardcode this for now, we upload everything as webp
      { "Content-Type": "image/webp" }
    );
  }
  async function deleteAttachment(a) {
    return minio.removeObject(STORAGE_BUCKET, objectName(a));
  }
  async function process2(a, history) {
    try {
      const results = await prisma.$transaction(async (prisma2) => {
        const meme = memes.fromAttachment(a);
        const dbMeme = await prisma2.meme.upsert({ create: meme, update: meme, where: { id: meme.id } });
        const dbA = await prisma2.attachment.upsert({ create: a, update: a, where: { id: a.id } });
        log.debug({ msg: "pending addition to db", dbMeme });
        const upload = await storeAttachment(a);
        log.debug({ msg: "successfully uploaded", upload });
        await memes.persist(dbMeme);
        log.info(dbMeme, "new meme");
        return { meme: dbMeme, upload, attachment: dbA };
      }, { timeout: 6e4 });
    } catch (e) {
      if (e.meta?.modelName !== "Meme" || e.meta?.target?.[0] !== "attachmentId") {
        log.error(e, "failed to process attachment");
        log.error(_.omit(a, ["vision"]), "failed to process attachment");
      }
      await deleteAttachment(a);
    }
    const newHistory = await prisma.channelHistory.update({ data: history, where: { id: history.id } });
    log.debug({ newHistory }, "updated channel history");
    return true;
  }
  await ensureStorageBucketExists();
  return {
    process: process2
  };
}

// src/scraper.ts
function convertSnowflakeToDate(snowflake) {
  const DISCORD_EPOCH = 14200704e5;
  const milliseconds = global.BigInt(snowflake) >> 22n;
  return new Date(Number(milliseconds) + DISCORD_EPOCH);
}
async function scraper(discordAuth, vision, minio, prisma, baseLog, attachments) {
  const headers = {
    "Authorization": discordAuth
  };
  async function fetchChannelMessages(channelId) {
    const url = `https://discord.com/api/v9/channels/${channelId}/messages&limit=100`;
    const res = await axios2.get(url, { headers });
    return res.data;
  }
  async function fetchChannelMessagesAfter(channelId, lastMessageId) {
    const url = `https://discord.com/api/v9/channels/${channelId}/messages?after=${lastMessageId}&limit=100`;
    const res = await axios2.get(url, { headers });
    return res.data;
  }
  async function fetchChannelMessagesBefore(channelId, earliestMessageId) {
    const url = `https://discord.com/api/v9/channels/${channelId}/messages?before=${earliestMessageId}&limit=100`;
    const res = await axios2.get(url, { headers });
    return res.data;
  }
  async function fetchGuildDetails(guildId) {
    const url = `https://discord.com/api/v9/guilds/${guildId}`;
    const res = await axios2.get(url, { headers });
    return _2.omit(res.data, ["emojis", "stickers", "roles", "features"]);
  }
  async function fetchChannel(channelId) {
    const url = `https://discord.com/api/v9/channels/${channelId}`;
    const res = await axios2.get(url, { headers });
    return res.data;
  }
  async function detectText(attachment) {
    if (!IMAGE_TYPES.includes(attachment.contentType))
      return {};
    const annotateRequest = {
      image: {
        source: {
          imageUri: attachment.url
        }
      },
      features: [{ type: "TEXT_DETECTION" }]
    };
    const res = await vision.annotateImage(annotateRequest);
    return res[0] || {};
  }
  async function fetchMissingVision() {
    const attachments2 = await prisma.attachment.findMany({ where: { vision: {}, contentType: { in: IMAGE_TYPES } } });
    baseLog.info({ attachments: attachments2.length }, "missing vision attachments");
  }
  async function processChannel(channelId) {
    const channel = await fetchChannel(channelId);
    const guild = await fetchGuildDetails(channel.guild_id);
    const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId });
    async function filterAttachments(messages) {
      const acceptedContentTypes = [
        ...IMAGE_TYPES
        // ...VIDEO_TYPES,
      ];
      const validAttachments = messages.filter((m) => m.attachments).flatMap((m) => m.attachments.map((a) => [a, m])).filter(([a, m]) => acceptedContentTypes.includes(a.content_type));
      log.debug({ validAttachments: validAttachments.length }, "attachments found in messages");
      return Promise.all(validAttachments.map(async ([a, m]) => {
        const attachment = {
          id: BigInt(a.id),
          msgId: BigInt(m.id),
          channelId: BigInt(m.channel_id),
          url: a.url,
          contentType: a.content_type,
          // create this ahead of time to associate with our copy of the media
          memeId: crypto.randomUUID(),
          vision: {},
          // quick hack so that detectText can receive a full attachment
          guild,
          channel,
          message: m,
          postedAt: convertSnowflakeToDate(a.id),
          // ts asks for these to be included but they don't need to be set
          createdAt: void 0,
          updatedAt: void 0
        };
        return { ...attachment, vision: await detectText(attachment) };
      }));
    }
    async function* fetchNew(channelId2, log2, latestMessageId) {
      const messages = (await fetchChannelMessagesAfter(channelId2, latestMessageId)).reverse();
      if (messages.length === 0) {
        log2.info("reached latest message");
      }
      log2.debug({ messages: messages.length, latestMessageId }, "fetched new messages");
      yield* await filterAttachments(messages);
    }
    async function* fetchOld(channelId2, log2, earliestMessageId) {
      const messages = await fetchChannelMessagesBefore(channelId2, earliestMessageId);
      if (messages.length === 0) {
        log2.info("reached beginning of history");
      }
      log2.debug({ messages: messages.length, earliestMessageId }, "fetched older messages");
      yield* await filterAttachments(messages);
    }
    const ensureHistory = async () => {
      const stored = await prisma.channelHistory.findFirst({ where: { id: channelId } });
      if (stored)
        return stored;
      const lastMessageId = channel.last_message_id;
      return await prisma.channelHistory.create({
        data: { latestMessageId: lastMessageId, earliestMessageId: lastMessageId, id: channelId }
      });
    };
    const history = await ensureHistory();
    async function processLoop() {
      const history2 = await ensureHistory();
      const newAttachments = fetchNew(channelId, log, history2.latestMessageId);
      const oldAttachments = fetchOld(channelId, log, history2.earliestMessageId);
      let oldCount = 0;
      let newCount = 0;
      for await (const attachment of newAttachments) {
        log.debug({ msg: "scraped new attachment", attachment: _2.omit(attachment, ["guild", "message", "channel", "vision"]) });
        const newHistory = { id: history2.id, latestMessageId: attachment.msgId };
        await attachments.process(attachment, newHistory);
        oldCount++;
      }
      for await (const attachment of oldAttachments) {
        log.debug({ msg: "scraped old attachment", attachment: _2.omit(attachment, ["guild", "message", "channel", "vision"]) });
        const newHistory = { id: history2.id, earliestMessageId: attachment.msgId };
        await attachments.process(attachment, newHistory);
        newCount++;
      }
      log.info({ oldCount, newCount }, "processed attachments");
      const shouldWait = newCount === 0 && oldCount === 0;
      const PROCESS_DELAY_MS = 1e3 * 60;
      if (shouldWait) {
        log.info({ PROCESS_DELAY_MS }, "waiting for new messages");
        await setTimeout(PROCESS_DELAY_MS);
      }
      return processLoop();
    }
    await processLoop();
  }
  return {
    processChannel
  };
}

// src/memes.ts
import _3 from "lodash";
var STORAGE_BUCKET2 = process.env.STORAGE_BUCKET;
function memeUrl(a) {
  const STORAGE_URL = process.env.STORAGE_URL;
  const u = new URL(`${STORAGE_BUCKET2}/${objectName(a)}`, `https://${STORAGE_URL}`);
  return u.href;
}
async function memeService(baseLog, prisma, meili) {
  const log = baseLog.child({ service: "meme" });
  function fromAttachment(a) {
    const text = a.vision.fullTextAnnotation?.text || "";
    const tags = [
      a.guild.name,
      a.channel.name,
      a.message.author.username
    ];
    const m = {
      id: a.memeId,
      url: memeUrl(a),
      text,
      tags,
      attachmentId: a.id,
      postedAt: a.postedAt,
      createdAt: void 0,
      updatedAt: void 0
    };
    return m;
  }
  async function ensureMeiliIndex() {
    const task = await meili.createIndex("memes", { primaryKey: "id" });
    const result = await meili.waitForTask(task.taskUid);
    log.debug(result);
    const index = await meili.index("memes");
    await index.updateSortableAttributes(["postedAt"]);
    return index;
  }
  async function resetMeili() {
    const delTask = await meili.deleteIndex("memes");
    const delResult = await meili.waitForTask(delTask.taskUid);
    const index = await ensureMeiliIndex();
    const memes = (await prisma.meme.findMany()).map((m) => ({ ...m, postedAt: m.postedAt.valueOf() }));
    for await (const batch of _3.chunk(memes, 1e3)) {
      const task = await index.updateDocuments(batch);
      const res = await meili.waitForTask(task.taskUid);
      log.info({ res }, "added meme batch");
    }
  }
  async function persist(m) {
    const index = meili.index("memes");
    const doc = { ...m, postedAt: m.postedAt.valueOf() };
    return index.updateDocuments([doc]);
  }
  await ensureMeiliIndex();
  return {
    ensureMeiliIndex,
    resetMeili,
    fromAttachment,
    persist
  };
}

// index.ts
BigInt.prototype.toJSON = function bigIntStringify() {
  return this.toString();
};
var exec = promisify(execCb);
async function main() {
  const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default;
  const log = pino({ base: {} });
  const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS });
  const prisma = new PrismaClient();
  const minio = new Minio.Client({
    endPoint: process.env.STORAGE_URL,
    accessKey: process.env.STORAGE_ACCESS_KEY,
    secretKey: process.env.STORAGE_SECRET_KEY
  });
  const meili = new MeiliSearch({
    host: process.env.MEILI_URL,
    apiKey: process.env.MEILI_MASTER_KEY
  });
  const memes = await memeService(log, prisma, meili);
  const attachments = await attachmentService(log, minio, prisma, memes);
  const scraper2 = await scraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments);
  const CHANNEL_IDS = [
    367936426962190336n,
    // TPM shitposts
    367936467592413194n,
    // tpm screenshots
    1028511952277864501n,
    // tpm sludge
    725835129326665768n,
    // machinegirl memes
    1095060717117374687n
    // aella memes
  ];
  async function migrateDb() {
    const output = await exec("prisma migrate deploy");
    log.info(output);
  }
  await migrateDb();
  await Promise.all(CHANNEL_IDS.map((id) => scraper2.processChannel(id)));
}
main().then(() => true).catch((e) => console.error(e));