ntr/surf_megastructure
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mapsdl

Browse Source
This commit is contained in:
Nathan Rashleigh 2026-02-09 23:21:40 +11:00
parent cf65959148
commit f66c505082
3 changed files with 723 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/mapsdl/Dockerfile
View File

@ -2,7 +2,7 @@ FROM denoland/deno:debian-2.1.4
# Install archive extraction tools
RUN apt-get update && apt-get install -y \
unrar \
unar \
unzip \
bzip2 \
&& rm -rf /var/lib/apt/lists/*
@ -13,19 +13,23 @@ WORKDIR /app
# Copy the script
COPY mapsdl.ts .
# Run as UID 1000 to match cssds (steam user)
RUN groupadd -g 1000 steam && useradd -u 1000 -g steam -m steam
# Create directories for credentials and data
RUN mkdir -p /app/credentials /maps /tmp/mapsdl
RUN mkdir -p /app/credentials /maps /tmp/mapsdl && \
chown steam:steam /maps /tmp/mapsdl
# Cache Deno dependencies
RUN deno cache --allow-scripts=npm:* mapsdl.ts
RUN deno cache --allow-import mapsdl.ts
# Set default environment variables
ENV MAPS_DIR=/maps
ENV TEMP_DIR=/tmp/mapsdl
ENV GOOGLE_APPLICATION_CREDENTIALS=/app/credentials/service-account.json
# Run as non-root user
USER deno
# Run as same UID as cssds
USER steam
# Default command runs dry-run mode
CMD ["run", "--allow-net", "--allow-read", "--allow-write", "--allow-env", "--allow-sys", "mapsdl.ts", "--dry-run"]
CMD ["run", "--allow-import", "--allow-net", "--allow-read", "--allow-write", "--allow-env", "--allow-sys", "mapsdl.ts", "--dry-run"]

586
src/mapsdl/mapsdl-all.ts Executable file
View File

@ -0,0 +1,586 @@
// #!/usr/bin/env -S deno run --allow-net --allow-read --allow-write --allow-env --allow-sys
// this script should be run with deno
// use async stdlib fns where possible
import { parse } from "https://deno.land/std@0.210.0/flags/mod.ts";
import { join } from "https://deno.land/std@0.210.0/path/mod.ts";
import { type CredentialsClient, Drive } from "https://googleapis.deno.dev/v1/drive:v3.ts";
import { SignJWT, importPKCS8 } from "https://deno.land/x/jose@v4.14.4/index.ts";
// Service account auth that exchanges JWT for an OAuth2 access token
class ServiceAccountAuth implements CredentialsClient {
#clientEmail: string;
#privateKey: string;
#tokenUri: string;
#scope: string;
#accessToken: string | null = null;
#tokenExpiry = 0;
constructor(json: { client_email: string; private_key: string; token_uri?: string }, scope: string) {
this.#clientEmail = json.client_email;
this.#privateKey = json.private_key;
this.#tokenUri = json.token_uri ?? "https://oauth2.googleapis.com/token";
this.#scope = scope;
}
async getRequestHeaders(_url?: string): Promise<Record<string, string>> {
if (!this.#accessToken || Date.now() / 1000 > this.#tokenExpiry - 60) {
await this.#refreshToken();
}
return { Authorization: `Bearer ${this.#accessToken}` };
}
async #refreshToken(): Promise<void> {
const now = Math.floor(Date.now() / 1000);
const key = await importPKCS8(this.#privateKey, "RS256");
const assertion = await new SignJWT({
iss: this.#clientEmail,
scope: this.#scope,
aud: this.#tokenUri,
})
.setProtectedHeader({ alg: "RS256", typ: "JWT" })
.setIssuedAt(now)
.setExpirationTime(now + 3600)
.sign(key);
const resp = await fetch(this.#tokenUri, {
method: "POST",
headers: { "Content-Type": "application/x-www-form-urlencoded" },
body: new URLSearchParams({
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
assertion,
}),
});
if (!resp.ok) {
const body = await resp.text();
throw new Error(`Token exchange failed (${resp.status}): ${body}`);
}
const data = await resp.json();
this.#accessToken = data.access_token;
this.#tokenExpiry = now + (data.expires_in ?? 3600);
}
}
// Configuration
const CONFIG = {
sqlUrl: "https://raw.githubusercontent.com/bhopppp/Shavit-Surf-Timer/master/sql/surfzones.sql",
mapsDir: Deno.env.get("MAPS_DIR") || "/home/ntr/surf_megastructure/data/cssds/cstrike/maps",
tempDir: Deno.env.get("TEMP_DIR") || "/tmp/mapsdl",
serviceAccountPath: Deno.env.get("GOOGLE_APPLICATION_CREDENTIALS") || "./credentials/service-account.json",
gdriveFolders: {
aToK: "17QJ-Wzk9eMHKZqX227HkPCg9_Vmrf9h-",
lToZ: "1f3Oe65BngrSxTPKHAt6MEwK0FTsDbUsO",
},
concurrency: 5,
};
interface GDriveFile {
id: string;
name: string;
size: string;
}
interface Manifest {
total: number;
found: number;
missing: string[];
notInGDrive: string[];
}
// Step 1: Download and parse SQL file for map names
async function downloadSurfZonesSQL(): Promise<string> {
console.log(`[DL] Downloading SQL file from ${CONFIG.sqlUrl}...`);
const response = await fetch(CONFIG.sqlUrl);
if (!response.ok) {
throw new Error(`Failed to download SQL: ${response.statusText}`);
}
return await response.text();
}
function extractMapNamesFromSQL(sqlContent: string): Set<string> {
console.log("[>>] Extracting map names from SQL...");
const mapNames = new Set<string>();
// Match INSERT statements and extract map names (surf_*)
// Handles both single and multi-value INSERT statements
const insertRegex = /INSERT INTO\s+`?(?:map)?zones`?\s+.*?VALUES\s*\((.*?)\);/gis;
for (const match of sqlContent.matchAll(insertRegex)) {
const valuesSection = match[1];
// Extract all string literals from the VALUES section
const stringRegex = /'([^']+)'/g;
for (const stringMatch of valuesSection.matchAll(stringRegex)) {
const value = stringMatch[1];
// Only keep values that start with 'surf_'
if (value.startsWith("surf_")) {
mapNames.add(value);
}
}
}
console.log(`[OK] Found ${mapNames.size} unique surf maps in SQL`);
return mapNames;
}
// Step 2: Check which maps exist locally
async function getLocalMaps(): Promise<Set<string>> {
console.log(`[DIR] Scanning local maps directory: ${CONFIG.mapsDir}`);
const localMaps = new Set<string>();
try {
for await (const entry of Deno.readDir(CONFIG.mapsDir)) {
if (entry.isFile && entry.name.endsWith(".bsp")) {
// Remove .bsp extension and add to set
const mapName = entry.name.replace(/\.bsp$/, "");
localMaps.add(mapName);
}
}
} catch (error) {
if (error instanceof Deno.errors.NotFound) {
console.warn(`[!] Maps directory not found: ${CONFIG.mapsDir}`);
await Deno.mkdir(CONFIG.mapsDir, { recursive: true });
} else {
throw error;
}
}
console.log(`[OK] Found ${localMaps.size} local maps`);
return localMaps;
}
// Step 3: Build manifest of missing maps
function buildManifest(
requiredMaps: Set<string>,
localMaps: Set<string>,
): Manifest {
console.log("[STATS] Building manifest of missing maps...");
const missing: string[] = [];
for (const mapName of requiredMaps) {
if (!localMaps.has(mapName)) {
missing.push(mapName);
}
}
return {
total: requiredMaps.size,
found: localMaps.size,
missing,
notInGDrive: [], // Will be populated after checking GDrive
};
}
// Google Drive API client with service account
class GoogleDriveClient {
private drive!: Drive;
private auth!: CredentialsClient;
async authenticate(): Promise<void> {
console.log("[AUTH] Authenticating with Google Drive API...");
const keyFile = await Deno.readTextFile(CONFIG.serviceAccountPath);
this.auth = new ServiceAccountAuth(
JSON.parse(keyFile),
"https://www.googleapis.com/auth/drive.readonly",
);
this.drive = new Drive(this.auth);
console.log("[OK] Authentication successful");
}
async listFiles(folderId: string): Promise<GDriveFile[]> {
const files: GDriveFile[] = [];
let pageToken: string | undefined;
do {
const response = await this.drive.filesList({
q: `'${folderId}' in parents and trashed=false`,
pageSize: 1000,
pageToken,
});
for (const f of response.files ?? []) {
if (f.id && f.name) {
files.push({ id: f.id, name: f.name, size: String(f.size ?? "") });
}
}
pageToken = response.nextPageToken ?? undefined;
} while (pageToken);
return files;
}
async downloadFile(fileId: string, destPath: string): Promise<void> {
const url = `https://www.googleapis.com/drive/v3/files/${fileId}?alt=media`;
const headers = await this.auth.getRequestHeaders(url);
const response = await fetch(url, { headers });
if (!response.ok) {
throw new Error(`Download failed: ${response.statusText}`);
}
const bytes = new Uint8Array(await response.arrayBuffer());
await Deno.writeFile(destPath, bytes);
}
}
// Step 5: Search Google Drive for missing maps
async function searchGDriveForMaps(
client: GoogleDriveClient,
manifest: Manifest,
): Promise<Map<string, GDriveFile>> {
console.log("\n[CLOUD] Searching Google Drive folders...");
// List files from both folders
const [filesAK, filesLZ] = await Promise.all([
client.listFiles(CONFIG.gdriveFolders.aToK),
client.listFiles(CONFIG.gdriveFolders.lToZ),
]);
const allFiles = [...filesAK, ...filesLZ];
console.log(`[OK] Found ${allFiles.length} files in Google Drive`);
console.log(` - A-K folder: ${filesAK.length} files`);
console.log(` - L-Z folder: ${filesLZ.length} files`);
// Build map of normalized names to GDrive files
const gdriveMap = new Map<string, GDriveFile>();
for (const file of allFiles) {
// Normalize: remove archive extensions (.rar, .bz2, .zip, .bsp)
const normalized = file.name
.replace(/\.bsp\.bz2$/i, "")
.replace(/\.(rar|bz2|zip|bsp)$/i, "")
.toLowerCase()
.trim();
// Only add if it looks like a map name (starts with surf_)
if (normalized.startsWith("surf_")) {
gdriveMap.set(normalized, file);
}
}
// Debug: Show first 5 files from each source
if (Deno.args.includes("--verbose")) {
console.log("\n[>>] Debug - First 5 GDrive files (normalized):");
Array.from(gdriveMap.keys()).slice(0, 5).forEach(name => {
console.log(` - "${name}"`);
});
console.log("\n[>>] Debug - First 5 SQL maps:");
manifest.missing.slice(0, 5).forEach(name => {
console.log(` - "${name.toLowerCase().trim()}"`);
});
}
// Check which missing maps are available in GDrive
const availableMaps = new Map<string, GDriveFile>();
for (const mapName of manifest.missing) {
const normalized = mapName.toLowerCase().trim();
if (gdriveMap.has(normalized)) {
availableMaps.set(mapName, gdriveMap.get(normalized)!);
} else {
manifest.notInGDrive.push(mapName);
}
}
console.log(`\n[STATS] Cross-reference results:`);
console.log(` [OK] Available to download: ${availableMaps.size} maps`);
console.log(` [X] Not in Google Drive: ${manifest.notInGDrive.length} maps`);
if (manifest.notInGDrive.length > 0) {
console.log(`\n[!] Maps not found in Google Drive (showing first 10):`);
manifest.notInGDrive.slice(0, 10).forEach(map => {
console.log(` - ${map}`);
});
if (manifest.notInGDrive.length > 10) {
console.log(` ... and ${manifest.notInGDrive.length - 10} more`);
}
}
return availableMaps;
}
// Cross-device safe move (copy + delete)
async function moveFile(src: string, dest: string): Promise<void> {
try {
await Deno.rename(src, dest);
} catch (e) {
if ((e as Error).message.includes("os error 18")) {
await Deno.copyFile(src, dest);
await Deno.remove(src);
} else {
throw e;
}
}
}
// Step 6: Download and extract maps
async function downloadAndExtractMaps(
client: GoogleDriveClient,
mapsToDownload: Map<string, GDriveFile>,
): Promise<{ success: number; failed: number }> {
console.log(`\n[DL] Downloading ${mapsToDownload.size} maps...`);
// Create temp directory
await Deno.mkdir(CONFIG.tempDir, { recursive: true });
const stats = { success: 0, failed: 0 };
let completed = 0;
// Process maps with concurrency control
const entries = Array.from(mapsToDownload.entries());
for (let i = 0; i < entries.length; i += CONFIG.concurrency) {
const batch = entries.slice(i, i + CONFIG.concurrency);
await Promise.all(
batch.map(async ([mapName, gdriveFile]) => {
try {
completed++;
console.log(`[${completed}/${mapsToDownload.size}] Downloading ${mapName}...`);
const archivePath = join(CONFIG.tempDir, gdriveFile.name);
const bspPath = join(CONFIG.mapsDir, `${mapName}.bsp`);
// Download archive
await client.downloadFile(gdriveFile.id, archivePath);
// Extract based on file type
if (gdriveFile.name.endsWith(".rar")) {
await extractRAR(archivePath, mapName, CONFIG.mapsDir);
await Deno.remove(archivePath); // Cleanup archive
} else if (gdriveFile.name.endsWith(".bz2")) {
await decompressBZ2(archivePath, bspPath);
await Deno.remove(archivePath); // Cleanup archive
} else if (gdriveFile.name.endsWith(".zip")) {
await extractZIP(archivePath, mapName, CONFIG.mapsDir);
await Deno.remove(archivePath); // Cleanup archive
} else if (gdriveFile.name.endsWith(".bsp")) {
// If already a BSP, just move it
await moveFile(archivePath, bspPath);
} else {
throw new Error(`Unknown archive format: ${gdriveFile.name}`);
}
// Verify it's a valid BSP file
await verifyBSP(bspPath);
stats.success++;
console.log(` [OK] ${mapName} downloaded and extracted`);
} catch (error) {
stats.failed++;
console.error(` [X] Failed to download ${mapName}: ${(error as Error).message}`);
}
})
);
}
return stats;
}
async function extractRAR(archivePath: string, mapName: string, destDir: string): Promise<void> {
// Use system unrar command to extract
const tempExtractDir = join(CONFIG.tempDir, `extract_${mapName}`);
await Deno.mkdir(tempExtractDir, { recursive: true });
const process = new Deno.Command("unar", {
args: ["-f", "-o", tempExtractDir, archivePath],
stdout: "piped",
stderr: "piped",
});
const { code, stderr } = await process.output();
if (code !== 0) {
const errorMsg = new TextDecoder().decode(stderr);
throw new Error(`unar failed: ${errorMsg}`);
}
// Find the .bsp file in the extracted contents
const bspFile = await findBSPFile(tempExtractDir, mapName);
if (!bspFile) {
throw new Error(`No BSP file found in archive for ${mapName}`);
}
// Move the BSP file to the maps directory
await moveFile(bspFile, join(destDir, `${mapName}.bsp`));
// Cleanup temp extraction directory
await Deno.remove(tempExtractDir, { recursive: true });
}
async function extractZIP(archivePath: string, mapName: string, destDir: string): Promise<void> {
// Use system unzip command
const tempExtractDir = join(CONFIG.tempDir, `extract_${mapName}`);
await Deno.mkdir(tempExtractDir, { recursive: true });
const process = new Deno.Command("unzip", {
args: ["-o", archivePath, "-d", tempExtractDir],
stdout: "piped",
stderr: "piped",
});
const { code, stderr } = await process.output();
if (code !== 0) {
const errorMsg = new TextDecoder().decode(stderr);
throw new Error(`unzip failed: ${errorMsg}`);
}
// Find the .bsp file
const bspFile = await findBSPFile(tempExtractDir, mapName);
if (!bspFile) {
throw new Error(`No BSP file found in archive for ${mapName}`);
}
// Move the BSP file to the maps directory
await moveFile(bspFile, join(destDir, `${mapName}.bsp`));
// Cleanup
await Deno.remove(tempExtractDir, { recursive: true });
}
async function decompressBZ2(archivePath: string, outputPath: string): Promise<void> {
// Use system bunzip2 command for decompression
const process = new Deno.Command("bunzip2", {
args: ["-c", archivePath],
stdout: "piped",
});
const { code, stdout } = await process.output();
if (code !== 0) {
throw new Error(`bunzip2 failed with exit code ${code}`);
}
await Deno.writeFile(outputPath, stdout);
}
async function findBSPFile(dir: string, mapName: string): Promise<string | null> {
// Recursively search for .bsp file matching the map name
for await (const entry of Deno.readDir(dir)) {
const fullPath = join(dir, entry.name);
if (entry.isFile && entry.name.endsWith(".bsp")) {
// Check if filename matches (case-insensitive)
const baseName = entry.name.replace(/\.bsp$/i, "").toLowerCase();
if (baseName === mapName.toLowerCase()) {
return fullPath;
}
} else if (entry.isDirectory) {
// Recursively search subdirectories
const found = await findBSPFile(fullPath, mapName);
if (found) return found;
}
}
return null;
}
async function verifyBSP(bspPath: string): Promise<void> {
const file = await Deno.open(bspPath, { read: true });
try {
const buffer = new Uint8Array(4);
await file.read(buffer);
const signature = new TextDecoder().decode(buffer);
if (signature !== "VBSP") {
throw new Error(`Invalid BSP signature: ${signature}`);
}
// Check file size (should be at least 1MB for a valid surf map)
const stat = await Deno.stat(bspPath);
if (stat.size < 1024 * 1024) {
throw new Error("BSP file too small - possibly corrupted");
}
} finally {
file.close();
}
}
// Main execution
async function main() {
const args = parse(Deno.args);
console.log("[*] Surf Map Downloader\n");
console.log("━".repeat(60));
try {
// Step 1: Download and parse SQL
const sqlContent = await downloadSurfZonesSQL();
const requiredMaps = extractMapNamesFromSQL(sqlContent);
// Step 2: Get local maps
const localMaps = await getLocalMaps();
// Step 3: Build manifest
const manifest = buildManifest(requiredMaps, localMaps);
console.log("\n[INFO] Summary:");
console.log(` Total maps in SQL: ${manifest.total}`);
console.log(` Already downloaded: ${manifest.found}`);
console.log(` Missing: ${manifest.missing.length}`);
if (manifest.missing.length === 0) {
console.log("\n[*] All maps are already downloaded!");
Deno.exit(0);
}
// Step 4 & 5: Search Google Drive
const client = new GoogleDriveClient();
await client.authenticate();
const mapsToDownload = await searchGDriveForMaps(client, manifest);
if (mapsToDownload.size === 0) {
console.log("\n[!] No maps available to download from Google Drive");
Deno.exit(0);
}
// Dry run check
if (args["dry-run"]) {
console.log("\n[END] Dry run complete. Use without --dry-run to download.");
Deno.exit(0);
}
// Step 6: Download maps
const stats = await downloadAndExtractMaps(client, mapsToDownload);
// Final report
console.log("\n" + "━".repeat(60));
console.log("[OK] Download Complete!\n");
console.log(`[DL] Successfully downloaded: ${stats.success} maps`);
console.log(`[X] Failed: ${stats.failed} maps`);
console.log(`[!] Not in Google Drive: ${manifest.notInGDrive.length} maps`);
console.log("━".repeat(60));
Deno.exit(stats.failed > 0 ? 1 : 0);
} catch (error) {
console.error(`\n[X] Fatal error: ${(error as Error).message}`);
if (args.verbose) {
console.error((error as Error).stack);
}
Deno.exit(1);
}
}
// Run if executed directly
if (import.meta.main) {
main();
}

227
src/mapsdl/mapsdl.ts
View File

@ -3,20 +3,83 @@
// this script should be run with deno
// use async stdlib fns where possible
// Polyfill Node.js globals for googleapis
import process from "node:process";
if (!globalThis.process) {
globalThis.process = process;
}
import { parse } from "https://deno.land/std@0.210.0/flags/mod.ts";
import { join } from "https://deno.land/std@0.210.0/path/mod.ts";
import { google } from "npm:googleapis@131";
import { GoogleAuth } from "npm:google-auth-library@9";
import { type CredentialsClient, Drive } from "https://googleapis.deno.dev/v1/drive:v3.ts";
import { SignJWT, importPKCS8 } from "https://deno.land/x/jose@v4.14.4/index.ts";
// Service account auth that exchanges JWT for an OAuth2 access token
class ServiceAccountAuth implements CredentialsClient {
#clientEmail: string;
#privateKey: string;
#tokenUri: string;
#scope: string;
#accessToken: string | null = null;
#tokenExpiry = 0;
constructor(json: { client_email: string; private_key: string; token_uri?: string }, scope: string) {
this.#clientEmail = json.client_email;
this.#privateKey = json.private_key;
this.#tokenUri = json.token_uri ?? "https://oauth2.googleapis.com/token";
this.#scope = scope;
}
async getRequestHeaders(_url?: string): Promise<Record<string, string>> {
if (!this.#accessToken || Date.now() / 1000 > this.#tokenExpiry - 60) {
await this.#refreshToken();
}
return { Authorization: `Bearer ${this.#accessToken}` };
}
async #refreshToken(): Promise<void> {
const now = Math.floor(Date.now() / 1000);
const key = await importPKCS8(this.#privateKey, "RS256");
const assertion = await new SignJWT({
iss: this.#clientEmail,
scope: this.#scope,
aud: this.#tokenUri,
})
.setProtectedHeader({ alg: "RS256", typ: "JWT" })
.setIssuedAt(now)
.setExpirationTime(now + 3600)
.sign(key);
const resp = await fetch(this.#tokenUri, {
method: "POST",
headers: { "Content-Type": "application/x-www-form-urlencoded" },
body: new URLSearchParams({
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
assertion,
}),
});
if (!resp.ok) {
const body = await resp.text();
throw new Error(`Token exchange failed (${resp.status}): ${body}`);
}
const data = await resp.json();
this.#accessToken = data.access_token;
this.#tokenExpiry = now + (data.expires_in ?? 3600);
}
}
// Configuration
const CONFIG = {
sqlUrl: "https://raw.githubusercontent.com/bhopppp/Shavit-Surf-Timer/master/sql/surfzones.sql",
maps: [
"surf_utopia_njv",
"surf_kitsune",
"surf_lt_omnific",
"surf_pantheon",
"surf_kitsune2",
"surf_kepler",
"surf_anzchamps",
"surf_deathstar",
"surf_ebony",
"surf_fornax",
"surf_garden",
"surf_in_space",
],
mapsDir: Deno.env.get("MAPS_DIR") || "/home/ntr/surf_megastructure/data/cssds/cstrike/maps",
tempDir: Deno.env.get("TEMP_DIR") || "/tmp/mapsdl",
serviceAccountPath: Deno.env.get("GOOGLE_APPLICATION_CREDENTIALS") || "./credentials/service-account.json",
@ -40,46 +103,7 @@ interface Manifest {
notInGDrive: string[];
}
// Step 1: Download and parse SQL file for map names
async function downloadSurfZonesSQL(): Promise<string> {
console.log(`[DL] Downloading SQL file from ${CONFIG.sqlUrl}...`);
const response = await fetch(CONFIG.sqlUrl);
if (!response.ok) {
throw new Error(`Failed to download SQL: ${response.statusText}`);
}
return await response.text();
}
function extractMapNamesFromSQL(sqlContent: string): Set<string> {
console.log("[>>] Extracting map names from SQL...");
const mapNames = new Set<string>();
// Match INSERT statements and extract map names (surf_*)
// Handles both single and multi-value INSERT statements
const insertRegex = /INSERT INTO\s+`?(?:map)?zones`?\s+.*?VALUES\s*\((.*?)\);/gis;
for (const match of sqlContent.matchAll(insertRegex)) {
const valuesSection = match[1];
// Extract all string literals from the VALUES section
const stringRegex = /'([^']+)'/g;
for (const stringMatch of valuesSection.matchAll(stringRegex)) {
const value = stringMatch[1];
// Only keep values that start with 'surf_'
if (value.startsWith("surf_")) {
mapNames.add(value);
}
}
}
console.log(`[OK] Found ${mapNames.size} unique surf maps in SQL`);
return mapNames;
}
// Step 2: Check which maps exist locally
// Step 1: Check which maps exist locally
async function getLocalMaps(): Promise<Set<string>> {
console.log(`[DIR] Scanning local maps directory: ${CONFIG.mapsDir}`);
@ -129,68 +153,57 @@ function buildManifest(
};
}
// Step 4: Google Drive API client with service account
// Google Drive API client with service account
class GoogleDriveClient {
private drive: any;
private drive!: Drive;
private auth!: CredentialsClient;
async authenticate(): Promise<void> {
console.log("[AUTH] Authenticating with Google Drive API...");
const auth = new GoogleAuth({
keyFile: CONFIG.serviceAccountPath,
scopes: ["https://www.googleapis.com/auth/drive.readonly"],
});
this.drive = google.drive({ version: "v3", auth });
const keyFile = await Deno.readTextFile(CONFIG.serviceAccountPath);
this.auth = new ServiceAccountAuth(
JSON.parse(keyFile),
"https://www.googleapis.com/auth/drive.readonly",
);
this.drive = new Drive(this.auth);
console.log("[OK] Authentication successful");
}
async listFiles(folderId: string): Promise<GDriveFile[]> {
const files: GDriveFile[] = [];
let pageToken: string | null = null;
let pageToken: string | undefined;
do {
const response = await this.drive.files.list({
const response = await this.drive.filesList({
q: `'${folderId}' in parents and trashed=false`,
fields: "nextPageToken, files(id, name, size)",
pageSize: 1000,
pageToken: pageToken || undefined,
pageToken,
});
files.push(...(response.data.files || []));
pageToken = response.data.nextPageToken || null;
for (const f of response.files ?? []) {
if (f.id && f.name) {
files.push({ id: f.id, name: f.name, size: String(f.size ?? "") });
}
}
pageToken = response.nextPageToken ?? undefined;
} while (pageToken);
return files;
}
async downloadFile(fileId: string, destPath: string): Promise<void> {
const response = await this.drive.files.get(
{ fileId, alt: "media" },
{ responseType: "stream" }
);
const url = `https://www.googleapis.com/drive/v3/files/${fileId}?alt=media`;
const headers = await this.auth.getRequestHeaders(url);
const response = await fetch(url, { headers });
const file = await Deno.open(destPath, { create: true, write: true });
if (!response.ok) {
throw new Error(`Download failed: ${response.statusText}`);
}
// Convert Node.js stream to web stream for Deno
const webStream = new ReadableStream({
async start(controller) {
response.data.on("data", (chunk: Uint8Array) => {
controller.enqueue(chunk);
});
response.data.on("end", () => {
controller.close();
});
response.data.on("error", (err: Error) => {
controller.error(err);
});
},
});
await webStream.pipeTo(file.writable);
const bytes = new Uint8Array(await response.arrayBuffer());
await Deno.writeFile(destPath, bytes);
}
}
@ -236,7 +249,7 @@ async function searchGDriveForMaps(
console.log(` - "${name}"`);
});
console.log("\n[>>] Debug - First 5 SQL maps:");
console.log("\n[>>] Debug - First 5 wishlist maps:");
manifest.missing.slice(0, 5).forEach(name => {
console.log(` - "${name.toLowerCase().trim()}"`);
});
@ -272,6 +285,20 @@ async function searchGDriveForMaps(
return availableMaps;
}
// Cross-device safe move (copy + delete)
async function moveFile(src: string, dest: string): Promise<void> {
try {
await Deno.rename(src, dest);
} catch (e) {
if ((e as Error).message.includes("os error 18")) {
await Deno.copyFile(src, dest);
await Deno.remove(src);
} else {
throw e;
}
}
}
// Step 6: Download and extract maps
async function downloadAndExtractMaps(
client: GoogleDriveClient,
@ -315,7 +342,7 @@ async function downloadAndExtractMaps(
await Deno.remove(archivePath); // Cleanup archive
} else if (gdriveFile.name.endsWith(".bsp")) {
// If already a BSP, just move it
await Deno.rename(archivePath, bspPath);
await moveFile(archivePath, bspPath);
} else {
throw new Error(`Unknown archive format: ${gdriveFile.name}`);
}
@ -327,7 +354,7 @@ async function downloadAndExtractMaps(
console.log(` [OK] ${mapName} downloaded and extracted`);
} catch (error) {
stats.failed++;
console.error(` [X] Failed to download ${mapName}: ${error.message}`);
console.error(` [X] Failed to download ${mapName}: ${(error as Error).message}`);
}
})
);
@ -341,8 +368,8 @@ async function extractRAR(archivePath: string, mapName: string, destDir: string)
const tempExtractDir = join(CONFIG.tempDir, `extract_${mapName}`);
await Deno.mkdir(tempExtractDir, { recursive: true });
const process = new Deno.Command("unrar", {
args: ["x", "-o+", archivePath, tempExtractDir],
const process = new Deno.Command("unar", {
args: ["-f", "-o", tempExtractDir, archivePath],
stdout: "piped",
stderr: "piped",
});
@ -351,7 +378,7 @@ async function extractRAR(archivePath: string, mapName: string, destDir: string)
if (code !== 0) {
const errorMsg = new TextDecoder().decode(stderr);
throw new Error(`unrar failed: ${errorMsg}`);
throw new Error(`unar failed: ${errorMsg}`);
}
// Find the .bsp file in the extracted contents
@ -362,7 +389,7 @@ async function extractRAR(archivePath: string, mapName: string, destDir: string)
}
// Move the BSP file to the maps directory
await Deno.rename(bspFile, join(destDir, `${mapName}.bsp`));
await moveFile(bspFile, join(destDir, `${mapName}.bsp`));
// Cleanup temp extraction directory
await Deno.remove(tempExtractDir, { recursive: true });
@ -394,7 +421,7 @@ async function extractZIP(archivePath: string, mapName: string, destDir: string)
}
// Move the BSP file to the maps directory
await Deno.rename(bspFile, join(destDir, `${mapName}.bsp`));
await moveFile(bspFile, join(destDir, `${mapName}.bsp`));
// Cleanup
await Deno.remove(tempExtractDir, { recursive: true });
@ -468,9 +495,9 @@ async function main() {
console.log("━".repeat(60));
try {
// Step 1: Download and parse SQL
const sqlContent = await downloadSurfZonesSQL();
const requiredMaps = extractMapNamesFromSQL(sqlContent);
// Step 1: Load wishlist
const requiredMaps = new Set(CONFIG.maps);
console.log(`[OK] Wishlist: ${requiredMaps.size} maps`);
// Step 2: Get local maps
const localMaps = await getLocalMaps();
@ -479,7 +506,7 @@ async function main() {
const manifest = buildManifest(requiredMaps, localMaps);
console.log("\n[INFO] Summary:");
console.log(` Total maps in SQL: ${manifest.total}`);
console.log(` Total maps in wishlist: ${manifest.total}`);
console.log(` Already downloaded: ${manifest.found}`);
console.log(` Missing: ${manifest.missing.length}`);
@ -519,9 +546,9 @@ async function main() {
Deno.exit(stats.failed > 0 ? 1 : 0);
} catch (error) {
console.error(`\n[X] Fatal error: ${error.message}`);
console.error(`\n[X] Fatal error: ${(error as Error).message}`);
if (args.verbose) {
console.error(error.stack);
console.error((error as Error).stack);
}
Deno.exit(1);
}