main.ts

#!/usr/bin/env -S deno run --allow-import=jsr.io:443 --allow-read --allow-write --allow-net

import { Command, ValidationError } from "./deps/cliffy/command.ts";
import { AsyncSemaphore } from "./deps/esfx/async_semaphore.ts";
import { HttpReader, terminateWorkers, ZipReader } from "./deps/zip_js.ts";

import { updateFromDatasette } from "./datasette.ts";
import { delay } from "./delay.ts";
import { signal } from "./interrupt_signal.ts";
import { log } from "./log.ts";
import { loadLevels, orchardURL } from "./orchard.ts";
import { retry } from "./retry.ts";
import { checkZip, writeEntry } from "./unzip.ts";

const { min } = Math;
const {
  options: {
    yeeted,
    database,
    concurrency,
    approvedOnly,
    maxFiles,
    maxSize,
    dryRun,
    orchard,
  },
  args: [output],
} = await new Command()
  .name("levelsync")
  .usage("[options] <output>")
  .description(`
    Automatically download Rhythm Doctor levels.
  `)
  .type("positive-integer", ({ label, name, value }) => {
    if (!/^0*[1-9]\d*$/.test(value)) {
      throw new ValidationError(
        `${label} "${name}" must be a positive integer, but got "${value}".`,
      );
    }
    return Number(value);
  })
  .type("url", ({ label, name, value }) => {
    const parse = URL.parse(value);
    if (!parse) {
      throw new ValidationError(
        `${label} "${name}" must be a URL, but got "${value}".`,
      );
    }
    return parse.href;
  })
  .option(
    "-y, --yeeted <path:file>",
    "Path of where to store removed levels.",
  )
  .option(
    "-d, --database <path:file>",
    "Path of where to cache the level database.",
    { default: "./orchard.db" },
  )
  .option(
    "-c, --concurrency <number:positive-integer>",
    "Number of levels to download concurrently.",
    { default: 1 },
  )
  .option(
    "-a, --approved-only",
    "Only download peer reviewed levels.",
  )
  .option(
    "--max-files <number:positive-integer>",
    "Limit on the number of files in each level.",
    { default: 10000 },
  )
  .option(
    "--max-size <bytes:positive-integer>",
    "Limit on the uncompressed size of each level.",
    { default: 500000000 },
  )
  .option(
    "-n, --dry-run",
    "Do not actually add or remove levels.",
  )
  .option(
    "--orchard <url:url>",
    "URL of the level database.",
    { default: orchardURL },
  )
  .arguments("<output:file>")
  .error((error, cmd) => {
    cmd.showHelp();
    console.error(
      "%cerror%c:",
      "color: red; font-weight: bold",
      "",
      error.message,
    );
    Deno.exit(2);
  })
  .parse();
Deno.mkdirSync(output, { recursive: true });
const lock = `${output}/.levelsync.lock`;
try {
  Deno.writeTextFileSync(lock, `${Deno.pid}`, { createNew: true });
} catch (e) {
  if (!(e instanceof Deno.errors.AlreadyExists)) {
    throw e;
  }
  let pid: number | undefined;
  try {
    const text = Deno.readTextFileSync(lock);
    if (/^\d+$/.test(text)) {
      pid = Number(text);
    }
  } catch {
    // ignored
  }
  log.warn(
    `Another instance of levelsync${
      pid === undefined ? "" : ` (pid ${pid})`
    } is already running or was erroneously terminated. Manually remove '${lock}' to continue anyway.`,
  );
  Deno.exit(4);
}
addEventListener("unload", () => Deno.removeSync(lock));
const toAdd = await (async () => {
  try {
    await updateFromDatasette(orchard, database, {
      onRetry: (e, n) =>
        log.warn(`Cannot update level database (${n} retries left): ${e}`),
      signal,
    });
    return loadLevels(database);
  } catch (e) {
    log.error(`Cannot update level database: ${e}`);
    Deno.exit(3);
  }
})();
if (approvedOnly) {
  for (const [hash, { approved }] of toAdd) {
    if (!approved) {
      toAdd.delete(hash);
    }
  }
}
const toRemove: { name: string; id: string }[] = [];
try {
  for await (const { name } of Deno.readDir(output)) {
    let haveId: string;
    try {
      haveId = await Deno.readTextFile(`${output}/${name}/.levelsync`);
    } catch {
      continue;
    }
    const wantId = toAdd.get(name)?.id;
    if (haveId === wantId) {
      toAdd.delete(name);
    } else {
      toRemove.push({ name, id: haveId || name });
    }
  }
} catch (e) {
  log.error(`Cannot read existing levels: ${e}`);
  Deno.exit(3);
}
if (
  !dryRun && toRemove.length >= 20 &&
  !confirm(`About to remove ${toRemove.length} levels. Continue?`)
) {
  log.warn(`Refusing to remove ${toRemove.length} levels.`);
  Deno.exit(3);
}
let error = false;
await Promise.all(toRemove.map(async ({ name, id }) => {
  log.step("Remove", id);
  if (dryRun) {
    return;
  }
  try {
    if (yeeted !== undefined) {
      await Deno.mkdir(yeeted, { recursive: true });
      await Deno.remove(`${output}/${name}/.levelsync`);
      await Deno.rename(`${output}/${name}`, `${yeeted}/${name}`);
    } else {
      await Deno.remove(`${output}/${name}`, { recursive: true });
    }
  } catch (e) {
    log.error(`Cannot remove ${name}: ${e}`);
    error = true;
  }
}));
if (error) {
  Deno.exit(1);
}
let maxFilesHelp = " (consider raising the limit using the --max-files option)";
let maxSizeHelp = " (consider raising the limit using the --max-size option)";
const semaphore = new AsyncSemaphore(concurrency);
try {
  await Promise.all(
    Array.from(toAdd, async ([hash, { id, url }]) => {
      let started = false;
      try {
        await retry(async () => {
          await semaphore.wait();
          try {
            signal.throwIfAborted();
            if (!started) {
              log.step("Download", `${id} (${url})`);
              started = true;
            }
            if (dryRun) {
              return;
            }
            const zipReader = new ZipReader(
              new HttpReader(url, { preventHeadRequest: true, signal }),
              { signal },
            );
            const result = await checkZip(zipReader, {
              fileCount: maxFiles,
              uncompressedSize: maxSize,
            });
            if (result.error !== null) {
              switch (result.error) {
                case "file-count":
                  log.warn(
                    `Refusing to download ${id}: Number of files (${result.fileCount}) exceeds limit${maxFilesHelp}`,
                  );
                  maxFilesHelp = "";
                  break;
                case "uncompressed-size":
                  log.warn(
                    `Refusing to download ${id}: Uncompressed size (${result.uncompressedSize} bytes) exceeds limit${maxSizeHelp}`,
                  );
                  maxSizeHelp = "";
                  break;
                case "name":
                  log.warn(
                    `Refusing to download ${id}: Invalid filename '${result.name}'`,
                  );
                  break;
              }
              return;
            }
            const tempDir = await Deno.makeTempDir({
              dir: output,
              prefix: hash,
            });
            try {
              await Deno.writeTextFile(`${tempDir}/.levelsync`, id, {
                createNew: true,
              });
              for (const { name, entry } of result.checkedEntries) {
                await writeEntry(`${tempDir}/${name}`, entry);
              }
              if (Deno.build.os !== "windows") {
                await Deno.chmod(tempDir, 0o755);
              }
              await Deno.rename(tempDir, `${output}/${hash}`);
            } catch (e) {
              await Deno.remove(tempDir, { recursive: true });
              throw e;
            }
          } finally {
            semaphore.release();
          }
        }, async (error, retries) => {
          if (retries === 10) {
            throw error;
          }
          signal.throwIfAborted();
          log.warn(
            `Cannot download ${id} (${10 - retries} retries left): ${error}`,
          );
          await delay(min(2 ** retries, 600) * 1000, { signal });
        });
      } catch (e) {
        if (started) {
          log.error(`Cannot download ${id}: ${e}`);
        }
        error = true;
      }
    }),
  );
} finally {
  await terminateWorkers();
}
if (error) {
  Deno.exit(1);
}