module-downloader.js

const fs = require("fs");
const path = require("path");
const https = require("https");

// Try F: drive first, fallback to local directory if not accessible
let DOWNLOAD_DIR = "F:/npm_packages";
const FALLBACK_DIR = path.join(__dirname, "npm_packages");
const BASE_URL = "https://registry.npmjs.org";
const MAX_VERSIONS = 200;

// Function to validate and set up download directory
function setupDownloadDirectory() {
  // First try the F: drive
  try {
    if (!fs.existsSync(DOWNLOAD_DIR)) {
      fs.mkdirSync(DOWNLOAD_DIR, { recursive: true });
    }
    // Test if we can write to the directory
    const testFile = path.join(DOWNLOAD_DIR, ".test-write");
    fs.writeFileSync(testFile, "test");
    fs.unlinkSync(testFile);
    console.log(`Using download directory: ${DOWNLOAD_DIR}`);
    return DOWNLOAD_DIR;
  } catch (error) {
    console.warn(`F: drive not accessible: ${error.message}`);
    console.log(`Falling back to local directory: ${FALLBACK_DIR}`);

    // Fallback to local directory
    try {
      if (!fs.existsSync(FALLBACK_DIR)) {
        fs.mkdirSync(FALLBACK_DIR, { recursive: true });
      }
      // Test if we can write to the fallback directory
      const testFile = path.join(FALLBACK_DIR, ".test-write");
      fs.writeFileSync(testFile, "test");
      fs.unlinkSync(testFile);
      DOWNLOAD_DIR = FALLBACK_DIR;
      console.log(`Using fallback download directory: ${DOWNLOAD_DIR}`);
      return DOWNLOAD_DIR;
    } catch (fallbackError) {
      console.error(
        `Cannot create or write to fallback directory: ${fallbackError.message}`
      );
      process.exit(1);
    }
  }
}

// Function to safely create directory with retry logic
function safeMkdirSync(dirPath, maxRetries = 3) {
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      if (!fs.existsSync(dirPath)) {
        fs.mkdirSync(dirPath, { recursive: true });
      }
      return true;
    } catch (error) {
      if (attempt === maxRetries) {
        throw error;
      }
      console.warn(
        `Attempt ${attempt} failed to create directory ${dirPath}: ${error.message}`
      );
      // Wait a bit before retrying
      const waitTime = Math.min(1000 * attempt, 5000); // Exponential backoff, max 5 seconds
      console.log(`Retrying in ${waitTime}ms...`);
      require("util").promisify(setTimeout)(waitTime);
    }
  }
}

function readPackagesFromFile() {
  try {
    const content = fs.readFileSync(
      path.join(__dirname, "output-no-dup.txt"),
      "utf-8"
    );
    return content
      .split("\n")
      .map((line) => line.trim())
      .filter(
        (line) => line && !line.startsWith("#") && !line.startsWith("//")
      );
  } catch (error) {
    console.error("Error reading output-no-dup.txt:", error.message);
    process.exit(1);
  }
}

async function processPackages() {
  const packages = readPackagesFromFile();
  console.log(`Found ${packages.length} packages to process`);

  // Set up download directory
  setupDownloadDirectory();

  for (const packageName of packages) {
    try {
      console.log(`\nStarting processing for ${packageName}`);
      await processPackage(packageName);
      console.log(`Finished processing ${packageName}`);
    } catch (error) {
      console.error(`Error processing ${packageName}:`, error.message);
      // Continue with next package instead of stopping
    }
  }

  console.log("\nAll packages processed!");
}

async function processPackage(packageName) {
  const packageDir = path.join(DOWNLOAD_DIR, ...packageName.split("/"));
  const packageJsonPath = path.join(packageDir, "package.json");

  // Skip if package is already fully processed
  if (fs.existsSync(packageJsonPath)) {
    console.log(`Skipping ${packageName} (already processed)`);
    return;
  }

  // Create package directory if it doesn't exist with retry logic
  try {
    safeMkdirSync(packageDir);
  } catch (error) {
    throw new Error(
      `Failed to create directory for ${packageName}: ${error.message}`
    );
  }

  // Fetch package metadata from npm registry
  const metadata = await fetchPackageMetadata(packageName);

  if (!metadata || !metadata.versions) {
    console.error(`No versions found for ${packageName}`);
    return;
  }

  // Sort versions by publish date (newest first) and limit to MAX_VERSIONS
  const sortedVersions = Object.keys(metadata.versions)
    .map((version) => ({
      version,
      time: new Date(metadata.time[version] || metadata.time.created || 0),
    }))
    .sort((a, b) => b.time - a.time)
    .map((item) => item.version)
    .slice(0, MAX_VERSIONS);

  // Prepare package.json structure
  const packageJson = {
    name: packageName,
    versions: {},
    time: {},
    "dist-tags": metadata["dist-tags"] || {},
    _id: metadata._id || packageName,
    readme: metadata.readme || "",
    readmeFilename: metadata.readmeFilename || "README.md",
    _attachments: {},
    totalVersions: Object.keys(metadata.versions).length,
    downloadedVersions: sortedVersions.length,
  };

  console.log(`Processing ${sortedVersions.length} versions of ${packageName}`);

  // Process each version
  for (const version of sortedVersions) {
    const versionData = metadata.versions[version];

    // Add version metadata to package.json
    packageJson.versions[version] = {
      ...versionData,
      _id: versionData._id || `${packageName}@${version}`,
      dist: versionData.dist || {},
    };

    // Add publish timestamp
    if (metadata.time[version]) {
      packageJson.time[version] = metadata.time[version];
    }

    // Download tarball if it doesn't exist
    if (versionData.dist && versionData.dist.tarball) {
      const fileName = `${packageName.replace(/\//g, "-")}-${version}.tgz`;
      const filePath = path.join(packageDir, fileName);

      if (!fs.existsSync(filePath)) {
        try {
          await downloadFile(versionData.dist.tarball, filePath);
          packageJson._attachments[fileName] = {
            shasum: versionData.dist.shasum,
            version: version,
          };
        } catch (downloadError) {
          console.warn(
            `Failed to download ${fileName}: ${downloadError.message}`
          );
          // Continue processing other versions
        }
      }
    }
  }

  // Write the complete package.json
  try {
    fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2));
    console.log(`Generated package.json for ${packageName}`);
  } catch (writeError) {
    throw new Error(
      `Failed to write package.json for ${packageName}: ${writeError.message}`
    );
  }
}

function fetchPackageMetadata(packageName) {
  return new Promise((resolve, reject) => {
    const url = `${BASE_URL}/${packageName.replace(/\//g, "%2F")}`;
    https
      .get(url, (res) => {
        if (res.statusCode === 404) {
          resolve(null);
          return;
        }
        if (res.statusCode !== 200) {
          reject(
            new Error(
              `Failed to fetch metadata for ${packageName}: ${res.statusCode}`
            )
          );
          return;
        }

        let data = "";
        res.on("data", (chunk) => (data += chunk));
        res.on("end", () => {
          try {
            resolve(JSON.parse(data));
          } catch (e) {
            reject(e);
          }
        });
      })
      .on("error", reject);
  });
}

function downloadFile(url, filePath) {
  return new Promise((resolve, reject) => {
    const file = fs.createWriteStream(filePath);
    https
      .get(url, (response) => {
        if (response.statusCode !== 200) {
          reject(
            new Error(`Failed to download ${url}: ${response.statusCode}`)
          );
          return;
        }
        response.pipe(file);
        file.on("finish", () => {
          file.close(resolve);
        });
      })
      .on("error", (err) => {
        fs.unlink(filePath, () => reject(err));
      });
  });
}

processPackages().catch(console.error);