scrap.js

import * as puppeteer from 'puppeteer';
import * as cheerio from 'cheerio';
import { sleep, parseDate } from './util.js';

export async function scrapeNewsList(url, source) {
  if(source === 'CryptoNews'){
    return await scrapeNewsListCN(url);
  } else if(source.includes('CT-')){
    return await scrapeNewsListCT(url);
  }
  return null;
}

export async function scrapeNewsItem(url, source) {
  if(source === 'CryptoNews'){
    return await scrapeNewsItemCN(url);
  } else if(source.includes('CT-')){
    return await scrapeNewsItemCT(url);
  }
  return null;
}

async function scrapeNewsListCN(url) {
  const $ = await readUrl(url);
  if(!$){
    return null;
  }

  const results = [];

  $('div.post-loop').each((index, row) => {
    const link = $(row).find('a.post-loop__link')[0];
    const meta = $(row).find('time.post-loop__date')[0];
    if(link && meta){
      const item = { title: $(link).text().trim(), time: $(meta).attr('datetime'), url: $(link).attr('href') };
      item.timeObj = parseDate(item.time);
      results.push(item);
    }
  });

  return results;
}

async function scrapeNewsListCT(url) {
  const $ = await readUrl(url, 'div.markets-header', false);
  if(!$){
    return null;
  }

  const results = [];

  $('article.post-card-inline').each((index, row) => {
    const link = $(row).find('a.post-card-inline__figure-link')[0];
    const title = $(row).find('span.post-card-inline__title')[0];
    const time = $(row).find('time.post-card-inline__date')[0];
    if(link && title && time){
      const item = { title: $(title).text().trim(), time: /*$(time).attr('datetime') + ' ' + */$(time).text().trim(), url: 'https://cointelegraph.com' + $(link).attr('href') };
      item.timeObj = parseDate(item.time);
      results.push(item);
    }
  });

  return results;
}

async function scrapeNewsItemCN(url) {
  const $ = await readUrl(url);
  if(!$){
    return null;
  }

  const article = $('article.post-detail')[0];
  if(article){
    const result = {};
    const author = $(article).find('a.author-list__link')[0];
    const editor = $(article).find('a.author-list__link')[1];
    if(author){
      result.author = $(author).text().trim();
    }
    if(editor){
      result.editor = $(editor).text().trim();
    }

    const content = $(article).find('div.post-detail__content.blocks')[0];
    if(content){
      $(content).find('div.wp-block-rank-math-toc-block').remove();
      $(content).find('figure.wp-block-image').remove();
      $(content).find('div.cn-block-related-link').remove();
      $(content).find('h2#fa-qs').remove();
      $(content).find('div.rank-math-block').remove();

      // result.contentOrg = content;
      // result.contentHTML = $(content).html();
      result.content = $(content).text().replace(/\\[nrtbf]|[\s]+/g, ' ').trim();
    }

    return result;
  }

  return null;
}

async function scrapeNewsItemCT(url) {
  const $ = await readUrl(url, 'aa', false);
  if(!$){
    return null;
  }

  const article = $('article.post__article')[0];
  if(article){
    const result = {};
    const author = $(article).find('a.post-meta__author-link')[0];
    if(author){
      result.author = $(author).text().trim();
    }

    const content = $(article).find('div.post__content-wrapper div.post-content')[0];
    if(content){
      $(content).find('figure').remove();
      $(content).find('style').remove();
      $(content).find('a.text-banner').remove();
      $(content).find('button[data-testid="close-ad-button"]').remove();
      $(content).find('img').remove();
      $(content).find('div.cn-block-related-link').remove();
      $(content).find('form').remove();

      // result.contentOrg = content;
      // result.contentHTML = $(content).html();
      result.content = $(content).text().replace(/\\[nrtbf]|[\s]+/g, ' ').trim();
    }

    return result;
  }

  return null;
}

async function readUrl(url, selectorToWait = null, launchHeadless = true){
  let $;
  let browser;

  try {
    browser = await puppeteer.launch({
      headless: launchHeadless, // Run in headless mode
    });

    const page = await browser.newPage();
    await page.goto(url, { waitUntil: 'networkidle2' });
    if(selectorToWait){
      // await page.waitForSelector(selectorToWait);
      await sleep(5000); // Sleep for 3 seconds
    }
    const html = await page.content();
    // console.log(html);
    $ = cheerio.load(html);
  } catch (error) {
    console.error('Error scraping the table:', error);
  } finally {
    if (browser) await browser.close();
  }

  return $;
}