aggregate-data

#!/usr/bin/env ruby

require 'nokogiri'
require 'csv'
require 'net/http'
require 'micro-optparse'

opts = Parser.new do |p|
  p.banner = "Terrible MissHK parser"
  p.version = "2019.09.08"

  p.option :refresh_cache, "force refresh of cache"
  p.option :year, "year", :default => 2017
end.process!

year = opts[:year]

header = []
csv = []
files = []

if opts[:refresh_cache]
  (1..15).each do |num|
    source = Net::HTTP.get(URI("http://misshk.tvb.com/#{year}/contestant.php?ID=#{num}"))
    title = Nokogiri::HTML(source).xpath("//title").children.to_s.split("-")[1].strip

    # terrible wide space
    file_name = "#{year}-#{title.gsub(/\s+|\t+|　/, "-")}.php"
    files << file_name

    File.open(file_name, 'w') { |file| file.write(source) }
  end
end

files = Dir.glob("#{year}-*.php")
files.each do |file|
  f = File.open(file)

  p = Nokogiri::HTML(f)

  hash = {}

  div = p.css("div.cname")
  div.each do |d|
    hash["name"] = div.text
  end

  spans = p.xpath("//span")
  spans.each_with_index do |s,i|
    next if i % 2 == 0 || s.text == "Prev"
    if spans[i + 1]
      hash[spans[i].text.split('︰')[0]] = spans[i + 1].text
    end
  end

  header = hash.keys
  csv << hash.values.to_csv
end
csv.insert(0, header.to_csv)

puts csv

f = File.open("contestant-data.csv", "w")
f.write(csv.join(""))
f.close