Merge pull request #18 from ciatph/dev

v1.0.3

Author: ciatph

src/01_recommendations/index.js

@@ -1,5 +1,6 @@
 require('dotenv').config()
 const path = require('path')
+const { CsvToFireStore } = require('csv-firestore')
 const SeasonalTab = require('./src/classes/seasonaltab')
 const TendayTab = require('./src/classes/tendaytab')
 const SpecialTab = require('./src/classes/specialtab')
@@ -16,6 +17,9 @@ const main = async () => {
   // Excel file path
   const filePath = path.join(__dirname, process.env.EXCEL_FILENAME)
 
+  // Firestore documents upload handler
+  const handler = new CsvToFireStore()
+
   // Excel tabs column names definitions
   const excelTabs = [
     new SeasonalTab(),
@@ -57,7 +61,16 @@ const main = async () => {
   if (upload) {
     try {
       data.forEach((item, index) => {
+        // Simple merged JSON data
         query.push(uploadToFirestore('n_list_crop_recommendations', item.recommendations.type, item.recommendations))
+
+        // Upload each recommendation row to a Document
+        // Path: /n_list_crop_recommendations_{type}
+        query.push(handler.firestoreUpload(
+          `n_list_crop_recommendations_${item.recommendations.type}`,
+          true,
+          item.recommendations.data
+        ))
       })
 
       // Upload data to Firestore

src/01_recommendations/src/classes/excelsheetdef.js

@@ -1,3 +1,5 @@
+const { CROP_STAGE_LABELS } = require('../../../02_crop_calendar/constants')
+
 class ExcelTabDefinition {
   /** Excel sheet tab number */
   excelTabNumber = -1
@@ -21,6 +23,13 @@ class ExcelTabDefinition {
     PRACTICE: 'practice',
     PRACTICE_TAGALOG: 'practice_tagalog'
   }
+
+  NORMAL_CROPSTAGE_CODES = {
+    'Newly Planted': Object.keys(CROP_STAGE_LABELS).find(key => CROP_STAGE_LABELS[key] === 'Newly Planted'),
+    'Vegetative/Reproductive': Object.keys(CROP_STAGE_LABELS).find(key => CROP_STAGE_LABELS[key] === 'Vegetative/Reproductive'),
+    Maturing: Object.keys(CROP_STAGE_LABELS).find(key => CROP_STAGE_LABELS[key] === 'Maturing'),
+    'Preparation Stage': Object.keys(CROP_STAGE_LABELS).find(key => CROP_STAGE_LABELS[key] === 'Preparation Stage')
+  }
 }
 
 module.exports = ExcelTabDefinition

src/01_recommendations/src/classes/seasonaltab.js

@@ -18,6 +18,22 @@ class SeasonalTab extends ExcelTabDefinition {
       __EMPTY_2: this.NORMAL_COLUMN_NAMES.PRACTICE,
       __EMPTY_3: this.NORMAL_COLUMN_NAMES.PRACTICE_TAGALOG
     }
+
+    /** Normalized weather forecast text */
+    this.NORMAL_FORECAST_NAMES = {
+      'WAY BELOW NORMAL': 'Way Below Normal',
+      'BELOW NORMAL': 'Below Normal',
+      'NEAR NORMAL': 'Near normal',
+      'ABOVE NORMAL': 'Above Normal'
+    }
+
+    /** Normalized weather forecast codes */
+    this.NORMAL_FORECAST_CODES = {
+      'WAY BELOW NORMAL': 'wb_normal',
+      'BELOW NORMAL': 'b_normal',
+      'NEAR NORMAL': 'near_normal',
+      'ABOVE NORMAL': 'above_normal'
+    }
   }
 }
 

src/01_recommendations/src/constants.js

@@ -1,7 +1,7 @@
 const RECOMMEDATIONS_TYPE = {
   SEASONAL: 'seasonal',
   TENDAY: 'tenday',
-  SPECIAL: 'sepcial'
+  SPECIAL: 'special'
 }
 
 module.exports = {

src/01_recommendations/src/extract.js

@@ -1,4 +1,5 @@
 const XLSXWrapper = require('../../lib/xlsxwrapper')
+const { RECOMMEDATIONS_TYPE } = require('./constants')
 
 /**
  * Extract normalized recommendations data and other metadata from an excel sheet tab
@@ -28,8 +29,38 @@ module.exports.extractExcelData = (ExcelTab, excelFilePath) => {
       const obj = {}
 
       for (const key in ExcelTab.EXCEL_COLUMN_NAMES) {
-        obj[ExcelTab.EXCEL_COLUMN_NAMES[key]] = item[key] || ''
-        obj[ExcelTab.EXCEL_COLUMN_NAMES[key]] = obj[ExcelTab.EXCEL_COLUMN_NAMES[key]].trim()
+        let value = item[key] || ''
+        value = value.trim()
+
+        switch (ExcelTab.type) {
+          case RECOMMEDATIONS_TYPE.SEASONAL:
+            // Normalize the seasonal forecast text
+            if (ExcelTab.EXCEL_COLUMN_NAMES[key] === ExcelTab.NORMAL_COLUMN_NAMES.FORECAST) {
+              value = ExcelTab.NORMAL_FORECAST_CODES[value]
+            }
+            break
+          default: break
+        }
+
+        // Insert <span> in <li>
+        switch (ExcelTab.EXCEL_COLUMN_NAMES[key]) {
+          case ExcelTab.NORMAL_COLUMN_NAMES.PRACTICE:
+          case ExcelTab.NORMAL_COLUMN_NAMES.PRACTICE_TAGALOG:
+          case ExcelTab.NORMAL_COLUMN_NAMES.IMPACT:
+          case ExcelTab.NORMAL_COLUMN_NAMES.IMPACT_TAGALOG:
+            value = value.replace(/<li>/g, '<li><span>')
+            value = value.replace(/<\/li>/g, '</span></li>')
+            break
+          default:
+            break
+        }
+
+        // Normalize the crop stage name - use crop calendar codes
+        if (ExcelTab.EXCEL_COLUMN_NAMES[key] === ExcelTab.NORMAL_COLUMN_NAMES.CROP_STAGE) {
+          value = ExcelTab.NORMAL_CROPSTAGE_CODES[value]
+        }
+
+        obj[ExcelTab.EXCEL_COLUMN_NAMES[key]] = value
       }
 
       list.push(obj)
@@ -38,6 +69,28 @@ module.exports.extractExcelData = (ExcelTab, excelFilePath) => {
     return list
   }, [])
 
+  // Normalize the special recommendations' farm operations
+  if (ExcelTab.type === RECOMMEDATIONS_TYPE.SPECIAL) {
+    recommendations.data = recommendations.data.reduce((list, item, index) => {
+      const farmoperation = item[ExcelTab.NORMAL_COLUMN_NAMES.FARM_OPERATION]
+
+      // Expand the merged farm operations vertically
+      if (farmoperation.includes('/') && farmoperation !== 'Planting/Transplanting') {
+        const farmoperations = farmoperation.split('/').map(operations => operations.trim())
+
+        farmoperations.forEach((operation) => {
+          const temp = { ...item }
+          temp[ExcelTab.NORMAL_COLUMN_NAMES.FARM_OPERATION] = operation
+          list.push(temp)
+        })
+      } else {
+        list.push(item)
+      }
+
+      return list
+    }, [])
+  }
+
   // List unique crop stages
   const cropstages = recommendations.data.map(x => x[ExcelTab.NORMAL_COLUMN_NAMES.CROP_STAGE])
     .filter((x, i, a) => a.indexOf(x) === i)

src/02_crop_calendar/index.js

@@ -3,16 +3,16 @@ const path = require('path')
 const CroppingCalendar = require('./cropping_calendar')
 const { uploadToFirestore } = require('../lib/uploadtofirestore')
 
-// Path: /n_cropping_calendar_merged/{province}.data[]
+// Path: /n_cropping_calendar_lite/{province}.data[]
 const main = async () => {
   const handler = new CroppingCalendar(path.resolve(__dirname, process.env.CSV_FILENAME))
   const upload = false
   const write = true
 
   // Cropping Calendar-specific tables and firestore collection names
   const newTables = {
-    provinces: 'n_provinces',
-    municipalities: 'n_municipalities',
+    // provinces: 'n_provinces',
+    // municipalities: 'n_municipalities',
     crops: 'n_crops',
     crop_stages: 'n_crop_stages'
   }
@@ -30,7 +30,8 @@ const main = async () => {
           group[province] = []
         }
 
-        const obj = {}
+        const obj = { province }
+
         for (const key in row) {
           if (!['id', 'province'].includes(key)) {
             obj[key] = row[key].trim()
@@ -41,7 +42,7 @@ const main = async () => {
         return { ...group }
       }, {})
 
-      console.log('\nUploading data to firestore...')
+      console.log('\nUploading list data to firestore...')
       const query = []
 
       // Upload full collections
@@ -63,11 +64,29 @@ const main = async () => {
         logs += `${province}: ${data[province].length} items\n`
 
         // Upload query
-        query.push(uploadToFirestore('n_cropping_calendar_merged', province, { data: data[province] }))
+        query.push(uploadToFirestore('n_cropping_calendar_lite', province, { data: data[province] }))
       }
 
       console.log(logs)
-      console.log('Uploading data to Firestore...')
+      console.log('Uploading calendar data to Firestore...')
+
+      // Upload list data as documents
+      console.log('Uploading lists in a single Firestore document...')
+
+      // Upload the list of provinces and respective municipalities for use as constant, static values
+      query.push(uploadToFirestore('constant_data', 'provinces', {
+        data: handler.provinces.reduce((list, province, index) => {
+          const temp = { id: province.id, label: province.name }
+
+          temp.municipalities = handler.municipalities.filter((municipality) => municipality.province === province.name)
+            .sort((a, b) => a.name > b.name ? 1 : (a.name < b.name) ? -1 : 0)
+            .map((municipality, id) => ({ id, label: municipality.name }))
+
+          list.push(temp)
+          return list
+        }, [])
+      }))
+
       await Promise.all(query)
       console.log('Upload success!')
     }

src/lib/uploadtofirestore/index.js

@@ -9,11 +9,13 @@ const firestore = new FirestoreData()
  *    - {Object[]} data - crop recommendations mapped to crop stages, farm operations and other keys
  *    - {String} type - type of recommendations. One of RECOMMEDATIONS_TYPE.
  *    - {String} description - Brief text description describing the nature of data
+ * @param {Object} metadata - Key-value pairs description and other information about the data
  * @returns {Timestamp} Firestore timestamp of successful data upload
  */
-module.exports.uploadToFirestore = async (collectionName, docName, jsonData) => {
+module.exports.uploadToFirestore = async (collectionName, docName, jsonData, metadata) => {
   // CSV and Firestore handler
-  jsonData.date_created = firestore.admin.firestore.Timestamp.now()
+  jsonData.metadata = metadata ?? {}
+  jsonData.metadata.date_created = firestore.admin.firestore.Timestamp.now()
 
   try {
     // Upload data to Firestore