Fixed so that any usfm project gets converted

functions/convert_usfm2html/module.json

@@ -2,7 +2,7 @@
     "name": "usfm2html",
     "version": "2",
     "type": "converter",
-    "resource_types": ["bible", "ulb", "udb", "reg"],
+    "resource_types": ["bible", "ulb", "udb", "reg", "other"],
     "input_format": ["usfm"],
     "output_format": ["html"],
     "options": [],

libraries/client/client_webhook.py

@@ -7,7 +7,7 @@
 from datetime import datetime, timedelta
 from libraries.general_tools.file_utils import unzip, write_file, add_contents_to_zip, remove_tree
 from libraries.general_tools.url_utils import download_file
-from libraries.resource_container.ResourceContainer import RC, BIBLE_RESOURCE_TYPES
+from libraries.resource_container.ResourceContainer import RC
 from libraries.client.preprocessors import do_preprocess
 from libraries.models.manifest import TxManifest
 from libraries.models.module import TxModule
@@ -395,7 +395,7 @@ def send_request_to_linter(self, job, linter, commit_url, extra_payload=None):
         }
         if extra_payload:
             payload.update(extra_payload)
-        if job.resource_type in BIBLE_RESOURCE_TYPES or job.resource_type == 'obs':
+        if job.input_format == 'usfm' or job.resource_type == 'obs':
             # Need to give the massaged source since it maybe was in chunks originally
             payload['source_url'] = job.source
         else:
@@ -468,11 +468,13 @@ def get_converter_module(self, job):
         :param TxJob job:
         :return TxModule:
         """
-        return TxModule.query().filter(TxModule.type=='converter') \
+        converters = TxModule.query().filter(TxModule.type=='converter') \
             .filter(TxModule.input_format.contains(job.input_format)) \
-            .filter(TxModule.output_format.contains(job.output_format)) \
-            .filter(TxModule.resource_types.contains(job.resource_type)) \
-            .first()
+            .filter(TxModule.output_format.contains(job.output_format))
+        converter = converters.filter(TxModule.resource_types.contains(job.resource_type)).first()
+        if not converter:
+            converter = converters.filter(TxModule.resource_types.contains('other')).first()
+        return converter
 
     def get_linter_module(self, job):
         """
@@ -484,4 +486,4 @@ def get_linter_module(self, job):
         linter = linters.filter(TxModule.resource_types.contains(job.resource_type)).first()
         if not linter:
             linter = linters.filter(TxModule.resource_types.contains('other')).first()
-        return linter
+        return linter

libraries/client/preprocessors.py

@@ -7,14 +7,13 @@
 from libraries.door43_tools.bible_books import BOOK_NUMBERS, BOOK_NAMES, BOOK_CHAPTER_VERSES
 from libraries.general_tools.file_utils import write_file, read_file
 from libraries.resource_container.ResourceContainer import RC
-from libraries.resource_container.ResourceContainer import BIBLE_RESOURCE_TYPES
 
 
 def do_preprocess(rc, repo_dir, output_dir):
     if rc.resource.identifier == 'obs':
         App.logger.debug("do_preprocess: using ObsPreprocessor")
         preprocessor = ObsPreprocessor(rc, repo_dir, output_dir)
-    elif rc.resource.identifier in BIBLE_RESOURCE_TYPES:
+    elif rc.resource.file_ext == 'usfm':
         App.logger.debug("do_preprocess: using BiblePreprocessor")
         preprocessor = BiblePreprocessor(rc, repo_dir, output_dir)
     elif rc.resource.identifier == 'ta':

libraries/converters/usfm2html_converter.py

@@ -9,19 +9,11 @@
 from libraries.general_tools.file_utils import write_file, remove_tree, get_files
 from converter import Converter
 from usfm_tools.transform import UsfmTransform
-from libraries.resource_container.ResourceContainer import BIBLE_RESOURCE_TYPES
 
 
 class Usfm2HtmlConverter(Converter):
 
     def convert(self):
-        if self.resource in BIBLE_RESOURCE_TYPES:
-            self.convert_bible()
-            return True
-        else:
-            return False
-
-    def convert_bible(self):
         App.logger.debug('Processing the Bible USFM files')
 
         # find the first directory that has usfm files.
@@ -74,3 +66,4 @@ def convert_bible(self):
                 except:
                     pass
         self.log.info('Finished processing Bible USFM files.')
+        return True

libraries/door43_tools/templaters.py

@@ -7,7 +7,6 @@
 from libraries.general_tools.file_utils import write_file
 from libraries.resource_container.ResourceContainer import RC
 from libraries.general_tools.file_utils import load_yaml_object
-from libraries.resource_container.ResourceContainer import BIBLE_RESOURCE_TYPES
 from libraries.app.app import App
 
 
@@ -17,9 +16,7 @@ def do_template(resource_type, source_dir, output_dir, template_file):
 
 
 def init_template(resource_type, source_dir, output_dir, template_file):
-    if resource_type in BIBLE_RESOURCE_TYPES:
-        templater = BibleTemplater(resource_type, source_dir, output_dir, template_file)
-    elif resource_type == 'obs':
+    if resource_type == 'obs':
         templater = ObsTemplater(resource_type, source_dir, output_dir, template_file)
     elif resource_type == 'ta':
         templater = TaTemplater(resource_type, source_dir, output_dir, template_file)
@@ -30,7 +27,7 @@ def init_template(resource_type, source_dir, output_dir, template_file):
     elif resource_type == 'tn':
         templater = TnTemplater(resource_type, source_dir, output_dir, template_file)
     else:
-        templater = Templater(resource_type, source_dir, output_dir, template_file)
+        templater = BibleTemplater(resource_type, source_dir, output_dir, template_file)
     return templater
 
 
@@ -51,6 +48,7 @@ def __init__(self, resource_type, source_dir, output_dir, template_file):
         self.titles = {}
         self.chapters = {}
         self.book_codes = {}
+        self.classes = []
 
     def run(self):
         # get the resource container
@@ -59,8 +57,8 @@ def run(self):
             self.template_html = template_file.read()
             soup = BeautifulSoup(self.template_html, 'html.parser')
             soup.body['class'] = soup.body.get('class', []) + [self.resource_type]
-            if self.resource_type in BIBLE_RESOURCE_TYPES and self.resource_type != 'bible':
-                soup.body['class'] = soup.body.get('class', []) + ['bible']
+            if self.classes:
+                soup.body['class'] = soup.body.get('class', []) + self.classes
             self.template_html = unicode(soup)
         self.apply_template()
         return True
@@ -456,6 +454,7 @@ def build_page_nav(self, filename=None):
 class BibleTemplater(Templater):
     def __init__(self, *args, **kwargs):
         super(BibleTemplater, self).__init__(*args, **kwargs)
+        self.classes = ['bible']
 
     def get_page_navigation(self):
         for fname in self.files:

libraries/resource_container/ResourceContainer.py

@@ -7,8 +7,6 @@
 from datetime import datetime
 from glob import glob
 
-BIBLE_RESOURCE_TYPES = ['ulb', 'udb', 'bible', 'reg']
-
 resource_map = {
     'udb': {
         'title': 'Unlocked Dynamic Bible',
@@ -353,11 +351,11 @@ def file_ext(self):
     def type(self):
         if 'type' in self.resource and isinstance(self.resource['type'], basestring):
             return self.resource['type'].lower()
-        elif self.identifier in BIBLE_RESOURCE_TYPES:
+        elif self.file_ext == 'usfm':
             if len(self.rc.usfm_files()):
                 return 'bundle'
             else:
-                return resource_map[self.identifier]['type']
+                return 'book'
         elif self.identifier in resource_map:
             return resource_map[self.identifier]['type']
         else:

requirements.txt

@@ -9,7 +9,7 @@ markdown==2.6.8
 markdown2==2.3.4
 future==0.16.0
 pyparsing==2.1.10
-usfm-tools==0.0.19
+usfm-tools==0.0.22
 mock==2.0.0
 sphinx==1.5.2
 sphinx-autobuild==0.6.0

setup.py

@@ -82,7 +82,7 @@ def read(f_name):
         'markdown2==2.3.4',
         'future==0.16.0',
         'pyparsing==2.1.10',
-        'usfm-tools==0.0.19',
+        'usfm-tools==0.0.22',
         'PyYAML==3.12',
         'pymysql==0.7.11',
         'sqlalchemy==1.2.0b2',

test-setup.py

@@ -70,7 +70,7 @@
         'markdown2==2.3.4',
         'future==0.16.0',
         'pyparsing==2.1.10',
-        'usfm-tools==0.0.19',
+        'usfm-tools==0.0.22',
         'mock',  # travis reports syntax error in mock setup.cfg if we give version
         'moto==1.0.1',
         'PyYAML==3.12',

tests/converter_tests/test_usfm2html_converter.py

@@ -187,16 +187,6 @@ def test_bad_source(self):
         self.assertFalse(result['success'])
         self.assertEqual(result['errors'], [u'Conversion process ended abnormally: Failed to download bad_source'])
 
-    def test_bad_resource(self):
-        """This tests giving a bad resource type to the converter"""
-        zip_file = self.resources_dir + "/51-PHP.zip"
-        zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file)
-        with closing(Usfm2HtmlConverter('', 'bad_resource')) as tx:
-            tx.input_zip_file = zip_file
-            result = tx.run()
-        self.assertFalse(result['success'])
-        self.assertEqual(result['errors'], ['Resource bad_resource currently not supported.'])
-
     #
     # helpers
     #