Fixes various instances of d43 links

.devcontainer/Dockerfile

@@ -0,0 +1,10 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.155.1/containers/python-3/.devcontainer/base.Dockerfile
+
+# [Choice] Python version: 3, 3.9, 3.8, 3.7, 3.6
+FROM ubuntu:18.04
+
+RUN apt update && apt install -y python python-pip build-essential git
+
+RUN groupadd --gid 2000 vscode && useradd --uid 2000 --gid 2000 -m vscode
+
+RUN pip install "pylint<2.0.0"

.devcontainer/devcontainer.json

@@ -0,0 +1,32 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.155.1/containers/python-3
+{
+	"name": "Python 2",
+	"build": {
+		"dockerfile": "Dockerfile",
+		"context": "..",
+		"args": { 
+		}
+	},
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": { 
+		"terminal.integrated.shell.linux": "/bin/bash",
+		"python.linting.enabled": true,
+		"python.linting.pylintEnabled": true,
+	},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-python.python"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "pip install -r requirements.txt",
+
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode"
+}

.gitignore

@@ -1,3 +1,6 @@
+# IDE
+*.swp
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.travis.yml

@@ -18,18 +18,3 @@ after_success:
 env:
   global:
     - PYTHONDONTWRITEBYTECODE=true
-
-deploy:
-- provider: script
-  skip_cleanup: true
-  script:
-  - scripts/deploy.sh && scripts/register_modules.sh https://dev-api.door43.org && scripts/integration_test.sh
-  on:
-    branch: develop
-- provider: script
-  skip_cleanup: true
-  script:
-  - scripts/deploy.sh && scripts/register_modules.sh https://api.door43.org && scripts/integration_test.sh
-  on:
-    branch: master
-

functions/markdown_linster/function.json → functions/markdown_linter/function.json

@@ -1,6 +1,6 @@
 {
   "description": "Markdown linter in node.js. This is called in checkers/markdown_checker.py by lambda invoke",
-  "runtime": "nodejs6.10",
+  "runtime": "nodejs10.x",
   "hooks": {
     "build": "npm install",
     "clean": "rm -rf node_modules"

libraries/aws_tools/dynamodb_handler.py

@@ -7,7 +7,7 @@
 
 class DynamoDBHandler(object):
 
-    def __init__(self, table_name, aws_access_key_id=None, aws_secret_access_key=None, aws_region_name='us-west-2'):
+    def __init__(self, table_name, aws_access_key_id=None, aws_secret_access_key=None, aws_region_name=None):
         self.table_name = table_name
         self.aws_access_key_id = aws_access_key_id
         self.aws_secret_access_key = aws_secret_access_key

libraries/aws_tools/lambda_handler.py

@@ -4,7 +4,7 @@
 
 
 class LambdaHandler(object):
-    def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, aws_region_name='us-west-2'):
+    def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, aws_region_name=None):
         self.aws_access_key_id = aws_access_key_id
         self.aws_secret_access_key = aws_secret_access_key
         self.aws_region_name = aws_region_name

libraries/aws_tools/s3_handler.py

@@ -9,7 +9,7 @@
 
 class S3Handler(object):
     def __init__(self, bucket_name=None, aws_access_key_id=None, aws_secret_access_key=None,
-                 aws_region_name='us-west-2'):
+                 aws_region_name=None):
         self.bucket_name = bucket_name
         self.aws_access_key_id = aws_access_key_id
         self.aws_secret_access_key = aws_secret_access_key

libraries/client/client_converter_callback.py

@@ -42,7 +42,7 @@ def process_callback(self):
         self.job = TxJob.get(job_id)
 
         if not self.job:
-            error = 'No job found for job_id = {0}, identifier = {0}'.format(job_id, self.identifier)
+            error = 'No job found for job_id = {0}, identifier = {1}'.format(job_id, self.identifier)
             App.logger.error(error)
             raise Exception(error)
 

libraries/client/client_webhook.py

@@ -140,7 +140,7 @@ def process_webhook(self):
         job.source = self.source_url_base + "/" + file_key
         job.cdn_bucket = App.cdn_bucket
         job.cdn_file = 'tx/job/{0}.zip'.format(job.job_id)
-        job.output = 'https://{0}/{1}'.format(App.cdn_bucket, job.cdn_file)
+        job.output = 'http://{0}.s3-{1}.amazonaws.com/{2}'.format(App.cdn_bucket, App.aws_region_name, job.cdn_file)
         job.callback = App.api_url + '/client/callback'
         job.output_format = 'html'
         job.links = {
@@ -218,7 +218,7 @@ def process_webhook(self):
                         book_job.job_id = self.get_unique_job_id()
                         book_job.identifier = '{0}/{1}/{2}/{3}'.format(book_job.job_id, book_count, i, book)
                         book_job.cdn_file = 'tx/job/{0}.zip'.format(book_job.job_id)
-                        book_job.output = 'https://{0}/{1}'.format(App.cdn_bucket, book_job.cdn_file)
+                        book_job.output = 'http://{0}.s3-{1}.amazonaws.com/{2}'.format(App.cdn_bucket, App.aws_region_name, book_job.cdn_file)
                         book_job.links = {
                             "href": "{0}/tx/job/{1}".format(App.api_url, book_job.job_id),
                             "rel": "self",
@@ -487,14 +487,23 @@ def get_converter_module(self, job):
             converter = converters.filter(TxModule.resource_types.contains('other')).first()
         return converter
 
+    def query_linters(self, resource_type = None, input_format = None):
+        query = TxModule.query().filter(TxModule.type == 'linter') 
+        if input_format:
+            query = query.filter(TxModule.input_format.contains(input_format))
+        if resource_type:
+            query = query.filter(TxModule.resource_types.contains(resource_type))
+        return query
+
     def get_linter_module(self, job):
         """
         :param TxJob job:
         :return TxModule:
         """
-        linters = TxModule.query().filter(TxModule.type=='linter') \
-            .filter(TxModule.input_format.contains(job.input_format))
-        linter = linters.filter(TxModule.resource_types.contains(job.resource_type)).first()
+        linter = self.query_linters(job.resource_type, job.input_format).first() or \
+            self.query_linters('other', job.input_format).first()
         if not linter:
-            linter = linters.filter(TxModule.resource_types.contains('other')).first()
+            App.logger.debug("No matching linter for job {0}".format(job.__dict__))
+            linters = [row.__dict__ for row in self.query_linters()]
+            App.logger.debug("Available linters: {0}".format(linters))
         return linter

libraries/client/converters.py

@@ -0,0 +1,66 @@
+import os
+import json
+import re
+import codecs
+from libraries.app.app import App
+
+
+def txt2md(rootdir="."):
+    """
+    Converts txt files to markdown
+    """
+    proccessed = False
+    for dir, subdir, files in os.walk(rootdir):
+        for fname in files:
+            filepath = os.path.join(dir, fname)
+
+            if os.path.splitext(fname)[1] == ".txt":
+                with open(filepath, "r") as data_file:
+                    # if content of the file starts from the valid json character
+                    # then it's a json file
+                    content = data_file.read().decode('utf-8')
+
+                    if re.match(r"^\[|^\{", content):
+                        try:
+                            data = json.loads(content)
+                            md = ""
+                            for elm in data:
+                                if "title" in elm and "body" in elm:
+                                    md += "# " + elm["title"] + "\n\n"
+                                    md += elm["body"] + "\n\n"
+
+                            md_filepath = re.sub(r"\.txt$", ".md", filepath)
+                            with codecs.open(md_filepath, "w", encoding="utf-8") as md_file:
+                                md_file.write(md)
+
+                            proccessed = True
+                        except BaseException as e:
+                            App.logger.debug('Error: {0}'.format(e.message))
+
+                if os.path.isfile(filepath):
+                    os.remove(filepath)
+
+    return proccessed
+
+
+def txt2usfm(rootdir="."):
+    """
+    Converts txt files to usfm
+    """
+    proccessed = False
+    for dir, subdir, files in os.walk(rootdir):
+        for fname in files:
+            filepath = os.path.join(dir, fname)
+
+            if os.path.splitext(fname)[1] == ".txt":
+                with open(filepath, "r") as data_file:
+                    # if content of the file starts from the valid usfm chapter or verse tag
+                    # then it's a usfm file
+                    if re.match(r"^[\s]*\\c|^[\s]*\\v", data_file.read()):
+                        proccessed = True
+
+                if proccessed and os.path.isfile(filepath):
+                    usfm_filepath = re.sub(r"\.txt$", ".usfm", filepath)
+                    os.rename(filepath, usfm_filepath)
+
+    return proccessed