Merge pull request #108 from berttejeda/develop

Develop

Author: berttejeda

HISTORY.md

@@ -2,6 +2,15 @@
 History
 =======
 
+## Release 2019-11-11 v1.3.6
+
+* Refactor git diff command and embedded inventory file logic [330d5de](https://github.com/berttejeda/ansible-taskrunner/commit/330d5de010758e6467312afa827863ba2388e073)
+* Skip deleted files when syncing local git folder to remote [64943b1](https://github.com/berttejeda/ansible-taskrunner/commit/64943b1c9c62fc8192d6d35e17db5002dfb995a9)
+
+## Release 2019-11-04 v1.3.5
+
+* Catch potential crash due to unexpected parameter mapping (python 2.x) [3f37e3e](https://github.com/berttejeda/ansible-taskrunner/commit/3f37e3e6c3bfa6cafc66fa5db5514f81e91879a2)
+
 ## Release 2019-11-04 v1.3.4
 
 * Import missing logging module [56b3ca4](https://github.com/berttejeda/ansible-taskrunner/commit/56b3ca4292a189d169937efc5d8c3a7dbcfb0563)

ansible_taskrunner/cli.py

@@ -83,7 +83,7 @@
 
 # Private variables
 __author__ = 'etejeda'
-__version__ = '1.3.4'
+__version__ = '1.3.6'
 __program_name__ = 'tasks'
 
 # Logging

ansible_taskrunner/libs/providers/ansible.py

@@ -54,9 +54,12 @@ def options(func):
         option = click.option('---debug', type=str, help='Start task run with ansible in debug mode',
                               default=False, required=False)
         func = option(func)
-        option = click.option('---inventory', help='Override embedded inventory specification',
+        option = click.option('---inventory', help='Specify inventory path if not using embedded inventory',
                               required=False)
         func = option(func)
+        option = click.option('---inventory-tmp-dir', help='Override path where we create embedded temporary inventory',
+                              required=False)
+        func = option(func)        
         return func
 
     def invoke_bastion_mode(self, bastion_settings, invocation, remote_command, kwargs):
@@ -148,8 +151,12 @@ def invoke_bastion_mode(self, bastion_settings, invocation, remote_command, kwar
                 sys.exit(1)                
         logger.info('Checking for locally changed files ...')
         if loc_is_git:
-            cmd = 'git diff-index --name-only HEAD -- && git ls-files --others --exclude-standard'
-            local_changed = os.popen(cmd).readlines()
+            # List modified and untracked files
+            changed_cmd = '''git diff-index HEAD --name-status'''
+            changed_files = [f.strip().split('\t')[1] for f in os.popen(changed_cmd).readlines() if not f.startswith('D\t')]
+            untracked_cmd = '''git ls-files --others --exclude-standard'''
+            untracked_files = [f.strip() for f in os.popen(untracked_cmd).readlines()]
+            local_changed = changed_files + untracked_files
         else:
             # If local path is not a git repo then
             # we'll only sync files in the current working directory
@@ -160,6 +167,7 @@ def invoke_bastion_mode(self, bastion_settings, invocation, remote_command, kwar
         logger.info('Checking for remotely changed files ...')
         no_clobber = settings.get('at_no_clobber')
         if rem_is_git:
+            remote_changed_cmd = '''{} | awk '$1 != "D" {{print $2}}' && {}'''.format(changed_cmd, untracked_cmd)
             remote_changed = remote_sub_process.call(remote_dir, cmd)
             if remote_changed:
                 if no_clobber:
@@ -182,12 +190,6 @@ def invoke_bastion_mode(self, bastion_settings, invocation, remote_command, kwar
             remote_path = os.path.normpath(_remote_path).replace('\\','/')
             logger.debug('Syncing {} to remote {}'.format(file_path, remote_path))
             sftp_sync.to_remote(file_path, remote_path)
-        # remote_command = ' '.join([a for a in sys.argv if a != '---bastion-mode'][1:])
-        # tasks_file_override = invocation.get('tasks_file_override')
-        # if tasks_file_override:
-        #     remote_command = 'tasks -f {} {}'.format(tasks_file_override, remote_command)
-        # else:
-        #     remote_command = 'tasks {}'.format(remote_command)
         remote_command_result = remote_sub_process.call(remote_dir, remote_command, stdout_listen=True)
         if remote_command_result.returncode > 0:
             logger.error('Remote command failed with: %s' % ' '.join(remote_command_result.stderr))
@@ -217,7 +219,10 @@ def invocation(self,
             'ansible_playbook_command', 'ansible-playbook')
         # Embedded inventory logic
         embedded_inventory = False
+        # Employ an exit trap if we're using bastion mode
         trap = ''
+        # Where to create the temporary inventory (if applicable)
+        inventory_dir = kwargs.get('_inventory_tmp_dir') or yaml_vars.get('inventory_dir')
         inventory_input = kwargs.get('_inventory')
         embedded_inventory_string = yaml_vars.get('inventory')
         if not inventory_input and not embedded_inventory_string:
@@ -233,12 +238,13 @@ def invocation(self,
                     trap = 'trap "rm -f %s" EXIT' % ans_inv_fp            
         else:
             if bastion_settings.get('enabled'):
-                ans_inv_fp = '/tmp/ansible.inventory.%s.tmp.ini' % str(uuid.uuid4())
+                inventory_dir = '/tmp' if not inventory_dir else inventory_dir
+                ans_inv_fp = '{}/ansible.inventory.{}.tmp.ini'.format(inventory_dir, uuid.uuid4())
                 ans_inv_fso_desc = None
                 if not debug:
                     trap = 'trap "rm -f %s" EXIT' % ans_inv_fp
             else:
-                ans_inv_fso_desc, ans_inv_fp = mkstemp(prefix='ansible-inventory', suffix='.tmp.ini')                 
+                ans_inv_fso_desc, ans_inv_fp = mkstemp(prefix='ansible-inventory', suffix='.tmp.ini', dir=inventory_dir)                 
                 logger.info("No inventory specified")
                 logger.info("Created temporary inventory file %s (normally deleted after run)" % ans_inv_fp)
             inventory_input = embedded_inventory_string

ansible_taskrunner/libs/sshutil/sync.py

@@ -20,6 +20,7 @@
 try:
     import paramiko
     from paramiko import SSHClient, ssh_exception
+    from libs.sshutil.scp import SCPException
 except ImportError as e:
     print('Error in %s ' % os.path.basename(self_file_name))
     print('Failed to import at least one required module')
@@ -49,12 +50,15 @@ def create_parent_dirs(self, remote_path):
                 self.sftp_obj.stat(directory)
 
     def to_remote(self, local_path, remote_path):
-        logger.debug("Lcl Sync Target {}".format(local_path))
-        logger.debug("Rmt Sync Target {}".format(remote_path))
+        logger.debug("Loc Sync Target {}".format(local_path))
+        logger.debug("Rem Sync Target {}".format(remote_path))
         if os.path.exists(local_path):
-            self.create_parent_dirs(remote_path)
-            self.scp.put(local_path, remote_path=remote_path, preserve_times=True, recursive=True)
-            logger.debug("Successfully copied to remote.")
+            try:
+                self.create_parent_dirs(remote_path)
+                self.scp.put(local_path, remote_path=remote_path, preserve_times=True, recursive=True)
+                logger.debug('Sync ok for %s' % local_path)
+            except SCPException as e:
+                logger.error('Failed to sync {} to remote {} - error was {}'.format(local_path, remote_path, e))                
         else:
             logger.warning("Skipping %s as it does not exist" % local_path.strip())
         return

setup.cfg

@@ -3,7 +3,7 @@ name = ansible_taskrunner
 author = Engelbert Tejeda
 author_email = berttejeda@gmail.com
 description = ansible-playbook wrapper with YAML-abstracted python click cli options
-version: 1.3.4
+version: 1.3.6
 url = https://github.com/berttejeda/ansible_taskrunner
 keywords =
   ansible