Upgrade Ansible to 2.14.0

.gitignore

@@ -21,3 +21,4 @@ __pycache__/
 /test.*
 /callback_plugins/clean.py
 .vscode/
+

ansible.cfg

@@ -1,13 +1,16 @@
 [defaults]
 remote_user =                   root
 host_key_checking =             no
+library =                       library
 
 retry_files_enabled =           no
+local_tmp =                     tmp/.ansible/tmp
 
 forks =                         20
 internal_poll_interval =        0.001
 
-stdout_callback =               yaml
+stdout_callback =               default
+result_format =                 yaml
 deprecation_warnings =          yes
 show_custom_stats =             yes
 display_failed_stderr =         yes
@@ -21,7 +24,8 @@ fact_caching_connection =       tmp/fact_cache
 fact_caching_timeout =          2700
 fact_caching_prefix =           mageops-facts
 
-callback_enabled =            timer, profile_tasks
+callbacks_enabled =           timer, profile_tasks
+vars_plugins =                ./vars_plugins
 
 collections_scan_sys_path = False
 

group_vars/all.yml

@@ -1161,25 +1161,41 @@ magento_core_config_settings_default:
     path: "web/secure/base_media_url"
     value: "https://{{ aws_cloudfront_distribution_domain | default('') }}/"
     default: "{{ aws_cloudfront_domain_aliases | default([]) | length > 0 }}"
-    enabled: "{{ aws_cloudfront_distribution_domain is string and aws_cloudfront_distribution_create }}"
+    enabled: >-
+      {{
+        (aws_cloudfront_distribution_domain | default('', true) | string | length) > 0
+        and (aws_cloudfront_distribution_create | bool)
+      }}
 
   - name: Set secure static url to cloudfront domain
     path: "web/secure/base_static_url"
     value: "https://{{ aws_cloudfront_distribution_domain | default('') }}/static/"
     default: "{{ aws_cloudfront_domain_aliases | default([]) | length > 0 }}"
-    enabled: "{{ aws_cloudfront_distribution_domain is string and aws_cloudfront_distribution_create }}"
+    enabled: >-
+      {{
+        (aws_cloudfront_distribution_domain | default('', true) | string | length) > 0
+        and (aws_cloudfront_distribution_create | bool)
+      }}
 
   - name: Set unsecure media url to cloudfront domain
     path: "web/secure/base_media_url"
     value: "https://{{ aws_cloudfront_distribution_domain | default('') }}/"
     default: "{{ aws_cloudfront_domain_aliases | default([]) | length > 0 }}"
-    enabled: "{{ aws_cloudfront_distribution_domain is string and aws_cloudfront_distribution_create }}"
+    enabled: >-
+      {{
+        (aws_cloudfront_distribution_domain | default('', true) | string | length) > 0
+        and (aws_cloudfront_distribution_create | bool)
+      }}
 
   - name: Set unsecure static url to cloudfront domain
     path: "web/secure/base_static_url"
     value: "https://{{ aws_cloudfront_distribution_domain | default('') }}/static/"
     default: "{{ aws_cloudfront_domain_aliases | default([]) | length > 0 }}"
-    enabled: "{{ aws_cloudfront_distribution_domain is string and aws_cloudfront_distribution_create }}"
+    enabled: >-
+      {{
+        (aws_cloudfront_distribution_domain | default('', true) | string | length) > 0
+        and (aws_cloudfront_distribution_create | bool)
+      }}
 
 
 # Extra core config settings to be set at deploy,
@@ -1507,7 +1523,7 @@ nginx_version: "1.10.3"
 nginx_user: "nginx"
 nginx_group: "nginx"
 
-nginx_worker_processes: "{{ ansible_processor_vcpus|default(ansible_processor_count + 1) }}"
+nginx_worker_processes: "{{ (ansible_facts | default({})).get('processor_vcpus', (((ansible_facts | default({})).get('processor_count', 0) | int) + 1)) }}"
 nginx_worker_connections: "1024"
 
 nginx_multi_accept: "off"
@@ -1712,22 +1728,34 @@ varnish_strip_params:
   - "gdftrk"
   - "_ga" # google analytics
   - "_gl" # google analytics
-  - "mc_[^=&]+"
+  - "mc_[^=&]+" # mailchimp
   - "trk_[^=&]+"
   - "dm_i"
   - "fbclid" # facebook
+  - "igshid" # instagram
   - "ttclid" # tiktok
+  - "twclid" # twitter
+  - "li_fat_id" # linkedin
+  - "epik" # pinterest
+  - "si" # spotify
   - "wbraid" # google
   - "gad_[^=&]+" # google ads
   - "adposition" # google ads
   - "srsltid" # google
   - "gbraid" # google
+  - "dclid" # google display & video 360
   - "msclkid" # microsoft
   - "sc_[^=&]+" # emarsys
   - "tduid" # tradedoubler
+  - "_ke" # klaviyo
+  - "ml_[^=&]+" # mailerlite
+  - "hsa_[^=&]+" # hubspot
+  - "__hs[^=&]+" # hubspot
+  - "hsCtaTracking" # hubspot
   - "{{ https_termination_redirect_source_domain_param }}"
   - "[0-9]+" # common bot cache busting pattern
 
+
 varnish_debug_request_info_header_name: "{{ mageops_debug_http_header_prefix }}-Info-Varnish"
 
 varnish_allowed_extensions_default:

inventory/aws_ec2.yml

@@ -2,6 +2,8 @@ plugin: aws_ec2
 
 strict: False
 
+hostvars_prefix: aws_
+
 regions: eu-central-1
 
 filters:
@@ -11,58 +13,58 @@ filters:
     - pending
 
 keyed_groups:
-  - key: tags.Name
+  - key: aws_tags.Name
     prefix: "node"
     separator: "_"
 
-  - key: tags.AppName
+  - key: aws_tags.AppName
     prefix: "app"
     separator: "_"
 
-  - key: tags.AppId
+  - key: aws_tags.AppId
     prefix: "app"
     separator: "_"
 
-  - key: tags.Role
+  - key: aws_tags.Role
     prefix: ""
     separator: ""
 
-  - key: tags.RoleBuilder
+  - key: aws_tags.RoleBuilder
     prefix: "builder"
     separator: "_"
 
-  - key: tags.RoleExtra
+  - key: aws_tags.RoleExtra
     prefix: "extra"
     separator: "_"
 
-  - key: tags.RoleSearch
+  - key: aws_tags.RoleSearch
     prefix: ""
     separator: ""
 
-  - key: tags.RoleCache
+  - key: aws_tags.RoleCache
     prefix: ""
     separator: ""
 
-  - key: tags.RoleHttpCache
+  - key: aws_tags.RoleHttpCache
     prefix: ""
     separator: ""
 
-  - key: tags.RoleLoadBalancer
+  - key: aws_tags.RoleLoadBalancer
     prefix: ""
     separator: ""
 
-  - key: tags.RoleDatabase
+  - key: aws_tags.RoleDatabase
     prefix: ""
     separator: ""
 
-  - key: tags.RoleStorage
+  - key: aws_tags.RoleStorage
     prefix: ""
     separator: ""
 
-  - key: tags.RoleMessageQueue
+  - key: aws_tags.RoleMessageQueue
     prefix: ""
     separator: ""
 
-  - key: tags.TraitImmutable
+  - key: aws_tags.TraitImmutable
     prefix: ""
-    separator: ""
+    separator: ""

library/cloudfront_distribution.py

@@ -1097,10 +1097,8 @@
   sample: abcd1234-1234-abcd-abcd-abcd12345678
 '''
 
-from ansible.module_utils._text import to_text, to_native
+from ansible.module_utils.common.text.converters import to_text, to_native
 from ansible.module_utils.aws.core import AnsibleAWSModule
-from ansible.module_utils.aws.cloudfront_facts import CloudFrontFactsServiceManager
-from ansible.module_utils.ec2 import get_aws_connection_info
 from ansible.module_utils.ec2 import ec2_argument_spec, boto3_conn, compare_aws_tags
 from ansible.module_utils.ec2 import camel_dict_to_snake_dict, ansible_dict_to_boto3_tag_list
 from ansible.module_utils.ec2 import snake_dict_to_camel_dict, boto3_tag_list_to_ansible_dict
@@ -1119,7 +1117,6 @@
 except ImportError:
     pass
 
-
 def change_dict_key_name(dictionary, old_key, new_key):
     if old_key in dictionary:
         dictionary[new_key] = dictionary.get(old_key)
@@ -1236,13 +1233,52 @@ def update_tags(client, module, existing_tags, valid_tags, purge_tags, arn):
     return changed
 
 
+class CloudFrontFactsServiceManager(object):
+    """
+    Local CloudFront facts manager used by this patched module.
+    Keep this implementation here instead of importing collection helpers whose
+    location and return shape changed across amazon.aws releases.
+    """
+
+    def __init__(self, module, client):
+        self.module = module
+        self.client = client
+
+    def get_distribution(self, distribution_id):
+        try:
+            return self.client.get_distribution(Id=distribution_id)
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+            self.module.fail_json_aws(e, msg="Error describing distribution")
+
+    def list_distributions(self, keyed=True):
+        try:
+            paginator = self.client.get_paginator('list_distributions')
+            result = paginator.paginate().build_full_result()
+            distribution_list = result.get('DistributionList', {}).get('Items', [])
+            if not keyed:
+                return distribution_list
+            return self.keyed_list_helper(distribution_list)
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+            self.module.fail_json_aws(e, msg="Error listing distributions")
+
+    def keyed_list_helper(self, list_to_key):
+        keyed_list = dict()
+        for item in list_to_key:
+            distribution_id = item.get('Id')
+            aliases = item.get('Aliases', {}).get('Items', [])
+            for alias in aliases:
+                keyed_list.update({alias: item})
+            keyed_list.update({distribution_id: item})
+        return keyed_list
+
+
 class CloudFrontValidationManager(object):
     """
     Manages Cloudfront validations
     """
 
-    def __init__(self, module):
-        self.__cloudfront_facts_mgr = CloudFrontFactsServiceManager(module)
+    def __init__(self, module, client):
+        self.__cloudfront_facts_mgr = CloudFrontFactsServiceManager(module, client)
         self.module = module
         self.__default_distribution_enabled = True
         self.__default_http_port = 80
@@ -1885,10 +1921,88 @@ def main():
         ]
     )
 
-    region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True)
-    client = boto3_conn(module, conn_type='client', resource='cloudfront', region=region, endpoint=ec2_url, **aws_connect_kwargs)
+    # Newer amazon.aws releases changed where and how AWS connection helpers are
+    # exposed. Instead of depending on whichever helper signature happens to be
+    # installed on the controller, keep the connection extraction here so this
+    # patched local module stays close to upstream behavior across versions.
+    params = module.params
+
+    endpoint_url = params.get('endpoint_url')
+    if endpoint_url is None:
+        endpoint_url = params.get('aws_endpoint_url')
+    if endpoint_url is None:
+        endpoint_url = params.get('ec2_url')
+
+    access_key = params.get('access_key')
+    if access_key is None:
+        access_key = params.get('aws_access_key')
+    if access_key is None:
+        access_key = params.get('aws_access_key_id')
+
+    secret_key = params.get('secret_key')
+    if secret_key is None:
+        secret_key = params.get('aws_secret_key')
+    if secret_key is None:
+        secret_key = params.get('aws_secret_access_key')
+
+    session_token = params.get('session_token')
+    if session_token is None:
+        session_token = params.get('aws_session_token')
+    if session_token is None:
+        session_token = params.get('security_token')
+
+    region = params.get('region')
+    profile_name = params.get('profile')
+    validate_certs = params.get('validate_certs')
+    ca_bundle = params.get('aws_ca_bundle')
+    aws_config = params.get('aws_config')
+
+    if profile_name and (access_key or secret_key or session_token):
+        module.fail_json(msg='Passing both a profile and access tokens is not supported.')
+
+    if not access_key:
+        access_key = None
+    if not secret_key:
+        secret_key = None
+    if not session_token:
+        session_token = None
+
+    if profile_name:
+        aws_connect_kwargs = dict(
+            aws_access_key_id=None,
+            aws_secret_access_key=None,
+            aws_session_token=None,
+            profile_name=profile_name,
+        )
+    else:
+        aws_connect_kwargs = dict(
+            aws_access_key_id=access_key,
+            aws_secret_access_key=secret_key,
+            aws_session_token=session_token,
+        )
+
+    if validate_certs and ca_bundle:
+        aws_connect_kwargs['verify'] = ca_bundle
+    else:
+        aws_connect_kwargs['verify'] = validate_certs
+
+    if aws_config is not None:
+        aws_connect_kwargs['aws_config'] = botocore.config.Config(**aws_config)
+
+    for key, value in list(aws_connect_kwargs.items()):
+        if isinstance(value, bytes):
+            aws_connect_kwargs[key] = str(value, 'utf-8', 'strict')
+
+    client = boto3_conn(
+        module,
+        conn_type='client',
+        resource='cloudfront',
+        region=region,
+        endpoint=endpoint_url,
+        **aws_connect_kwargs
+    )
 
-    validation_mgr = CloudFrontValidationManager(module)
+    validation_mgr = CloudFrontValidationManager(module, client)
 
     state = module.params.get('state')
     caller_reference = module.params.get('caller_reference')
@@ -2005,4 +2119,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    main()