diff --git a/.ansible/.lock b/.ansible/.lock deleted file mode 100644 index e69de29bb..000000000 diff --git a/.gitignore b/.gitignore index bcbebd163..c66c0e024 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ __pycache__/ /test.* /callback_plugins/clean.py .vscode/ + diff --git a/ansible.cfg b/ansible.cfg index 607dc68e7..c33f915f0 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,13 +1,16 @@ [defaults] remote_user = root host_key_checking = no +library = library retry_files_enabled = no +local_tmp = tmp/.ansible/tmp forks = 20 internal_poll_interval = 0.001 -stdout_callback = yaml +stdout_callback = default +result_format = yaml deprecation_warnings = yes show_custom_stats = yes display_failed_stderr = yes @@ -21,7 +24,8 @@ fact_caching_connection = tmp/fact_cache fact_caching_timeout = 2700 fact_caching_prefix = mageops-facts -callback_enabled = timer, profile_tasks +callbacks_enabled = timer, profile_tasks +vars_plugins = ./vars_plugins collections_scan_sys_path = False diff --git a/group_vars/all.yml b/group_vars/all.yml index 8c5879ab4..cf3053565 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1161,25 +1161,41 @@ magento_core_config_settings_default: path: "web/secure/base_media_url" value: "https://{{ aws_cloudfront_distribution_domain | default('') }}/" default: "{{ aws_cloudfront_domain_aliases | default([]) | length > 0 }}" - enabled: "{{ aws_cloudfront_distribution_domain is string and aws_cloudfront_distribution_create }}" + enabled: >- + {{ + (aws_cloudfront_distribution_domain | default('', true) | string | length) > 0 + and (aws_cloudfront_distribution_create | bool) + }} - name: Set secure static url to cloudfront domain path: "web/secure/base_static_url" value: "https://{{ aws_cloudfront_distribution_domain | default('') }}/static/" default: "{{ aws_cloudfront_domain_aliases | default([]) | length > 0 }}" - enabled: "{{ aws_cloudfront_distribution_domain is string and aws_cloudfront_distribution_create }}" + enabled: >- + {{ + (aws_cloudfront_distribution_domain | default('', true) | string | length) > 0 + and (aws_cloudfront_distribution_create | bool) + }} - name: Set unsecure media url to cloudfront domain path: "web/secure/base_media_url" value: "https://{{ aws_cloudfront_distribution_domain | default('') }}/" default: "{{ aws_cloudfront_domain_aliases | default([]) | length > 0 }}" - enabled: "{{ aws_cloudfront_distribution_domain is string and aws_cloudfront_distribution_create }}" + enabled: >- + {{ + (aws_cloudfront_distribution_domain | default('', true) | string | length) > 0 + and (aws_cloudfront_distribution_create | bool) + }} - name: Set unsecure static url to cloudfront domain path: "web/secure/base_static_url" value: "https://{{ aws_cloudfront_distribution_domain | default('') }}/static/" default: "{{ aws_cloudfront_domain_aliases | default([]) | length > 0 }}" - enabled: "{{ aws_cloudfront_distribution_domain is string and aws_cloudfront_distribution_create }}" + enabled: >- + {{ + (aws_cloudfront_distribution_domain | default('', true) | string | length) > 0 + and (aws_cloudfront_distribution_create | bool) + }} # Extra core config settings to be set at deploy, @@ -1507,7 +1523,7 @@ nginx_version: "1.10.3" nginx_user: "nginx" nginx_group: "nginx" -nginx_worker_processes: "{{ ansible_processor_vcpus|default(ansible_processor_count + 1) }}" +nginx_worker_processes: "{{ (ansible_facts | default({})).get('processor_vcpus', (((ansible_facts | default({})).get('processor_count', 0) | int) + 1)) }}" nginx_worker_connections: "1024" nginx_multi_accept: "off" @@ -1712,22 +1728,34 @@ varnish_strip_params: - "gdftrk" - "_ga" # google analytics - "_gl" # google analytics - - "mc_[^=&]+" + - "mc_[^=&]+" # mailchimp - "trk_[^=&]+" - "dm_i" - "fbclid" # facebook + - "igshid" # instagram - "ttclid" # tiktok + - "twclid" # twitter + - "li_fat_id" # linkedin + - "epik" # pinterest + - "si" # spotify - "wbraid" # google - "gad_[^=&]+" # google ads - "adposition" # google ads - "srsltid" # google - "gbraid" # google + - "dclid" # google display & video 360 - "msclkid" # microsoft - "sc_[^=&]+" # emarsys - "tduid" # tradedoubler + - "_ke" # klaviyo + - "ml_[^=&]+" # mailerlite + - "hsa_[^=&]+" # hubspot + - "__hs[^=&]+" # hubspot + - "hsCtaTracking" # hubspot - "{{ https_termination_redirect_source_domain_param }}" - "[0-9]+" # common bot cache busting pattern + varnish_debug_request_info_header_name: "{{ mageops_debug_http_header_prefix }}-Info-Varnish" varnish_allowed_extensions_default: diff --git a/inventory/aws_ec2.yml b/inventory/aws_ec2.yml index f5f0acb14..bef29916d 100644 --- a/inventory/aws_ec2.yml +++ b/inventory/aws_ec2.yml @@ -2,6 +2,8 @@ plugin: aws_ec2 strict: False +hostvars_prefix: aws_ + regions: eu-central-1 filters: @@ -11,58 +13,58 @@ filters: - pending keyed_groups: - - key: tags.Name + - key: aws_tags.Name prefix: "node" separator: "_" - - key: tags.AppName + - key: aws_tags.AppName prefix: "app" separator: "_" - - key: tags.AppId + - key: aws_tags.AppId prefix: "app" separator: "_" - - key: tags.Role + - key: aws_tags.Role prefix: "" separator: "" - - key: tags.RoleBuilder + - key: aws_tags.RoleBuilder prefix: "builder" separator: "_" - - key: tags.RoleExtra + - key: aws_tags.RoleExtra prefix: "extra" separator: "_" - - key: tags.RoleSearch + - key: aws_tags.RoleSearch prefix: "" separator: "" - - key: tags.RoleCache + - key: aws_tags.RoleCache prefix: "" separator: "" - - key: tags.RoleHttpCache + - key: aws_tags.RoleHttpCache prefix: "" separator: "" - - key: tags.RoleLoadBalancer + - key: aws_tags.RoleLoadBalancer prefix: "" separator: "" - - key: tags.RoleDatabase + - key: aws_tags.RoleDatabase prefix: "" separator: "" - - key: tags.RoleStorage + - key: aws_tags.RoleStorage prefix: "" separator: "" - - key: tags.RoleMessageQueue + - key: aws_tags.RoleMessageQueue prefix: "" separator: "" - - key: tags.TraitImmutable + - key: aws_tags.TraitImmutable prefix: "" - separator: "" \ No newline at end of file + separator: "" diff --git a/library/cloudfront_distribution.py b/library/cloudfront_distribution.py index bafbc02e8..b715776d8 100644 --- a/library/cloudfront_distribution.py +++ b/library/cloudfront_distribution.py @@ -1097,10 +1097,8 @@ sample: abcd1234-1234-abcd-abcd-abcd12345678 ''' -from ansible.module_utils._text import to_text, to_native +from ansible.module_utils.common.text.converters import to_text, to_native from ansible.module_utils.aws.core import AnsibleAWSModule -from ansible.module_utils.aws.cloudfront_facts import CloudFrontFactsServiceManager -from ansible.module_utils.ec2 import get_aws_connection_info from ansible.module_utils.ec2 import ec2_argument_spec, boto3_conn, compare_aws_tags from ansible.module_utils.ec2 import camel_dict_to_snake_dict, ansible_dict_to_boto3_tag_list from ansible.module_utils.ec2 import snake_dict_to_camel_dict, boto3_tag_list_to_ansible_dict @@ -1119,7 +1117,6 @@ except ImportError: pass - def change_dict_key_name(dictionary, old_key, new_key): if old_key in dictionary: dictionary[new_key] = dictionary.get(old_key) @@ -1236,13 +1233,52 @@ def update_tags(client, module, existing_tags, valid_tags, purge_tags, arn): return changed +class CloudFrontFactsServiceManager(object): + """ + Local CloudFront facts manager used by this patched module. + Keep this implementation here instead of importing collection helpers whose + location and return shape changed across amazon.aws releases. + """ + + def __init__(self, module, client): + self.module = module + self.client = client + + def get_distribution(self, distribution_id): + try: + return self.client.get_distribution(Id=distribution_id) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + self.module.fail_json_aws(e, msg="Error describing distribution") + + def list_distributions(self, keyed=True): + try: + paginator = self.client.get_paginator('list_distributions') + result = paginator.paginate().build_full_result() + distribution_list = result.get('DistributionList', {}).get('Items', []) + if not keyed: + return distribution_list + return self.keyed_list_helper(distribution_list) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + self.module.fail_json_aws(e, msg="Error listing distributions") + + def keyed_list_helper(self, list_to_key): + keyed_list = dict() + for item in list_to_key: + distribution_id = item.get('Id') + aliases = item.get('Aliases', {}).get('Items', []) + for alias in aliases: + keyed_list.update({alias: item}) + keyed_list.update({distribution_id: item}) + return keyed_list + + class CloudFrontValidationManager(object): """ Manages Cloudfront validations """ - def __init__(self, module): - self.__cloudfront_facts_mgr = CloudFrontFactsServiceManager(module) + def __init__(self, module, client): + self.__cloudfront_facts_mgr = CloudFrontFactsServiceManager(module, client) self.module = module self.__default_distribution_enabled = True self.__default_http_port = 80 @@ -1885,10 +1921,88 @@ def main(): ] ) - region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True) - client = boto3_conn(module, conn_type='client', resource='cloudfront', region=region, endpoint=ec2_url, **aws_connect_kwargs) + # Newer amazon.aws releases changed where and how AWS connection helpers are + # exposed. Instead of depending on whichever helper signature happens to be + # installed on the controller, keep the connection extraction here so this + # patched local module stays close to upstream behavior across versions. + params = module.params + + endpoint_url = params.get('endpoint_url') + if endpoint_url is None: + endpoint_url = params.get('aws_endpoint_url') + if endpoint_url is None: + endpoint_url = params.get('ec2_url') + + access_key = params.get('access_key') + if access_key is None: + access_key = params.get('aws_access_key') + if access_key is None: + access_key = params.get('aws_access_key_id') + + secret_key = params.get('secret_key') + if secret_key is None: + secret_key = params.get('aws_secret_key') + if secret_key is None: + secret_key = params.get('aws_secret_access_key') + + session_token = params.get('session_token') + if session_token is None: + session_token = params.get('aws_session_token') + if session_token is None: + session_token = params.get('security_token') + + region = params.get('region') + profile_name = params.get('profile') + validate_certs = params.get('validate_certs') + ca_bundle = params.get('aws_ca_bundle') + aws_config = params.get('aws_config') + + if profile_name and (access_key or secret_key or session_token): + module.fail_json(msg='Passing both a profile and access tokens is not supported.') + + if not access_key: + access_key = None + if not secret_key: + secret_key = None + if not session_token: + session_token = None + + if profile_name: + aws_connect_kwargs = dict( + aws_access_key_id=None, + aws_secret_access_key=None, + aws_session_token=None, + profile_name=profile_name, + ) + else: + aws_connect_kwargs = dict( + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + aws_session_token=session_token, + ) + + if validate_certs and ca_bundle: + aws_connect_kwargs['verify'] = ca_bundle + else: + aws_connect_kwargs['verify'] = validate_certs + + if aws_config is not None: + aws_connect_kwargs['aws_config'] = botocore.config.Config(**aws_config) + + for key, value in list(aws_connect_kwargs.items()): + if isinstance(value, bytes): + aws_connect_kwargs[key] = str(value, 'utf-8', 'strict') + + client = boto3_conn( + module, + conn_type='client', + resource='cloudfront', + region=region, + endpoint=endpoint_url, + **aws_connect_kwargs + ) - validation_mgr = CloudFrontValidationManager(module) + validation_mgr = CloudFrontValidationManager(module, client) state = module.params.get('state') caller_reference = module.params.get('caller_reference') @@ -2005,4 +2119,4 @@ def main(): if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/library/cloudfront_distribution_info_compat.py b/library/cloudfront_distribution_info_compat.py new file mode 100644 index 000000000..c47055331 --- /dev/null +++ b/library/cloudfront_distribution_info_compat.py @@ -0,0 +1,219 @@ +#!/usr/bin/python +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +DOCUMENTATION = r''' +--- +module: cloudfront_distribution_info_compat +short_description: Retrieve CloudFront distribution information (compat module) +description: + - Lightweight replacement for cloudfront distribution info gathering. + - Intended to avoid collection incompatibilities between community.aws and amazon.aws. +requirements: + - boto3 + - botocore +options: + region: + description: + - AWS region to use when creating the CloudFront client. + - CloudFront is global, but region can still be provided for credential/session consistency. + type: str + summary: + description: + - Return summary of distributions with tags. + type: bool + default: false + distribution: + description: + - Return details for a specific distribution. + type: bool + default: false + distribution_id: + description: + - Distribution ID used when C(distribution=true). + type: str + profile: + description: + - AWS credential profile name. + type: str + aws_access_key: + description: + - AWS access key ID. + aliases: [aws_access_key_id] + type: str + aws_secret_key: + description: + - AWS secret access key. + aliases: [aws_secret_access_key] + type: str + aws_session_token: + description: + - AWS session token. + aliases: [security_token] + type: str + endpoint_url: + description: + - Alternate CloudFront endpoint URL. + aliases: [aws_endpoint_url, ec2_url] + type: str + validate_certs: + description: + - Toggle TLS certificate validation. + type: bool + default: true + aws_ca_bundle: + description: + - CA bundle path for TLS verification. + type: path +author: + - MageOps +''' + +EXAMPLES = r''' +- name: Get CloudFront distributions summary + cloudfront_distribution_info_compat: + summary: true + register: cloudfront_info + +- name: Get a specific CloudFront distribution + cloudfront_distribution_info_compat: + distribution: true + distribution_id: E1234567890 + register: cloudfront_distribution +''' + +RETURN = r''' +cloudfront: + description: CloudFront info compatible with existing role usage. + returned: always + type: dict +''' + +from ansible.module_utils.basic import AnsibleModule + + +def build_cloudfront_client(module): + try: + import boto3 + except ImportError: + module.fail_json(msg='boto3 is required for cloudfront_distribution_info_compat') + + session_kwargs = {} + profile = module.params.get('profile') + access_key = module.params.get('aws_access_key') + secret_key = module.params.get('aws_secret_key') + session_token = module.params.get('aws_session_token') + + if profile: + session_kwargs['profile_name'] = profile + if access_key: + session_kwargs['aws_access_key_id'] = access_key + if secret_key: + session_kwargs['aws_secret_access_key'] = secret_key + if session_token: + session_kwargs['aws_session_token'] = session_token + + session = boto3.session.Session(**session_kwargs) + + client_kwargs = {} + region = module.params.get('region') + endpoint_url = module.params.get('endpoint_url') + validate_certs = module.params.get('validate_certs') + aws_ca_bundle = module.params.get('aws_ca_bundle') + + if region: + client_kwargs['region_name'] = region + if endpoint_url: + client_kwargs['endpoint_url'] = endpoint_url + if aws_ca_bundle: + client_kwargs['verify'] = aws_ca_bundle + elif not validate_certs: + client_kwargs['verify'] = False + + return session.client('cloudfront', **client_kwargs) + + +def list_distributions_with_tags(module, client): + distributions = [] + paginator = client.get_paginator('list_distributions') + + for page in paginator.paginate(): + page_items = page.get('DistributionList', {}).get('Items', []) + for dist in page_items: + # Keep AWS payload shape, but normalize aliases and append tag dict as expected by role filtering. + distribution = dict(dist) + aliases = distribution.get('Aliases', {}) + if isinstance(aliases, dict): + distribution['Aliases'] = list(aliases.get('Items', [])) + + tags = {} + arn = distribution.get('ARN') + if arn: + tags_response = client.list_tags_for_resource(Resource=arn) + tag_items = tags_response.get('Tags', {}).get('Items', []) + for tag in tag_items: + key = tag.get('Key') + if key is not None: + tags[key] = tag.get('Value') + distribution['Tags'] = tags + distributions.append(distribution) + + return distributions + + +def main(): + argument_spec = dict( + region=dict(type='str', required=False), + summary=dict(type='bool', default=False), + distribution=dict(type='bool', default=False), + distribution_id=dict(type='str', required=False), + profile=dict(type='str', required=False), + aws_access_key=dict(type='str', required=False, aliases=['aws_access_key_id']), + aws_secret_key=dict(type='str', required=False, aliases=['aws_secret_access_key'], no_log=True), + aws_session_token=dict(type='str', required=False, aliases=['security_token'], no_log=True), + endpoint_url=dict(type='str', required=False, aliases=['aws_endpoint_url', 'ec2_url']), + validate_certs=dict(type='bool', default=True), + aws_ca_bundle=dict(type='path', required=False), + ) + + module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=True) + + summary = module.params.get('summary') + distribution = module.params.get('distribution') + distribution_id = module.params.get('distribution_id') + + # Match community.aws behavior: summary if no selection provided. + if not summary and not distribution: + summary = True + + if distribution and not distribution_id: + module.fail_json(msg='distribution_id is required when distribution=true') + + try: + client = build_cloudfront_client(module) + except Exception as exc: + module.fail_json(msg='Failed to initialize CloudFront client: {0}'.format(exc)) + + result = dict(changed=False, msg='Retrieved CloudFront info.', cloudfront={}) + + try: + if summary: + result['cloudfront']['summary'] = dict( + distributions=list_distributions_with_tags(module, client), + ) + + if distribution: + dist = client.get_distribution(Id=distribution_id) + # Preserve convenient access path used by existing role code. + result['cloudfront']['result'] = dist + result['cloudfront'][distribution_id] = dist + except Exception as exc: + module.fail_json(msg='Failed to gather CloudFront info: {0}'.format(exc)) + + module.exit_json(**result) + + +if __name__ == '__main__': + main() diff --git a/library/ec2_launch_template.py b/library/ec2_launch_template.py index c401c822f..621182b53 100644 --- a/library/ec2_launch_template.py +++ b/library/ec2_launch_template.py @@ -382,7 +382,7 @@ from uuid import uuid4 -from ansible.module_utils._text import to_text +from ansible.module_utils.common.text.converters import to_text from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict from ansible.module_utils.common.dict_transformations import snake_dict_to_camel_dict diff --git a/requirements-galaxy.yml b/requirements-galaxy.yml index 90b73254e..89a176330 100644 --- a/requirements-galaxy.yml +++ b/requirements-galaxy.yml @@ -1,32 +1,30 @@ roles: -- src: geerlingguy.mysql - version: 4.3.5 -- src: geerlingguy.composer -- src: geerlingguy.ntp - version: 2.4.0 -- src: zauberpony.mysql-query - -- src: pinkeen.postfix - version: v1.1 + - src: geerlingguy.mysql + version: 6.3.1 + - src: geerlingguy.composer + - src: geerlingguy.ntp + version: 4.0.0 + - src: zauberpony.mysql-query + version: v0.6.1 + - src: pinkeen.postfix + version: v1.1 collections: -- name: community.aws - version: 5.5.1 -- name: amazon.aws - version: 5.5.2 -- name: community.mysql -- name: ansible.utils - version: 2.10.3 -- name: community.general - version: 7.2.0 -- name: ansible.posix - version: 1.5.4 -- name: community.rabbitmq - version: 1.2.3 -- name: community.crypto - version: 2.18.0 -- name: ansible.netcommon - version: 5.1.2 -- community.crypto -- ansible.posix -- containers.podman + - name: amazon.aws + version: 11.1.0 + - name: community.mysql + version: 4.1.0 + - name: ansible.utils + version: 6.0.1 + - name: community.general + version: 12.4.0 + - name: ansible.posix + version: 2.1.0 + - name: community.rabbitmq + version: 1.6.0 + - name: community.crypto + version: 3.1.1 + - name: ansible.netcommon + version: 8.4.0 + - name: containers.podman + version: 1.19.0 diff --git a/requirements-python.txt b/requirements-python.txt index 470cd9b40..e7e4c0016 100644 --- a/requirements-python.txt +++ b/requirements-python.txt @@ -1,14 +1,19 @@ -ansible>=6,<7 +ansible>=13,<14 # some tasks call aws command on localhost # make sure this is BEFORE boto3 and boto -awscli +awscli>=1,<2 # needed for inventory and aws modules -boto3 +boto3>=1.34.0 # required by ansible ec2 module -boto +botocore>=1.34.0 + +# boto3/botocore imports can break in mixed Homebrew + user-site environments +# when these transitive dependencies are missing from the controller interpreter. +python-dateutil>=2.8.2 +six>=1.16.0 # needed for python `ipaddr` filter ipaddress @@ -28,22 +33,24 @@ netaddr git+https://github.com/mageops/PyMySQL.git@dev-ssl-by-default#egg=PyMySQL # needed for running docker (e.g. building aws lambda artifacts) -docker-py +docker # required for awscli -PyYAML<5.3 +#PyYAML<5.3 # required for `cs.ansible-plugins` role jq -# required if host running ansible have selinux installed -selinux +# Do not install the PyPI `selinux` package in the controller venv. +# With Python 3.12 it expects matching system SELinux bindings for the same +# interpreter version and can break Ansible startup on Jenkins before any +# playbook code runs. Target hosts install `python3-libselinux` when needed. # distro 1.7.0 breaks fact discovery in ansible for some nodes # distro introduced breaking change that changed package # from using single file to directory module, this isn't compatible with # import used by our ansible version # We are pinning this to previous version until we upgrade to never ansible -distro<1.7.0 +#distro<1.7.0 # Dummy comment to force reinstallation of dependencies _1 diff --git a/roles/cs.ansible-plugins/filter_plugins/transform.py b/roles/cs.ansible-plugins/filter_plugins/transform.py index 6bd5c8f3f..9f5e15ccc 100644 --- a/roles/cs.ansible-plugins/filter_plugins/transform.py +++ b/roles/cs.ansible-plugins/filter_plugins/transform.py @@ -3,7 +3,7 @@ __metaclass__ = type from ansible.errors import AnsibleError, AnsibleFilterError -from ansible.module_utils._text import to_native +from ansible.module_utils.common.text.converters import to_native import json diff --git a/roles/cs.aws-ami-facts/tasks/find-app-node-ami.yml b/roles/cs.aws-ami-facts/tasks/find-app-node-ami.yml index fabfbffa5..520658a50 100644 --- a/roles/cs.aws-ami-facts/tasks/find-app-node-ami.yml +++ b/roles/cs.aws-ami-facts/tasks/find-app-node-ami.yml @@ -7,6 +7,7 @@ ami_facts_app_node_filters, ami_facts_app_node_tag_filters) }} vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" ami_facts_app_node_tag_filters: "{{ ami_facts_app_node_tags | prefix_keys('tag:') }}" register: ami_app_node_facts diff --git a/roles/cs.aws-ami-facts/tasks/find-clean-base-ami.yml b/roles/cs.aws-ami-facts/tasks/find-clean-base-ami.yml index d481d5c89..237f267a3 100644 --- a/roles/cs.aws-ami-facts/tasks/find-clean-base-ami.yml +++ b/roles/cs.aws-ami-facts/tasks/find-clean-base-ami.yml @@ -3,6 +3,8 @@ amazon.aws.ec2_ami_info: region: "{{ aws_region }}" filters: "{{ ami_facts_common_filters | combine(ami_facts_clean_base_filters) }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" register: ami_clean_base_facts - name: Fail if no clean base AMI is found diff --git a/roles/cs.aws-autoscaling-facts/tasks/main.yml b/roles/cs.aws-autoscaling-facts/tasks/main.yml index 55a5ca14b..93a43c514 100644 --- a/roles/cs.aws-autoscaling-facts/tasks/main.yml +++ b/roles/cs.aws-autoscaling-facts/tasks/main.yml @@ -1,5 +1,5 @@ - name: Get ASG app information - ec2_asg_info: + amazon.aws.autoscaling_group_info: region: "{{ aws_region }}" tags: >- {{ @@ -15,10 +15,12 @@ ) ) }} + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" register: _aws_asg_app - name: Get ASG app-extra information - ec2_asg_info: + amazon.aws.autoscaling_group_info: region: "{{ aws_region }}" tags: >- {{ @@ -35,10 +37,12 @@ ) ) }} + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" register: _aws_asg_app_extra - name: Get ASG app-pio information - ec2_asg_info: + amazon.aws.autoscaling_group_info: region: "{{ aws_region }}" tags: >- {{ aws_tags_default | combine( @@ -46,6 +50,8 @@ aws_tags_node_pio, ) }} + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" register: _aws_asg_app_pio - name: Show found ASG diff --git a/roles/cs.aws-autoscaling-triggers/tasks/main.yml b/roles/cs.aws-autoscaling-triggers/tasks/main.yml index 4386e0abd..59f1ce2b4 100644 --- a/roles/cs.aws-autoscaling-triggers/tasks/main.yml +++ b/roles/cs.aws-autoscaling-triggers/tasks/main.yml @@ -28,7 +28,7 @@ register: aws_autoscaling_event - name: Allow autoscaling event handler lambda to be executed by CloudWatch Events - lambda_policy: + amazon.aws.lambda_policy: state: present region: "{{ aws_region }}" function_name: "{{ function_name }}" diff --git a/roles/cs.aws-autoscaling/tasks/create.yml b/roles/cs.aws-autoscaling/tasks/create.yml index 12204437e..47e2bcd97 100644 --- a/roles/cs.aws-autoscaling/tasks/create.yml +++ b/roles/cs.aws-autoscaling/tasks/create.yml @@ -1,11 +1,11 @@ - name: Create Launch Template Configuration - ec2_launch_template: + amazon.aws.ec2_launch_template: state: present name: "{{ autoscaling_launch_template_name }}" region: "{{ aws_region }}" image_id: "{{ autoscaling_instance_ami_id }}" instance_type: "{{ autoscaling_instance_type }}" - iam_instance_profile : "{{ autoscaling_instance_iam_role }}" + iam_instance_profile: "{{ autoscaling_instance_iam_role }}" key_name: "{{ aws_ec2_ssh_key_name }}" network_interfaces: - groups: "{{ autoscaling_instance_security_groups }}" @@ -21,7 +21,7 @@ aws_ec2_user_launch_script: "{{ autoscaling_instance_start_script }}" - name: Create AutoScaling Group - ec2_asg: + amazon.aws.autoscaling_group: state: present name: "{{ autoscaling_asg_name }}" region: "{{ aws_region }}" @@ -43,7 +43,6 @@ # This ansible module does not handle instance replacement properly # so we just update the ASG configuration here and will perform the # replacement via AWS API in the next steps. - replace_instances: [] vars: asg_name_tags: - Name: "{{ autoscaling_asg_name }}" @@ -56,7 +55,7 @@ and autoscaling_asg_desired_capacity > 0 - name: Create ASG Termination Lifecycle Hook - ec2_asg_lifecycle_hook: + community.aws.autoscaling_lifecycle_hook: state: present region: "{{ aws_region }}" autoscaling_group_name: "{{ autoscaling_asg_name }}" diff --git a/roles/cs.aws-autoscaling/tasks/delete.yml b/roles/cs.aws-autoscaling/tasks/delete.yml index 34ece283c..d18ec1206 100644 --- a/roles/cs.aws-autoscaling/tasks/delete.yml +++ b/roles/cs.aws-autoscaling/tasks/delete.yml @@ -1,5 +1,5 @@ - name: Remove AutoScaling Group - ec2_asg: + amazon.aws.autoscaling_group: state: absent name: "{{ autoscaling_asg_name }}" region: "{{ aws_region }}" diff --git a/roles/cs.aws-cloudfront-facts/tasks/main.yml b/roles/cs.aws-cloudfront-facts/tasks/main.yml index 52e78c95c..d877b78a6 100644 --- a/roles/cs.aws-cloudfront-facts/tasks/main.yml +++ b/roles/cs.aws-cloudfront-facts/tasks/main.yml @@ -1,35 +1,40 @@ - name: Get list of cloudfront distributions - community.aws.cloudfront_distribution_info: + cloudfront_distribution_info_compat: region: "{{ aws_region }}" summary: yes + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" register: cloudfront_distributions_info -- name: Set list of cloudfront distributions - set_fact: - aws_cloudfront_distributions: "{{ cloudfront_distributions_info.cloudfront.summary.distributions }}" +- name: Set list of cloudfront distributions from module result + ansible.builtin.set_fact: + aws_cloudfront_distributions: "{{ cloudfront_distributions_info.cloudfront.summary.distributions | default([]) }}" - name: Filter cloudfront distribution list by tags - set_fact: + ansible.builtin.set_fact: aws_cloudfront_distributions: "{{ aws_cloudfront_distributions | json_query(distributions_tag_filter_query) }}" vars: - distributions_tag_filter_query: "[?{% for k, v in aws_cloudfront_distribution_tags.items() -%}Tags.{{ k }} == '{{ v }}'{% if not loop.last %} && {% endif %}{% endfor %}]" + distributions_tag_filter_query: >- + [?{% for k, v in aws_cloudfront_distribution_tags.items() -%} + Tags.{{ k }} == '{{ v }}'{% if not loop.last %} && {% endif %} + {% endfor %}] - name: Warn when more than one distribution has been found - debug: + ansible.builtin.debug: msg: | Warning! More than one matching cloudfront distribution found, using first one. Found: {{ aws_cloudfront_distributions | map(attribute='Comment') | join(', ') }} when: aws_cloudfront_distributions | length > 1 - name: Set facts about project's cloudfront distribution - set_fact: + ansible.builtin.set_fact: aws_cloudfront_distribution: "{{ aws_cloudfront_distributions | first }}" aws_cloudfront_distribution_id: "{{ (aws_cloudfront_distributions | first).Id }}" aws_cloudfront_distribution_domain: "{{ (aws_cloudfront_distributions | first).DomainName }}" when: aws_cloudfront_distributions | length > 0 - name: Show gathered cloudfront info - debug: + ansible.builtin.debug: msg: | Using Cloudfront Distribution: {{ aws_cloudfront_distribution_id }} with domain {{ aws_cloudfront_distribution_domain }} when: aws_cloudfront_distributions | length > 0 diff --git a/roles/cs.aws-cloudfront/tasks/setup-distribution-lb-media.yml b/roles/cs.aws-cloudfront/tasks/setup-distribution-lb-media.yml index 8a3345f29..786edf829 100644 --- a/roles/cs.aws-cloudfront/tasks/setup-distribution-lb-media.yml +++ b/roles/cs.aws-cloudfront/tasks/setup-distribution-lb-media.yml @@ -1,14 +1,55 @@ -- name: Set up Cloudfront Distribution - # This ansible module is so FUBAR (ans so is the AWS CF API) that I'd - # recommend to not touch this unless you're in for an all-nighter. /FS +- name: Get current cloudfront distribution config + cloudfront_distribution_info_compat: + region: "{{ aws_region }}" + distribution: true + distribution_id: "{{ aws_cloudfront_distribution_id }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + register: cloudfront_current_distribution_info + when: aws_cloudfront_distribution_id is defined + +- name: Set current cloudfront origin list + ansible.builtin.set_fact: + cloudfront_current_origins: "{{ cloudfront_current_distribution_info.cloudfront.result.Distribution.DistributionConfig.Origins.Items | default([]) }}" + when: aws_cloudfront_distribution_id is defined + +- name: Initialize current cloudfront origin list + ansible.builtin.set_fact: + cloudfront_current_origins: "{{ cloudfront_current_origins | default([]) }}" + +- name: Resolve cloudfront origin domain name + ansible.builtin.set_fact: + cloudfront_origin_domain_name: >- + {{ + ( + lb_domain is defined and lb_domain | length > 0 + ) + | ternary( + lb_domain, + ( + cloudfront_current_origins + | selectattr('OriginPath', 'equalto', '/media') + | map(attribute='DomainName') + | first + | default( + ( + cloudfront_current_origins + | map(attribute='DomainName') + | first + ), + true + ) + | default(magento_hostname, true) + ) + ) + }} + +- name: Set up cloudfront distribution cloudfront_distribution: state: present enabled: true - # The ID is mandatory if the distro already exists (ansible_bug) - # Vide: https://github.com/ansible/ansible/issues/45043 distribution_id: "{{ aws_cloudfront_distribution_id | default(omit, true) }}" - # Caller_reference has to be ommited if distro exists (related to the above bug) caller_reference: "{{ aws_cloudfront_distribution_id | default(false, true) | ternary(omit, aws_cloudfront_distribution_caller_id) }}" comment: "{{ mageops_project }}-{{ mageops_environment }}-media" @@ -58,39 +99,37 @@ origins: - id: "WebsiteRoot" - domain_name: "{{ magento_hostname }}" + domain_name: "{{ cloudfront_origin_domain_name }}" custom_headers: - header_name: X-MageOps-Cloudfront-Origin header_value: WebsiteMedia - id: "WebsiteMedia" - domain_name: "{{ magento_hostname }}" + domain_name: "{{ cloudfront_origin_domain_name }}" origin_path: "/media" custom_headers: - header_name: X-MageOps-Cloudfront-Origin header_value: WebsiteMedia - # To maintain backwards compatibility with the previous setup # all CDN requests go by default to "/media" website path. default_cache_behavior: >- - {{ - cloudfront_distribution_cache_behaviour_defaults | combine({ - "target_origin_id": "WebsiteMedia" - }) - }} + {{ + cloudfront_distribution_cache_behaviour_defaults | combine({ + "target_origin_id": "WebsiteMedia" + }) + }} cache_behaviors: # Requests to Magento Static Cache should go to website root # to maintain the original URL - >- - {{ - cloudfront_distribution_cache_behaviour_defaults | combine({ - "target_origin_id": "WebsiteRoot", - "path_pattern": "static/*" - }) - }} - + {{ + cloudfront_distribution_cache_behaviour_defaults | combine({ + "target_origin_id": "WebsiteRoot", + "path_pattern": "static/*" + }) + }} vars: cloudfront_distribution_cache_behaviour_defaults: default_ttl: 86400 @@ -118,4 +157,26 @@ smooth_streaming: false compress: false - + ansible_python_interpreter: "{{ ansible_playbook_python }}" + register: aws_cloudfront_distribution_result + +- name: Set cloudfront distribution id fact + ansible.builtin.set_fact: + aws_cloudfront_distribution_id: "{{ aws_cloudfront_distribution_result.id }}" + when: + - aws_cloudfront_distribution_result is defined + - aws_cloudfront_distribution_result.id is defined + +- name: Set cloudfront distribution arn fact + ansible.builtin.set_fact: + aws_cloudfront_distribution_arn: "{{ aws_cloudfront_distribution_result.arn }}" + when: + - aws_cloudfront_distribution_result is defined + - aws_cloudfront_distribution_result.arn is defined + +- name: Set cloudfront distribution domain fact + ansible.builtin.set_fact: + aws_cloudfront_distribution_domain: "{{ aws_cloudfront_distribution_result.domain_name }}" + when: + - aws_cloudfront_distribution_result is defined + - aws_cloudfront_distribution_result.domain_name is defined diff --git a/roles/cs.aws-cloudfront/tasks/setup-distribution-s3-media.yml b/roles/cs.aws-cloudfront/tasks/setup-distribution-s3-media.yml index 653e49cb7..cbf84e35d 100644 --- a/roles/cs.aws-cloudfront/tasks/setup-distribution-s3-media.yml +++ b/roles/cs.aws-cloudfront/tasks/setup-distribution-s3-media.yml @@ -1,36 +1,162 @@ -# Note: This is a legacy way of doing things. When this was developed -# ansible did not have a cloudfront distribution module at all. -# EFS: This is kept for now not to break things for the S3 media setup (and to save work) -# and should be rewritten ten same way EFS setup is done now. -- name: Create temporary distribution config build directory - tempfile: - state: directory - suffix: build - register: aws_cloudfront_distribution_temp_dir - -- name: Build cloudfront distribution config - template: - src: s3-media-distribution-config.json - dest: "{{ aws_cloudfront_distribution_temp_dir.path }}/config.json" - -- name: Create cloudfront distribution - shell: "aws cloudfront create-distribution --distribution-config=file://{{ aws_cloudfront_distribution_temp_dir.path }}/config.json" - when: not aws_cloudfront_distribution | default(false, true) - register: aws_cloudfront_create_distribution_cmd - -- name: Update cloudfront distribution - shell: "aws cloudfront update-distribution --distribution-config=file://{{ aws_cloudfront_distribution_temp_dir.path }}/config.json --id={{ aws_cloudfront_distribution.Id }} --if-match={{ aws_cloudfront_distribution.ETag }}" - when: aws_cloudfront_distribution | default(false, true) - register: aws_cloudfront_update_distribution_cmd - -- name: Set cloudfront distribution id fact - set_fact: - aws_cloudfront_distribution_id: "{{ (aws_cloudfront_create_distribution_cmd.stdout | from_json).Distribution.Id }}" - when: aws_cloudfront_create_distribution_cmd is changed - -- name: Ensure distribution is tagged +- name: Set up cloudfront distribution cloudfront_distribution: state: present - distribution_id: "{{ aws_cloudfront_distribution_id }}" + enabled: true + + distribution_id: "{{ aws_cloudfront_distribution_id | default(omit, true) }}" + caller_reference: "{{ aws_cloudfront_distribution_id | default(false, true) | ternary(omit, aws_cloudfront_distribution_caller_id) }}" + + comment: "{{ mageops_project }}-{{ mageops_environment }}-media" tags: "{{ aws_cloudfront_ditribution_tags }}" - when: aws_cloudfront_distribution_id is defined \ No newline at end of file + + # Purge everything as there is no manual modification permitted + purge_tags: yes + purge_cache_behaviors: yes + purge_custom_error_responses: yes + purge_aliases: yes + purge_origins: yes + + # Do not slow the deploy + wait: no + + # Our nginx ingress supports HTTP2 + http_version: http2 + + price_class: PriceClass_100 + ipv6_enabled: no + + aliases: "{{ aws_cloudfront_domain_aliases | default(omit, true) }}" + + viewer_certificate: >- + {{ + aws_cloudfront_custom_certificate_arn | default(false, true) | ternary( + { + "acm_certificate_arn": aws_cloudfront_custom_certificate_arn + }, + omit + ) + }} + + # Disable 5xx response caching (at least try to) + custom_error_responses: + - error_code: 502 + error_caching_min_ttl: 1 + response_page_path: "" + response_code: "" + + - error_code: 503 + error_caching_min_ttl: 1 + response_page_path: "" + response_code: "" + + - error_code: 504 + error_caching_min_ttl: 1 + response_page_path: "" + response_code: "" + + origins: + - id: "S3Media" + domain_name: "{{ aws_cloudfront_website_endpoint }}" + custom_origin_config: + origin_protocol_policy: http-only + origin_ssl_protocols: + - TLSv1 + - TLSv1.1 + - TLSv1.2 + origin_read_timeout: 30 + http_port: 80 + https_port: 443 + origin_keepalive_timeout: 5 + + - id: "Website" + domain_name: "{{ magento_hostname }}" + custom_origin_config: + origin_protocol_policy: http-only + origin_ssl_protocols: + - TLSv1 + - TLSv1.1 + - TLSv1.2 + origin_read_timeout: 30 + http_port: 80 + https_port: 443 + origin_keepalive_timeout: 5 + + # To maintain backwards compatibility with the previous setup + # all CDN requests go by default to the S3 media origin. + default_cache_behavior: >- + {{ + cloudfront_distribution_cache_behaviour_defaults | combine({ + "target_origin_id": "S3Media", + "forwarded_values": { + "query_string": false, + "cookies": {"forward": "none"}, + "headers": ["Origin"] + } + }, recursive=True) + }} + + cache_behaviors: + # Requests to Magento static cache should go to website root + # to maintain the original URL. + - >- + {{ + cloudfront_distribution_cache_behaviour_defaults | combine({ + "target_origin_id": "Website", + "path_pattern": "static/*", + "forwarded_values": { + "query_string": false, + "cookies": {"forward": "none"}, + "headers": [] + } + }, recursive=True) + }} + vars: + cloudfront_distribution_cache_behaviour_defaults: + default_ttl: 86400 + + min_ttl: 0 + max_ttl: 31536000 + + viewer_protocol_policy: allow-all + field_level_encryption_id: "" + + allowed_methods: + items: + - HEAD + - GET + - OPTIONS + cached_methods: + - HEAD + - GET + - OPTIONS + + forwarded_values: + query_string: false + cookies: + forward: "none" + headers: [] + + smooth_streaming: false + compress: false + register: aws_cloudfront_distribution_result + +- name: Set cloudfront distribution id fact + ansible.builtin.set_fact: + aws_cloudfront_distribution_id: "{{ aws_cloudfront_distribution_result.id }}" + when: + - aws_cloudfront_distribution_result is defined + - aws_cloudfront_distribution_result.id is defined + +- name: Set cloudfront distribution arn fact + ansible.builtin.set_fact: + aws_cloudfront_distribution_arn: "{{ aws_cloudfront_distribution_result.arn }}" + when: + - aws_cloudfront_distribution_result is defined + - aws_cloudfront_distribution_result.arn is defined + +- name: Set cloudfront distribution domain fact + ansible.builtin.set_fact: + aws_cloudfront_distribution_domain: "{{ aws_cloudfront_distribution_result.domain_name }}" + when: + - aws_cloudfront_distribution_result is defined + - aws_cloudfront_distribution_result.domain_name is defined diff --git a/roles/cs.aws-cloudfront/templates/s3-media-distribution-config.json b/roles/cs.aws-cloudfront/templates/s3-media-distribution-config.json deleted file mode 100644 index c5de6d3cd..000000000 --- a/roles/cs.aws-cloudfront/templates/s3-media-distribution-config.json +++ /dev/null @@ -1,214 +0,0 @@ -{ - "Comment": "{{ mageops_project }}-{{ mageops_environment }}-media", - "CacheBehaviors": { - "Items": [ - { - "FieldLevelEncryptionId": "", - "TrustedSigners": { - "Enabled": false, - "Quantity": 0 - }, - "LambdaFunctionAssociations": { - "Quantity": 0 - }, - "TargetOriginId": "Website", - "ViewerProtocolPolicy": "allow-all", - "ForwardedValues": { - "Headers": { - "Quantity": 0 - }, - "Cookies": { - "Forward": "none" - }, - "QueryStringCacheKeys": { - "Quantity": 0 - }, - "QueryString": false - }, - "MaxTTL": 31536000, - "PathPattern": "static/*", - "SmoothStreaming": false, - "DefaultTTL": 86400, - "AllowedMethods": { - "Items": [ - "HEAD", - "GET", - "OPTIONS" - ], - "CachedMethods": { - "Items": [ - "HEAD", - "GET", - "OPTIONS" - ], - "Quantity": 3 - }, - "Quantity": 3 - }, - "MinTTL": 0, - "Compress": false - } - ], - "Quantity": 1 - }, - "IsIPV6Enabled": false, - "Logging": { - "Bucket": "", - "Prefix": "", - "Enabled": false, - "IncludeCookies": false - }, - "WebACLId": "", - "Origins": { - "Items": [ - { - "OriginPath": "", - "CustomOriginConfig": { - "OriginSslProtocols": { - "Items": [ - "TLSv1", - "TLSv1.1", - "TLSv1.2" - ], - "Quantity": 3 - }, - "OriginProtocolPolicy": "http-only", - "OriginReadTimeout": 30, - "HTTPPort": 80, - "HTTPSPort": 443, - "OriginKeepaliveTimeout": 5 - }, - "CustomHeaders": { - "Quantity": 0 - }, - "Id": "S3Media", - "DomainName": "{{ aws_cloudfront_website_endpoint }}" - }, - { - "OriginPath": "", - "CustomOriginConfig": { - "OriginSslProtocols": { - "Items": [ - "TLSv1", - "TLSv1.1", - "TLSv1.2" - ], - "Quantity": 3 - }, - "OriginProtocolPolicy": "http-only", - "OriginReadTimeout": 30, - "HTTPPort": 80, - "HTTPSPort": 443, - "OriginKeepaliveTimeout": 5 - }, - "CustomHeaders": { - "Quantity": 0 - }, - "Id": "Website", - "DomainName": "{{ magento_hostname }}" - } - ], - "Quantity": 2 - }, - "DefaultRootObject": "", - "PriceClass": "PriceClass_100", - "Enabled": true, - "DefaultCacheBehavior": { - "FieldLevelEncryptionId": "", - "TrustedSigners": { - "Enabled": false, - "Quantity": 0 - }, - "LambdaFunctionAssociations": { - "Quantity": 0 - }, - "TargetOriginId": "S3Media", - "ViewerProtocolPolicy": "allow-all", - "ForwardedValues": { - "Headers": { - "Items": [ - "Origin" - ], - "Quantity": 1 - }, - "Cookies": { - "Forward": "none" - }, - "QueryStringCacheKeys": { - "Quantity": 0 - }, - "QueryString": false - }, - "MaxTTL": 31536000, - "SmoothStreaming": false, - "DefaultTTL": 86400, - "AllowedMethods": { - "Items": [ - "HEAD", - "GET", - "OPTIONS" - ], - "CachedMethods": { - "Items": [ - "HEAD", - "GET", - "OPTIONS" - ], - "Quantity": 3 - }, - "Quantity": 3 - }, - "MinTTL": 0, - "Compress": false - }, - "CallerReference": "{{ aws_cloudfront_distribution_caller_id }}", - "ViewerCertificate": { - {% if aws_cloudfront_custom_certificate_arn %} - "SSLSupportMethod": "sni-only", - "ACMCertificateArn": "{{ aws_cloudfront_custom_certificate_arn }}", - "MinimumProtocolVersion": "TLSv1.1_2016", - "Certificate": "{{ aws_cloudfront_custom_certificate_arn }}", - "CertificateSource": "acm" - {% else %} - "CloudFrontDefaultCertificate": true, - "MinimumProtocolVersion": "TLSv1", - "CertificateSource": "cloudfront" - {% endif %} - }, - "CustomErrorResponses": { - "Items": [ - { - "ErrorCode": 502, - "ResponsePagePath": "", - "ResponseCode": "", - "ErrorCachingMinTTL": 1 - }, - { - "ErrorCode": 503, - "ResponsePagePath": "", - "ResponseCode": "", - "ErrorCachingMinTTL": 1 - }, - { - "ErrorCode": 504, - "ResponsePagePath": "", - "ResponseCode": "", - "ErrorCachingMinTTL": 1 - } - ], - "Quantity": 3 - }, - "HttpVersion": "http2", - "Restrictions": { - "GeoRestriction": { - "RestrictionType": "none", - "Quantity": 0 - } - }, - "Aliases": { - {% if aws_cloudfront_domain_aliases | length > 0 %} - "Items": ["{{ aws_cloudfront_domain_aliases | join('", "') }}"], - {% endif %} - "Quantity": {{ aws_cloudfront_domain_aliases | length }} - } -} diff --git a/roles/cs.aws-create-ami/tasks/main.yml b/roles/cs.aws-create-ami/tasks/main.yml index c0e1a064b..0bed10031 100644 --- a/roles/cs.aws-create-ami/tasks/main.yml +++ b/roles/cs.aws-create-ami/tasks/main.yml @@ -8,7 +8,7 @@ register: ami_builder_instance - name: Create AMI - ec2_ami: + amazon.aws.ec2_ami: region: "{{ aws_region }}" state: present description: "{{ ami_description }}" @@ -60,7 +60,7 @@ release_id: "{{ aws_ami_release_info.release_id | default(none) }}" - name: Save built AMI info into a file - when: aws_ami_build_save_info_file_path | default(false, true) + when: (aws_ami_build_save_info_file_path | default('', true) | string | length) > 0 copy: dest: "{{ aws_ami_build_save_info_file_path }}" content: "{{ aws_create_ami_info | to_nice_json }}" diff --git a/roles/cs.aws-ec2-cleanup/tasks/main.yml b/roles/cs.aws-ec2-cleanup/tasks/main.yml index 6acad51b6..ce9aedc79 100644 --- a/roles/cs.aws-ec2-cleanup/tasks/main.yml +++ b/roles/cs.aws-ec2-cleanup/tasks/main.yml @@ -36,17 +36,17 @@ - name: Keep all default versions set_fact: - _to_keep: "{{ _to_keep + item.LaunchTemplateVersions | json_query(query) }}" + _to_keep: "{{ _to_keep + item.LaunchTemplateVersions | json_query(launch_template_version_query) }}" vars: - query: "[?DefaultVersion]" + launch_template_version_query: "[?DefaultVersion]" loop: "{{ _launch_template_version_output.stdout_lines | map('from_json') | list }}" when: aws_ec2_cleanup_lt_to_keep > 0 - name: Keep last N versions set_fact: - _to_keep: "{{ _to_keep + item.LaunchTemplateVersions | json_query(query) }}" + _to_keep: "{{ _to_keep + item.LaunchTemplateVersions | json_query(launch_template_version_query) }}" vars: - query: "[:{{aws_ec2_cleanup_lt_to_keep}}]" + launch_template_version_query: "[:{{aws_ec2_cleanup_lt_to_keep}}]" loop: "{{ _launch_template_version_output.stdout_lines | map('from_json') | list }}" when: aws_ec2_cleanup_lt_to_keep > 0 @@ -125,15 +125,15 @@ when: aws_ec2_cleanup_lt_to_keep > 0 - name: Delete whole LT - ec2_launch_template: - name: "{{item.name | quote }}" + amazon.aws.ec2_launch_template: + name: "{{ item.name | quote }}" state: absent region: "{{ aws_region }}" loop: "{{ _remove_template_version }}" when: aws_ec2_cleanup_lt_to_keep == 0 -- name : Remove obsoled AMIs - ec2_ami: +- name: Remove obsoled AMIs + amazon.aws.ec2_ami: image_id: "{{ item }}" state: absent delete_snapshot: True diff --git a/roles/cs.aws-efs/tasks/main.yml b/roles/cs.aws-efs/tasks/main.yml index 8091fd98e..f2c8449e4 100644 --- a/roles/cs.aws-efs/tasks/main.yml +++ b/roles/cs.aws-efs/tasks/main.yml @@ -1,6 +1,8 @@ - name: Perform AWS API tasks delegate_to: localhost become: no + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" block: - name: Create EFS efs: @@ -54,15 +56,14 @@ # EFS Once created takes a really long time to become available # hence the retries - name: Configure root mount - mount: - boot: yes + ansible.posix.mount: src: "{{ _aws_efs.efs.file_system_id }}:/" path: "{{ efs_root_mountpoint }}" fstype: efs - state: mounted opts: "defaults,{{ efs_mount_opts }}" + state: mounted register: efs_root_mount - until: efs_root_mount is not failed + until: efs_root_mount is succeeded retries: 300 delay: 30 @@ -96,14 +97,15 @@ - "{{ _efs_local_mountpoints.results }}" - name: Configure mounts - mount: - boot: yes - src: "{{ _aws_efs.efs.file_system_id }}:{{ item.remote_path }}" - path: "{{ item.local_mountpoint }}" + ansible.posix.mount: + src: "{{ _aws_efs.efs.file_system_id }}:{{ efs_mount.remote_path }}" + path: "{{ efs_mount.local_mountpoint }}" fstype: efs - state: mounted opts: "defaults,{{ efs_mount_opts }}" - with_items: "{{ efs_mounts }}" + state: mounted + loop: "{{ efs_mounts }}" + loop_control: + loop_var: efs_mount # Warning, this has to be done post-mount! - name: Ensure mount permissions diff --git a/roles/cs.aws-iam-employee-access/tasks/main.yml b/roles/cs.aws-iam-employee-access/tasks/main.yml index ed72c62c7..fd90fc395 100644 --- a/roles/cs.aws-iam-employee-access/tasks/main.yml +++ b/roles/cs.aws-iam-employee-access/tasks/main.yml @@ -39,4 +39,4 @@ state: present managed_policy: - arn:aws:iam::aws:policy/AdministratorAccess - when: aws_iam_master_account_id | default(false, true) \ No newline at end of file + when: aws_iam_master_account_id is defined and (aws_iam_master_account_id | string | length > 0) diff --git a/roles/cs.aws-lambda-import/tasks/main.yml b/roles/cs.aws-lambda-import/tasks/main.yml index 748bd0152..a5a71acaf 100644 --- a/roles/cs.aws-lambda-import/tasks/main.yml +++ b/roles/cs.aws-lambda-import/tasks/main.yml @@ -1,4 +1,3 @@ - - name: Download lambda deploy packages get_url: url: "{{ aws_lambda_import_scaling_deploy_package_url }}" @@ -6,31 +5,36 @@ when: magento_aws_ondemand_import_instance_enable - name: Check if lambda function exists - community.aws.lambda_info: + amazon.aws.lambda_info: region: "{{ aws_region }}" register: _lambda_facts when: magento_aws_ondemand_import_instance_enable - name: Delete lambda function if it exists with different runtime - lambda: + amazon.aws.lambda: name: "{{ aws_lambda_handle_import_scaling_name }}" state: absent region: "{{ aws_region }}" when: magento_aws_ondemand_import_instance_enable and _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_handle_import_scaling_name] is defined and _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_handle_import_scaling_name].runtime != aws_lambda_handle_import_scaling_runtime - name: Register import scaling handler lambda - lambda: + amazon.aws.lambda: name: "{{ aws_lambda_handle_import_scaling_name }}" - state: "{% if magento_aws_ondemand_import_instance_enable %}present{% else %}absent{% endif %}" - zip_file: "{% if magento_aws_ondemand_import_instance_enable %}{{ aws_lambda_import_scaling_deploy_package_path }}{% else %}omit{% endif %}" + state: "{{ 'present' if magento_aws_ondemand_import_instance_enable else 'absent' }}" + zip_file: >- + {{ + aws_lambda_import_scaling_deploy_package_path + if magento_aws_ondemand_import_instance_enable + else omit + }} runtime: "{{ aws_lambda_handle_import_scaling_runtime }}" role: "arn:aws:iam::{{ aws_account_id }}:role/{{ aws_iam_role_import_scaling_lambda_execution }}" handler: import_scaling_handler.handle region: "{{ aws_region }}" timeout: 120 environment_variables: - ASG_NAME: "{{ mageops_app_name }}-extra-app" - CHECK_ENDPOINT: "{{ magento_base_url }}rest/V1/importer/is_runner_needed" + ASG_NAME: "{{ mageops_app_name }}-extra-app" + CHECK_ENDPOINT: "{{ magento_base_url }}rest/V1/importer/is_runner_needed" register: handle_import_scaling_lambda - name: Define eventrule target @@ -45,17 +49,22 @@ cloudwatchevent_rule: name: "handle-import-instance-scaling-{{ mageops_app_name }}" region: "{{ aws_region }}" - state: "{% if magento_aws_ondemand_import_instance_enable %}present{% else %}absent{% endif %}" + state: "{{ 'present' if magento_aws_ondemand_import_instance_enable else 'absent' }}" schedule_expression: "{{ aws_import_scaling_lambda_schedule }}" - targets: "{% if magento_aws_ondemand_import_instance_enable %}{{ handle_import_scaling_lambda_eventrule_targets }}{% else %}[]{% endif %}" + targets: "{{ handle_import_scaling_lambda_eventrule_targets if magento_aws_ondemand_import_instance_enable else [] }}" register: import_scaling_schedule_event - name: Allow autoscaling event handler lambda to be executed by CloudWatch Events - lambda_policy: - state: "{% if magento_aws_ondemand_import_instance_enable %}present{% else %}absent{% endif %}" + amazon.aws.lambda_policy: + state: "{{ 'present' if magento_aws_ondemand_import_instance_enable else 'absent' }}" region: "{{ aws_region }}" function_name: "{{ aws_lambda_handle_import_scaling_name }}" action: "lambda:InvokeFunction" principal: "events.amazonaws.com" statement_id: "AWSEvents-importScalingHandler-{{ mageops_app_name }}" - source_arn: "{% if magento_aws_ondemand_import_instance_enable %}{{ import_scaling_schedule_event.rule.arn }}{% else %}omit{% endif %}" + source_arn: >- + {{ + import_scaling_schedule_event.rule.arn + if magento_aws_ondemand_import_instance_enable + else omit + }} diff --git a/roles/cs.aws-lambda-node-coordinator/tasks/main.yml b/roles/cs.aws-lambda-node-coordinator/tasks/main.yml index 13e24dcf0..dc2f58482 100644 --- a/roles/cs.aws-lambda-node-coordinator/tasks/main.yml +++ b/roles/cs.aws-lambda-node-coordinator/tasks/main.yml @@ -4,19 +4,19 @@ dest: "{{ aws_lambda_node_coordinator_package_path }}" - name: Check if lambda function exists - community.aws.lambda_info: + amazon.aws.lambda_info: region: "{{ aws_region }}" register: _lambda_facts - name: Delete lambda function if it exists with different runtime - lambda: + amazon.aws.lambda: name: "{{ aws_lambda_handle_node_coordinator_autoscaling_event_name }}" state: absent region: "{{ aws_region }}" when: _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_handle_node_coordinator_autoscaling_event_name] is defined and _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_handle_node_coordinator_autoscaling_event_name].runtime != aws_lambda_node_coordinator_runtime - name: Register lambda handler for coordinating nodes - lambda: + amazon.aws.lambda: name: "{{ aws_lambda_handle_node_coordinator_autoscaling_event_name }}" state: present zip_file: "{{ aws_lambda_node_coordinator_package_path }}" diff --git a/roles/cs.aws-lambda-varnish/tasks/create-autoscaling-handler-lambda.yml b/roles/cs.aws-lambda-varnish/tasks/create-autoscaling-handler-lambda.yml index 4bb479769..3af7f3123 100644 --- a/roles/cs.aws-lambda-varnish/tasks/create-autoscaling-handler-lambda.yml +++ b/roles/cs.aws-lambda-varnish/tasks/create-autoscaling-handler-lambda.yml @@ -1,25 +1,50 @@ - name: Transform instance filters to format used in Lambda set_fact: - varnish_instances_filter_lambda: "{{ varnish_instances_aws_tags | prefix_keys('tag:') | dict2items | map('json_query', _jq_map_ec2_instance_filter)|list }}" - varnish_backend_instances_filter_lambda: "{{ varnish_backend_instances_aws_tags | prefix_keys('tag:') | dict2items | map('json_query', _jq_map_ec2_instance_filter)|list }}" - varnish_extra_instances_filter_lambda: "{{ varnish_extra_instances_aws_tags | prefix_keys('tag:') | dict2items | map('json_query', _jq_map_ec2_instance_filter)|list }}" + varnish_instances_filter_lambda: >- + {{ + varnish_instances_aws_tags + | prefix_keys('tag:') + | dict2items + | map('json_query', _jq_map_ec2_instance_filter) + | list + }} + varnish_backend_instances_filter_lambda: >- + {{ + varnish_backend_instances_aws_tags + | prefix_keys('tag:') + | dict2items + | map('json_query', _jq_map_ec2_instance_filter) + | list + }} + varnish_extra_instances_filter_lambda: >- + {{ + varnish_extra_instances_aws_tags + | prefix_keys('tag:') + | dict2items + | map('json_query', _jq_map_ec2_instance_filter) + | list + }} vars: _jq_map_ec2_instance_filter: '{Name: key, Values: [value]}' - name: Check if lambda function exists - community.aws.lambda_info: + amazon.aws.lambda_info: region: "{{ aws_region }}" register: _lambda_facts - name: Delete lambda function if it exists with different runtime - lambda: + amazon.aws.lambda: name: "{{ aws_lambda_handle_varnish_autoscaling_event_name }}" state: absent region: "{{ aws_region }}" - when: _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_handle_varnish_autoscaling_event_name] is defined and _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_handle_varnish_autoscaling_event_name].runtime != aws_lambda_varnish_runtime + when: + - _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_handle_varnish_autoscaling_event_name] is defined + - >- + _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_handle_varnish_autoscaling_event_name].runtime + != aws_lambda_varnish_runtime - name: Register lambda handler for autoscaling event - lambda: + amazon.aws.lambda: name: "{{ aws_lambda_handle_varnish_autoscaling_event_name }}" state: present zip_file: "{{ aws_lambda_varnish_deploy_package_path }}" @@ -29,15 +54,15 @@ region: "{{ aws_region }}" timeout: 120 environment_variables: - VARNISH_INSTANCE_FILTER: "{{ varnish_instances_filter_lambda | to_json }}" - BACKEND_INSTANCE_FILTER: "{{ varnish_backend_instances_filter_lambda | to_json }}" - EXTRA_INSTANCE_FILTER: "{{ varnish_extra_instances_filter_lambda | to_json }}" - KEY_BUCKET: "{{ aws_lambda_update_varnish_vcl_function_s3_bucket_name }}" - KEY_NAME: "{{ aws_lambda_update_varnish_vcl_function_ssh_key_name }}" - SSH_USERNAME: "{{ varnish_manager_user }}" - ASG_PREFIX: "{{ mageops_app_name }}" - TERMINATE_HOOK: "{{ aws_app_node_webnodedown_hook_name }}" - UPDATE_LAMBDA_NAME: "{{ aws_lambda_update_varnish_backends_name }}" + VARNISH_INSTANCE_FILTER: "{{ varnish_instances_filter_lambda | to_json }}" + BACKEND_INSTANCE_FILTER: "{{ varnish_backend_instances_filter_lambda | to_json }}" + EXTRA_INSTANCE_FILTER: "{{ varnish_extra_instances_filter_lambda | to_json }}" + KEY_BUCKET: "{{ aws_lambda_update_varnish_vcl_function_s3_bucket_name }}" + KEY_NAME: "{{ aws_lambda_update_varnish_vcl_function_ssh_key_name }}" + SSH_USERNAME: "{{ varnish_manager_user }}" + ASG_PREFIX: "{{ mageops_app_name }}" + TERMINATE_HOOK: "{{ aws_app_node_webnodedown_hook_name }}" + UPDATE_LAMBDA_NAME: "{{ aws_lambda_update_varnish_backends_name }}" register: handle_autoscaling_event_lambda - name: Register autoscaling trigger diff --git a/roles/cs.aws-lambda-varnish/tasks/create-update-varnish-backends-lambda.yml b/roles/cs.aws-lambda-varnish/tasks/create-update-varnish-backends-lambda.yml index 935f3150b..dc3287d84 100644 --- a/roles/cs.aws-lambda-varnish/tasks/create-update-varnish-backends-lambda.yml +++ b/roles/cs.aws-lambda-varnish/tasks/create-update-varnish-backends-lambda.yml @@ -5,22 +5,26 @@ varnish_backend_probe_endpoint: "{{ varnish_backend_probe_endpoint }}" varnish_backend_max_conns: "{{ varnish_backend_max_conns }}" varnish_backend_first_byte_timeout: "{{ varnish_backend_first_byte_timeout }}" - gtm_enabled: "{{ gtm_enabled | default(false) }}" + gtm_enabled: "{{ gtm_enabled | default(false) | bool }}" - name: Check if lambda function exists - community.aws.lambda_info: + amazon.aws.lambda_info: region: "{{ aws_region }}" register: _lambda_facts - name: Delete lambda function if it exists with different runtime - lambda: + amazon.aws.lambda: name: "{{ aws_lambda_update_varnish_backends_name }}" state: absent region: "{{ aws_region }}" - when: _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_update_varnish_backends_name] is defined and _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_update_varnish_backends_name].runtime != aws_lambda_varnish_runtime + when: + - _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_update_varnish_backends_name] is defined + - >- + _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_update_varnish_backends_name].runtime + != aws_lambda_varnish_runtime - name: Register lambda handler for updating varnish backends - lambda: + amazon.aws.lambda: name: "{{ aws_lambda_update_varnish_backends_name }}" state: present zip_file: "{{ aws_lambda_varnish_deploy_package_path }}" @@ -33,5 +37,5 @@ - "{{ aws_security_group_lambda_ssh_id }}" vpc_subnet_ids: "{{ aws_vpc_subnet_ids }}" environment_variables: - BACKEND_TEMPLATE: "{{ lookup('file', 'roles/cs.varnish/templates/vcl/backends.vcl.j2')|to_json }}" + BACKEND_TEMPLATE: "{{ lookup('file', 'roles/cs.aws-lambda-varnish/templates/backends.vcl.j2') | to_json }}" BACKEND_TEMPLATE_VARS: "{{ varnish_backend_template_vars | to_json }}" diff --git a/roles/cs.aws-lambda-varnish/templates/backends.vcl.j2 b/roles/cs.aws-lambda-varnish/templates/backends.vcl.j2 new file mode 100644 index 000000000..99abb0b80 --- /dev/null +++ b/roles/cs.aws-lambda-varnish/templates/backends.vcl.j2 @@ -0,0 +1,50 @@ +probe app_probe {.request = + "HEAD {{ varnish_backend_probe_endpoint }} HTTP/1.1" + "Host: localhost" + "Connection: close"; + .interval = 1s;.timeout = 1s;.threshold = 2;.window = 3; +} +{% if gtm_enabled | default(false) %} +probe gtm_probe {.request = + "GET /healthy HTTP/1.1" + "Host: localhost" + "Connection: close"; + .interval = 1s;.timeout = 1s;.threshold = 2;.window = 3; +} +{% endif %} +{% for instance in varnish_backend_instances_app %} +backend app{{ instance.private_ip_address | replace('.','') }} {.host="{{ instance.private_ip_address }}";.port="{{ varnish_backend_port }}";.max_connections={{ varnish_backend_max_conns }};.probe=app_probe;.first_byte_timeout={{ varnish_backend_first_byte_timeout }};.connect_timeout=5s;.between_bytes_timeout=60s;} +{% endfor %} +{% for instance in varnish_backend_instances_extra %} +backend ext{{ instance.private_ip_address | replace('.','') }} {.host="{{ instance.private_ip_address }}";.port="{{ varnish_backend_port }}";.max_connections={{ varnish_backend_max_conns }};.probe=app_probe;.first_byte_timeout={{ varnish_backend_first_byte_timeout }};.connect_timeout=5s;.between_bytes_timeout=60s;} +{% endfor %} +{% if gtm_enabled | default(false) %} +{% for instance in varnish_backend_instances_app %} +backend gtm_app{{ instance.private_ip_address | replace('.','') }} {.host="{{ instance.private_ip_address }}";.port="8080";.max_connections={{ varnish_backend_max_conns }};.probe=gtm_probe;.first_byte_timeout={{ varnish_backend_first_byte_timeout }};.connect_timeout=5s;.between_bytes_timeout=60s;} +backend gtm_previewapp{{ instance.private_ip_address | replace('.','') }} {.host="{{ instance.private_ip_address }}";.port="8081";.probe=gtm_probe;.max_connections={{ varnish_backend_max_conns }};.first_byte_timeout={{ varnish_backend_first_byte_timeout }};.connect_timeout=5s;.between_bytes_timeout=60s;} +{% endfor %} +{% for instance in varnish_backend_instances_extra %} +backend gtm_ext{{ instance.private_ip_address | replace('.','') }} {.host="{{ instance.private_ip_address }}";.port="8080";.max_connections={{ varnish_backend_max_conns }};.probe=gtm_probe;.first_byte_timeout={{ varnish_backend_first_byte_timeout }};.connect_timeout=5s;.between_bytes_timeout=60s;} +backend gtm_previewext{{ instance.private_ip_address | replace('.','') }} {.host="{{ instance.private_ip_address }}";.port="8081";.probe=gtm_probe;.max_connections={{ varnish_backend_max_conns }};.first_byte_timeout={{ varnish_backend_first_byte_timeout }};.connect_timeout=5s;.between_bytes_timeout=60s;} +{% endfor %} +{% endif %} +sub backends_init { + new app_director = directors.round_robin(); +{% for instance in varnish_backend_instances_app %} + app_director.add_backend(app{{ instance.private_ip_address | replace('.','') }}); +{% endfor %} +{% if gtm_enabled | default(false) %} + new gtm_director = directors.round_robin(); +{% for instance in varnish_backend_instances_app %} + gtm_director.add_backend(gtm_app{{ instance.private_ip_address | replace('.','') }}); +{% endfor %} + new gtm_preview_director = directors.round_robin(); +{% for instance in varnish_backend_instances_app %} + gtm_preview_director.add_backend(gtm_previewapp{{ instance.private_ip_address | replace('.','') }}); +{% endfor %} +{% endif %} + new extra_director = directors.round_robin(); +{% for instance in varnish_backend_instances_extra %} + extra_director.add_backend(ext{{ instance.private_ip_address | replace('.','') }}); +{% endfor %} +} diff --git a/roles/cs.aws-logs-slack/tasks/main.yml b/roles/cs.aws-logs-slack/tasks/main.yml index 08a3b94c8..4fa35d81f 100644 --- a/roles/cs.aws-logs-slack/tasks/main.yml +++ b/roles/cs.aws-logs-slack/tasks/main.yml @@ -27,19 +27,19 @@ format: zip - name: Check if lambda function exists - community.aws.lambda_info: + amazon.aws.lambda_info: region: "{{ aws_region }}" register: _lambda_facts - name: Delete lambda function if it exists with different runtime - lambda: + amazon.aws.lambda: name: "{{ aws_lambda_slack_exceptions_forwarder_name }}" state: absent region: "{{ aws_region }}" when: _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_slack_exceptions_forwarder_name] is defined and _lambda_facts.ansible_facts.lambda_facts.function[aws_lambda_slack_exceptions_forwarder_name].runtime != aws_lambda_slack_exceptions_forwarder_runtime - name: Create slack log forwarder lambda function - lambda: + amazon.aws.lambda: name: "{{ aws_lambda_slack_exceptions_forwarder_name }}" state: present runtime: "{{ aws_lambda_slack_exceptions_forwarder_runtime }}" @@ -53,7 +53,7 @@ SLACK_CHANNEL: "{{ aws_logs_slack_channel }}" - name: Allow slack log forwarder lambda function to be invoked by CloudWatch Logs - lambda_policy: + amazon.aws.lambda_policy: state: present region: "{{ aws_region }}" function_name: "{{ aws_lambda_slack_exceptions_forwarder_name }}" @@ -66,8 +66,19 @@ # Function ARN has to be constructed manually, because the one from register contains version and prohibits the call from working - name: Attach function to exception report group - local_action: - module: "shell aws logs put-subscription-filter --region {{ aws_region }} --log-group-name /{{ mageops_project }}/{{ mageops_environment }}/{{ item.group }} --filter-name send_to_slack --filter-pattern {{ item.filter | default('')|quote }} --destination-arn arn:aws:lambda:{{ aws_region }}:{{ aws_account_id }}:function:{{ aws_lambda_slack_exceptions_forwarder_name }}" + delegate_to: localhost + become: no + shell: >- + aws logs put-subscription-filter + --region {{ aws_region }} + --log-group-name /{{ mageops_project }}/{{ mageops_environment }}/{{ item.group }} + --filter-name send_to_slack + --filter-pattern {{ item.filter | default('') | quote }} + --destination-arn arn:aws:lambda:{{ aws_region }}:{{ aws_account_id }}:function:{{ aws_lambda_slack_exceptions_forwarder_name }} + register: aws_logs_slack_subscription_filter + retries: 12 + delay: 5 + until: "'rc' in aws_logs_slack_subscription_filter and aws_logs_slack_subscription_filter.rc == 0" with_items: "{{ aws_logs_slack_log_groups }}" - name: Fetch existing log groups diff --git a/roles/cs.aws-node-ami-builder/tasks/main.yml b/roles/cs.aws-node-ami-builder/tasks/main.yml index 7e3ca1dde..ece2ee9d8 100644 --- a/roles/cs.aws-node-ami-builder/tasks/main.yml +++ b/roles/cs.aws-node-ami-builder/tasks/main.yml @@ -33,8 +33,8 @@ wait_timeout: "{{ builder_create_instance_timeout }}" # This has zero costs on current-gen instances ebs_optimized: yes - network: - assign_public_ip: yes + network_interfaces: + - assign_public_ip: true security_groups: "{{ builder_instance_security_groups }}" vars: builder_instance_name_tags: diff --git a/roles/cs.aws-node-facts/tasks/main.yml b/roles/cs.aws-node-facts/tasks/main.yml index 42d0d49ae..77dd98455 100644 --- a/roles/cs.aws-node-facts/tasks/main.yml +++ b/roles/cs.aws-node-facts/tasks/main.yml @@ -1,18 +1,40 @@ +- name: Print EC2 instance lookup filters for nodes + debug: + msg: + node: "{{ node.name }}" + filters: "{{ aws_node_facts_base_filters | combine(aws_node_facts_base_tags | combine(node.node_tags) | prefix_keys('tag:')) }}" + loop: + - name: app_builder + node_tags: "{{ aws_tags_role_app_builder }}" + - name: persistent + node_tags: "{{ aws_tags_role_persistent }}" + - name: varnish_loadbalancer + node_tags: "{{ aws_tags_role_loadbalancer | combine(aws_tags_role_varnish_http_cache) }}" + - name: elasticsearch + node_tags: "{{ aws_tags_role_elasticsearch }}" + - name: redis + node_tags: "{{ aws_tags_role_redis_cache }}" + loop_control: + loop_var: node + label: "{{ node.name }}" + - name: Get EC2 instance information for nodes - ec2_instance_info: + amazon.aws.ec2_instance_info: region: "{{ aws_region }}" - filters: "{{ aws_node_facts_base_filters | combine(aws_node_facts_base_tags | combine(node.tags) | prefix_keys('tag:')) }}" + filters: "{{ aws_node_facts_base_filters | combine(aws_node_facts_base_tags | combine(node.node_tags) | prefix_keys('tag:')) }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" loop: - name: app_builder - tags: "{{ aws_tags_role_app_builder }}" + node_tags: "{{ aws_tags_role_app_builder }}" - name: persistent - tags: "{{ aws_tags_role_persistent }}" + node_tags: "{{ aws_tags_role_persistent }}" - name: varnish_loadbalancer - tags: "{{ aws_tags_role_loadbalancer | combine(aws_tags_role_varnish_http_cache) }}" + node_tags: "{{ aws_tags_role_loadbalancer | combine(aws_tags_role_varnish_http_cache) }}" - name: elasticsearch - tags: "{{ aws_tags_role_elasticsearch }}" + node_tags: "{{ aws_tags_role_elasticsearch }}" - name: redis - tags: "{{ aws_tags_role_redis_cache }}" + node_tags: "{{ aws_tags_role_redis_cache }}" loop_control: loop_var: node label: "{{ node.name }}" @@ -23,7 +45,25 @@ aws_nodes_info: >- {{ aws_nodes_info | default({}) | combine({ - result.node.name: result.instances | first | default(false) + result.node.name: ( + ( + result.instances + | selectattr('state.name', 'equalto', 'running') + | sort(attribute='launch_time') + | list + ) + | last + | default( + ( + result.instances + | sort(attribute='launch_time') + | list + ) + | last + | default(false), + true + ) + ) }) }} loop: "{{ aws_node_instance_facts.results | default([]) }}" @@ -33,7 +73,15 @@ - name: Print info about running nodes debug: - msg: "{{ item.key }} instance: {% if item.value %}{{ item.value.instance_id }} ({{ item.value.private_ip_address }} / {{ item.value.public_ip_address | default('N/A') }}){% else %}not found{% endif %}" + msg: >- + {{ item.key }} instance: + {% if item.value %} + {{ item.value.instance_id | default('unknown') }} + ({{ item.value.private_ip_address | default('N/A') }} / + {{ item.value.public_ip_address | default('N/A') }}) + {% else %} + not found + {% endif %} loop: "{{ aws_nodes_info | default({}) | dict2items }}" loop_control: label: "{{ item.key }}" @@ -52,31 +100,49 @@ - name: Set varnish host to varnish loadbalancer set_fact: mageops_varnish_host: "{{ aws_nodes_info.varnish_loadbalancer.private_ip_address }}" - when: varnish_as_loadbalancer and aws_nodes_info.varnish_loadbalancer + when: + - varnish_as_loadbalancer + - aws_nodes_info.varnish_loadbalancer is defined + - aws_nodes_info.varnish_loadbalancer is not sameas false - name: Set elasticsearch host if not predefined set_fact: mageops_elasticsearch_host: "{{ aws_nodes_info.elasticsearch.private_ip_address }}" - when: not mageops_elasticsearch_host | default(false, true) and aws_nodes_info.elasticsearch + when: + - (mageops_elasticsearch_host | default('', true) | string | length) == 0 + - aws_nodes_info.elasticsearch is defined + - aws_nodes_info.elasticsearch is not sameas false - name: Set opensearch host if not predefined set_fact: mageops_opensearch_host: "{{ aws_nodes_info.elasticsearch.private_ip_address }}" - when: not mageops_opensearch_host | default(false, true) and aws_nodes_info.elasticsearch + when: + - (mageops_opensearch_host | default('', true) | string | length) == 0 + - aws_nodes_info.elasticsearch is defined + - aws_nodes_info.elasticsearch is not sameas false # TODO: Find RabbitMQ through tags, right now we don't have option to separate it anyway - name: Set RabbitMQ host if not predefined set_fact: mageops_rabbitmq_host: "{{ aws_nodes_info.elasticsearch.private_ip_address }}" - when: not mageops_rabbitmq_host | default(false, true) and aws_nodes_info.elasticsearch + when: + - (mageops_rabbitmq_host | default('', true) | string | length) == 0 + - aws_nodes_info.elasticsearch is defined + - aws_nodes_info.elasticsearch is not sameas false - name: Set redis host if not predefined set_fact: mageops_redis_host: "{{ aws_nodes_info.redis.private_ip_address }}" - when: not mageops_redis_host | default(false, true) and aws_nodes_info.redis + when: + - (mageops_redis_host | default('', true) | string | length) == 0 + - aws_nodes_info.redis is defined + - aws_nodes_info.redis is not sameas false - name: Set redis sessions host if not predefined set_fact: mageops_redis_sessions_host: "{{ aws_nodes_info.redis.private_ip_address }}" - when: not mageops_redis_sessions_host | default(false, true) and aws_nodes_info.redis + when: + - (mageops_redis_sessions_host | default('', true) | string | length) == 0 + - aws_nodes_info.redis is defined + - aws_nodes_info.redis is not sameas false diff --git a/roles/cs.aws-node-persistent/tasks/main.yml b/roles/cs.aws-node-persistent/tasks/main.yml index b900a84c0..d4bb8eede 100644 --- a/roles/cs.aws-node-persistent/tasks/main.yml +++ b/roles/cs.aws-node-persistent/tasks/main.yml @@ -2,10 +2,18 @@ amazon.aws.ec2_instance: region: "{{ aws_region }}" detailed_monitoring: no - # 2 lines below because of https://github.com/ansible/ansible/issues/20867 - # otherwise state: running would be ok. - state: "{{ aws_persistent_instance_id | ternary('running', omit) }}" - exact_count: "{{ aws_persistent_instance_id | ternary(omit, 1) }}" + state: running + instance_ids: "{{ [aws_persistent_instance_id] }}" + wait: yes + wait_timeout: 1500 + when: (aws_persistent_instance_id | default('', true) | string | length) > 0 + register: aws_persistent_ec2 + +- name: Create persistent node instance + amazon.aws.ec2_instance: + region: "{{ aws_region }}" + detailed_monitoring: no + exact_count: 1 key_name: "{{ aws_ec2_ssh_key_name }}" vpc_subnet_id: "{{ aws_persistent_node_vpc_subnet_id }}" image: @@ -36,8 +44,8 @@ wait_timeout: 1500 # This has zero costs on current-gen instances ebs_optimized: yes - network: - assign_public_ip: yes + network_interfaces: + - assign_public_ip: true security_groups: >- {{ aws_persistent_node_security_groups @@ -49,19 +57,35 @@ [] ) }} + when: (aws_persistent_instance_id | default('', true) | string | length) == 0 vars: aws_ec2_user_launch_script: "{{ aws_persistent_node_launch_script_extra }}" aws_persistent_node_name_tags: Name: "{{ aws_persistent_node_instance_name }}" register: aws_persistent_ec2 +- name: Gather persistent node instance information + amazon.aws.ec2_instance_info: + region: "{{ aws_region }}" + instance_ids: >- + {{ + ( + aws_persistent_ec2.instances | default([]) + | map(attribute='instance_id') + | list + ) + if (aws_persistent_ec2.instances is defined) + else [aws_persistent_instance_id] + }} + register: aws_persistent_ec2_info + - name: Ensure persistent disks are gp3 volumes when: aws_persistent_node_ebs_volume_type == 'gp3' and aws_persistent_node_ebs_gp3_convert include_role: name: cs.aws-ebs-vol vars: aws_ebs_volume_modify_ids: >- - {{ ( aws_persistent_ec2.instances ) + {{ ( aws_persistent_ec2_info.instances ) | map(attribute='block_device_mappings') | flatten | map(attribute='ebs') @@ -75,7 +99,9 @@ wait_for: port: 22 host: "{{ item.public_ip_address }}" - loop: '{{ aws_persistent_ec2.instances }}' + delay: 10 + timeout: 900 + loop: '{{ aws_persistent_ec2_info.instances }}' loop_control: label: "{{ item.public_dns_name }}" @@ -85,13 +111,13 @@ groupname: - persistent - "{{ mageops_app_host_group }}" - loop: '{{ aws_persistent_ec2.instances }}' + loop: '{{ aws_persistent_ec2_info.instances }}' loop_control: label: "{{ item.public_ip_address }}" - name: Assign elastic IP to persistent node ec2_eip: region: "{{ aws_region }}" - device_id: "{{ aws_persistent_ec2.instances[0].instance_id }}" + device_id: "{{ aws_persistent_ec2_info.instances[0].instance_id }}" ip: "{{ aws_persistent_node_elastic_ip }}" - when: aws_persistent_node_elastic_ip | default(false, true) + when: (aws_persistent_node_elastic_ip | default('', true) | string | length) > 0 diff --git a/roles/cs.aws-node-varnish/tasks/main.yml b/roles/cs.aws-node-varnish/tasks/main.yml index 1c7585e24..a04149b65 100644 --- a/roles/cs.aws-node-varnish/tasks/main.yml +++ b/roles/cs.aws-node-varnish/tasks/main.yml @@ -2,10 +2,18 @@ amazon.aws.ec2_instance: region: "{{ aws_region }}" detailed_monitoring: no - # 2 lines below because of https://github.com/ansible/ansible/issues/20867 - # otherwise state: running would be ok. - state: "{{ aws_varnish_instance_id | ternary('running', omit) }}" - exact_count: "{{ aws_varnish_instance_id | ternary(omit, 1) }}" + state: running + instance_ids: "{{ [aws_varnish_instance_id] }}" + wait: yes + wait_timeout: 1500 + when: (aws_varnish_instance_id | default('', true) | string | length) > 0 + register: aws_varnish_ec2 + +- name: Create varnish node instance + amazon.aws.ec2_instance: + region: "{{ aws_region }}" + detailed_monitoring: no + exact_count: 1 key_name: "{{ aws_ec2_ssh_key_name }}" vpc_subnet_id: "{{ aws_varnish_node_vpc_subnet_id }}" image: @@ -34,22 +42,38 @@ wait_timeout: 1500 # This has zero costs on current-gen instances ebs_optimized: yes - network: - assign_public_ip: yes + network_interfaces: + - assign_public_ip: true security_groups: "{{ aws_varnish_node_security_groups + varnish_as_loadbalancer | ternary([aws_security_group_lb_id], []) }}" + when: (aws_varnish_instance_id | default('', true) | string | length) == 0 vars: aws_ec2_user_launch_script: "{{ aws_varnish_node_launch_script_extra }}" aws_varnish_node_name_tags: Name: "{{ aws_varnish_node_instance_name }}" register: aws_varnish_ec2 +- name: Gather varnish node instance information + amazon.aws.ec2_instance_info: + region: "{{ aws_region }}" + instance_ids: >- + {{ + ( + aws_varnish_ec2.instances | default([]) + | map(attribute='instance_id') + | list + ) + if (aws_varnish_ec2.instances is defined) + else [aws_varnish_instance_id] + }} + register: aws_varnish_ec2_info + - name: Ensure varnish disks are gp3 volumes when: aws_varnish_node_ebs_volume_type == 'gp3' and aws_varnish_node_ebs_gp3_convert include_role: name: cs.aws-ebs-vol vars: aws_ebs_volume_modify_ids: >- - {{ ( aws_varnish_ec2.instances ) + {{ ( aws_varnish_ec2_info.instances ) | map(attribute='block_device_mappings') | flatten | map(attribute='ebs') @@ -63,7 +87,9 @@ wait_for: port: 22 host: "{{ item.public_ip_address }}" - loop: '{{ aws_varnish_ec2.instances }}' + delay: 10 + timeout: 900 + loop: '{{ aws_varnish_ec2_info.instances }}' loop_control: label: "{{ item.public_dns_name }}" @@ -75,13 +101,13 @@ - builder_app - builder - "{{ mageops_app_host_group }}" - loop: '{{ aws_varnish_ec2.instances }}' + loop: '{{ aws_varnish_ec2_info.instances }}' loop_control: label: "{{ item.public_ip_address }}" - name: Assign elastic IP to varnish node ec2_eip: region: "{{ aws_region }}" - device_id: "{{ aws_varnish_ec2.instances[0].instance_id }}" + device_id: "{{ aws_varnish_ec2_info.instances[0].instance_id }}" ip: "{{ aws_varnish_node_elastic_ip }}" - when: aws_varnish_node_elastic_ip | default(false, true) + when: (aws_varnish_node_elastic_ip | default('', true) | string | length) > 0 diff --git a/roles/cs.aws-rds-facts/tasks/main.yml b/roles/cs.aws-rds-facts/tasks/main.yml index 8f1f43f8a..338ada52c 100644 --- a/roles/cs.aws-rds-facts/tasks/main.yml +++ b/roles/cs.aws-rds-facts/tasks/main.yml @@ -1,6 +1,8 @@ - name: Get list of RDS instances community.aws.rds_instance_info: region: "{{ aws_region }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" register: rds_instances_info - name: Set list of rds instances @@ -34,4 +36,8 @@ - name: Set MySQL host to the RDS instance set_fact: mageops_mysql_host: "{{ aws_rds_instance.endpoint.address }}" - when: aws_rds_create and aws_rds_instance | default(false, true) + when: + - aws_rds_create + - aws_rds_instance is defined + - aws_rds_instance.endpoint is defined + - aws_rds_instance.endpoint.address is defined diff --git a/roles/cs.aws-rds/tasks/main.yml b/roles/cs.aws-rds/tasks/main.yml index 38431a518..8c949e8b3 100644 --- a/roles/cs.aws-rds/tasks/main.yml +++ b/roles/cs.aws-rds/tasks/main.yml @@ -1,5 +1,9 @@ +--- +# collections: +# - amazon.aws + - name: Create RDS subnet group - rds_subnet_group: + amazon.aws.rds_subnet_group: state: present region: "{{ aws_region }}" name: "{{ aws_rds_subnet_group_name }}" @@ -7,13 +11,23 @@ subnets: "{{ aws_rds_subnet_group_subnets }}" - name: Enable collecting slow log - set_fact: - aws_rds_param_group_params: "{{ aws_rds_param_group_params | combine({'slow_query_log': 1, 'long_query_time': aws_rds_slowlog_time}, recursive=True) }}" - aws_rds_cloudwatch_logs_exports: "{{ aws_rds_cloudwatch_logs_exports + [ 'slowquery' ] }}" + ansible.builtin.set_fact: + aws_rds_param_group_params: >- + {{ + aws_rds_param_group_params + | combine( + { + 'slow_query_log': 1, + 'long_query_time': aws_rds_slowlog_time + }, + recursive=True + ) + }} + aws_rds_cloudwatch_logs_exports: "{{ aws_rds_cloudwatch_logs_exports + ['slowquery'] }}" when: aws_rds_slowlog_enabled - name: Create RDS parameter group - rds_param_group: + amazon.aws.rds_instance_param_group: state: present region: "{{ aws_region }}" name: "{{ aws_rds_param_group_name }}" @@ -23,23 +37,25 @@ params: "{{ aws_rds_param_group_params | default(omit) }}" - name: Create RDS encryption key - aws_kms: + amazon.aws.kms_key: alias: "{{ aws_rds_encryption_key_name }}" region: "{{ aws_region }}" tags: "{{ aws_tags_default | combine(aws_tags_kms_database) }}" when: aws_rds_storage_encrypt and aws_rds_dedicated_encryption_key -- name: Wait for active rds modifications to finish - rds_instance_info: +- name: Wait for active RDS modifications to finish + amazon.aws.rds_instance_info: region: "{{ aws_region }}" db_instance_identifier: "{{ aws_rds_instance_name }}" register: aws_rds_current retries: 600 delay: 5 - until: (aws_rds_current.instances|length) == 0 or aws_rds_current.instances[0].db_instance_status != "modifying" + until: >- + (aws_rds_current.instances | length) == 0 + or aws_rds_current.instances[0].db_instance_status != "modifying" - name: Check current RDS configuration - rds_instance_info: + amazon.aws.rds_instance_info: region: "{{ aws_region }}" db_instance_identifier: "{{ aws_rds_instance_name }}" register: aws_rds_current @@ -47,72 +63,82 @@ - name: Update RDS instance class before applying full configuration (workaround) block: - name: Update RDS instance class before applying full configuration (workaround) - shell: | - aws \ - --region '{{ aws_region }}' \ - rds modify-db-instance \ - --db-instance-identifier='{{ aws_rds_instance_name }}' \ - --db-instance-class='{{ aws_rds_instance_type }}' \ - --apply-immediately + ansible.builtin.command: + argv: + - aws + - --region + - "{{ aws_region }}" + - rds + - modify-db-instance + - --db-instance-identifier={{ aws_rds_instance_name }} + - --db-instance-class={{ aws_rds_instance_type }} + - --apply-immediately + - name: Wait for modification to start - rds_instance_info: + amazon.aws.rds_instance_info: region: "{{ aws_region }}" db_instance_identifier: "{{ aws_rds_instance_name }}" register: aws_rds_wait_start_result retries: 20 delay: 3 - until: aws_rds_wait_start_result.instances[0].db_instance_status == "modifying" or aws_rds_wait_start_result.instances[0].pending_modified_values.db_instance_class is not defined + until: >- + aws_rds_wait_start_result.instances[0].db_instance_status == "modifying" + or aws_rds_wait_start_result.instances[0].pending_modified_values.db_instance_class is not defined + - name: Wait for modification to finish - rds_instance_info: + amazon.aws.rds_instance_info: region: "{{ aws_region }}" db_instance_identifier: "{{ aws_rds_instance_name }}" register: aws_rds_wait_finish_result retries: 600 delay: 5 until: aws_rds_wait_finish_result.instances[0].db_instance_status == "available" - when: (aws_rds_current.instances|length) > 0 and aws_rds_current.instances[0].db_instance_class != aws_rds_instance_type - + when: + - (aws_rds_current.instances | length) > 0 + - aws_rds_current.instances[0].db_instance_class != aws_rds_instance_type # When we have known upgrade path: # we create temporary parameter group for specific db version # we upgrade database to next engine version -# then we procees with usual rds provisioning - this will switch to normal parameter group -# we remowe all created temporary parameter groups +# then we proceed with usual RDS provisioning - this will switch to normal parameter group +# we remove all created temporary parameter groups - name: Set upgrade plan - set_fact: - aws_rds_upgrade_plan: | - {{ aws_rds_upgrade_path[aws_rds_db_engine][ + ansible.builtin.set_fact: + aws_rds_upgrade_plan: >- + {{ + aws_rds_upgrade_path[aws_rds_db_engine][ + aws_rds_current.instances[0].engine_version.split('.')[0] + ~ '.' + ~ aws_rds_current.instances[0].engine_version.split('.')[1] + ][aws_rds_db_engine_version] + }} + when: + - aws_rds_upgrade_path[aws_rds_db_engine] is defined + - aws_rds_current.instances | length > 0 + - aws_rds_upgrade_path[aws_rds_db_engine][ + aws_rds_current.instances[0].engine_version.split('.')[0] + ~ '.' + ~ aws_rds_current.instances[0].engine_version.split('.')[1] + ] is defined + - aws_rds_upgrade_path[aws_rds_db_engine][ aws_rds_current.instances[0].engine_version.split('.')[0] - + "." - + aws_rds_current.instances[0].engine_version.split('.')[1] - ][aws_rds_db_engine_version] }} - when: | - aws_rds_upgrade_path[aws_rds_db_engine] is defined - and aws_rds_current.instances|length > 0 - and aws_rds_upgrade_path[aws_rds_db_engine][ - aws_rds_current.instances[0].engine_version.split('.')[0] - + "." - + aws_rds_current.instances[0].engine_version.split('.')[1] - ] is defined - and aws_rds_upgrade_path[aws_rds_db_engine][ - aws_rds_current.instances[0].engine_version.split('.')[0] - + "." - + aws_rds_current.instances[0].engine_version.split('.')[1] - ][aws_rds_db_engine_version] is defined + ~ '.' + ~ aws_rds_current.instances[0].engine_version.split('.')[1] + ][aws_rds_db_engine_version] is defined - name: Print upgrade plan - debug: + ansible.builtin.debug: msg: | Upgrade plan from {{ aws_rds_current.instances[0].engine_version }} to {{ aws_rds_db_engine_version }} {{ aws_rds_current.instances[0].engine_version }} --> {{ " --> ".join(aws_rds_upgrade_plan) }} - when: aws_rds_upgrade_plan|length > 0 + when: aws_rds_upgrade_plan | length > 0 - name: Upgrade MySQL/MariaDB RDS engine - include_tasks: upgrade_iteration.yml + ansible.builtin.include_tasks: upgrade_iteration.yml loop: "{{ aws_rds_upgrade_plan }}" - name: Create MySQL/MariaDB RDS - rds_instance: + amazon.aws.rds_instance: region: "{{ aws_region }}" publicly_accessible: "{{ aws_rds_publicly_accessible }}" db_instance_identifier: "{{ aws_rds_instance_name }}" @@ -133,27 +159,46 @@ tags: "{{ aws_tags_default | combine(aws_tags_role_database, aws_tags_role_mysql_database, aws_tags_node_mysql) }}" availability_zone: "{{ aws_rds_availability_zone }}" storage_encrypted: "{{ aws_rds_storage_encrypt }}" - kms_key_id: "{{ aws_rds_encryption_key_name if aws_rds_storage_encrypt and aws_rds_dedicated_encryption_key else omit }}" + kms_key_id: >- + {{ + aws_rds_encryption_key_name + if aws_rds_storage_encrypt and aws_rds_dedicated_encryption_key + else omit + }} enable_performance_insights: "{{ aws_rds_performance_insights }}" - performance_insights_retention_period: "{{ aws_rds_performance_insights_retention_period if aws_rds_performance_insights else omit }}" - performance_insights_kms_key_id: "{{ aws_rds_encryption_key_name if aws_rds_storage_encrypt and aws_rds_dedicated_encryption_key else omit }}" + performance_insights_retention_period: >- + {{ + aws_rds_performance_insights_retention_period + if aws_rds_performance_insights + else omit + }} + performance_insights_kms_key_id: >- + {{ + aws_rds_encryption_key_name + if aws_rds_storage_encrypt and aws_rds_dedicated_encryption_key + else omit + }} enable_cloudwatch_logs_exports: "{{ aws_rds_cloudwatch_logs_exports }}" - apply_immediately: yes - wait: yes + apply_immediately: true + wait: true - name: Wait for upgrade to finish - rds_instance_info: + amazon.aws.rds_instance_info: region: "{{ aws_region }}" db_instance_identifier: "{{ aws_rds_instance_name }}" register: _aws_rds_wait_upgrade_result retries: 600 delay: 5 - until: _aws_rds_wait_upgrade_result.instances[0].db_instance_status == "available" or _aws_rds_wait_upgrade_result.instances[0].db_instance_status == "Storage-optimization" - when: (aws_rds_current.instances|length) > 0 and aws_rds_current.instances[0].engine_version != aws_rds_db_engine_version + until: >- + _aws_rds_wait_upgrade_result.instances[0].db_instance_status == "available" + or _aws_rds_wait_upgrade_result.instances[0].db_instance_status == "Storage-optimization" + when: + - (aws_rds_current.instances | length) > 0 + - aws_rds_current.instances[0].engine_version != aws_rds_db_engine_version - name: Remove temporary parameter groups used for engine upgrade - rds_param_group: + amazon.aws.rds_instance_param_group: state: absent name: "{{ item }}" region: "{{ aws_region }}" - loop: "{{ aws_rds_temporary_parameter_groups }}" + loop: "{{ aws_rds_temporary_parameter_groups }}" \ No newline at end of file diff --git a/roles/cs.aws-rds/tasks/upgrade_iteration.yml b/roles/cs.aws-rds/tasks/upgrade_iteration.yml index 017418a43..7964f23f8 100644 --- a/roles/cs.aws-rds/tasks/upgrade_iteration.yml +++ b/roles/cs.aws-rds/tasks/upgrade_iteration.yml @@ -1,18 +1,18 @@ - name: Set upgrade parameters - set_fact: + ansible.builtin.set_fact: _aws_rds_upgrade_parameter_group: "{{ mageops_app_name }}-{{ item | replace('.', '-') }}-upgrade" - name: Schedule removal of temporary parameters - set_fact: - aws_rds_temporary_parameter_groups: "{{ aws_rds_temporary_parameter_groups + [ _aws_rds_upgrade_parameter_group ] }}" + ansible.builtin.set_fact: + aws_rds_temporary_parameter_groups: "{{ aws_rds_temporary_parameter_groups + [_aws_rds_upgrade_parameter_group] }}" - name: Show step info - debug: + ansible.builtin.debug: msg: | Upgrading to {{ aws_rds_db_engine }} {{ item }} - name: Create temporary RDS parameter group - rds_param_group: + amazon.aws.rds_instance_param_group: state: present region: "{{ aws_region }}" name: "{{ _aws_rds_upgrade_parameter_group }}" @@ -22,7 +22,7 @@ params: "{{ aws_rds_param_group_params | default(omit) }}" - name: Upgrade RDS - rds_instance: + amazon.aws.rds_instance: region: "{{ aws_region }}" publicly_accessible: "{{ aws_rds_publicly_accessible }}" db_instance_identifier: "{{ aws_rds_instance_name }}" @@ -38,28 +38,45 @@ backup_retention_period: "{{ aws_rds_backup_retention }}" preferred_backup_window: "{{ aws_rds_backup_window }}" preferred_maintenance_window: "{{ aws_rds_maintenance_window }}" - auto_minor_version_upgrade: yes - allow_major_version_upgrade: yes + auto_minor_version_upgrade: true + allow_major_version_upgrade: true tags: "{{ aws_tags_default | combine(aws_tags_role_database, aws_tags_role_mysql_database, aws_tags_node_mysql) }}" availability_zone: "{{ aws_rds_availability_zone }}" storage_encrypted: "{{ aws_rds_storage_encrypt }}" - kms_key_id: "{{ aws_rds_encryption_key_name if aws_rds_storage_encrypt and aws_rds_dedicated_encryption_key else omit }}" + kms_key_id: >- + {{ + aws_rds_encryption_key_name + if aws_rds_storage_encrypt and aws_rds_dedicated_encryption_key + else omit + }} enable_performance_insights: "{{ aws_rds_performance_insights }}" - performance_insights_retention_period: "{{ aws_rds_performance_insights_retention_period if aws_rds_performance_insights else omit }}" - performance_insights_kms_key_id: "{{ aws_rds_encryption_key_name if aws_rds_storage_encrypt and aws_rds_dedicated_encryption_key else omit }}" + performance_insights_retention_period: >- + {{ + aws_rds_performance_insights_retention_period + if aws_rds_performance_insights + else omit + }} + performance_insights_kms_key_id: >- + {{ + aws_rds_encryption_key_name + if aws_rds_storage_encrypt and aws_rds_dedicated_encryption_key + else omit + }} enable_cloudwatch_logs_exports: "{{ aws_rds_cloudwatch_logs_exports }}" - apply_immediately: yes - wait: yes + apply_immediately: true + wait: true -- name: Wait for AWS to acknowledge that rds upgrade is happening (workaround) - pause: +- name: Wait for AWS to acknowledge that RDS upgrade is happening (workaround) + ansible.builtin.pause: minutes: 1 -- name: Wait for active rds modifications to finish - rds_instance_info: +- name: Wait for active RDS modifications to finish + amazon.aws.rds_instance_info: region: "{{ aws_region }}" db_instance_identifier: "{{ aws_rds_instance_name }}" register: aws_rds_current retries: 600 delay: 5 - until: (aws_rds_current.instances|length) == 0 or aws_rds_current.instances[0].db_instance_status == "available" + until: >- + (aws_rds_current.instances | length) == 0 + or aws_rds_current.instances[0].db_instance_status == "available" diff --git a/roles/cs.aws-region-facts/tasks/main.yml b/roles/cs.aws-region-facts/tasks/main.yml index 5390e11fa..55fe4a79b 100644 --- a/roles/cs.aws-region-facts/tasks/main.yml +++ b/roles/cs.aws-region-facts/tasks/main.yml @@ -1,7 +1,11 @@ - name: Get AWS region - community.aws.aws_region_info: + amazon.aws.aws_region_info: region: "{{ aws_region }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" register: region_facts + delegate_to: localhost + become: no - name: Set region facts set_fact: @@ -13,7 +17,11 @@ - name: Get availaibility zones for region amazon.aws.aws_az_info: region: "{{ aws_region }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" register: az_facts + delegate_to: localhost + become: no - name: Set region AZ facts set_fact: @@ -26,7 +34,7 @@ set_fact: aws_preferred_availability_zone: "{{ aws_availability_zone_names | first }}" register: aws_selected_preferred_az - when: not aws_preferred_availability_zone | default(false, true) + when: (aws_preferred_availability_zone | default('', true) | string | length) == 0 - name: Print the chosen availability zone debug: diff --git a/roles/cs.aws-s3/tasks/main.yml b/roles/cs.aws-s3/tasks/main.yml index 922d8f056..8ca860bc7 100644 --- a/roles/cs.aws-s3/tasks/main.yml +++ b/roles/cs.aws-s3/tasks/main.yml @@ -42,15 +42,18 @@ https_termination_hosts | default([])|selectattr('server_name', 'defined')|map(attribute='server_name')|list + https_termination_hosts | default([])|selectattr('aliases', 'defined')|map(attribute='aliases')|list | flatten }} - when: mageops_https_termination_enable | default(false) + when: mageops_https_termination_enable | default(false) | bool - name: Collect load balancer host set_fact: aws_s3_media_bucket_cors_allowed_origins: "{{ aws_s3_media_bucket_cors_allowed_origins + [lb_domain] }}" - when: aws_elb_create | default(false) and lb_domain is defined and lb_domain | length + when: + - aws_elb_create | default(false) | bool + - lb_domain is defined + - (lb_domain | string | length) > 0 when: aws_s3_generate_allowed_origins - name: Apply CORS configuration for Media bucket - aws_s3_cors: + community.aws.s3_cors: region: "{{ aws_region }}" name: "{{ aws_s3_media_bucket }}" state: present diff --git a/roles/cs.aws-security-group-facts/defaults/main.yml b/roles/cs.aws-security-group-facts/defaults/main.yml index 37273106e..25db0577d 100644 --- a/roles/cs.aws-security-group-facts/defaults/main.yml +++ b/roles/cs.aws-security-group-facts/defaults/main.yml @@ -1,2 +1,14 @@ aws_sg_facts_filters: {} aws_sg_facts_tags: "{{ aws_tags_base }}" +aws_sg_facts_group_name_by_fact: + aws_security_group_efs: "{{ aws_security_group_efs_name }}" + aws_security_group_elasticsearch: "{{ aws_security_group_elasticsearch_name }}" + aws_security_group_redis: "{{ aws_security_group_redis_name }}" + aws_security_group_rds: "{{ aws_security_group_rds_name }}" + aws_security_group_app: "{{ aws_security_group_app_name }}" + aws_security_group_varnish: "{{ aws_security_group_varnish_name }}" + aws_security_group_lb: "{{ aws_security_group_lb_name }}" + aws_security_group_ssh: "{{ aws_security_group_ssh_name }}" + aws_security_group_lambda_ssh: "{{ aws_security_group_lambda_ssh_name }}" + aws_security_group_persistant: "{{ aws_security_group_persistant_name }}" +aws_sg_facts_group_names: "{{ aws_sg_facts_group_name_by_fact | dict2items | map(attribute='value') | list }}" diff --git a/roles/cs.aws-security-group-facts/tasks/main.yml b/roles/cs.aws-security-group-facts/tasks/main.yml index 022c21f6e..2599e5673 100644 --- a/roles/cs.aws-security-group-facts/tasks/main.yml +++ b/roles/cs.aws-security-group-facts/tasks/main.yml @@ -1,7 +1,18 @@ - name: Get info about security groups - amazon.aws.ec2_group_info: + amazon.aws.ec2_security_group_info: region: "{{ aws_region }}" - filters: "{{ aws_sg_facts_filters | combine (aws_sg_facts_tags | prefix_keys('tag:')) }}" + filters: "{{ aws_sg_facts_filters_resolved }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + aws_sg_facts_filters_resolved: >- + {{ + aws_sg_facts_filters + | combine({ + 'vpc-id': aws_vpc_id, + 'group-name': aws_sg_facts_group_names + }) + | combine(aws_sg_facts_tags | prefix_keys('tag:')) + }} register: ec2_group_facts - name: Set info about security groups @@ -9,30 +20,23 @@ aws_security_groups_info: "{{ ec2_group_facts.security_groups }}" cacheable: yes -- name: Set info for each security group +- name: Set info and id fact for each security group set_fact: - aws_security_group_efs: "{{ aws_security_groups_info | selectattr('group_name', 'equalto', aws_security_group_efs_name) | list | default([false], true) | first }}" - aws_security_group_elasticsearch: "{{ aws_security_groups_info | selectattr('group_name', 'equalto', aws_security_group_elasticsearch_name) | list | default([false], true) | first }}" - aws_security_group_redis: "{{ aws_security_groups_info | selectattr('group_name', 'equalto', aws_security_group_redis_name) | list | default([false], true) | first }}" - aws_security_group_rds: "{{ aws_security_groups_info | selectattr('group_name', 'equalto', aws_security_group_rds_name) | list | default([false], true) | first }}" - aws_security_group_app: "{{ aws_security_groups_info | selectattr('group_name', 'equalto', aws_security_group_app_name) | list | default([false], true) | first }}" - aws_security_group_varnish: "{{ aws_security_groups_info | selectattr('group_name', 'equalto', aws_security_group_varnish_name) | list | default([false], true) | first }}" - aws_security_group_lb: "{{ aws_security_groups_info | selectattr('group_name', 'equalto', aws_security_group_lb_name) | list | default([false], true) | first }}" - aws_security_group_ssh: "{{ aws_security_groups_info | selectattr('group_name', 'equalto', aws_security_group_ssh_name) | list | default([false], true) | first }}" - aws_security_group_lambda_ssh: "{{ aws_security_groups_info | selectattr('group_name', 'equalto', aws_security_group_lambda_ssh_name) | list | default([false], true) | first }}" - aws_security_group_persistant: "{{ aws_security_groups_info | selectattr('group_name', 'equalto', aws_security_group_persistant_name) | list | default([false], true) | first }}" - cacheable: yes - -- name: Set id fact for each security group - set_fact: - aws_security_group_efs_id: "{{ aws_security_group_efs.group_id | default(omit) }}" - aws_security_group_elasticsearch_id: "{{ aws_security_group_elasticsearch.group_id | default(omit) }}" - aws_security_group_redis_id: "{{ aws_security_group_redis.group_id | default(omit) }}" - aws_security_group_rds_id: "{{ aws_security_group_rds.group_id | default(omit) }}" - aws_security_group_app_id: "{{ aws_security_group_app.group_id | default(omit) }}" - aws_security_group_varnish_id: "{{ aws_security_group_varnish.group_id | default(omit) }}" - aws_security_group_lb_id: "{{ aws_security_group_lb.group_id | default(omit) }}" - aws_security_group_ssh_id: "{{ aws_security_group_ssh.group_id | default(omit) }}" - aws_security_group_lambda_ssh_id: "{{ aws_security_group_lambda_ssh.group_id | default(omit) }}" - aws_security_group_persistant_id: "{{ aws_security_group_persistant.group_id | default(omit) }}" + "{{ sg_fact_name }}": "{{ sg_info }}" + "{{ sg_fact_name }}_id": "{{ sg_info.group_id | default(omit) }}" cacheable: yes + vars: + sg_fact_name: "{{ sg_fact.key }}" + sg_group_name: "{{ sg_fact.value }}" + sg_info: >- + {{ + aws_security_groups_info + | selectattr('group_name', 'equalto', sg_group_name) + | list + | default([false], true) + | first + }} + loop: "{{ aws_sg_facts_group_name_by_fact | dict2items }}" + loop_control: + loop_var: sg_fact + label: "{{ sg_fact.key }}" diff --git a/roles/cs.aws-security-group/tasks/main.yml b/roles/cs.aws-security-group/tasks/main.yml index 98e3e50d1..674eee193 100644 --- a/roles/cs.aws-security-group/tasks/main.yml +++ b/roles/cs.aws-security-group/tasks/main.yml @@ -1,5 +1,5 @@ - name: Create security group for Varnish lambda - ec2_group: + amazon.aws.ec2_security_group: name: "{{ aws_security_group_lambda_ssh_name }}" description: "{{ mageops_app_name }} Varnish backend update lambda security group" region: "{{ aws_region }}" @@ -11,7 +11,7 @@ register: aws_security_group_lambda_ssh - name: Create SSH group for access from the office - ec2_group: + amazon.aws.ec2_security_group: name: "{{ aws_security_group_ssh_name }}" description: "{{ mageops_app_name }} SSH access security group" region: "{{ aws_region }}" @@ -27,7 +27,7 @@ register: aws_security_group_ssh - name: Create security group for load balancer - ec2_group: + amazon.aws.ec2_security_group: name: "{{ aws_security_group_lb_name }}" description: "{{ mageops_app_name }} Load balancer security group" region: "{{ aws_region }}" @@ -47,13 +47,12 @@ Name: "{{ aws_security_group_lb_name }}" register: aws_security_group_lb - - name: Create security group for Varnish - ec2_group: + amazon.aws.ec2_security_group: name: "{{ aws_security_group_varnish_name }}" description: "{{ mageops_app_name }} - allows for connection between Varnish servers" region: "{{ aws_region }}" - purge_rules: no + purge_rules: false rules: "{{ aws_security_group_varnish_rules_base + aws_security_group_varnish_rules + aws_security_group_varnish_extra_rules }}" vpc_id: "{{ aws_vpc_id }}" tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}" @@ -73,11 +72,17 @@ register: aws_security_group_varnish - name: Create security group for webnodes - ec2_group: + amazon.aws.ec2_security_group: name: "{{ aws_security_group_app_name }}" description: "{{ mageops_app_name }} Webnodes security group" region: "{{ aws_region }}" - rules: "{{ aws_security_group_app_rules_base + aws_security_group_app_rules + aws_security_group_app_extra_rules + (aws_security_group_app_gtm_rules if gtm_enabled | default(false) else []) }}" + rules: >- + {{ + aws_security_group_app_rules_base + + aws_security_group_app_rules + + aws_security_group_app_extra_rules + + (aws_security_group_app_gtm_rules if gtm_enabled | default(false) else []) + }} vpc_id: "{{ aws_vpc_id }}" tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}" vars: @@ -96,8 +101,8 @@ group_name: "{{ aws_security_group_varnish_name }}" register: aws_security_group_app -- name: Create security group for persistant node - ec2_group: +- name: Create security group for persistent node + amazon.aws.ec2_security_group: name: "{{ aws_security_group_persistant_name }}" description: "{{ mageops_app_name }} persistent node" region: "{{ aws_region }}" @@ -106,50 +111,97 @@ rules: "{{ aws_security_group_persistant_rules_base + aws_security_group_persistant_rules + aws_security_group_persistant_extra_rules }}" vars: ec2_sg_tags: - Name: "{{ aws_security_group_persistant_name }}" + Name: "{{ aws_security_group_persistant_name }}" aws_security_group_persistant_rules_base: - # TODO: We have reached SG limits - that's why rabbitmq - # is added to persistant SG. The whole SG approach needs to be - # rethinked and refactored - probably we should use subnets - # instead. - proto: tcp ports: - "{{ rabbitmq_amqp_port }}" - "{{ rabbitmq_http_port }}" group_name: "{{ aws_security_group_app_name }}" - register: aws_security_group_persistant - name: Create security group for RDS - ec2_group: + amazon.aws.ec2_security_group: name: "{{ aws_security_group_rds_name }}" description: "{{ mageops_app_name }} RDS security group" region: "{{ aws_region }}" + vpc_id: "{{ aws_vpc_id }}" + tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}" + vars: + ec2_sg_tags: + Name: "{{ aws_security_group_rds_name }}" + register: aws_security_group_rds + when: aws_security_group_rds_create + +- name: Allow app security group to access RDS + amazon.aws.ec2_security_group: + name: "{{ aws_security_group_rds_name }}" + description: "{{ mageops_app_name }} RDS security group" + region: "{{ aws_region }}" + purge_rules: false rules: - proto: tcp ports: 3306 - group_name: "{{ [aws_security_group_app_name] + aws_security_group_rds_access_extra_groups | default([]) }}" + group_id: "{{ aws_security_group_app.group_id }}" + vpc_id: "{{ aws_vpc_id }}" + tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}" + vars: + ec2_sg_tags: + Name: "{{ aws_security_group_rds_name }}" + when: + - aws_security_group_rds_create + - aws_security_group_app.group_id is defined + +- name: Allow extra security groups to access RDS + amazon.aws.ec2_security_group: + name: "{{ aws_security_group_rds_name }}" + description: "{{ mageops_app_name }} RDS security group" + region: "{{ aws_region }}" + purge_rules: false + rules: - proto: tcp ports: 3306 - cidr_ip: "{{ mageops_trusted_cidr_blocks }}" + group_name: "{{ item }}" vpc_id: "{{ aws_vpc_id }}" tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}" vars: ec2_sg_tags: Name: "{{ aws_security_group_rds_name }}" - register: aws_security_group_rds + loop: "{{ aws_security_group_rds_access_extra_groups | default([]) }}" + loop_control: + label: "{{ item }}" + when: aws_security_group_rds_create + +- name: Allow trusted CIDRs to access RDS + amazon.aws.ec2_security_group: + name: "{{ aws_security_group_rds_name }}" + description: "{{ mageops_app_name }} RDS security group" + region: "{{ aws_region }}" + purge_rules: false + rules: + - proto: tcp + ports: 3306 + cidr_ip: "{{ item }}" + vpc_id: "{{ aws_vpc_id }}" + tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}" + vars: + ec2_sg_tags: + Name: "{{ aws_security_group_rds_name }}" + loop: "{{ mageops_trusted_cidr_blocks | default([]) }}" + loop_control: + label: "{{ item }}" when: aws_security_group_rds_create - name: Create security group for Redis - ec2_group: + amazon.aws.ec2_security_group: name: "{{ aws_security_group_redis_name }}" description: "{{ mageops_app_name }} Redis security group" region: "{{ aws_region }}" rules: - proto: tcp ports: - - "{{ mageops_redis_port }}" - - "{{ mageops_redis_sessions_port }}" + - "{{ mageops_redis_port }}" + - "{{ mageops_redis_sessions_port }}" group_name: "{{ aws_security_group_app_name }}" vpc_id: "{{ aws_vpc_id }}" tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}" @@ -160,13 +212,13 @@ when: aws_security_group_redis_create - name: Create security group for Search - ec2_group: + amazon.aws.ec2_security_group: name: "{{ aws_security_group_elasticsearch_name }}" description: "{{ mageops_app_name }} search security group" region: "{{ aws_region }}" rules: - proto: tcp - ports: "{{ (mageops_elasticsearch_opensearch_flavor == 'elasticsearch') | ternary(elasticsearch_http_port , opensearch_http_port) }}" + ports: "{{ (mageops_elasticsearch_opensearch_flavor == 'elasticsearch') | ternary(elasticsearch_http_port, opensearch_http_port) }}" group_name: "{{ aws_security_group_app_name }}" - proto: tcp ports: "{{ (mageops_elasticsearch_opensearch_flavor == 'elasticsearch') | ternary(elasticsearch_http_port, opensearch_http_port) }}" @@ -180,17 +232,19 @@ when: aws_security_group_elastic_create - name: Create security group for EFS - ec2_group: + amazon.aws.ec2_security_group: name: "{{ aws_security_group_efs_name }}" description: "{{ mageops_app_name }} EFS security group" region: "{{ aws_region }}" - purge_rules: no + purge_rules: false rules: - proto: tcp - ports: [2049] + ports: + - 2049 group_name: "{{ aws_security_group_app_name }}" - proto: tcp - ports: [2049] + ports: + - 2049 group_name: "{{ aws_security_group_persistant_name }}" vpc_id: "{{ aws_vpc_id }}" tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}" @@ -200,20 +254,20 @@ register: aws_security_group_efs - name: Allow app to access varnish - ec2_group: + amazon.aws.ec2_security_group: name: "{{ aws_security_group_varnish_name }}" description: "{{ mageops_app_name }} - allows for connection between Varnish servers" region: "{{ aws_region }}" - purge_rules: no + purge_rules: false rules: - proto: tcp ports: >- {{ - [ mageops_varnish_port ] - + mageops_xdebug_proxy_remote_connection_to_loadbalancer | ternary( - [ php_fpm_debug_pool_xdebug_remote_port ], - [] - ) + [mageops_varnish_port] + + ( + mageops_xdebug_proxy_remote_connection_to_loadbalancer + | ternary([php_fpm_debug_pool_xdebug_remote_port], []) + ) }} group_name: "{{ aws_security_group_app_name }}" vpc_id: "{{ aws_vpc_id }}" diff --git a/roles/cs.aws-ses/tasks/main.yml b/roles/cs.aws-ses/tasks/main.yml index 199dbeee6..90ecd48ee 100644 --- a/roles/cs.aws-ses/tasks/main.yml +++ b/roles/cs.aws-ses/tasks/main.yml @@ -16,8 +16,7 @@ register: _aws_sns_complaints - name: Add e-mail to SES - #community.aws.aws_ses_identity: - aws_ses_identity: + community.aws.ses_identity: identity: "{{ ses_email }}" region: "{{ aws_region }}" state: present @@ -28,11 +27,10 @@ topic: "{{ _aws_sns_bounce.sns_arn }}" include_headers: False feedback_forwarding: False - when: ses_email | default(false, true) + when: (ses_email | default('', true) | string | length) > 0 - name: Add domain to SES - #community.aws.aws_ses_identity: - aws_ses_identity: + community.aws.ses_identity: identity: "{{ ses_domain }}" region: "{{ aws_region }}" state: present @@ -43,7 +41,7 @@ topic: "{{ _aws_sns_bounce.sns_arn }}" include_headers: no feedback_forwarding: False - when: ses_domain | default(false, true) + when: (ses_domain | default('', true) | string | length) > 0 # Could be falled back with # "{{ ses_domain | default(ses_email.split('@')[1]) }}" diff --git a/roles/cs.aws-vpc-facts/tasks/main.yml b/roles/cs.aws-vpc-facts/tasks/main.yml index b96370072..190cf938f 100644 --- a/roles/cs.aws-vpc-facts/tasks/main.yml +++ b/roles/cs.aws-vpc-facts/tasks/main.yml @@ -3,4 +3,4 @@ when: aws_vpc_id is not defined - name: Get VPC subnets info - include_tasks: subnets.yml \ No newline at end of file + include_tasks: subnets.yml diff --git a/roles/cs.aws-vpc-facts/tasks/network.yml b/roles/cs.aws-vpc-facts/tasks/network.yml index f097049a2..467f98415 100644 --- a/roles/cs.aws-vpc-facts/tasks/network.yml +++ b/roles/cs.aws-vpc-facts/tasks/network.yml @@ -1,7 +1,9 @@ - name: Get info about VPC networks amazon.aws.ec2_vpc_net_info: region: "{{ aws_region }}" - filters: "{{ aws_vpc_facts_net_filters | combine (aws_vpc_facts_net_tags | prefix_keys('tag:')) }}" + filters: "{{ aws_vpc_facts_net_filters | combine(aws_vpc_facts_net_tags | prefix_keys('tag:')) }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" register: vpc_net_facts - name: Fail if no VPC network has been found diff --git a/roles/cs.aws-vpc-facts/tasks/subnets.yml b/roles/cs.aws-vpc-facts/tasks/subnets.yml index ff1cf2dde..9121faf27 100644 --- a/roles/cs.aws-vpc-facts/tasks/subnets.yml +++ b/roles/cs.aws-vpc-facts/tasks/subnets.yml @@ -1,9 +1,9 @@ - - name: Get info about VPC subnets amazon.aws.ec2_vpc_subnet_info: region: "{{ aws_region }}" - filters: "{{ aws_vpc_facsts_subnet_filters_base | combine (aws_vpc_facts_subnet_filters, aws_vpc_facts_subnet_tags | prefix_keys('tag:')) }}" + filters: "{{ aws_vpc_facsts_subnet_filters_base | combine(aws_vpc_facts_subnet_filters, aws_vpc_facts_subnet_tags | prefix_keys('tag:')) }}" vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" aws_vpc_facsts_subnet_filters_base: vpc-id: "{{ aws_vpc_id }}" register: vpc_subnet_facts diff --git a/roles/cs.aws-vpc/tasks/main.yml b/roles/cs.aws-vpc/tasks/main.yml index c7573449c..5c59fb44a 100644 --- a/roles/cs.aws-vpc/tasks/main.yml +++ b/roles/cs.aws-vpc/tasks/main.yml @@ -1,49 +1,54 @@ -- name: Create VPC network - ec2_vpc_net: - state: present - region: "{{ aws_region }}" - cidr_block: "{{ aws_vpc_subnet_prefix }}.0.0/16" - name: "{{ aws_vpc_name }}" - tags: "{{ aws_tags_default }}" - register: vpc_net +- name: Create VPC resources + when: aws_vpc_create | bool + block: + - name: Ensure VPC exists + amazon.aws.ec2_vpc_net: + name: "{{ aws_vpc_name }}" + cidr_block: "{{ aws_vpc_subnet_prefix }}.0.0/16" + region: "{{ aws_region }}" + tags: "{{ aws_tags_default }}" + state: present + register: vpc_net -- name: Create internet gateway in VPC - ec2_vpc_igw: - state: present - region: "{{ aws_region }}" - tags: "{{ aws_tags_default | combine(vpc_igw_tags) }}" - vpc_id: "{{ vpc_net.vpc.id }}" - vars: - vpc_igw_tags: - Name: "{{ mageops_app_name }}" + - name: Create internet gateway in VPC + ec2_vpc_igw: + state: present + region: "{{ aws_region }}" + tags: "{{ aws_tags_default | combine(vpc_igw_tags) }}" + vpc_id: "{{ vpc_net.vpc.id }}" + vars: + vpc_igw_tags: + Name: "{{ mageops_app_name }}" -- name: Create public subnets in VPC - ec2_vpc_subnet: - state: present - region: "{{ aws_region }}" - vpc_id: "{{ vpc_net.vpc.id }}" - tags: "{{ aws_tags_default | combine(vpc_subnet_tags) }}" - az: "{{ az_name }}" - cidr: "{{ aws_vpc_subnet_prefix }}.{{ az_index + 1 }}.0/25" - vars: - vpc_subnet_tags: - Name: "{{ aws_vpc_name }}-public-{{ az_name[-2:] }}" - loop: "{{ aws_availability_zone_names }}" - loop_control: - loop_var: az_name - index_var: az_index - register: vpc_subnet - -- name: Create route table - ec2_vpc_route_table: - state: present - region: "{{ aws_region }}" - tags: "{{ aws_tags_default | combine(vpc_route_table_tags) }}" - vpc_id: "{{ vpc_net.vpc.id }}" - subnets: "{{ vpc_subnet.results | map(attribute='subnet') | map(attribute='cidr_block') | list }}" - routes: "{{ aws_vpc_routes_default + aws_vpc_routes_extra | default([]) }}" - vars: - vpc_route_table_tags: - Name: "{{ aws_vpc_name }}-routes" + - name: Create public subnets in VPC + ec2_vpc_subnet: + state: present + region: "{{ aws_region }}" + vpc_id: "{{ vpc_net.vpc.id }}" + tags: "{{ aws_tags_default | combine(vpc_subnet_tags) }}" + az: "{{ az_name }}" + cidr: "{{ aws_vpc_subnet_prefix }}.{{ az_index + 1 }}.0/25" + vars: + vpc_subnet_tags: + Name: "{{ aws_vpc_name }}-public-{{ az_name[-2:] }}" + loop: "{{ aws_availability_zone_names }}" + loop_control: + loop_var: az_name + index_var: az_index + register: vpc_subnet + - name: Create route table + ec2_vpc_route_table: + state: present + region: "{{ aws_region }}" + tags: "{{ aws_tags_default | combine(vpc_route_table_tags) }}" + vpc_id: "{{ vpc_net.vpc.id }}" + subnets: "{{ vpc_subnet.results | map(attribute='subnet') | map(attribute='cidr_block') | list }}" + routes: "{{ aws_vpc_routes_default + aws_vpc_routes_extra | default([]) }}" + vars: + vpc_route_table_tags: + Name: "{{ aws_vpc_name }}-routes" +- name: Resolve existing VPC facts + ansible.builtin.include_role: + name: cs.aws-vpc-facts diff --git a/roles/cs.blackfire/tasks/main.yml b/roles/cs.blackfire/tasks/main.yml index b59309731..17ebefab0 100644 --- a/roles/cs.blackfire/tasks/main.yml +++ b/roles/cs.blackfire/tasks/main.yml @@ -51,4 +51,7 @@ shell: "mageopscli set_feature_flag blackfire_apm {{ blackfire_enable_apm | ternary('yes', 'no') }}" when: aws_use - when: blackfire_install and blackfire_server_id and blackfire_server_token + when: + - blackfire_install | bool + - (blackfire_server_id | default('', true) | string | length) > 0 + - (blackfire_server_token | default('', true) | string | length) > 0 diff --git a/roles/cs.deploy/tasks/90-emergency-link-back.yml b/roles/cs.deploy/tasks/90-emergency-link-back.yml index 02e3e4b84..214fc29c5 100644 --- a/roles/cs.deploy/tasks/90-emergency-link-back.yml +++ b/roles/cs.deploy/tasks/90-emergency-link-back.yml @@ -7,4 +7,4 @@ become: yes become_user: "{{ deploy_app_user }}" ignore_errors: yes - when: deploy_current_release_path | default(false, true) + when: (deploy_current_release_path | default('', true) | string | length) > 0 diff --git a/roles/cs.deploy/tasks/91-emergency-rename.yml b/roles/cs.deploy/tasks/91-emergency-rename.yml index 7e99f88c3..dcf8e1fb2 100644 --- a/roles/cs.deploy/tasks/91-emergency-rename.yml +++ b/roles/cs.deploy/tasks/91-emergency-rename.yml @@ -1,8 +1,6 @@ - name: "Failed deploy! Damage control: Rename the deployed release directory to indicate it's broken" command: "mv '{{ deploy_next_release_path }}' '{{ deploy_next_release_path }}.failed'" - args: - warn: no become: yes become_user: "{{ deploy_app_user }}" ignore_errors: yes - when: deploy_next_release_path | default(false, true) \ No newline at end of file + when: (deploy_next_release_path | default('', true) | string | length) > 0 diff --git a/roles/cs.deploy/tasks/main.yml b/roles/cs.deploy/tasks/main.yml index 620572818..d84cf9471 100644 --- a/roles/cs.deploy/tasks/main.yml +++ b/roles/cs.deploy/tasks/main.yml @@ -10,7 +10,7 @@ vars: deploy_hook_name: "Pre install" deploy_hook_actions: "{{ deploy_pre_actions }}" - when: deploy_pre_actions | length + when: (deploy_pre_actions | length) > 0 - import_tasks: 03-install-artifact.yml when: deploy_install_new_release @@ -25,7 +25,7 @@ vars: deploy_hook_name: "Post install" deploy_hook_actions: "{{ deploy_install_actions }}" - when: deploy_install_actions | length + when: (deploy_install_actions | length) > 0 always: - import_tasks: 06-cleanup-post-install.yml @@ -35,16 +35,22 @@ vars: deploy_hook_name: "Custom cleanup" deploy_hook_actions: "{{ deploy_cleanup_actions }}" - when: deploy_cleanup_actions | length + when: (deploy_cleanup_actions | length) > 0 rescue: + - set_fact: + deploy_failed_task_name: "{{ ansible_failed_task.name | default('unknown') }}" + deploy_failed_result_msg: "{{ ansible_failed_result.msg | default('') }}" + deploy_failed_result_stderr: "{{ ansible_failed_result.stderr | default('') }}" + deploy_failed_result_stdout: "{{ ansible_failed_result.stdout | default('') }}" + - import_tasks: 91-emergency-rename.yml - include_tasks: hook/hook-actions.yml vars: deploy_hook_name: "Emergency installation abort" deploy_hook_actions: "{{ deploy_abort_install_actions }}" - when: deploy_abort_install_actions | length + when: (deploy_abort_install_actions | length) > 0 ignore_errors: true - set_fact: @@ -52,7 +58,13 @@ - name: Force fail if install has been aborted fail: - msg: "{{ deploy_post_install_abort_msg }}" + msg: |- + {{ deploy_post_install_abort_msg }} + + Failed task: {{ deploy_failed_task_name | default('unknown') }} + Error: {{ deploy_failed_result_msg | default('') }} + Stderr: {{ deploy_failed_result_stderr | default('') }} + Stdout: {{ deploy_failed_result_stdout | default('') }} when: deploy_emergency_install_aborted is defined and deploy_emergency_install_aborted - name: Perform release deploy tasks @@ -66,7 +78,7 @@ vars: deploy_hook_name: "After release" deploy_hook_actions: "{{ deploy_release_actions }}" - when: deploy_release_actions | length + when: (deploy_release_actions | length) > 0 - import_tasks: 08-post-deploy.yml @@ -74,9 +86,15 @@ vars: deploy_hook_name: "post" deploy_hook_actions: "{{ deploy_post_actions }}" - when: deploy_post_actions | length + when: (deploy_post_actions | length) > 0 rescue: + - set_fact: + deploy_failed_task_name: "{{ ansible_failed_task.name | default('unknown') }}" + deploy_failed_result_msg: "{{ ansible_failed_result.msg | default('') }}" + deploy_failed_result_stderr: "{{ ansible_failed_result.stderr | default('') }}" + deploy_failed_result_stdout: "{{ ansible_failed_result.stdout | default('') }}" + - import_tasks: 90-emergency-link-back.yml - import_tasks: 91-emergency-rename.yml @@ -84,7 +102,7 @@ vars: deploy_hook_name: "Emergency release abort" deploy_hook_actions: "{{ deploy_abort_release_actions }}" - when: deploy_abort_release_actions | length + when: (deploy_abort_release_actions | length) > 0 ignore_errors: true - set_fact: @@ -92,7 +110,11 @@ - name: Force fail if release has been aborted fail: - msg: "{{ deploy_post_release_abort_msg }}" - when: deploy_emergency_release_aborted is defined and deploy_emergency_release_aborted - + msg: |- + {{ deploy_post_release_abort_msg }} + Failed task: {{ deploy_failed_task_name | default('unknown') }} + Error: {{ deploy_failed_result_msg | default('') }} + Stderr: {{ deploy_failed_result_stderr | default('') }} + Stdout: {{ deploy_failed_result_stdout | default('') }} + when: deploy_emergency_release_aborted is defined and deploy_emergency_release_aborted diff --git a/roles/cs.elasticsearch/tasks/main.yml b/roles/cs.elasticsearch/tasks/main.yml index 4dced25e8..65cbe39c7 100644 --- a/roles/cs.elasticsearch/tasks/main.yml +++ b/roles/cs.elasticsearch/tasks/main.yml @@ -149,12 +149,15 @@ loop: "{{ elasticsearch_plugins }}" notify: Restart elasticsearch +- name: Gather service facts + service_facts: + - name: Ensure opensearch is stopped and disabled service: name: opensearch state: stopped enabled: no - ignore_errors: true + when: "'opensearch.service' in ansible_facts.services" - name: Ensure elasticsearch is started and enabled service: diff --git a/roles/cs.gtm/tasks/main.yml b/roles/cs.gtm/tasks/main.yml index 0d59ba144..3498235a0 100644 --- a/roles/cs.gtm/tasks/main.yml +++ b/roles/cs.gtm/tasks/main.yml @@ -19,6 +19,7 @@ image: gcr.io/cloud-tagging-10302018/gtm-cloud-image:stable state: started restart_policy: always + healthcheck_failure_action: kill publish: - "8080:8080" env: @@ -36,6 +37,7 @@ image: gcr.io/cloud-tagging-10302018/gtm-cloud-image:stable state: started restart_policy: always + healthcheck_failure_action: kill publish: - "8081:8080" env: diff --git a/roles/cs.magento-configure/defaults/main/app-etc.yml b/roles/cs.magento-configure/defaults/main/app-etc.yml index a4a5b0f83..365d71736 100644 --- a/roles/cs.magento-configure/defaults/main/app-etc.yml +++ b/roles/cs.magento-configure/defaults/main/app-etc.yml @@ -80,6 +80,10 @@ magento_app_etc_config: install: date: "Tue, 11 Nov 2016 11:11:00 +0000" +magento_app_etc_config_lazy_resize: + lazy_resize: + secret : "{{ lazy_resize_secret }}" + magento_mysql_ssl_required: db: connection: diff --git a/roles/cs.magento-configure/defaults/main/general.yml b/roles/cs.magento-configure/defaults/main/general.yml index c4315aa7e..d23ce196b 100644 --- a/roles/cs.magento-configure/defaults/main/general.yml +++ b/roles/cs.magento-configure/defaults/main/general.yml @@ -82,7 +82,7 @@ magento_scd_strategy: quick # We need a t3.small builder or frequently it will run out of memory # So setting the default at 1 upon consideration -magento_scd_parallel_jobs: "{{ ansible_processor_vcpus }}" +magento_scd_parallel_jobs: "{{ ansible_facts['processor_vcpus'] }}" magento_scd_excluded_themes: [] # Tip: When using custom themes exclude luma diff --git a/roles/cs.magento-configure/tasks/000-prepare-runtime-config.yml b/roles/cs.magento-configure/tasks/000-prepare-runtime-config.yml index b6c50c7dd..ae695a484 100644 --- a/roles/cs.magento-configure/tasks/000-prepare-runtime-config.yml +++ b/roles/cs.magento-configure/tasks/000-prepare-runtime-config.yml @@ -21,7 +21,7 @@ set_fact: magento_scd_containerized_tasks: "{{ magento_scd_containerized_tasks + [magento_magepack_js_bundling_task] }}" - when: magento_scd_advanced_js_bundling and magento_scd_advanced_js_bundling_strategy == 'magepack' + when: (magento_scd_advanced_js_bundling | bool) and magento_scd_advanced_js_bundling_strategy == 'magepack' - name: Configure baler JS bundling task hook block: @@ -61,16 +61,49 @@ - name: Ensure that Magento's js minification, merging and bundling is disabled set_fact: magento_core_config_settings: "{{ magento_core_config_settings + magento_baler_js_bundling_core_config }}" - when: magento_scd_advanced_js_bundling and magento_scd_advanced_js_bundling_strategy == 'baler' + when: (magento_scd_advanced_js_bundling | bool) and magento_scd_advanced_js_bundling_strategy == 'baler' - name: Check if database is initialized - command: mysql {{ mageops_mysql_require_ssl | ternary("--ssl-mode=REQUIRED", "") }} -N --batch -u {{ mageops_app_mysql_user|quote }} -p{{ mageops_app_mysql_pass|quote }} -h {{ mageops_mysql_host|quote }} -e "SHOW TABLES FROM `{{ mageops_app_mysql_db }}` LIKE 'admin_user';" + command: > + mysql {{ mageops_mysql_require_ssl | ternary("--ssl-mode=REQUIRED", "") }} + -N --batch + -u {{ mageops_app_mysql_user | quote }} + -p{{ mageops_app_mysql_pass | quote }} + -h {{ mageops_mysql_host | quote }} + -e "SHOW TABLES FROM `{{ mageops_app_mysql_db }}` LIKE 'admin_user';" changed_when: false register: admins +- name: Check if Magento default website exists + command: > + mysql {{ mageops_mysql_require_ssl | ternary("--ssl-mode=REQUIRED", "") }} + -N --batch + -u {{ mageops_app_mysql_user | quote }} + -p{{ mageops_app_mysql_pass | quote }} + -h {{ mageops_mysql_host | quote }} + -e "SELECT COUNT(*) FROM `{{ mageops_app_mysql_db }}`.`store_website` WHERE is_default = 1;" + changed_when: false + failed_when: false + register: websites + - name: Set database existence fact set_fact: - database_exists: "{{ admins.stdout_lines | length == 1 and admins.stdout_lines[0] == 'admin_user' }}" + magento_db_has_admin_user_table: "{{ admins.stdout_lines | length == 1 and admins.stdout_lines[0] == 'admin_user' }}" + magento_db_has_default_website: "{{ (websites.rc | default(1)) == 0 and (websites.stdout | default('0') | trim | int) > 0 }}" + +- name: Set effective Magento database initialization state + set_fact: + database_exists: "{{ (magento_db_has_admin_user_table | bool) and (magento_db_has_default_website | bool) }}" + +- name: Warn on partially initialized Magento database + debug: + msg: >- + Magento database appears partially initialized: + `admin_user` table exists but default website is missing. + Fresh install will be attempted. + when: + - magento_db_has_admin_user_table | bool + - not (magento_db_has_default_website | bool) - name: Get ElasticSuite version shell: "cat {{ magento_release_dir }}/composer.lock | jq '.packages | .[] | select(.name == \"smile/elasticsuite\") | .version'" diff --git a/roles/cs.magento-configure/tasks/035-clean-old-static-content.yml b/roles/cs.magento-configure/tasks/035-clean-old-static-content.yml index ecdd145ca..8494608a7 100644 --- a/roles/cs.magento-configure/tasks/035-clean-old-static-content.yml +++ b/roles/cs.magento-configure/tasks/035-clean-old-static-content.yml @@ -6,8 +6,6 @@ ### - name: Make sure previous Mangeto static cache files are gone shell: "[[ ! -d '{{ magento_static_cache_dir }}' ]] || find '{{ magento_static_cache_dir }}' -xdev -type f -exec rm -vf {} \\;" - args: - warn: no changed_when: _magento_install_clean_statics.stdout | trim | length > 0 register: _magento_install_clean_statics retries: 5 diff --git a/roles/cs.magento-configure/tasks/037-wait-for-services.yml b/roles/cs.magento-configure/tasks/037-wait-for-services.yml index 450ecb063..2cb6ad837 100644 --- a/roles/cs.magento-configure/tasks/037-wait-for-services.yml +++ b/roles/cs.magento-configure/tasks/037-wait-for-services.yml @@ -2,35 +2,39 @@ - name: Wait for Elasticsearch shell: curl -u {{ mageops_elasticsearch_username | default('elastic')}}:{{mageops_elasticsearch_password | default('changeme')}} -sfk 'http://{{ mageops_elasticsearch_host }}:{{ elasticsearch_http_port }}' - args: - warn: no retries: 15 delay: 5 register: result - until: result.rc == 0 - when: mageops_elasticsearch_host | default(false, true) and not mageops_skip_elasticsearch_check and mageops_elasticsearch_opensearch_flavor == "elasticsearch" + until: "'rc' in result and result.rc == 0" + when: + - (mageops_elasticsearch_host | default('', true) | string | length) > 0 + - not (mageops_skip_elasticsearch_check | bool) + - mageops_elasticsearch_opensearch_flavor == "elasticsearch" - name: Wait for Opensearch shell: curl -u {{ mageops_opensearch_username | default('elastic')}}:{{mageops_elasticsearch_password | default('changeme')}} -sfk 'http://{{ mageops_opensearch_host }}:{{ opensearch_http_port }}' - args: - warn: no retries: 15 delay: 5 register: result - until: result.rc == 0 - when: mageops_openearch_host | default(false, true) and mageops_elasticsearch_opensearch_flavor == "opensearch" + until: "'rc' in result and result.rc == 0" + when: + - (mageops_opensearch_host | default('', true) | string | length) > 0 + - mageops_elasticsearch_opensearch_flavor == "opensearch" - name: Wait for Redis cache host shell: echo -ne "PING\r\n" | nc {{ mageops_redis_host }} {{ mageops_redis_port }} retries: 15 delay: 5 register: result - until: result.rc == 0 - when: magento_redis_cache + until: "'rc' in result and result.rc == 0" + when: + - magento_redis_cache | bool + - (mageops_redis_host | default('', true) | string | length) > 0 - name: Wait for Redis session host shell: echo -ne "PING\r\n" | nc {{ mageops_redis_sessions_host }} {{ mageops_redis_sessions_port }} retries: 15 delay: 5 register: result - until: result.rc == 0 + until: "'rc' in result and result.rc == 0" + when: (mageops_redis_sessions_host | default('', true) | string | length) > 0 diff --git a/roles/cs.magento-configure/tasks/040-install-magento.yml b/roles/cs.magento-configure/tasks/040-install-magento.yml index cc7f1f865..fab29f3b7 100644 --- a/roles/cs.magento-configure/tasks/040-install-magento.yml +++ b/roles/cs.magento-configure/tasks/040-install-magento.yml @@ -1,6 +1,6 @@ - name: Install Magento (fresh) block: - - name: Install Magento + - name: Install Magento (fresh) shell: > php {{ magento_release_dir }}/bin/magento setup:install \ '--backend-frontname={{ magento_admin_path }}' \ diff --git a/roles/cs.magento-configure/tasks/075-compile-di.yml b/roles/cs.magento-configure/tasks/075-compile-di.yml index 0b6769e2a..3a8e3539e 100644 --- a/roles/cs.magento-configure/tasks/075-compile-di.yml +++ b/roles/cs.magento-configure/tasks/075-compile-di.yml @@ -1,3 +1,119 @@ +- name: Read effective Magento database config from env.php + become: yes + become_user: "{{ magento_user }}" + command: + argv: + - php + - -r + - | + $env = include $argv[1]; + $db = $env['db']['connection']['default'] ?? []; + $prefix = $env['db']['table_prefix'] ?? ''; + echo json_encode([ + 'host' => (string) ($db['host'] ?? ''), + 'name' => (string) ($db['dbname'] ?? ''), + 'user' => (string) ($db['username'] ?? ''), + 'password' => (string) ($db['password'] ?? ''), + 'table_prefix' => (string) $prefix, + ]); + - "{{ magento_release_dir }}/app/etc/env.php" + changed_when: false + register: _magento_env_db_raw + +- name: Parse effective Magento database config from env.php + ansible.builtin.set_fact: + _magento_effective_db: "{{ _magento_env_db_raw.stdout | from_json }}" + +- name: Validate effective Magento database config from env.php + ansible.builtin.assert: + that: + - (_magento_effective_db.host | default('') | trim | length) > 0 + - (_magento_effective_db.user | default('') | trim | length) > 0 + - (_magento_effective_db.name | default('') | trim | length) > 0 + - (_magento_effective_db.host | regex_search('\\{\\{')) is none + - (_magento_effective_db.user | regex_search('\\{\\{')) is none + - (_magento_effective_db.name | regex_search('\\{\\{')) is none + - (_magento_effective_db.table_prefix | default('') | regex_search('\\{\\{')) is none + fail_msg: >- + Invalid or unresolved Magento DB config in env.php. + host=`{{ _magento_effective_db.host | default('') }}`, + user=`{{ _magento_effective_db.user | default('') }}`, + db=`{{ _magento_effective_db.name | default('') }}`, + table_prefix=`{{ _magento_effective_db.table_prefix | default('') }}`. + +- name: Check Magento default website count before DI compile + command: > + mysql {{ mageops_mysql_require_ssl | ternary("--ssl-mode=REQUIRED", "") }} + -N --batch + -u {{ _magento_effective_db.user | quote }} + -p{{ _magento_effective_db.password | quote }} + -h {{ _magento_effective_db.host | quote }} + -e "SELECT COUNT(*) FROM `{{ _magento_effective_db.name }}`.`{{ _magento_effective_db.table_prefix | default('') }}store_website` WHERE is_default = 1;" + changed_when: false + failed_when: false + register: _magento_default_website_count_before_compile + +- name: Restore default website marker when missing before DI compile + block: + - name: Select first website id for default marker recovery + command: > + mysql {{ mageops_mysql_require_ssl | ternary("--ssl-mode=REQUIRED", "") }} + -N --batch + -u {{ _magento_effective_db.user | quote }} + -p{{ _magento_effective_db.password | quote }} + -h {{ _magento_effective_db.host | quote }} + -e "SELECT website_id FROM `{{ _magento_effective_db.name }}`.`{{ _magento_effective_db.table_prefix | default('') }}store_website` ORDER BY website_id ASC LIMIT 1;" + changed_when: false + failed_when: false + register: _magento_first_website_id + + - name: Ensure Magento website exists for default marker recovery + ansible.builtin.assert: + that: + - (_magento_first_website_id.rc | default(1)) == 0 + - (_magento_first_website_id.stdout | default('') | trim | length) > 0 + fail_msg: >- + Magento database has no website rows; cannot set default website before DI compile. + mysql rc={{ _magento_first_website_id.rc | default('n/a') }}, + stdout=`{{ _magento_first_website_id.stdout | default('') | trim }}`, + stderr=`{{ _magento_first_website_id.stderr | default('') | trim }}`. + + - name: Set first website as default before DI compile + command: > + mysql {{ mageops_mysql_require_ssl | ternary("--ssl-mode=REQUIRED", "") }} + -N --batch + -u {{ _magento_effective_db.user | quote }} + -p{{ _magento_effective_db.password | quote }} + -h {{ _magento_effective_db.host | quote }} + -e "UPDATE `{{ _magento_effective_db.name }}`.`{{ _magento_effective_db.table_prefix | default('') }}store_website` SET is_default = 0; UPDATE `{{ _magento_effective_db.name }}`.`{{ _magento_effective_db.table_prefix | default('') }}store_website` SET is_default = 1 WHERE website_id = {{ _magento_first_website_id.stdout | trim | int }};" + changed_when: true + when: + - (_magento_default_website_count_before_compile.rc | default(1)) == 0 + - (_magento_default_website_count_before_compile.stdout | default('0') | trim | int) == 0 + +- name: Re-check Magento default website count before DI compile + command: > + mysql {{ mageops_mysql_require_ssl | ternary("--ssl-mode=REQUIRED", "") }} + -N --batch + -u {{ _magento_effective_db.user | quote }} + -p{{ _magento_effective_db.password | quote }} + -h {{ _magento_effective_db.host | quote }} + -e "SELECT COUNT(*) FROM `{{ _magento_effective_db.name }}`.`{{ _magento_effective_db.table_prefix | default('') }}store_website` WHERE is_default = 1;" + changed_when: false + failed_when: false + register: _magento_default_website_count_after_recovery + +- name: Validate default website state before DI compile + ansible.builtin.assert: + that: + - (_magento_default_website_count_after_recovery.rc | default(1)) == 0 + - (_magento_default_website_count_after_recovery.stdout | default('0') | trim | int) > 0 + fail_msg: >- + Magento database has no default website before DI compile. + mysql rc={{ _magento_default_website_count_after_recovery.rc | default('n/a') }}, + stdout=`{{ _magento_default_website_count_after_recovery.stdout | default('') | trim }}`, + stderr=`{{ _magento_default_website_count_after_recovery.stderr | default('') | trim }}`. + - name: Compile Magento DI become: yes become_user: "{{ magento_user }}" diff --git a/roles/cs.magento-configure/tasks/080-core-config.yml b/roles/cs.magento-configure/tasks/080-core-config.yml index 371969b96..e479262af 100644 --- a/roles/cs.magento-configure/tasks/080-core-config.yml +++ b/roles/cs.magento-configure/tasks/080-core-config.yml @@ -2,12 +2,12 @@ - name: Add varnish configuration for core_config if needed vars: _extra_items: - - name: Set varnish URI for warmup crawler - path: "cache_warmer_crawler/general/varnish_uri" - value: "{% if magento_varnish_host %}http://{{ magento_varnish_host }}:{{ magento_varnish_port }}{% endif %}" + - name: Set varnish URI for warmup crawler + path: "cache_warmer_crawler/general/varnish_uri" + value: "{% if magento_varnish_host %}http://{{ magento_varnish_host }}:{{ magento_varnish_port }}{% endif %}" set_fact: magento_core_config_settings: "{{ magento_core_config_settings + _extra_items }}" - when: magento_varnish_host | default(false, true) + when: (magento_varnish_host | default('', true) | string | length) > 0 - name: Ensure core config database settings values delegate_to: localhost @@ -26,7 +26,12 @@ scope = "default" ON DUPLICATE KEY UPDATE value = "{{ magento_db_setting.value | quote }}" - when: not magento_db_setting.default | default(false) and magento_db_setting.value | default(false) is string and magento_db_setting.enabled | default(true) + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + when: + - not (magento_db_setting.default | default(false) | bool) + - magento_db_setting.value | default(false) is string + - magento_db_setting.enabled | default(true) | bool loop: "{{ magento_core_config_settings }}" loop_control: loop_var: magento_db_setting @@ -46,7 +51,12 @@ value = "{{ magento_db_setting.value | quote }}", scope_id = 0, scope = "default" - when: magento_db_setting.default | default(false) and magento_db_setting.value | default(false) is string and magento_db_setting.enabled | default(true) + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + when: + - magento_db_setting.default | default(false) | bool + - magento_db_setting.value | default(false) is string + - magento_db_setting.enabled | default(true) | bool loop: "{{ magento_core_config_settings }}" loop_control: loop_var: magento_db_setting @@ -63,6 +73,8 @@ DELETE FROM core_config_data WHERE path = "{{ magento_db_setting_path | quote }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" loop: "{{ magento_core_config_settings_to_remove }}" loop_control: loop_var: magento_db_setting_path diff --git a/roles/cs.magento-configure/tasks/action/configure-env.yml b/roles/cs.magento-configure/tasks/action/configure-env.yml index e4fa1de78..654a850b6 100644 --- a/roles/cs.magento-configure/tasks/action/configure-env.yml +++ b/roles/cs.magento-configure/tasks/action/configure-env.yml @@ -1,127 +1,160 @@ - name: Configure Magento environment (env.php) block: - - name: Enable HTTP cache - when: magento_http_cache_enable - set_fact: - magento_app_etc_config: >- - {{ magento_app_etc_config - | combine(magento_app_etc_config_http_cache, recursive=true) }} - - - name: Enable ElasticSuite configuration - when: elasticsuite_version | default(false) and (mageops_elasticsearch_opensearch_flavor == "elasticsearch") - set_fact: - magento_app_etc_config: >- - {{ magento_app_etc_config - | combine(magento_app_etc_config_elasticsuite, recursive=true) }} - - - name: Enable ElasticSuite configuration with Opensearch - when: elasticsuite_version | default(false) and (mageops_elasticsearch_opensearch_flavor == "opensearch") - set_fact: - magento_app_etc_config: >- - {{ magento_app_etc_config - | combine(magento_app_etc_config_opensearch, recursive=true) }} - - - name: Configure Redis caching - when: magento_redis_cache - block: - - name: Enable default Redis cache - when: not magento_redis_cache_l2 + - name: Enable HTTP cache + when: magento_http_cache_enable | bool set_fact: magento_app_etc_config: >- {{ magento_app_etc_config - | combine(magento_app_etc_config_cache_default_redis, recursive=true) }} + | combine(magento_app_etc_config_http_cache, recursive=true) }} - - name: Enable 2-level default Redis cache - when: magento_redis_cache_l2 + - name: Enable lazy resize configuration + when: (lazy_resize_secret | default('', true) | string | length) > 0 set_fact: magento_app_etc_config: >- {{ magento_app_etc_config - | combine(magento_app_etc_config_cache_default_redis_l2, recursive=true) }} + | combine(magento_app_etc_config_lazy_resize, recursive=true) }} - - name: Configure Redis page cache + - name: Enable ElasticSuite configuration + when: + - (elasticsuite_version | default('', true) | string | length) > 0 + - mageops_elasticsearch_opensearch_flavor == "elasticsearch" set_fact: magento_app_etc_config: >- {{ magento_app_etc_config - | combine(magento_app_etc_config_cache_page_redis, recursive=true) }} - - - name: Enable RabbitMQ queue configuration - when: magento_rabbitmq_queue - set_fact: - magento_app_etc_config: >- - {{ magento_app_etc_config - | combine(magento_app_etc_config_queue_rabbitmq, recursive=true) }} - - - name: Adjust configuration for consumer workers - when: magento_consumer_workers_enable - set_fact: - magento_app_etc_config: >- - {{ magento_app_etc_config - | combine(magento_app_etc_config_consumer_workers, recursive=true) }} - - - name: Adjust configuration for cron consumers - when: magento_cron_consumers_enable - set_fact: - magento_app_etc_config: >- - {{ magento_app_etc_config - | combine(magento_app_etc_config_cron_consumers, recursive=true) }} - - - name: Configure X-Magento-Vary cookie signing - when: magento_vary_sign - set_fact: - magento_app_etc_config: >- - {{ magento_app_etc_config - | combine(magento_app_etc_config_cookie_sign, recursive=true) }} - - - name: Enable mysql ssl requirement - when: mageops_mysql_require_ssl - set_fact: - magento_app_etc_config: >- - {{ magento_app_etc_config - | combine(magento_mysql_ssl_required, recursive=true) }} - - - name: Set extra options - set_fact: - magento_app_etc_config: >- - {{ magento_app_etc_config - | combine(magento_app_etc_config_extra, recursive=true) }} - - - name: Config graphql salt - when: magento_graphql_salt - set_fact: - magento_app_etc_config: >- - {{ magento_app_etc_config - | combine(magento_app_etc_config_graphql_salt, recursive=true) }} - - - name: Export configuration as PHP code - command: - stdin: "{{ magento_app_etc_config | to_json }}" - argv: - - php - - -r - - >- - echo " 0 + - mageops_elasticsearch_opensearch_flavor == "opensearch" + set_fact: + magento_app_etc_config: >- + {{ magento_app_etc_config + | combine(magento_app_etc_config_opensearch, recursive=true) }} + + - name: Configure Redis caching + when: magento_redis_cache | bool + block: + - name: Enable default Redis cache + when: not (magento_redis_cache_l2 | bool) + set_fact: + magento_app_etc_config: >- + {{ magento_app_etc_config + | combine(magento_app_etc_config_cache_default_redis, recursive=true) }} + + - name: Enable 2-level default Redis cache + when: magento_redis_cache_l2 | bool + set_fact: + magento_app_etc_config: >- + {{ magento_app_etc_config + | combine(magento_app_etc_config_cache_default_redis_l2, recursive=true) }} + + - name: Configure Redis page cache + set_fact: + magento_app_etc_config: >- + {{ magento_app_etc_config + | combine(magento_app_etc_config_cache_page_redis, recursive=true) }} + + - name: Enable RabbitMQ queue configuration + when: magento_rabbitmq_queue | bool + set_fact: + magento_app_etc_config: >- + {{ magento_app_etc_config + | combine(magento_app_etc_config_queue_rabbitmq, recursive=true) }} + + - name: Adjust configuration for consumer workers + when: magento_consumer_workers_enable | bool + set_fact: + magento_app_etc_config: >- + {{ magento_app_etc_config + | combine(magento_app_etc_config_consumer_workers, recursive=true) }} + + - name: Adjust configuration for cron consumers + when: magento_cron_consumers_enable | bool + set_fact: + magento_app_etc_config: >- + {{ magento_app_etc_config + | combine(magento_app_etc_config_cron_consumers, recursive=true) }} + + - name: Configure X-Magento-Vary cookie signing + when: magento_vary_sign | bool + set_fact: + magento_app_etc_config: >- + {{ magento_app_etc_config + | combine(magento_app_etc_config_cookie_sign, recursive=true) }} + + - name: Enable mysql ssl requirement + when: mageops_mysql_require_ssl | bool + set_fact: + magento_app_etc_config: >- + {{ magento_app_etc_config + | combine(magento_mysql_ssl_required, recursive=true) }} + + - name: Set extra options + set_fact: + magento_app_etc_config: >- + {{ magento_app_etc_config + | combine(magento_app_etc_config_extra, recursive=true) }} + + - name: Normalize Magento DB connection values + set_fact: + magento_app_etc_config: >- + {{ + magento_app_etc_config + | combine( + { + 'db': { + 'connection': { + 'default': { + 'host': mageops_mysql_host, + 'dbname': mageops_app_mysql_db, + 'username': mageops_app_mysql_user, + 'password': mageops_app_mysql_pass, + } + } + } + }, + recursive=true + ) + }} + + - name: Config graphql salt + when: magento_graphql_salt | bool + set_fact: + magento_app_etc_config: >- + {{ magento_app_etc_config + | combine(magento_app_etc_config_graphql_salt, recursive=true) }} + + - name: Export configuration as PHP code + command: + stdin: "{{ magento_app_etc_config | to_json }}" + argv: + - php + - -r + - >- + echo " 0 diff --git a/roles/cs.magento-shared-storage/tasks/main.yml b/roles/cs.magento-shared-storage/tasks/main.yml index 8cfa8f204..85d79f136 100644 --- a/roles/cs.magento-shared-storage/tasks/main.yml +++ b/roles/cs.magento-shared-storage/tasks/main.yml @@ -58,4 +58,4 @@ efs_mount_watchdog_cron_enable: "{{ magento_media_storage_strategy == 'aws-s3' }}" - name: Clean up shared Magento asset cache - include: cleanup-static-cache-releases.yml + include_tasks: cleanup-static-cache-releases.yml diff --git a/roles/cs.mysql-configure/meta/main.yml b/roles/cs.mysql-configure/meta/main.yml index 80b0ecea1..1902c6a35 100644 --- a/roles/cs.mysql-configure/meta/main.yml +++ b/roles/cs.mysql-configure/meta/main.yml @@ -1,3 +1,3 @@ dependencies: - role: cs.aws-rds-facts - when: aws_use | default(false) + when: aws_use | default(false) | bool diff --git a/roles/cs.mysql-configure/tasks/create-db.yml b/roles/cs.mysql-configure/tasks/create-db.yml index 008dc5cd4..f231ed123 100644 --- a/roles/cs.mysql-configure/tasks/create-db.yml +++ b/roles/cs.mysql-configure/tasks/create-db.yml @@ -11,7 +11,7 @@ login_host: "{{ mageops_mysql_host }}" login_user: "{{ mageops_mysql_root_user }}" login_password: "{{ mageops_mysql_root_pass }}" - user: "{{ mageops_app_mysql_user }}" + name: "{{ mageops_app_mysql_user }}" password: "{{ mageops_app_mysql_pass }}" host: "%" state: present @@ -22,7 +22,7 @@ login_host: "{{ mageops_mysql_host }}" login_user: "{{ mageops_mysql_root_user }}" login_password: "{{ mageops_mysql_root_pass }}" - user: "{{ mageops_app_mysql_user }}" + name: "{{ mageops_app_mysql_user }}" password: "{{ mageops_app_mysql_pass }}" host: "localhost" state: present diff --git a/roles/cs.new-relic/tasks/main.yml b/roles/cs.new-relic/tasks/main.yml index 99f4f086a..d8bc51a40 100644 --- a/roles/cs.new-relic/tasks/main.yml +++ b/roles/cs.new-relic/tasks/main.yml @@ -1,3 +1,8 @@ +- name: Import New Relic GPG key + ansible.builtin.rpm_key: + key: https://download.newrelic.com/php_agent/NEWRELIC_RPM_BD2E199C.public + state: present + - name: Install new relic repo dnf: name: "{{ new_relic_repo_url }}" diff --git a/roles/cs.nginx-https-termination/tasks/000-facts.yml b/roles/cs.nginx-https-termination/tasks/000-facts.yml index ff33902b3..8b1e21318 100644 --- a/roles/cs.nginx-https-termination/tasks/000-facts.yml +++ b/roles/cs.nginx-https-termination/tasks/000-facts.yml @@ -19,19 +19,58 @@ set_fact: https_termination_vhosts: "{{ https_termination_vhosts | default({}) | combine({ vhost_derived.name: vhost_derived }) }}" vars: + host_server_name_raw: "{{ host.server_name | default('*') }}" + host_server_name_resolved: >- + {%- set ns = namespace(value=host_server_name_raw) -%} + {%- if ns.value is string -%} + {%- for _ in range(5) -%} + {%- for var_name in ns.value | regex_findall('\\{\\{\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*\\}\\}') -%} + {%- set replacement = ((hostvars[inventory_hostname] | default({})).get(var_name, lookup('vars', var_name, default='{{ ' ~ var_name ~ ' }}')) | string) -%} + {%- set ns.value = ns.value | regex_replace('\\{\\{\\s*' ~ var_name ~ '\\s*\\}\\}', replacement) -%} + {%- endfor -%} + {%- endfor -%} + {%- endif -%} + {{ ns.value }} + host_aliases_resolved: >- + {%- set aliases = [] -%} + {%- for alias in host.aliases | default([]) -%} + {%- set ns = namespace(value=alias) -%} + {%- if ns.value is string -%} + {%- for _ in range(5) -%} + {%- for var_name in ns.value | regex_findall('\\{\\{\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*\\}\\}') -%} + {%- set replacement = ((hostvars[inventory_hostname] | default({})).get(var_name, lookup('vars', var_name, default='{{ ' ~ var_name ~ ' }}')) | string) -%} + {%- set ns.value = ns.value | regex_replace('\\{\\{\\s*' ~ var_name ~ '\\s*\\}\\}', replacement) -%} + {%- endfor -%} + {%- endfor -%} + {%- endif -%} + {%- set _ = aliases.append(ns.value) -%} + {%- endfor -%} + {{ aliases }} + host_name_raw: "{{ host.name | default(host_server_name_resolved) }}" + host_name_resolved: >- + {%- set ns = namespace(value=host_name_raw) -%} + {%- if ns.value is string -%} + {%- for _ in range(5) -%} + {%- for var_name in ns.value | regex_findall('\\{\\{\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*\\}\\}') -%} + {%- set replacement = ((hostvars[inventory_hostname] | default({})).get(var_name, lookup('vars', var_name, default='{{ ' ~ var_name ~ ' }}')) | string) -%} + {%- set ns.value = ns.value | regex_replace('\\{\\{\\s*' ~ var_name ~ '\\s*\\}\\}', replacement) -%} + {%- endfor -%} + {%- endfor -%} + {%- endif -%} + {{ ns.value }} vhost_derived: params: "{{ host }}" - label: "{{ host.name }}" - name: "{{ host.name | trim | lower | regex_replace('[^a-z0-9.-]+', '_') }}" + label: "{{ host_name_resolved }}" + name: "{{ host_name_resolved | trim | lower | regex_replace('[^a-z0-9.-]+', '_') }}" - conf_path: "{{ nginx_confd_dir }}/050-vhost-{{ host.name }}.conf" + conf_path: "{{ nginx_confd_dir }}/050-vhost-{{ host_name_resolved }}.conf" conf_custom_block: "{{ host.extra_config | default(false, true) }}" upstream_uri: "{{ host.upstream | default(https_termination_default_upstream_uri, true) }}" upstream_proxy_http: "{{ host.proxy_http | default(https_termination_proxy_http_port) }}" - server_name: "{{ host.server_name | default('*') }}" - server_aliases: "{{ host.aliases | default([]) }}" + server_name: "{{ host_server_name_resolved }}" + server_aliases: "{{ host_aliases_resolved }}" server_aliases_redirect: "{{ host.redirect_aliases | default(false) }}" server_aliases_redirect_include_domain: "{{ redirect_aliases_include_domain | default(false) }}" server_custom_block: "{{ host.extra_vhost_config | default(false, true) }}" @@ -124,7 +163,7 @@ '127.0.0.1', '::1', 'localhost', - ansible_hostname, + ansible_facts['hostname'] | default(''), inventory_hostname ] loop: "{{ https_termination_vhosts.values() | list }}" @@ -137,7 +176,7 @@ - name: Handle missing ACME gracefully - when: not (https_termination_crt_acme_email | default(false, true)) + when: (https_termination_crt_acme_email | default('', true) | string | length) == 0 block: - name: Set ACME e-mail to a dummy fallback value set_fact: diff --git a/roles/cs.nginx-https-termination/tasks/034-certificates-acme.yml b/roles/cs.nginx-https-termination/tasks/034-certificates-acme.yml index b75b8cd77..4c96e4678 100644 --- a/roles/cs.nginx-https-termination/tasks/034-certificates-acme.yml +++ b/roles/cs.nginx-https-termination/tasks/034-certificates-acme.yml @@ -7,13 +7,22 @@ set_fact: certbot_args_default: >- {{ - [ '/usr/bin/certbot', 'certonly' ] - + certbot_args_base - + https_termination_crt_acme_email | default(false, true) | ternary(certbot_args_email, []) - + https_termination_crt_acme_staging | default(false, true) | ternary( + [ '/usr/bin/certbot', 'certonly' ] + + certbot_args_base + + ( + (https_termination_crt_acme_email | default('', true) | string | length > 0) + | ternary(certbot_args_email, []) + ) + + ( + (https_termination_crt_acme_staging | bool) + | ternary( [ '--staging' ], - https_termination_crt_acme_directory | default(false, true) | ternary(https_termination_crt_acme_directory, []) + ( + (https_termination_crt_acme_directory | default('', true) | string | length > 0) + | ternary(certbot_args_acme_server, []) + ) ) + ) }} vars: certbot_args_base: [ @@ -41,16 +50,33 @@ name: nginx state: reloaded +- name: Validate certbot domains for vhost + ansible.builtin.assert: + that: + - (_vhost_domains | string | length) > 0 + - (_vhost_domains | regex_search('\\{\\{')) is none + - (_vhost_domains | regex_search('\\}\\}')) is none + fail_msg: >- + Vhost `{{ vhost.name }}` has unresolved domain template(s): `{{ _vhost_domains }}`. + Ensure referenced variables are defined and resolved before ACME run. + vars: + _vhost_domains: "{{ ([vhost.server_name] + (vhost.server_aliases | default([], true))) | join(',') }}" + when: vhost.crt_do_acme + loop: "{{ https_termination_vhosts.values() | list }}" + loop_control: + label: "{{ vhost.name }} -> {{ _vhost_domains }}" + loop_var: vhost + - name: Run certbot for vhost command: argv: "{{ certbot_args_default + certbot_args_vhost }}" vars: certbot_args_vhost: [ --webroot, - --webroot-path, "{{ vhost.crt_acme_webroot_dir }}", - --domains, "{{ ([vhost.server_name] + vhost.server_aliases | default([], true)) | join(',') }}", - --cert-name, "{{ vhost.name }}", - --post-hook, "/usr/bin/systemctl reload nginx" + --webroot-path, "{{ vhost.crt_acme_webroot_dir }}", + --domains, "{{ ([vhost.server_name] + vhost.server_aliases | default([], true)) | join(',') }}", + --cert-name, "{{ vhost.name }}", + --post-hook, "/usr/bin/systemctl reload nginx" ] when: vhost.crt_do_acme loop: "{{ https_termination_vhosts.values() | list }}" @@ -58,11 +84,31 @@ label: "{{ vhost.name }} -> {{ (certbot_args_default + certbot_args_vhost) | join(' ') }}" loop_var: vhost register: https_termination_certbot_run + changed_when: true + failed_when: false + +- name: Fail with certbot output when certificate request fails + fail: + msg: | + Certbot failed for {{ certbot_result.vhost.name }}. + Command: {{ certbot_result.cmd | default('unknown') }} + RC: {{ certbot_result.rc | default('unknown') }} + STDOUT: + {{ certbot_result.stdout | default('') }} + STDERR: + {{ certbot_result.stderr | default('') }} + loop: "{{ https_termination_certbot_run.results | default([]) }}" + loop_control: + loop_var: certbot_result + label: "{{ certbot_result.vhost.name | default('unknown-vhost') }}" + when: + - certbot_result.skipped is not defined or not certbot_result.skipped + - (certbot_result.rc | default(0)) != 0 - name: Symlink latest ACME certificates as live ones file: state: link - src: "{{ vhost[key.src] }}" + src: "{{ vhost[key.src] }}" dest: "{{ vhost[key.dst] }}" when: vhost.crt_do_acme vars: @@ -88,6 +134,7 @@ path: /etc/sysconfig/certbot regexp: '^CERTBOT_ARGS=' line: 'CERTBOT_ARGS="--allow-subset-of-names"' + mode: "0644" create: yes notify: Reload systemctl daemon diff --git a/roles/cs.nginx-magento/tasks/001-vhost.yml b/roles/cs.nginx-magento/tasks/001-vhost.yml index 1a6aa0a25..ef812b96a 100644 --- a/roles/cs.nginx-magento/tasks/001-vhost.yml +++ b/roles/cs.nginx-magento/tasks/001-vhost.yml @@ -30,5 +30,9 @@ - "{{ nginx_magento_vhost_conf_path }}" - "{{ nginx_magento_fcgi_upstream_conf_path }}" +- name: Remove deprecated nginx configs + include_role: + name: cs.nginx + tasks_from: 099-cleanup.yml diff --git a/roles/cs.packages/tasks/main.yml b/roles/cs.packages/tasks/main.yml index c2b08eee9..41664aedb 100644 --- a/roles/cs.packages/tasks/main.yml +++ b/roles/cs.packages/tasks/main.yml @@ -12,16 +12,18 @@ dnf: name: "{{ packages_remove | default([]) }}" state: absent - when: packages_remove is defined and packages_remove | length + when: + - packages_remove is defined + - (packages_remove | length) > 0 - name: Store information that package maintenance was done copy: - content: "{{ ansible_date_time.epoch }}" + content: "{{ ansible_facts['date_time']['epoch'] }}" dest: "{{ packages_maintenance_marker_path }}" when: >- packages_maintenance_last_timestamp.content is not defined or ( ( - packages_maintenance_last_timestamp.content | int - ansible_date_time.epoch | int + packages_maintenance_last_timestamp.content | int - ansible_facts['date_time']['epoch'] | int ) > packages_maintenance_interval | int ) diff --git a/roles/cs.php-fpm/tasks/001-daemon.yml b/roles/cs.php-fpm/tasks/001-daemon.yml index 487158a6a..013fafe88 100644 --- a/roles/cs.php-fpm/tasks/001-daemon.yml +++ b/roles/cs.php-fpm/tasks/001-daemon.yml @@ -48,3 +48,11 @@ group: root mode: u=rwx,g=rx,o=rx when: php_fpm_weekly_restart + +- name: Install php-fpm logrotate configuration + template: + src: php-fpm.logrotate + dest: /etc/logrotate.d/php-fpm + owner: root + group: root + mode: "0644" diff --git a/roles/cs.php-fpm/tasks/main.yml b/roles/cs.php-fpm/tasks/main.yml index 93efd0ccd..67e6530ca 100644 --- a/roles/cs.php-fpm/tasks/main.yml +++ b/roles/cs.php-fpm/tasks/main.yml @@ -1,5 +1,5 @@ - debug: - msg: "Detected: {{ ansible_memtotal_mb }}M of memory and {{ ansible_processor_vcpus }} vcpus" + msg: "Detected: {{ ansible_facts['memtotal_mb'] }}M of memory and {{ ansible_facts['processor_vcpus'] }} vcpus" - name: Calculate workers set_fact: @@ -7,14 +7,14 @@ {{ ( ( - ( ansible_memtotal_mb * 1024 * 1024 ) + ( ansible_facts['memtotal_mb'] * 1024 * 1024 ) - ( mageops_app_node_reserved_memory | human_to_bytes ) ) / ( php_http_request_memory_limit | human_to_bytes ) ) | int }} - php_fpm_pm_max_children_by_cpu: "{{ ansible_processor_vcpus * 8 | int }}" + php_fpm_pm_max_children_by_cpu: "{{ ansible_facts['processor_vcpus'] * 8 | int }}" - name: Calculate default workers set_fact: diff --git a/roles/cs.php-fpm/templates/php-fpm.logrotate b/roles/cs.php-fpm/templates/php-fpm.logrotate new file mode 100644 index 000000000..6729dd1d7 --- /dev/null +++ b/roles/cs.php-fpm/templates/php-fpm.logrotate @@ -0,0 +1,14 @@ +{{ php_fpm_log_dir_path }}/*.log { + rotate 8 + size 128M + missingok + notifempty + compress + compresscmd /usr/bin/zstd + uncompresscmd /usr/bin/unzstd + compressext .zstd + compressoptions -19 + copytruncate + create 0660 root root + su root root +} diff --git a/roles/cs.php-tideways/tasks/main.yml b/roles/cs.php-tideways/tasks/main.yml index e3ceefbbd..6c34bc235 100644 --- a/roles/cs.php-tideways/tasks/main.yml +++ b/roles/cs.php-tideways/tasks/main.yml @@ -1,5 +1,7 @@ - name: Enable tideways - when: tideways_enable and tideways_api_key | default(false, true) + when: + - tideways_enable | bool + - (tideways_api_key | default('', true) | string | length) > 0 block: - name: Install tideways repo GPG keys rpm_key: diff --git a/roles/cs.pio/defaults/main.yml b/roles/cs.pio/defaults/main.yml index 84e5b1ea7..90dbbef30 100644 --- a/roles/cs.pio/defaults/main.yml +++ b/roles/cs.pio/defaults/main.yml @@ -29,6 +29,9 @@ pio_filesystem: "{{ (magento_media_storage_strategy == 'aws-s3') | ternary('s3', pio_resize_strategy: Contain # Url to pio-worker rpm +# Optional package version used when pio_package_url_rpm contains +# `{{ pio_package_version }}` (legacy `{{ pio_package_verison }}` is also supported) +pio_package_version: pio_package_url_rpm: x86_64: aarch64: diff --git a/roles/cs.pio/tasks/install_rpm.yml b/roles/cs.pio/tasks/install_rpm.yml index 13c68637a..bd3f7a002 100644 --- a/roles/cs.pio/tasks/install_rpm.yml +++ b/roles/cs.pio/tasks/install_rpm.yml @@ -4,10 +4,26 @@ suffix: pio-worker-install register: _temp_dir +- name: Resolve pio-worker package version + ansible.builtin.set_fact: + _pio_package_version: "{{ pio_package_version | default(pio_package_verison | default(''), true) }}" + +- name: Resolve pio-worker package URL + ansible.builtin.set_fact: + _pio_worker_rpm_url: >- + {{ + pio_package_url_rpm[ansible_facts['architecture']] + | replace('{{ pio_package_version }}', _pio_package_version) + | replace('{{pio_package_version}}', _pio_package_version) + | replace('{{ pio_package_verison }}', _pio_package_version) + | replace('{{pio_package_verison}}', _pio_package_version) + }} + - name: Download pio-worker rpm get_url: - url: "{{ pio_package_url_rpm[ansible_architecture] }}" + url: "{{ _pio_worker_rpm_url }}" dest: "{{ _temp_dir.path }}/pio-worker.rpm" + mode: "0644" - name: Install pio-worker rpm dnf: diff --git a/roles/cs.redis/tasks/configure.yml b/roles/cs.redis/tasks/configure.yml index f5e7e2f88..3906fe214 100644 --- a/roles/cs.redis/tasks/configure.yml +++ b/roles/cs.redis/tasks/configure.yml @@ -119,18 +119,17 @@ name: "{{ redis_uninstalling_flavor }}" state: stopped enabled: false - when: services[redis_uninstalling_flavor+'.service']['status'] | default('not-found') != 'not-found' + when: ansible_facts['services'].get(redis_uninstalling_flavor + '.service', {}).get('status', 'not-found') != 'not-found' - name: Stop {{ redis_uninstalling_flavor }} session in case of migration service: name: "{{ redis_uninstalling_flavor }}-sessions" state: stopped enabled: false - when: mageops_redis_sessions_create and services[redis_uninstalling_flavor+'-sessions.service']['status'] | default('not-found') != 'not-found' + when: mageops_redis_sessions_create and ansible_facts['services'].get(redis_uninstalling_flavor + '-sessions.service', {}).get('status', 'not-found') != 'not-found' - name: Ensure Redis/valkey service is started service: name: "{{ redis_daemon }}" state: started enabled: yes - diff --git a/roles/cs.s3-mount/tasks/002-setup-mounts.yml b/roles/cs.s3-mount/tasks/002-setup-mounts.yml index 85c88b006..083fddd15 100644 --- a/roles/cs.s3-mount/tasks/002-setup-mounts.yml +++ b/roles/cs.s3-mount/tasks/002-setup-mounts.yml @@ -1,11 +1,13 @@ - name: Create S3 mount remote source dirs become: no - local_action: - module: aws_s3 + delegate_to: localhost + aws_s3: bucket: "{{ s3_mount_bucket.bucket.split(':')[0] }}" object: "{{ s3_mount_bucket.bucket.split(':')[1] }}" mode: create region: "{{ aws_region }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" when: s3_mount_bucket.bucket is search(':/') loop: "{{ s3_mount_buckets_resolved }}" loop_control: @@ -14,6 +16,7 @@ # `findmnt` returns 1 if there are not mounts yet - name: Get current S3 mounts shell: >- + set -o pipefail && findmnt -l -n -o TARGET,SOURCE,FSTYPE | awk '$2 ~ /^(s3fs|goofys)#/ || $2 ~ /^{{ s3_mount_rclone_remote_name }}:/ { print $1 "\t" $2 "\t" $3 }' register: s3fs_current_mounts_check @@ -40,29 +43,72 @@ - name: Set current S3 mountpoints set_fact: s3fs_current_mountpoints: "{{ s3_mount_current_entries | map(attribute='mountpoint') | list }}" + s3fs_current_mount_sources_by_mountpoint: >- + {{ + dict( + s3_mount_current_entries | map(attribute='mountpoint') + | zip(s3_mount_current_entries | map(attribute='source')) + ) + }} s3fs_target_mountpoints: "{{ s3_mount_buckets_resolved | map(attribute='mountpoint') | map('realpath') | list }}" -- name: Remove S3 mounts that should not be present - mount: - path: "{{ s3fs_mountpoint.rstrip('/') ~ '/' }}" - state: absent +- name: Unmount obsolete S3 mounts + ansible.posix.mount: + path: "{{ s3fs_mountpoint }}" + state: unmounted when: s3fs_mountpoint not in s3fs_target_mountpoints loop: "{{ s3fs_current_mountpoints }}" - register: s3fs_mount_removal - retries: 30 - delay: 1 - until: s3fs_mount_removal is not failed loop_control: loop_var: s3fs_mountpoint label: "{{ s3fs_mountpoint }}" -- name: Create S3 mount - mount: +- name: Remove obsolete S3 mount definitions from fstab + ansible.posix.mount: + path: "{{ s3fs_mountpoint }}" + state: absent_from_fstab + when: s3fs_mountpoint not in s3fs_target_mountpoints + loop: "{{ s3fs_current_mountpoints }}" + loop_control: + loop_var: s3fs_mountpoint + label: "{{ s3fs_mountpoint }}" + +- name: Ensure target S3 mountpoint directories exist + ansible.builtin.file: path: "{{ s3_mount_bucket.mountpoint }}" + state: directory + mode: "0755" + loop: "{{ s3_mount_buckets_resolved }}" + loop_control: + loop_var: s3_mount_bucket + +- name: Unmount target S3 mountpoints with stale sources + ansible.posix.mount: + path: "{{ s3_mount_bucket.mountpoint }}" + state: unmounted + when: >- + (s3fs_current_mount_sources_by_mountpoint.get(s3_mount_bucket.mountpoint | realpath, '')) + != s3_mount_bucket.mount_src + loop: "{{ s3_mount_buckets_resolved }}" + loop_control: + loop_var: s3_mount_bucket + +- name: Create S3 mounts + ansible.posix.mount: src: "{{ s3_mount_bucket.mount_src }}" + path: "{{ s3_mount_bucket.mountpoint }}" fstype: "{{ s3_mount_fstype }}" - opts: "_netdev,allow_other,auto,{{ s3_mount_opts | join(',') }}" + opts: >- + _netdev,allow_other,auto + {%- if s3_mount_opts is defined and (s3_mount_opts | string | length > 0) -%} + ,{{ + (s3_mount_opts is string) + | ternary(s3_mount_opts, (s3_mount_opts | join(','))) + }} + {%- endif -%} state: mounted + when: >- + (s3fs_current_mount_sources_by_mountpoint.get(s3_mount_bucket.mountpoint | realpath, '')) + != s3_mount_bucket.mount_src loop: "{{ s3_mount_buckets_resolved }}" loop_control: loop_var: s3_mount_bucket diff --git a/roles/cs.s3-mount/tasks/004-restore-premount-backups.yml b/roles/cs.s3-mount/tasks/004-restore-premount-backups.yml index 6b649a0c8..2f622a31f 100644 --- a/roles/cs.s3-mount/tasks/004-restore-premount-backups.yml +++ b/roles/cs.s3-mount/tasks/004-restore-premount-backups.yml @@ -1,10 +1,28 @@ - name: Bind mount rootfs to temp dir - mount: - path: "{{ _s3fs_premount_temp_dir.path }}" - src: "/" - opts: bind - fstype: none - state: mounted + block: + - name: Read backup root bind mount state + ansible.builtin.command: findmnt -n -o SOURCE,FSTYPE {{ _s3fs_premount_temp_dir.path }} + register: _s3_premount_bind_state + changed_when: false + failed_when: false + + - name: Ensure backup root bind mount is not occupied by another source + ansible.builtin.assert: + that: + - (_s3_premount_bind_state.rc | default(1)) != 0 + or ((_s3_premount_bind_state.stdout | default('') | trim).split() | first) == '/' + fail_msg: >- + Temporary premount backup path is already mounted from another source: + `{{ _s3_premount_bind_state.stdout | default('') | trim }}`. + + - name: Bind mount rootfs to temp dir + ansible.posix.mount: + src: / + path: "{{ _s3fs_premount_temp_dir.path }}" + fstype: none + opts: bind + state: ephemeral + when: (_s3_premount_bind_state.rc | default(1)) != 0 - name: Restore premount backup to new mountpoints shell: > @@ -29,4 +47,11 @@ - name: Print remount file restore log debug: - var: _s3fs_mountpoint_backup_restore_sync.stdout_lines + msg: >- + {{ + _s3fs_mountpoint_backup_restore_sync.results + | default([]) + | map(attribute='stdout_lines') + | list + | flatten + }} diff --git a/roles/cs.s3-mount/tasks/005-cleanup.yml b/roles/cs.s3-mount/tasks/005-cleanup.yml index fc6c064e0..86b6244db 100644 --- a/roles/cs.s3-mount/tasks/005-cleanup.yml +++ b/roles/cs.s3-mount/tasks/005-cleanup.yml @@ -1,10 +1,11 @@ - name: Make sure backup root bind mount is absent - mount: + ansible.posix.mount: path: "{{ _s3fs_premount_temp_dir.path }}" - src: "/" - state: absent + state: unmounted + register: _s3_cleanup_mount + failed_when: false - name: Remove backup mount temp dir file: path: "{{ _s3fs_premount_temp_dir.path }}" - state: absent \ No newline at end of file + state: absent diff --git a/roles/cs.s3-mount/tasks/main.yml b/roles/cs.s3-mount/tasks/main.yml index 8954ff3f7..1fc877612 100644 --- a/roles/cs.s3-mount/tasks/main.yml +++ b/roles/cs.s3-mount/tasks/main.yml @@ -9,6 +9,6 @@ - include_tasks: 003-setup-watchdog.yml when: s3_mount_backend == 's3fs' - include_tasks: 004-restore-premount-backups.yml - when: s3fs_premount_bind_mounts | length + when: (s3fs_premount_bind_mounts | length) > 0 always: - include_tasks: 005-cleanup.yml diff --git a/roles/cs.sshd/defaults/main.yml b/roles/cs.sshd/defaults/main.yml index c3438f76b..c40caa336 100644 --- a/roles/cs.sshd/defaults/main.yml +++ b/roles/cs.sshd/defaults/main.yml @@ -23,7 +23,7 @@ sshd_max_auth_tries: "{{ sshd_password_auth | ternary(1, 20) }}" # We could allow everything (sshd default) since user has shell anyway so # he can forward any port using other means, however, lets stick only to # the local network. -sshd_permit_open: "127.0.0.1:* {{ ansible_default_ipv4.network | default('0.0.0.0') }}:*" +sshd_permit_open: "127.0.0.1:* {{ ansible_facts['default_ipv4']['network'] | default('0.0.0.0') }}:*" # PITA: There is no `PermitListen` directive in OpenSSH 7.4 (latest on CentOS 7) # This option should be present since OpenSSH 7.8, so maybe in CentOS 8... @@ -89,6 +89,7 @@ sshd_config_default: SyslogFacility: AUTHPRIV TCPKeepAlive: "yes" UsePAM: "yes" + KexAlgorithms: "+sntrup761x25519-sha512@openssh.com" # Configuration applied to trusted networks # Note: None/null (`~`) means that the runtime config value or base config diff --git a/roles/cs.swap/tasks/main.yml b/roles/cs.swap/tasks/main.yml index ffa854bcb..392b52d16 100644 --- a/roles/cs.swap/tasks/main.yml +++ b/roles/cs.swap/tasks/main.yml @@ -18,7 +18,7 @@ line: "{{ swap_file_location }} none swap sw,pri={{ swap_file_priority }} 0 0" - name: Drop all vm caches # Prevent memory error when enabling swapfile - shell: echo 3 > /proc/sys/vm/drop_caches + shell: echo 3 > /proc/sys/vm/drop_caches changed_when: false - name: Enable swap file @@ -27,13 +27,11 @@ delay: 30 register: _enabled_swap_cmd_result until: _enabled_swap_cmd_result.rc == 0 - + - name: Set swappiness sysctl: name: vm.swappiness value: "{{ swap_swappiness }}" reload: yes state: present - when: ansible_swaptotal_mb < 1 - - + when: (ansible_facts['swaptotal_mb'] | default(0) | int) < 1 diff --git a/roles/cs.unison-project/meta/main.yml b/roles/cs.unison-project/meta/main.yml index 92ac98831..22758df3e 100644 --- a/roles/cs.unison-project/meta/main.yml +++ b/roles/cs.unison-project/meta/main.yml @@ -1,3 +1,3 @@ dependencies: - role: cs.unison - when: not unison_project_skip_install | default(False) \ No newline at end of file + when: not (unison_project_skip_install | default(false) | bool) diff --git a/roles/cs.varnish-manager/tasks/main.yml b/roles/cs.varnish-manager/tasks/main.yml index 9681c2238..d67351e35 100644 --- a/roles/cs.varnish-manager/tasks/main.yml +++ b/roles/cs.varnish-manager/tasks/main.yml @@ -37,6 +37,8 @@ object: "{{ varnish_manager_ssh_key_filename }}" mode: put retries: 3 + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" - name: Ensure sudoers configuration for Varnish Manager exists template: diff --git a/roles/cs.varnish/defaults/main.yml b/roles/cs.varnish/defaults/main.yml index c2aa2d918..fcca7d04c 100644 --- a/roles/cs.varnish/defaults/main.yml +++ b/roles/cs.varnish/defaults/main.yml @@ -67,7 +67,7 @@ varnish_thread_pool_stack: 128k # Max. number of thread pools that should be at most the number of vCPUs # See: https://varnish-cache.org/docs/6.0/reference/varnishd.html#thread-pool-max -varnish_thread_pools: "{{ ansible_processor_vcpus }}" +varnish_thread_pools: "{{ ansible_facts['processor_vcpus'] }}" # The min/max number of threads per pool varnish_thread_pool_max: "{{ [(varnish_thread_mem_quota | int / varnish_thread_mem_est | int / varnish_thread_pools | int) | int, 20] | max }}" @@ -161,6 +161,7 @@ varnish_vcl_backend_error_extra: "" #Send original URL to backend varnish_original_url: false varnish_strip_params: ['gclid'] +varnish_strip_params_extra: [] # To be used for advanced customizations varnish_url_replace: [] diff --git a/roles/cs.varnish/tasks/000-facts.yml b/roles/cs.varnish/tasks/000-facts.yml index 755fd3372..18d0cbd6b 100644 --- a/roles/cs.varnish/tasks/000-facts.yml +++ b/roles/cs.varnish/tasks/000-facts.yml @@ -9,8 +9,8 @@ --- - Total target system vCPUs: {{ ansible_processor_vcpus }} - Total target system memory: {{ ( ansible_memtotal_mb ~ 'MB' ) | human_to_bytes | human_readable }} + Total target system vCPUs: {{ ansible_facts['processor_vcpus'] }} + Total target system memory: {{ ( ansible_facts['memtotal_mb'] ~ 'MB' ) | human_to_bytes | human_readable }} --- @@ -34,9 +34,11 @@ - name: Get VPC IP addresses of backend app instances delegate_to: localhost become: no - ec2_instance_info: + amazon.aws.ec2_instance_info: region: "{{ aws_region }}" filters: "{{ varnish_backend_instances_filter_base | combine(varnish_backend_instances_aws_tags | prefix_keys('tag:')) }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" register: _varnish_backends_ec2_instance_info - name: Set Varnish backend instances if any are running (AWS) @@ -66,14 +68,16 @@ https_termination_hosts | default([])|selectattr('server_name', 'defined')|map(attribute='server_name')|list + https_termination_hosts | default([])|selectattr('aliases', 'defined')|map(attribute='aliases')|list | flatten }} - when: mageops_https_termination_enable | default(false) + when: mageops_https_termination_enable | default(false) | bool - name: Add Cloudfront domnains set_fact: varnish_media_cors_allowed_origins: | {{ varnish_media_cors_allowed_origins + - [ aws_cloudfront_distribution_domain ] + - aws_cloudfront_domain_aliases | default([]) + [aws_cloudfront_distribution_domain] + + (aws_cloudfront_domain_aliases | default([])) }} - when: aws_cloudfront_distribution_create | default(false) and aws_cloudfront_distribution_domain | default(false, true) + when: + - aws_cloudfront_distribution_create | default(false) | bool + - (aws_cloudfront_distribution_domain | default('', true) | string | length) > 0 when: varnish_generate_allowed_origins diff --git a/roles/cs.varnish/tasks/002-configure.yml b/roles/cs.varnish/tasks/002-configure.yml index 26320163a..fc3e36b89 100644 --- a/roles/cs.varnish/tasks/002-configure.yml +++ b/roles/cs.varnish/tasks/002-configure.yml @@ -1,14 +1,55 @@ # Prevents unneeded shmlog disk I/O - name: Mount VSL under tmpfs - mount: - path: /var/lib/varnish - src: none - fstype: tmpfs - opts: rw,auto,noatime - state: mounted - notify: - - Restart varnish - - Restart varnish-exporter + block: + - name: Ensure VSL tmpfs entry exists in fstab + ansible.builtin.lineinfile: + path: /etc/fstab + regexp: '^\s*none\s+/var/lib/varnish\s+tmpfs\s+' + line: 'none /var/lib/varnish tmpfs rw,auto,noatime 0 0' + notify: + - Restart varnish + - Restart varnish-exporter + + - name: Read current VSL mount state + ansible.builtin.command: findmnt -n -o FSTYPE,OPTIONS /var/lib/varnish + register: _varnish_vsl_mount_state + changed_when: false + failed_when: false + + - name: Set VSL mount state facts + ansible.builtin.set_fact: + _varnish_vsl_is_tmpfs: >- + {{ (_varnish_vsl_mount_state.rc | default(1)) == 0 + and ((_varnish_vsl_mount_state.stdout | default('') | trim).split(' ') | first) == 'tmpfs' }} + _varnish_vsl_has_noatime: >- + {{ (_varnish_vsl_mount_state.rc | default(1)) == 0 + and (_varnish_vsl_mount_state.stdout | default('') | regex_search('(^|,)noatime(,|$)')) is not none }} + + - name: Ensure VSL mount path is not occupied by non-tmpfs mount + ansible.builtin.assert: + that: + - (_varnish_vsl_mount_state.rc | default(1)) != 0 or (_varnish_vsl_is_tmpfs | bool) + fail_msg: >- + /var/lib/varnish is already mounted with non-tmpfs filesystem. + Existing mount details: `{{ _varnish_vsl_mount_state.stdout | default('') | trim }}`. + + - name: Mount VSL tmpfs from fstab + ansible.builtin.command: mount /var/lib/varnish + when: (_varnish_vsl_mount_state.rc | default(1)) != 0 + changed_when: true + notify: + - Restart varnish + - Restart varnish-exporter + + - name: Remount VSL tmpfs with noatime + ansible.builtin.command: mount -o remount,rw,noatime /var/lib/varnish + when: + - _varnish_vsl_is_tmpfs | bool + - not (_varnish_vsl_has_noatime | bool) + changed_when: true + notify: + - Restart varnish + - Restart varnish-exporter - name: Ensure varnish var directories always exist lineinfile: diff --git a/roles/cs.varnish/templates/vcl/acl.vcl.j2 b/roles/cs.varnish/templates/vcl/acl.vcl.j2 index bfc78d973..5b8889491 100644 --- a/roles/cs.varnish/templates/vcl/acl.vcl.j2 +++ b/roles/cs.varnish/templates/vcl/acl.vcl.j2 @@ -12,10 +12,10 @@ acl client_ip_whitelist { acl purge { "127.0.0.1"; {% for ip in varnish_purge_trusted_ips %} - {% if ip | ipaddr('network') %} - "{{ ip | ipaddr('network') }}"/{{ ip | ipaddr('prefix') }}; + {% if '/' in (ip | string) %} + "{{ (ip | string).split('/')[0] }}"/{{ (ip | string).split('/')[1] }}; {% else %} - "{{ ip | ipaddr('address') }}"; + "{{ ip }}"; {% endif %} {% endfor %} } @@ -23,10 +23,10 @@ acl purge { {% if varnish_do_not_expose_caching or varnish_secure_site %} acl trusted { {% for ip in varnish_trusted_ips %} - {% if ip | ipaddr('network') %} - "{{ ip | ipaddr('network') }}"/{{ ip | ipaddr('prefix') }}; + {% if '/' in (ip | string) %} + "{{ (ip | string).split('/')[0] }}"/{{ (ip | string).split('/')[1] }}; {% else %} - "{{ ip | ipaddr('address') }}"; + "{{ ip }}"; {% endif %} {% endfor %} } diff --git a/roles/cs.varnish/templates/vcl/backends.vcl.j2 b/roles/cs.varnish/templates/vcl/backends.vcl.j2 index 11a535486..b6a4df948 100644 --- a/roles/cs.varnish/templates/vcl/backends.vcl.j2 +++ b/roles/cs.varnish/templates/vcl/backends.vcl.j2 @@ -30,7 +30,7 @@ probe gtm_probe { {% for instance in (varnish_backend_instances_app + varnish_backend_instances_extra) %} -backend {{ instance.instance_id | replace('-','') }} { +backend app{{ instance.private_ip_address | replace('.','') }} { .host = "{{ instance.private_ip_address }}"; .port = "{{ varnish_backend_port }}"; .max_connections = {{ varnish_backend_max_conns }}; @@ -43,7 +43,7 @@ backend {{ instance.instance_id | replace('-','') }} { {% if (gtm_enabled | default(false)) %} {% for instance in (varnish_backend_instances_app + varnish_backend_instances_extra) %} -backend gtm_{{ instance.instance_id | replace('-','') }} { +backend gtm_{{ instance.private_ip_address | replace('.','') }} { .host = "{{ instance.private_ip_address }}"; .port = "8080"; .max_connections = {{ varnish_backend_max_conns }}; @@ -57,7 +57,7 @@ backend gtm_{{ instance.instance_id | replace('-','') }} { {% if (gtm_enabled | default(false)) %} {% for instance in (varnish_backend_instances_app + varnish_backend_instances_extra) %} -backend gtm_preview{{ instance.instance_id | replace('-','') }} { +backend gtm_preview{{ instance.private_ip_address | replace('.','') }} { .host = "{{ instance.private_ip_address }}"; .port = "8081"; .probe = gtm_probe; @@ -72,18 +72,18 @@ backend gtm_preview{{ instance.instance_id | replace('-','') }} { sub backends_init { new app_director = directors.round_robin(); {% for instance in varnish_backend_instances_app %} - app_director.add_backend({{ instance.instance_id | replace('-','') }}); + app_director.add_backend(app{{ instance.private_ip_address | replace('.','') }}); {% endfor %} {% if (gtm_enabled | default(false)) %} new gtm_director = directors.round_robin(); {% for instance in varnish_backend_instances_app %} - gtm_director.add_backend(gtm_{{ instance.instance_id | replace('-','') }}); + gtm_director.add_backend(gtm_{{ instance.private_ip_address | replace('.','') }}); {% endfor %} new gtm_preview_director = directors.round_robin(); {% for instance in varnish_backend_instances_app %} - gtm_preview_director.add_backend(gtm_preview{{ instance.instance_id | replace('-','') }}); + gtm_preview_director.add_backend(gtm_preview{{ instance.private_ip_address | replace('.','') }}); {% endfor %} {% endif %} @@ -91,6 +91,6 @@ sub backends_init { new extra_director = directors.round_robin(); {% for instance in varnish_backend_instances_extra %} - extra_director.add_backend({{ instance.instance_id | replace('-','') }}); + extra_director.add_backend(app{{ instance.private_ip_address | replace('.','') }}); {% endfor %} -} \ No newline at end of file +} diff --git a/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 b/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 index e9af13db4..17fc376b8 100644 --- a/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 +++ b/roles/cs.varnish/templates/vcl/subroutines/recv.vcl.j2 @@ -192,8 +192,8 @@ if (req.http.X-Blackfire-Query) { } } -# Bypass customer, shopping cart, checkout and search requests -if (req.url ~ "/customer" || req.url ~ "/checkout" || req.url ~ "/catalogsearch") { +# Bypass customer, shopping cart, checkout +if (req.url ~ "/customer" || req.url ~ "/checkout") { return (pass); } @@ -279,7 +279,7 @@ if (req.http.Accept-Encoding) { set req.http.X-Original-URL = req.url; {% endif %} -{% for param in varnish_strip_params %} +{% for param in (varnish_strip_params | default([])) + (varnish_strip_params_extra | default([])) %} # remove all forms of param: {{ param }} - The order is important! set req.url = regsuball(req.url,"&{{ param }}=[^&]+",""); # strips when QS = "?foo=bar¶m=AAA" or QS = "?foo=bar¶m=AAA&bar=baz" set req.url = regsuball(req.url,"\?{{ param }}=[^&]+&","?"); # strips when QS = "?param=AAA&foo=bar" diff --git a/roles/cs.varnish/vars/main.yml b/roles/cs.varnish/vars/main.yml index 2a3e6d021..0373ca00b 100644 --- a/roles/cs.varnish/vars/main.yml +++ b/roles/cs.varnish/vars/main.yml @@ -11,7 +11,7 @@ varnish_backend_instances_filter_base: varnish_storage: "malloc" # Total system memory in bytes -varnish_total_sys_mem: "{{ ansible_memtotal_mb * 1024 * 1024 }}" +varnish_total_sys_mem: "{{ ansible_facts['memtotal_mb'] * 1024 * 1024 }}" # Base amount of system memory dedicated to varnish used for calculating derived dynamic param defaults varnish_base_mem_quota: >- diff --git a/site.aws.yml b/site.aws.yml index 8b2db8ab0..a86b4fa25 100644 --- a/site.aws.yml +++ b/site.aws.yml @@ -26,4 +26,4 @@ import_playbook: site.step-70-cleanup.yml - name: Post Deploy Monitoring Maintenance - import_playbook: site.step-80-monitoring.yml \ No newline at end of file + import_playbook: site.step-80-monitoring.yml diff --git a/site.maintenance.aws-remove-all.yml b/site.maintenance.aws-remove-all.yml index ad91a42a4..dd3c02cae 100644 --- a/site.maintenance.aws-remove-all.yml +++ b/site.maintenance.aws-remove-all.yml @@ -1,5 +1,16 @@ - hosts: localhost connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + remove_asg: yes + remove_ec2: yes + remove_rds: yes + remove_efs: yes + remove_lambda: yes + remove_sg: yes + remove_vol: yes + remove_s3: no + dry_run_mode : no pre_tasks: - name: Get VPC data block: @@ -27,7 +38,7 @@ set_fact: _aws_asgs_to_remove: "{{ _aws_asgs | json_query('results[].{ name: auto_scaling_group_name } ') }}" - name: Remove ASGs - ec2_asg: + amazon.aws.autoscaling_group: name: "{{ item.name }}" region: "{{ aws_region }}" state: absent @@ -127,7 +138,7 @@ - name: Remove Lambdas block: - name: Remove Lambdas - lambda: + amazon.aws.lambda: name: "{{ item }}" state: absent region: "{{ aws_region }}" @@ -153,7 +164,7 @@ vars: query: "security_groups[?group_name!='default'].{id: group_id, name: group_name}" - name: Drop rules from SG (removes circular dependencies) - ec2_group: + amazon.aws.ec2_group: region: "{{ aws_region }}" state: present rules: [] @@ -163,7 +174,7 @@ with_items: "{{ _aws_groups_to_remove }}" when: not dry_run_mode - name: Drop Security Groups - ec2_group: + amazon.aws.ec2_group: region: "{{ aws_region }}" state: absent name: "{{ item.name }}" @@ -189,14 +200,3 @@ - role: cs.aws-ec2-cleanup aws_ec2_cleanup_lt_to_keep: 0 when: remove_asg and not dry_run_mode - - vars: - remove_asg: yes - remove_ec2: yes - remove_rds: yes - remove_efs: yes - remove_lambda: yes - remove_sg: yes - remove_vol: yes - remove_s3: no - dry_run_mode : no diff --git a/site.maintenance.aws-start.yml b/site.maintenance.aws-start.yml index de9883733..f570d8e2f 100644 --- a/site.maintenance.aws-start.yml +++ b/site.maintenance.aws-start.yml @@ -1,6 +1,12 @@ - hosts: localhost connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + start: yes + start_ec2: "{{ start }}" + start_asg: "{{ start }}" + start_rds: "{{ start }}" roles: - role: cs.aws-region-facts - role: cs.aws-rds-facts @@ -38,7 +44,7 @@ register: start_asg_facts - name: Scale up ASG - ec2_asg: + amazon.aws.autoscaling_group: name: "{{ item.name }}" region: "{{ aws_region }}" state: present @@ -60,7 +66,7 @@ register: start_asg_facts - name: Scale up ASG - ec2_asg: + amazon.aws.autoscaling_group: name: "{{ item.name }}" region: "{{ aws_region }}" state: present @@ -78,9 +84,3 @@ region: "{{ aws_region }}" db_instance_identifier: "{{ aws_rds_instance.db_instance_identifier }}" when: start_rds and aws_rds_instance is defined - - vars: - start: yes - start_ec2: "{{ start }}" - start_asg: "{{ start }}" - start_rds: "{{ start }}" diff --git a/site.maintenance.aws-stop.yml b/site.maintenance.aws-stop.yml index 8800ff49c..9dacb66e7 100644 --- a/site.maintenance.aws-stop.yml +++ b/site.maintenance.aws-stop.yml @@ -1,6 +1,12 @@ - hosts: localhost connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + stop: yes + stop_ec2: "{{ stop }}" + stop_asg: "{{ stop }}" + stop_rds: "{{ stop }}" roles: - role: cs.aws-region-facts - role: cs.aws-rds-facts @@ -18,7 +24,7 @@ register: stop_asg_facts - name: Scale down ASG to zero - ec2_asg: + amazon.aws.autoscaling_group: name: "{{ item.name }}" region: "{{ aws_region }}" state: present @@ -42,7 +48,7 @@ - name: Scale down all App Node ASGs when: stop_asg - ec2_asg: + amazon.aws.autoscaling_group: name: "{{ item.auto_scaling_group_name }}" region: "{{ aws_region }}" state: present @@ -59,9 +65,3 @@ region: "{{ aws_region }}" db_instance_identifier: "{{ aws_rds_instance.db_instance_identifier }}" when: stop_rds and aws_rds_instance is defined - - vars: - stop: yes - stop_ec2: "{{ stop }}" - stop_asg: "{{ stop }}" - stop_rds: "{{ stop }}" diff --git a/site.maintenance.blackfire.yml b/site.maintenance.blackfire.yml index 80557ed83..add433591 100644 --- a/site.maintenance.blackfire.yml +++ b/site.maintenance.blackfire.yml @@ -1,6 +1,10 @@ - hosts: localhost connection: local become: no + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + blackfire_urls: + - "{{ magento_base_url }}" tasks: - name: Get list of EC2 instances ec2_instance_info: @@ -34,6 +38,8 @@ - hosts: localhost connection: local become: no + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" tasks: - name: Blackfire profile with empty cache shell: "blackfire --json curl --samples=1 --metadata='app={{ mageops_app_name }}' --metadata='cache=no' {{ magento_base_url }}" @@ -54,6 +60,3 @@ vars_files: - vars/app/env.yml - vars/aws/env.yml - vars: - blackfire_urls: - - "{{ magento_base_url }}" diff --git a/site.maintenance.rds-gfs-backup.yml b/site.maintenance.rds-gfs-backup.yml index 8b354ddc6..e2fe52245 100644 --- a/site.maintenance.rds-gfs-backup.yml +++ b/site.maintenance.rds-gfs-backup.yml @@ -2,6 +2,8 @@ - hosts: localhost connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" roles: - role: cs.aws-rds-facts - role: cs.aws-rds-gfs-backup diff --git a/site.step-1-iam.yml b/site.step-1-iam.yml index e924fcd87..028859f94 100644 --- a/site.step-1-iam.yml +++ b/site.step-1-iam.yml @@ -1,6 +1,8 @@ - hosts: localhost connection: local gather_facts: false + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" roles: - role: cs.aws-iam - role: cs.aws-iam-employee-access diff --git a/site.step-10-infrastructure-aws.yml b/site.step-10-infrastructure-aws.yml index 1c4cb365b..e56c1f5af 100644 --- a/site.step-10-infrastructure-aws.yml +++ b/site.step-10-infrastructure-aws.yml @@ -1,6 +1,8 @@ - hosts: localhost connection: local gather_facts: false + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" pre_tasks: - name: Collect minimum facts (we need date time stuff) setup: diff --git a/site.step-15-varnish.yml b/site.step-15-varnish.yml index a2d82500d..ff3b3bd82 100644 --- a/site.step-15-varnish.yml +++ b/site.step-15-varnish.yml @@ -1,6 +1,19 @@ - import_playbook: site.common.group-current-hosts.yml - hosts: varnish:¤t:!immutable + gather_facts: no + pre_tasks: + - name: Wait for varnish node SSH to become available + ansible.builtin.wait_for: + host: "{{ ansible_host | default(inventory_hostname) }}" + port: 22 + delay: 5 + timeout: 900 + delegate_to: localhost + become: no + + - name: Gather facts for varnish node + setup: ~ roles: - role: cs.aws-node-facts delegate_to: localhost @@ -43,6 +56,7 @@ when: aws_use - role: cs.cron - role: geerlingguy.ntp + services: "{{ ansible_facts.services | default({}) }}" - role: cs.nginx - role: cs.nginx-url-blacklist nginx_blacklist_urls: "{{ nginx_blacklist_urls_default + nginx_blacklist_urls_project | default([]) }}" @@ -108,5 +122,6 @@ when: mageops_extra_tasks_varnish_node is defined vars: + ansible_python_interpreter: /usr/bin/python3.9 mageops_node_role: varnish magento_facts_detect_from_artifacts: yes diff --git a/site.step-20-persistent.yml b/site.step-20-persistent.yml index fd15df12a..0114a6762 100644 --- a/site.step-20-persistent.yml +++ b/site.step-20-persistent.yml @@ -1,6 +1,19 @@ - import_playbook: site.common.group-current-hosts.yml - hosts: persistent:¤t:!immutable + gather_facts: no + pre_tasks: + - name: Wait for persistent node SSH to become available + ansible.builtin.wait_for: + host: "{{ ansible_host | default(inventory_hostname) }}" + port: 22 + delay: 5 + timeout: 900 + delegate_to: localhost + become: no + + - name: Gather facts for persistent node + setup: ~ roles: - role: cs.aws-node-facts delegate_to: localhost @@ -44,6 +57,7 @@ when: aws_use - role: cs.cron - role: geerlingguy.ntp + services: "{{ ansible_facts.services | default({}) }}" - role: cs.opensearch when: mageops_elasticsearch_opensearch_flavor == "opensearch" - role: cs.elasticsearch @@ -106,5 +120,6 @@ when: mageops_extra_tasks_persistent_node is defined vars: + ansible_python_interpreter: /usr/bin/python3.9 mageops_node_role: persistent magento_facts_detect_from_artifacts: yes diff --git a/site.step-30-builder.yml b/site.step-30-builder.yml index 8db884cb4..2905bac2e 100644 --- a/site.step-30-builder.yml +++ b/site.step-30-builder.yml @@ -1,5 +1,7 @@ - hosts: localhost connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" roles: - role: cs.aws-security-group-facts - role: cs.aws-node-state diff --git a/site.step-40-app-node.yml b/site.step-40-app-node.yml index efdca46b3..ea9b749a5 100644 --- a/site.step-40-app-node.yml +++ b/site.step-40-app-node.yml @@ -3,10 +3,16 @@ - hosts: app:¤t:!immutable gather_facts: no pre_tasks: - - name: Wait for builder's SSH to become available - wait_for_connection: ~ + - name: Wait for builder node SSH to become available + ansible.builtin.wait_for: + host: "{{ ansible_host | default(inventory_hostname) }}" + port: 22 + delay: 5 + timeout: 900 + delegate_to: localhost + become: no - - name: Gather facts for first time + - name: Gather facts for app node setup: ~ roles: @@ -81,6 +87,7 @@ - role: cs.cron - role: geerlingguy.ntp + services: "{{ ansible_facts.services | default({}) }}" - role: cs.supervisor supervisor_programs: "{{ mageops_supervisor_programs_app_node }}" @@ -190,6 +197,7 @@ when: mageops_extra_tasks_app_node is defined vars: + ansible_python_interpreter: /usr/bin/python3.9 mageops_node_role: app mageops_app_node_nginx_blacklist_urls: "{{ nginx_blacklist_urls_default + nginx_blacklist_urls_project | default([]) }}" mageops_app_node_https_termination_enable: "{{ mageops_https_termination_enable and not varnish_standalone }}" diff --git a/site.step-45-app-deploy.yml b/site.step-45-app-deploy.yml index c00d5dd47..4619d3d5e 100644 --- a/site.step-45-app-deploy.yml +++ b/site.step-45-app-deploy.yml @@ -1,7 +1,20 @@ - import_playbook: site.common.group-current-hosts.yml - hosts: app:¤t:!immutable + gather_facts: no pre_tasks: + - name: Wait for app node SSH to become available + ansible.builtin.wait_for: + host: "{{ ansible_host | default(inventory_hostname) }}" + port: 22 + delay: 5 + timeout: 900 + delegate_to: localhost + become: no + + - name: Gather facts for app node + setup: ~ + - name: Release artifact sanity check failed fail: msg: | @@ -36,7 +49,7 @@ --------------------------------------------- - name: Set artifact build number fact if available - when: deploy_artifact_build_json_data.nr | default(false, true) + when: (deploy_artifact_build_json_data.nr | default('', true) | string | length) > 0 set_fact: mageops_artifact_build_nr: "{{ deploy_artifact_build_json_data.nr }}" @@ -52,8 +65,8 @@ - name: Gather deployment run facts set_fact: - deploy_release_id: "{{ ansible_date_time.iso8601_basic_short }}" - deploy_keep_current_release: "{{ deploy_skip_installation | default(false) and deploy_previous_release_exists }}" + deploy_release_id: "{{ ansible_facts['date_time']['iso8601_basic_short'] }}" + deploy_keep_current_release: "{{ (deploy_skip_installation | default(false) | bool) and (deploy_previous_release_exists | bool) }}" deploy_previous_release_id: "{{ deploy_previous_release_exists | ternary((_current_release_check.stat.lnk_target | default('')).rstrip('/').split('/') | last, false) }}" - name: "Skipping installation: reusing previous release" @@ -129,7 +142,7 @@ when: mageops_extra_tasks_deploy is defined and mageops_extra_tasks_deploy - name: Save deploy release information to file - when: deploy_release_save_info_file_path | default(false, true) + when: (deploy_release_save_info_file_path | default('', true) | string | length) > 0 copy: dest: "{{ deploy_release_save_info_file_path }}" content: "{{ deploy_release_info | to_nice_json }}" @@ -148,6 +161,7 @@ when: mageops_fstrim_enable vars: + ansible_python_interpreter: /usr/bin/python3.9 deploy_install_new_release: "{{ not deploy_keep_current_release }}" deploy_keep_current_releases: 2 deploy_copy_previous_release: yes diff --git a/site.step-50-ami.yml b/site.step-50-ami.yml index 3b45be39d..6275b0742 100644 --- a/site.step-50-ami.yml +++ b/site.step-50-ami.yml @@ -1,13 +1,15 @@ - hosts: localhost connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" roles: - role: cs.aws-node-facts - role: cs.aws-create-ami - ami_description: "{{ mageops_project | capitalize }} {{ mageops_environment | capitalize }} App Node ({{ ansible_date_time.iso8601 }})" + ami_description: "{{ mageops_project | capitalize }} {{ mageops_environment | capitalize }} App Node ({{ ansible_facts['date_time']['iso8601'] }})" ami_instance_id: "{{ aws_app_builder_node_instance.instance_id }}" - ami_name: "{{ mageops_app_name }}-app-{{ ansible_date_time.epoch }}" + ami_name: "{{ mageops_app_name }}-app-{{ ansible_facts['date_time']['epoch'] }}" ami_tags: "{{ aws_tags_role_app }}" aws_ami_build_ebs_volume_type: "{{ aws_app_node_ebs_volume_type | default(aws_ec2_ebs_volume_type, true) }}" @@ -22,4 +24,4 @@ aws_tags_role_app_builder ) }} - when: aws_ami_build_terminate_builder \ No newline at end of file + when: aws_ami_build_terminate_builder diff --git a/site.step-60-autoscaling.yml b/site.step-60-autoscaling.yml index 75309632a..79c9227ba 100644 --- a/site.step-60-autoscaling.yml +++ b/site.step-60-autoscaling.yml @@ -11,7 +11,7 @@ - block: - name: Enable Magento maintenance mode (if AMI release has DB migrations) command: "bin/magento maintenance:enable" - when: aws_ami_app_node_needs_db_migrations | default(true) + when: aws_ami_app_node_needs_db_migrations | default(true) | bool args: chdir: "{{ item.dir | default(magento_live_release_dir) }}" run_once: true @@ -35,11 +35,12 @@ - hosts: localhost connection: local vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" # Disable rolling updates when enabling maintenance mode to speed # the refresh up... autoscaling_rolling_instance_refresh_min_healthy_percent: >- {{ - aws_ami_app_node_needs_db_migrations | default(true) | ternary( + (aws_ami_app_node_needs_db_migrations | default(true) | bool) | ternary( 0, aws_autoscaling_rolling_instance_refresh_min_healthy_percent ) diff --git a/site.step-65-post-deploy.yml b/site.step-65-post-deploy.yml index 753410875..304725e66 100644 --- a/site.step-65-post-deploy.yml +++ b/site.step-65-post-deploy.yml @@ -45,4 +45,4 @@ - supervisord.service loop_control: loop_var: service_name - \ No newline at end of file + diff --git a/site.step-70-cleanup.yml b/site.step-70-cleanup.yml index 8becc7999..67f770638 100644 --- a/site.step-70-cleanup.yml +++ b/site.step-70-cleanup.yml @@ -1,5 +1,7 @@ - hosts: localhost connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" roles: - role: cs.aws-ec2-cleanup aws_ec2_cleanup_lt_to_keep: 3 diff --git a/site.step-80-monitoring.yml b/site.step-80-monitoring.yml index aa5ab4024..770c0214e 100644 --- a/site.step-80-monitoring.yml +++ b/site.step-80-monitoring.yml @@ -1,5 +1,7 @@ - hosts: localhost connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" roles: - role: cs.aws-region-facts - role: cs.aws-logs-slack diff --git a/vars_plugins/mageops_vars.py b/vars_plugins/mageops_vars.py index ec5586b18..acc4cda68 100644 --- a/vars_plugins/mageops_vars.py +++ b/vars_plugins/mageops_vars.py @@ -41,18 +41,38 @@ import os import re -from ansible import constants as C +from jinja2 import Environment +from jinja2.nativetypes import NativeEnvironment from ansible.errors import AnsibleParserError -from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text +from ansible.plugins.filter.core import FilterModule as CoreFilterModule from ansible.plugins.vars import BaseVarsPlugin from ansible.inventory.host import Host from ansible.inventory.group import Group +from ansible.plugins.test.core import TestModule as CoreTestModule from ansible.utils.vars import combine_vars CONFIG_GROUPS = ['all'] CONFIG_SUBDIR = 'vars' +GROUP_VARS_SUBDIR = 'group_vars' CONFIG_TYPES = ['global', 'project'] CONFIG_TYPE_FILES_CACHE = {} +YAML_EXTENSIONS = ('.yml', '.yaml') +MAX_TEMPLATE_PASSES = 5 +SIMPLE_VAR_PATTERN = re.compile(r'^\s*\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}\s*$') +FULL_TEMPLATE_PATTERN = re.compile(r'^\s*\{\{.*\}\}\s*$') + +# ansible-core newer than the legacy behavior no longer expands these `vars/*` +# strings reliably during later variable access. We keep a dedicated Jinja +# environment here and register Ansible core filters/tests explicitly so values +# from vars/project and vars/global can still reference other vars the same way +# older Ansible setups allowed. +JINJA_ENV = Environment() +JINJA_ENV.filters.update(CoreFilterModule().filters()) +JINJA_ENV.tests.update(CoreTestModule().tests()) +NATIVE_JINJA_ENV = NativeEnvironment() +NATIVE_JINJA_ENV.filters.update(CoreFilterModule().filters()) +NATIVE_JINJA_ENV.tests.update(CoreTestModule().tests()) def get_entity_name(entity): if isinstance(entity, Host): @@ -63,6 +83,122 @@ def get_entity_name(entity): return "unknown: %s" % (str(entity)) + +def should_skip_path(path): + return ( + re.search(r'/(tasks|certs|certificates|files|templates|resources|roles|playbooks)/', path) + or re.search(r'/(?:\.[^/]+|[^/]+~)(?:/|$)', path) + ) + + +def list_config_type_files(config_dir_path, config_type): + config_type_path = os.path.join(config_dir_path, config_type) + + if not os.path.exists(config_type_path) or should_skip_path(config_type_path): + return [] + + if os.path.isfile(config_type_path): + return [config_type_path] if config_type_path.endswith(YAML_EXTENSIONS) else [] + + found_files = [] + + for root, dirnames, filenames in os.walk(config_type_path): + dirnames[:] = [ + dirname for dirname in sorted(dirnames) + if not should_skip_path(os.path.join(root, dirname)) + ] + + for filename in sorted(filenames): + found = os.path.join(root, filename) + + if should_skip_path(found) or not found.endswith(YAML_EXTENSIONS): + continue + + found_files.append(found) + + return found_files + + +def load_yaml_file(loader, path): + if not os.path.exists(path) or should_skip_path(path) or not path.endswith(YAML_EXTENSIONS): + return {} + + data = loader.load_from_file(path, cache=True, unsafe=False) + + return data if data else {} + + +def load_group_vars_context(loader, base_dir): + group_vars_dir_path = os.path.join(base_dir, GROUP_VARS_SUBDIR) + context = {} + + for filename in ('all.yml', 'all.yaml'): + context = combine_vars(context, load_yaml_file(loader, os.path.join(group_vars_dir_path, filename))) + + return context + + +def resolve_templates_with_context(loader, base_variables, data): + rendered = data + + for _ in range(MAX_TEMPLATE_PASSES): + combined_variables = combine_vars(base_variables, rendered) + next_rendered = render_value(combined_variables, rendered) + + if next_rendered == rendered: + break + + rendered = next_rendered + + return rendered + + +def render_string(variables, value): + try: + if FULL_TEMPLATE_PATTERN.match(value): + # Preserve native types for values that are entirely a Jinja + # expression. Newer ansible-core stopped resolving these reliably + # for vars loaded from vars/*, but conditionals still need actual + # booleans instead of "True"/"False" strings. + return NATIVE_JINJA_ENV.from_string(value).render(variables) + + return JINJA_ENV.from_string(value).render(variables) + except Exception: + return value + + +def render_value(variables, value): + if isinstance(value, dict): + rendered_dict = {} + + for key, item in value.items(): + rendered_key = render_value(variables, key) + rendered_item = render_value(variables, item) + rendered_dict[rendered_key] = rendered_item + + return rendered_dict + + if isinstance(value, list): + rendered_list = [] + + for item in value: + rendered_list.append(render_value(variables, item)) + + return rendered_list + + if isinstance(value, tuple): + rendered_items = [] + + for item in value: + rendered_items.append(render_value(variables, item)) + + return tuple(rendered_items) + + if isinstance(value, str): + return render_string(variables, value) + + return value + class VarsModule(BaseVarsPlugin): REQUIRES_WHITELIST = False @@ -79,15 +215,30 @@ def get_vars(self, loader, path, entities, cache=True): if self._basedir is None or self._basedir == 'None': return {} - processable_entities = [entity for entity in entities if isinstance(entity, Group) and entity.get_name() in CONFIG_GROUPS] + processable_entities = [ + entity for entity in entities + if ( + isinstance(entity, Host) + or (isinstance(entity, Group) and entity.get_name() in CONFIG_GROUPS) + ) + ] - # We want to load vars only for the 'all' group + # We want to load vars only for regular inventory entities. if len(processable_entities) == 0: return {} # Skipping tasks templates and vars config_dir_path = to_text(os.path.realpath(to_bytes(os.path.join(self._basedir, CONFIG_SUBDIR)))) data = {} + # Older Ansible behavior effectively let vars loaded from `vars/*` + # reference values from `group_vars/all.yml`. Newer ansible-core leaves + # many such strings unresolved, so we explicitly seed the render + # context with group_vars/all before loading MageOps vars files. + entity_vars = load_group_vars_context(loader, self._basedir) + + for entity in processable_entities: + if hasattr(entity, 'get_vars'): + entity_vars = combine_vars(entity_vars, entity.get_vars()) try: if os.path.isdir(config_dir_path): @@ -102,18 +253,18 @@ def get_vars(self, loader, path, entities, cache=True): if cache and config_type in CONFIG_TYPE_FILES_CACHE: found_files = CONFIG_TYPE_FILES_CACHE[config_type] else: - found_files = loader.find_vars_files(config_dir_path, config_type, allow_dir=True) + found_files = list_config_type_files(config_dir_path, config_type) + if cache: + CONFIG_TYPE_FILES_CACHE[config_type] = found_files for found in found_files: - if re.search(r'/(tasks|certs|certificates|files|templates|resources|roles|playbooks)/', found): - continue - self._display.vv("Loading MageOps configuration file: %s" % (found)) - new_data = loader.load_from_file(found, cache=True, unsafe=True) + new_data = load_yaml_file(loader, found) if new_data: data = combine_vars(data, new_data) + data = resolve_templates_with_context(loader, entity_vars, data) else: self._display.v("Skipping non-existent MageOps configuration dir: %s" % (config_dir_path))