From 464099991876aef13e569d6abbbebc5793aefe33 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Fri, 17 Nov 2023 11:09:15 +0100 Subject: [PATCH 01/19] Add possibility to switch pkg manager with configuration option --- group_vars/all.yml | 5 +++++ roles/cs.switch-pkg-mgr/defaults/main.yml | 1 + .../files/dnf.conf | 0 roles/cs.switch-pkg-mgr/files/yum.conf | 7 +++++++ .../main.yml => cs.switch-pkg-mgr/tasks/dnf.yml} | 0 roles/cs.switch-pkg-mgr/tasks/main.yml | 12 ++++++++++++ roles/cs.switch-pkg-mgr/tasks/yum.yml | 9 +++++++++ site.step-15-varnish.yml | 3 ++- site.step-20-persistent.yml | 3 ++- site.step-40-app-node.yml | 3 ++- site.step-45-app-deploy.yml | 3 ++- 11 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 roles/cs.switch-pkg-mgr/defaults/main.yml rename roles/{cs.switch-to-dnf => cs.switch-pkg-mgr}/files/dnf.conf (100%) create mode 100644 roles/cs.switch-pkg-mgr/files/yum.conf rename roles/{cs.switch-to-dnf/tasks/main.yml => cs.switch-pkg-mgr/tasks/dnf.yml} (100%) create mode 100644 roles/cs.switch-pkg-mgr/tasks/main.yml create mode 100644 roles/cs.switch-pkg-mgr/tasks/yum.yml diff --git a/group_vars/all.yml b/group_vars/all.yml index 80dce4642..cf8dbc68d 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1820,6 +1820,11 @@ mageops_cli_features_dir: /usr/local/lib/mageops/features # Whether to perform full update mageops_packages_full_update: yes +# Package manager to use +mageops_pkg_mgr: + # Supported options: dnf, yum + centos7: dnf + # Packages that are ensured to be absent on all nodes mageops_packages_mirrorlist_countrycode: "de" diff --git a/roles/cs.switch-pkg-mgr/defaults/main.yml b/roles/cs.switch-pkg-mgr/defaults/main.yml new file mode 100644 index 000000000..e1274760e --- /dev/null +++ b/roles/cs.switch-pkg-mgr/defaults/main.yml @@ -0,0 +1 @@ +switch_pkg_mgr_option: "" diff --git a/roles/cs.switch-to-dnf/files/dnf.conf b/roles/cs.switch-pkg-mgr/files/dnf.conf similarity index 100% rename from roles/cs.switch-to-dnf/files/dnf.conf rename to roles/cs.switch-pkg-mgr/files/dnf.conf diff --git a/roles/cs.switch-pkg-mgr/files/yum.conf b/roles/cs.switch-pkg-mgr/files/yum.conf new file mode 100644 index 000000000..d04e79d4d --- /dev/null +++ b/roles/cs.switch-pkg-mgr/files/yum.conf @@ -0,0 +1,7 @@ +[main] +gpgcheck=True +installonly_limit=3 +clean_requirements_on_remove=True +best=False +fastestmirror=True +max_parallel_downloads=20 diff --git a/roles/cs.switch-to-dnf/tasks/main.yml b/roles/cs.switch-pkg-mgr/tasks/dnf.yml similarity index 100% rename from roles/cs.switch-to-dnf/tasks/main.yml rename to roles/cs.switch-pkg-mgr/tasks/dnf.yml diff --git a/roles/cs.switch-pkg-mgr/tasks/main.yml b/roles/cs.switch-pkg-mgr/tasks/main.yml new file mode 100644 index 000000000..b72899007 --- /dev/null +++ b/roles/cs.switch-pkg-mgr/tasks/main.yml @@ -0,0 +1,12 @@ +- name: Switch to dnf + include_tasks: dnf.yml + when: switch_pkg_mgr_option == 'dnf' + +- name: Switch to yum + include_tasks: yum.yml + when: switch_pkg_mgr_option == 'yum' + +- name: Fail if option is invalid + fail: + msg: "Invalid switch_pkg_mgr_option value: {{ switch_pkg_mgr_option }}" + when: switch_pkg_mgr_option != 'dnf' and switch_pkg_mgr_option != 'yum' and switch_pkg_mgr_option is not empty diff --git a/roles/cs.switch-pkg-mgr/tasks/yum.yml b/roles/cs.switch-pkg-mgr/tasks/yum.yml new file mode 100644 index 000000000..97b0b493c --- /dev/null +++ b/roles/cs.switch-pkg-mgr/tasks/yum.yml @@ -0,0 +1,9 @@ +- name: Set yum configuration + copy: + src: yum.conf + dest: /etc/yum.conf + +- name: Switch package manager to yum + set_fact: + ansible_facts: + pkg_mgr: yum diff --git a/site.step-15-varnish.yml b/site.step-15-varnish.yml index 2ca4a8ac3..a4b8084ea 100644 --- a/site.step-15-varnish.yml +++ b/site.step-15-varnish.yml @@ -6,7 +6,8 @@ delegate_to: localhost become: no when: aws_use - - role: cs.switch-to-dnf + - role: cs.switch-pkg-mgr + switch_pkg_mgr_option: "{{ mageops_pkg_mgr.centos7 }}" - role: cs.versionlock node_name: varnish versionlock_packages: "{{ versionlock_varnish_packages + versionlock_varnish_packages_base + versionlock_varnish_packages_extra }}" diff --git a/site.step-20-persistent.yml b/site.step-20-persistent.yml index 608d69830..30e74c5ad 100644 --- a/site.step-20-persistent.yml +++ b/site.step-20-persistent.yml @@ -16,7 +16,8 @@ delegate_facts: no become: no when: aws_use - - role: cs.switch-to-dnf + - role: cs.switch-pkg-mgr + switch_pkg_mgr_option: "{{ mageops_pkg_mgr.centos7 }}" - role: cs.versionlock node_name: persistent versionlock_packages: "{{ versionlock_persistent_packages + versionlock_persistent_packages_base + versionlock_persistent_packages_extra }}" diff --git a/site.step-40-app-node.yml b/site.step-40-app-node.yml index 4905cb169..45fb64b8f 100644 --- a/site.step-40-app-node.yml +++ b/site.step-40-app-node.yml @@ -22,7 +22,8 @@ become: no when: aws_use - - role: cs.switch-to-dnf + - role: cs.switch-pkg-mgr + switch_pkg_mgr_option: "{{ mageops_pkg_mgr.centos7 }}" - role: cs.mageops-cli-user mageops_cli_user: root diff --git a/site.step-45-app-deploy.yml b/site.step-45-app-deploy.yml index de473c551..97ef4edc3 100644 --- a/site.step-45-app-deploy.yml +++ b/site.step-45-app-deploy.yml @@ -87,7 +87,8 @@ become: no when: aws_use - - role: cs.switch-to-dnf + - role: cs.switch-pkg-mgr + switch_pkg_mgr_option: "{{ mageops_pkg_mgr.centos7 }}" - role: cs.deploy deploy_install_actions: - name: Configure Magento From 3a2c89a5cde7b7bc79c9fec62fc9d3683df538f7 Mon Sep 17 00:00:00 2001 From: Piotr Matras Date: Tue, 31 Oct 2023 07:53:57 +0100 Subject: [PATCH 02/19] feat: MOPS-326 queue consumers optimization --- roles/cs.magento-configure/defaults/main/app-etc.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/cs.magento-configure/defaults/main/app-etc.yml b/roles/cs.magento-configure/defaults/main/app-etc.yml index a130a3b1c..7c4680958 100644 --- a/roles/cs.magento-configure/defaults/main/app-etc.yml +++ b/roles/cs.magento-configure/defaults/main/app-etc.yml @@ -165,6 +165,9 @@ magento_app_etc_config_consumer_workers: cron_consumers_runner: cron_run: false max_messages: "{{ magento_consumer_workers_max_messages | default(500) }}" + queue: + consumers_wait_for_messages: 0 + only_spawn_when_message_available: 1 magento_app_etc_config_cron_consumers: cron_consumers_runner: From 9acebd3a54abe58b245e052094fd3ad495c7069c Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Tue, 12 Dec 2023 08:48:58 +0100 Subject: [PATCH 03/19] Add missing dependency to new relic role --- roles/cs.new-relic/meta/main.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 roles/cs.new-relic/meta/main.yml diff --git a/roles/cs.new-relic/meta/main.yml b/roles/cs.new-relic/meta/main.yml new file mode 100644 index 000000000..68605ed6d --- /dev/null +++ b/roles/cs.new-relic/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - role: cs.mageops-cli + when: aws_use From 9ac5381b2f809b6ccdfa497273bc5b4fe10cb995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3=C5=BAwiak?= Date: Tue, 28 Feb 2023 08:46:11 +0100 Subject: [PATCH 04/19] Deployment process finish without checking if Magento can pass the traffic. This commit check if WARMUP is accessible on app nodes with timeout (magento_node_wait_for_warmup_secs) --- group_vars/all.yml | 1 + roles/cs.wait-for-warmup/defaults/main.yml | 2 ++ roles/cs.wait-for-warmup/tasks/main.yml | 10 ++++++++++ site.step-80-monitoring.yml | 6 ++++++ 4 files changed, 19 insertions(+) create mode 100644 roles/cs.wait-for-warmup/defaults/main.yml create mode 100644 roles/cs.wait-for-warmup/tasks/main.yml diff --git a/group_vars/all.yml b/group_vars/all.yml index cf8dbc68d..cc1a9c204 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -955,6 +955,7 @@ magento_admin_user_lastname: Suite # Path to node warmup script executed at instance start relative to magento app root dir - never need to override... magento_node_warmup_script_path: /bin/node-warmup.sh +mageops_wait_for_warmup_secs: 600 # ------------------------------ # -------- Magento SCD -------- diff --git a/roles/cs.wait-for-warmup/defaults/main.yml b/roles/cs.wait-for-warmup/defaults/main.yml new file mode 100644 index 000000000..c270a16a7 --- /dev/null +++ b/roles/cs.wait-for-warmup/defaults/main.yml @@ -0,0 +1,2 @@ +wait_for_warmup_retries: "{{ (mageops_wait_for_warmup_secs / wait_for_warmup_delay)|int }}" +wait_for_warmup_delay: 5 diff --git a/roles/cs.wait-for-warmup/tasks/main.yml b/roles/cs.wait-for-warmup/tasks/main.yml new file mode 100644 index 000000000..43e2a261f --- /dev/null +++ b/roles/cs.wait-for-warmup/tasks/main.yml @@ -0,0 +1,10 @@ +- name: Deployment finished, waiting for magento warmup. + uri: + method: HEAD + url: "http://localhost/WARMUP" + status_code: 200 + register: _warmup_result + until: _warmup_result is not failed + retries: "{{ wait_for_warmup_retries }}" + delay: "{{ wait_for_warmup_delay }}" + changed_when: false diff --git a/site.step-80-monitoring.yml b/site.step-80-monitoring.yml index aa5ab4024..4bae517c2 100644 --- a/site.step-80-monitoring.yml +++ b/site.step-80-monitoring.yml @@ -1,3 +1,5 @@ +- import_playbook: site.common.group-current-hosts.yml + - hosts: localhost connection: local roles: @@ -5,3 +7,7 @@ - role: cs.aws-logs-slack when: aws_logs_slack_notifications - role: cs.aws-logs-retention +- hosts: app:¤t + roles: + - role: cs.wait-for-warmup + run_once: true From d233c8cffa7b07963ecd4b6ede842be617b5bb6c Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Wed, 13 Dec 2023 16:23:09 +0100 Subject: [PATCH 05/19] Resolve issue with basic 503 error page --- .../cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 b/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 index 252040bce..0f6a2600c 100644 --- a/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 +++ b/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 @@ -40,7 +40,14 @@ if (req.http.{{ varnish_bypass_request_info_header_name }}) { set resp.http.Cache-Control = "no-cache"; } -if (resp.status == 502 || resp.status == 504) { +if (req.http.{{ varnish_bypass_request_info_header_name }}) { + set resp.http.{{ varnish_bypass_request_info_header_name }} = + req.http.{{ varnish_bypass_request_info_header_name }} + "; Backend-Response-Delivered"; + set resp.http.Set-Cookie = "{{ varnish_bypass_request_cookie_name}} = {{ varnish_bypass_request_token }}; Secure"; + set resp.http.Cache-Control = "no-cache"; +} + +if (resp.status == 502 || resp.status == 503 || resp.status == 504) { return(synth(resp.status)); } From 58b9a2184a3160acd5b9ae1930f87684d662b79d Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Thu, 5 May 2022 14:37:30 +0200 Subject: [PATCH 06/19] Make sure that awscli is installed before boto ans boto3 Order of dependencies in requirements does matter and pip isn't smart enought to choice correct versions when there is conflict Also remove outdated cloudfront override --- library/cloudfront_distribution.py | 2008 ---------------------------- requirements-python.txt | 4 + 2 files changed, 4 insertions(+), 2008 deletions(-) delete mode 100644 library/cloudfront_distribution.py diff --git a/library/cloudfront_distribution.py b/library/cloudfront_distribution.py deleted file mode 100644 index 8e6907eaf..000000000 --- a/library/cloudfront_distribution.py +++ /dev/null @@ -1,2008 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2017 Ansible Project -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -### WARNING!!! PATCHED MODULE AHEAD! #### -# This module override an be removed once the PR is merged -# and we migrate to the appropriate version - 2.10.x at the soonest. -# Fixes: https://github.com/ansible/ansible/issues/68354 -# This file was taken from ansible 2.7.16 and patched with: -# https://github.com/ansible/ansible/pull/68355 -# /FS - -ANSIBLE_METADATA = {'metadata_version': '1.1', - 'status': ['preview'], - 'supported_by': 'community'} - -DOCUMENTATION = ''' ---- - -module: cloudfront_distribution - -short_description: create, update and delete aws cloudfront distributions. - -description: - - Allows for easy creation, updating and deletion of CloudFront distributions. - -requirements: - - boto3 >= 1.0.0 - - python >= 2.6 - -version_added: "2.5" - -author: - - Willem van Ketwich (@wilvk) - - Will Thames (@willthames) - -extends_documentation_fragment: - - aws - - ec2 - -options: - - state: - description: - - The desired state of the distribution - present - creates a new distribution or updates an existing distribution. - absent - deletes an existing distribution. - choices: ['present', 'absent'] - default: 'present' - - distribution_id: - description: - - The id of the cloudfront distribution. This parameter can be exchanged with I(alias) or I(caller_reference) and is used in conjunction with I(e_tag). - - e_tag: - description: - - A unique identifier of a modified or existing distribution. Used in conjunction with I(distribution_id). - Is determined automatically if not specified. - - caller_reference: - description: - - A unique identifier for creating and updating cloudfront distributions. Each caller reference must be unique across all distributions. e.g. a caller - reference used in a web distribution cannot be reused in a streaming distribution. This parameter can be used instead of I(distribution_id) - to reference an existing distribution. If not specified, this defaults to a datetime stamp of the format - 'YYYY-MM-DDTHH:MM:SS.ffffff'. - - tags: - description: - - Should be input as a dict() of key-value pairs. - Note that numeric keys or values must be wrapped in quotes. e.g. "Priority:" '1' - - purge_tags: - description: - - Specifies whether existing tags will be removed before adding new tags. When I(purge_tags=yes), existing tags are removed and I(tags) are added, if - specified. If no tags are specified, it removes all existing tags for the distribution. When I(purge_tags=no), existing tags are kept and I(tags) - are added, if specified. - default: 'no' - type: bool - - alias: - description: - - The name of an alias (CNAME) that is used in a distribution. This is used to effectively reference a distribution by its alias as an alias can only - be used by one distribution per AWS account. This variable avoids having to provide the I(distribution_id) as well as - the I(e_tag), or I(caller_reference) of an existing distribution. - - aliases: - description: - - A I(list[]) of domain name aliases (CNAMEs) as strings to be used for the distribution. Each alias must be unique across all distribution for the AWS - account. - - purge_aliases: - description: - - Specifies whether existing aliases will be removed before adding new aliases. When I(purge_aliases=yes), existing aliases are removed and I(aliases) - are added. - default: 'no' - type: bool - - default_root_object: - description: - - A config element that specifies the path to request when the user requests the origin. e.g. if specified as 'index.html', this maps to - www.example.com/index.html when www.example.com is called by the user. This prevents the entire distribution origin from being exposed at the root. - - default_origin_domain_name: - description: - - The domain name to use for an origin if no I(origins) have been specified. Should only be used on a first run of generating a distribution and not on - subsequent runs. Should not be used in conjunction with I(distribution_id), I(caller_reference) or I(alias). - - default_origin_path: - description: - - The default origin path to specify for an origin if no I(origins) have been specified. Defaults to empty if not specified. - - origins: - description: - - A config element that is a I(list[]) of complex origin objects to be specified for the distribution. Used for creating and updating distributions. - Each origin item comprises the attributes - I(id) - I(domain_name) (defaults to default_origin_domain_name if not specified) - I(origin_path) (defaults to default_origin_path if not specified) - I(custom_headers[]) - I(header_name) - I(header_value) - I(s3_origin_access_identity_enabled) - I(custom_origin_config) - I(http_port) - I(https_port) - I(origin_protocol_policy) - I(origin_ssl_protocols[]) - I(origin_read_timeout) - I(origin_keepalive_timeout) - - purge_origins: - description: Whether to remove any origins that aren't listed in I(origins) - default: false - - default_cache_behavior: - description: - - A config element that is a complex object specifying the default cache behavior of the distribution. If not specified, the I(target_origin_id) is - defined as the I(target_origin_id) of the first valid I(cache_behavior) in I(cache_behaviors) with defaults. - The default cache behavior comprises the attributes - I(target_origin_id) - I(forwarded_values) - I(query_string) - I(cookies) - I(forward) - I(whitelisted_names) - I(headers[]) - I(query_string_cache_keys[]) - I(trusted_signers) - I(enabled) - I(items[]) - I(viewer_protocol_policy) - I(min_ttl) - I(allowed_methods) - I(items[]) - I(cached_methods[]) - I(smooth_streaming) - I(default_ttl) - I(max_ttl) - I(compress) - I(lambda_function_associations[]) - I(lambda_function_arn) - I(event_type) - I(field_level_encryption_id) - - cache_behaviors: - description: - - A config element that is a I(list[]) of complex cache behavior objects to be specified for the distribution. The order - of the list is preserved across runs unless C(purge_cache_behavior) is enabled. - Each cache behavior comprises the attributes - I(path_pattern) - I(target_origin_id) - I(forwarded_values) - I(query_string) - I(cookies) - I(forward) - I(whitelisted_names) - I(headers[]) - I(query_string_cache_keys[]) - I(trusted_signers) - I(enabled) - I(items[]) - I(viewer_protocol_policy) - I(min_ttl) - I(allowed_methods) - I(items[]) - I(cached_methods[]) - I(smooth_streaming) - I(default_ttl) - I(max_ttl) - I(compress) - I(lambda_function_associations[]) - I(field_level_encryption_id) - - purge_cache_behaviors: - description: Whether to remove any cache behaviors that aren't listed in I(cache_behaviors). This switch - also allows the reordering of cache_behaviors. - default: false - - custom_error_responses: - description: - - A config element that is a I(list[]) of complex custom error responses to be specified for the distribution. This attribute configures custom http - error messages returned to the user. - Each custom error response object comprises the attributes - I(error_code) - I(reponse_page_path) - I(response_code) - I(error_caching_min_ttl) - - purge_custom_error_responses: - description: Whether to remove any custom error responses that aren't listed in I(custom_error_responses) - default: false - - comment: - description: - - A comment that describes the cloudfront distribution. If not specified, it defaults to a - generic message that it has been created with Ansible, and a datetime stamp. - - logging: - description: - - A config element that is a complex object that defines logging for the distribution. - The logging object comprises the attributes - I(enabled) - I(include_cookies) - I(bucket) - I(prefix) - - price_class: - description: - - A string that specifies the pricing class of the distribution. As per - U(https://aws.amazon.com/cloudfront/pricing/) - I(price_class=PriceClass_100) consists of the areas - United States - Canada - Europe - I(price_class=PriceClass_200) consists of the areas - United States - Canada - Europe - Hong Kong, Philippines, S. Korea, Singapore & Taiwan - Japan - India - I(price_class=PriceClass_All) consists of the areas - United States - Canada - Europe - Hong Kong, Philippines, S. Korea, Singapore & Taiwan - Japan - India - South America - Australia - choices: ['PriceClass_100', 'PriceClass_200', 'PriceClass_All'] - default: aws defaults this to 'PriceClass_All' - - enabled: - description: - - A boolean value that specifies whether the distribution is enabled or disabled. - default: 'yes' - type: bool - - viewer_certificate: - description: - - A config element that is a complex object that specifies the encryption details of the distribution. - Comprises the following attributes - I(cloudfront_default_certificate) - I(iam_certificate_id) - I(acm_certificate_arn) - I(ssl_support_method) - I(minimum_protocol_version) - I(certificate) - I(certificate_source) - - restrictions: - description: - - A config element that is a complex object that describes how a distribution should restrict it's content. - The restriction object comprises the following attributes - I(geo_restriction) - I(restriction_type) - I(items[]) - - web_acl_id: - description: - - The id of a Web Application Firewall (WAF) Access Control List (ACL). - - http_version: - description: - - The version of the http protocol to use for the distribution. - choices: [ 'http1.1', 'http2' ] - default: aws defaults this to 'http2' - - ipv6_enabled: - description: - - Determines whether IPv6 support is enabled or not. - type: bool - default: 'no' - - wait: - description: - - Specifies whether the module waits until the distribution has completed processing the creation or update. - type: bool - default: 'no' - - wait_timeout: - description: - - Specifies the duration in seconds to wait for a timeout of a cloudfront create or update. Defaults to 1800 seconds (30 minutes). - default: 1800 - -''' - -EXAMPLES = ''' - -# create a basic distribution with defaults and tags - -- cloudfront_distribution: - state: present - default_origin_domain_name: www.my-cloudfront-origin.com - tags: - Name: example distribution - Project: example project - Priority: '1' - -# update a distribution comment by distribution_id - -- cloudfront_distribution: - state: present - distribution_id: E1RP5A2MJ8073O - comment: modified by ansible cloudfront.py - -# update a distribution comment by caller_reference - -- cloudfront_distribution: - state: present - caller_reference: my cloudfront distribution 001 - comment: modified by ansible cloudfront.py - -# update a distribution's aliases and comment using the distribution_id as a reference - -- cloudfront_distribution: - state: present - distribution_id: E1RP5A2MJ8073O - comment: modified by cloudfront.py again - aliases: [ 'www.my-distribution-source.com', 'zzz.aaa.io' ] - -# update a distribution's aliases and comment using an alias as a reference - -- cloudfront_distribution: - state: present - caller_reference: my test distribution - comment: modified by cloudfront.py again - aliases: - - www.my-distribution-source.com - - zzz.aaa.io - -# update a distribution's comment and aliases and tags and remove existing tags - -- cloudfront_distribution: - state: present - distribution_id: E15BU8SDCGSG57 - comment: modified by cloudfront.py again - aliases: - - tested.com - tags: - Project: distribution 1.2 - purge_tags: yes - -# create a distribution with an origin, logging and default cache behavior - -- cloudfront_distribution: - state: present - caller_reference: unique test distribution id - origins: - - id: 'my test origin-000111' - domain_name: www.example.com - origin_path: /production - custom_headers: - - header_name: MyCustomHeaderName - header_value: MyCustomHeaderValue - default_cache_behavior: - target_origin_id: 'my test origin-000111' - forwarded_values: - query_string: true - cookies: - forward: all - headers: - - '*' - viewer_protocol_policy: allow-all - smooth_streaming: true - compress: true - allowed_methods: - items: - - GET - - HEAD - cached_methods: - - GET - - HEAD - logging: - enabled: true - include_cookies: false - bucket: mylogbucket.s3.amazonaws.com - prefix: myprefix/ - enabled: false - comment: this is a cloudfront distribution with logging - -# delete a distribution - -- cloudfront_distribution: - state: absent - caller_reference: replaceable distribution -''' - -RETURN = ''' -active_trusted_signers: - description: Key pair IDs that CloudFront is aware of for each trusted signer - returned: always - type: complex - contains: - enabled: - description: Whether trusted signers are in use - returned: always - type: bool - sample: false - quantity: - description: Number of trusted signers - returned: always - type: int - sample: 1 - items: - description: Number of trusted signers - returned: when there are trusted signers - type: list - sample: - - key_pair_id -aliases: - description: Aliases that refer to the distribution - returned: always - type: complex - contains: - items: - description: List of aliases - returned: always - type: list - sample: - - test.example.com - quantity: - description: Number of aliases - returned: always - type: int - sample: 1 -arn: - description: Amazon Resource Name of the distribution - returned: always - type: string - sample: arn:aws:cloudfront::123456789012:distribution/E1234ABCDEFGHI -cache_behaviors: - description: Cloudfront cache behaviors - returned: always - type: complex - contains: - items: - description: List of cache behaviors - returned: always - type: complex - contains: - allowed_methods: - description: Methods allowed by the cache behavior - returned: always - type: complex - contains: - cached_methods: - description: Methods cached by the cache behavior - returned: always - type: complex - contains: - items: - description: List of cached methods - returned: always - type: list - sample: - - HEAD - - GET - quantity: - description: Count of cached methods - returned: always - type: int - sample: 2 - items: - description: List of methods allowed by the cache behavior - returned: always - type: list - sample: - - HEAD - - GET - quantity: - description: Count of methods allowed by the cache behavior - returned: always - type: int - sample: 2 - compress: - description: Whether compression is turned on for the cache behavior - returned: always - type: bool - sample: false - default_ttl: - description: Default Time to Live of the cache behavior - returned: always - type: int - sample: 86400 - forwarded_values: - description: Values forwarded to the origin for this cache behavior - returned: always - type: complex - contains: - cookies: - description: Cookies to forward to the origin - returned: always - type: complex - contains: - forward: - description: Which cookies to forward to the origin for this cache behavior - returned: always - type: string - sample: none - whitelisted_names: - description: The names of the cookies to forward to the origin for this cache behavior - returned: when I(forward) is C(whitelist) - type: complex - contains: - quantity: - description: Count of cookies to forward - returned: always - type: int - sample: 1 - items: - description: List of cookies to forward - returned: when list is not empty - type: list - sample: my_cookie - headers: - description: Which headers are used to vary on cache retrievals - returned: always - type: complex - contains: - quantity: - description: Count of headers to vary on - returned: always - type: int - sample: 1 - items: - description: List of headers to vary on - returned: when list is not empty - type: list - sample: - - Host - query_string: - description: Whether the query string is used in cache lookups - returned: always - type: bool - sample: false - query_string_cache_keys: - description: Which query string keys to use in cache lookups - returned: always - type: complex - contains: - quantity: - description: Count of query string cache keys to use in cache lookups - returned: always - type: int - sample: 1 - items: - description: List of query string cache keys to use in cache lookups - returned: when list is not empty - type: list - sample: - lambda_function_associations: - description: Lambda function associations for a cache behavior - returned: always - type: complex - contains: - quantity: - description: Count of lambda function associations - returned: always - type: int - sample: 1 - items: - description: List of lambda function associations - returned: when list is not empty - type: list - sample: - - lambda_function_arn: arn:aws:lambda:123456789012:us-east-1/lambda/lambda-function - event_type: viewer-response - max_ttl: - description: Maximum Time to Live - returned: always - type: int - sample: 31536000 - min_ttl: - description: Minimum Time to Live - returned: always - type: int - sample: 0 - path_pattern: - description: Path pattern that determines this cache behavior - returned: always - type: string - sample: /path/to/files/* - smooth_streaming: - description: Whether smooth streaming is enabled - returned: always - type: bool - sample: false - target_origin_id: - description: Id of origin reference by this cache behavior - returned: always - type: string - sample: origin_abcd - trusted_signers: - description: Trusted signers - returned: always - type: complex - contains: - enabled: - description: Whether trusted signers are enabled for this cache behavior - returned: always - type: bool - sample: false - quantity: - description: Count of trusted signers - returned: always - type: int - sample: 1 - viewer_protocol_policy: - description: Policy of how to handle http/https - returned: always - type: string - sample: redirect-to-https - quantity: - description: Count of cache behaviors - returned: always - type: int - sample: 1 - -caller_reference: - description: Idempotency reference given when creating cloudfront distribution - returned: always - type: string - sample: '1484796016700' -comment: - description: Any comments you want to include about the distribution - returned: always - type: string - sample: 'my first cloudfront distribution' -custom_error_responses: - description: Custom error responses to use for error handling - returned: always - type: complex - contains: - items: - description: List of custom error responses - returned: always - type: complex - contains: - error_caching_min_ttl: - description: Mininum time to cache this error response - returned: always - type: int - sample: 300 - error_code: - description: Origin response code that triggers this error response - returned: always - type: int - sample: 500 - response_code: - description: Response code to return to the requester - returned: always - type: string - sample: '500' - response_page_path: - description: Path that contains the error page to display - returned: always - type: string - sample: /errors/5xx.html - quantity: - description: Count of custom error response items - returned: always - type: int - sample: 1 -default_cache_behavior: - description: Default cache behavior - returned: always - type: complex - contains: - allowed_methods: - description: Methods allowed by the cache behavior - returned: always - type: complex - contains: - cached_methods: - description: Methods cached by the cache behavior - returned: always - type: complex - contains: - items: - description: List of cached methods - returned: always - type: list - sample: - - HEAD - - GET - quantity: - description: Count of cached methods - returned: always - type: int - sample: 2 - items: - description: List of methods allowed by the cache behavior - returned: always - type: list - sample: - - HEAD - - GET - quantity: - description: Count of methods allowed by the cache behavior - returned: always - type: int - sample: 2 - compress: - description: Whether compression is turned on for the cache behavior - returned: always - type: bool - sample: false - default_ttl: - description: Default Time to Live of the cache behavior - returned: always - type: int - sample: 86400 - forwarded_values: - description: Values forwarded to the origin for this cache behavior - returned: always - type: complex - contains: - cookies: - description: Cookies to forward to the origin - returned: always - type: complex - contains: - forward: - description: Which cookies to forward to the origin for this cache behavior - returned: always - type: string - sample: none - whitelisted_names: - description: The names of the cookies to forward to the origin for this cache behavior - returned: when I(forward) is C(whitelist) - type: complex - contains: - quantity: - description: Count of cookies to forward - returned: always - type: int - sample: 1 - items: - description: List of cookies to forward - returned: when list is not empty - type: list - sample: my_cookie - headers: - description: Which headers are used to vary on cache retrievals - returned: always - type: complex - contains: - quantity: - description: Count of headers to vary on - returned: always - type: int - sample: 1 - items: - description: List of headers to vary on - returned: when list is not empty - type: list - sample: - - Host - query_string: - description: Whether the query string is used in cache lookups - returned: always - type: bool - sample: false - query_string_cache_keys: - description: Which query string keys to use in cache lookups - returned: always - type: complex - contains: - quantity: - description: Count of query string cache keys to use in cache lookups - returned: always - type: int - sample: 1 - items: - description: List of query string cache keys to use in cache lookups - returned: when list is not empty - type: list - sample: - lambda_function_associations: - description: Lambda function associations for a cache behavior - returned: always - type: complex - contains: - quantity: - description: Count of lambda function associations - returned: always - type: int - sample: 1 - items: - description: List of lambda function associations - returned: when list is not empty - type: list - sample: - - lambda_function_arn: arn:aws:lambda:123456789012:us-east-1/lambda/lambda-function - event_type: viewer-response - max_ttl: - description: Maximum Time to Live - returned: always - type: int - sample: 31536000 - min_ttl: - description: Minimum Time to Live - returned: always - type: int - sample: 0 - path_pattern: - description: Path pattern that determines this cache behavior - returned: always - type: string - sample: /path/to/files/* - smooth_streaming: - description: Whether smooth streaming is enabled - returned: always - type: bool - sample: false - target_origin_id: - description: Id of origin reference by this cache behavior - returned: always - type: string - sample: origin_abcd - trusted_signers: - description: Trusted signers - returned: always - type: complex - contains: - enabled: - description: Whether trusted signers are enabled for this cache behavior - returned: always - type: bool - sample: false - quantity: - description: Count of trusted signers - returned: always - type: int - sample: 1 - viewer_protocol_policy: - description: Policy of how to handle http/https - returned: always - type: string - sample: redirect-to-https -default_root_object: - description: The object that you want CloudFront to request from your origin (for example, index.html) - when a viewer requests the root URL for your distribution - returned: always - type: string - sample: '' -diff: - description: Difference between previous configuration and new configuration - returned: always - type: dict - sample: {} -domain_name: - description: Domain name of cloudfront distribution - returned: always - type: string - sample: d1vz8pzgurxosf.cloudfront.net -enabled: - description: Whether the cloudfront distribution is enabled or not - returned: always - type: bool - sample: true -http_version: - description: Version of HTTP supported by the distribution - returned: always - type: string - sample: http2 -id: - description: Cloudfront distribution ID - returned: always - type: string - sample: E123456ABCDEFG -in_progress_invalidation_batches: - description: The number of invalidation batches currently in progress - returned: always - type: int - sample: 0 -is_ipv6_enabled: - description: Whether IPv6 is enabled - returned: always - type: bool - sample: true -last_modified_time: - description: Date and time distribution was last modified - returned: always - type: string - sample: '2017-10-13T01:51:12.656000+00:00' -logging: - description: Logging information - returned: always - type: complex - contains: - bucket: - description: S3 bucket logging destination - returned: always - type: string - sample: logs-example-com.s3.amazonaws.com - enabled: - description: Whether logging is enabled - returned: always - type: bool - sample: true - include_cookies: - description: Whether to log cookies - returned: always - type: bool - sample: false - prefix: - description: Prefix added to logging object names - returned: always - type: string - sample: cloudfront/test -origins: - description: Origins in the cloudfront distribution - returned: always - type: complex - contains: - items: - description: List of origins - returned: always - type: complex - contains: - custom_headers: - description: Custom headers passed to the origin - returned: always - type: complex - contains: - quantity: - description: Count of headers - returned: always - type: int - sample: 1 - custom_origin_config: - description: Configuration of the origin - returned: always - type: complex - contains: - http_port: - description: Port on which HTTP is listening - returned: always - type: int - sample: 80 - https_port: - description: Port on which HTTPS is listening - returned: always - type: int - sample: 443 - origin_keepalive_timeout: - description: Keep-alive timeout - returned: always - type: int - sample: 5 - origin_protocol_policy: - description: Policy of which protocols are supported - returned: always - type: string - sample: https-only - origin_read_timeout: - description: Timeout for reads to the origin - returned: always - type: int - sample: 30 - origin_ssl_protocols: - description: SSL protocols allowed by the origin - returned: always - type: complex - contains: - items: - description: List of SSL protocols - returned: always - type: list - sample: - - TLSv1 - - TLSv1.1 - - TLSv1.2 - quantity: - description: Count of SSL protocols - returned: always - type: int - sample: 3 - domain_name: - description: Domain name of the origin - returned: always - type: string - sample: test-origin.example.com - id: - description: ID of the origin - returned: always - type: string - sample: test-origin.example.com - origin_path: - description: Subdirectory to prefix the request from the S3 or HTTP origin - returned: always - type: string - sample: '' - quantity: - description: Count of origins - returned: always - type: int - sample: 1 -price_class: - description: Price class of cloudfront distribution - returned: always - type: string - sample: PriceClass_All -restrictions: - description: Restrictions in use by Cloudfront - returned: always - type: complex - contains: - geo_restriction: - description: Controls the countries in which your content is distributed. - returned: always - type: complex - contains: - quantity: - description: Count of restrictions - returned: always - type: int - sample: 1 - items: - description: List of country codes allowed or disallowed - returned: always - type: list - sample: xy - restriction_type: - description: Type of restriction - returned: always - type: string - sample: blacklist -status: - description: Status of the cloudfront distribution - returned: always - type: string - sample: InProgress -tags: - description: Distribution tags - returned: always - type: dict - sample: - Hello: World -viewer_certificate: - description: Certificate used by cloudfront distribution - returned: always - type: complex - contains: - acm_certificate_arn: - description: ARN of ACM certificate - returned: when certificate comes from ACM - type: string - sample: arn:aws:acm:us-east-1:123456789012:certificate/abcd1234-1234-1234-abcd-123456abcdef - certificate: - description: Reference to certificate - returned: always - type: string - sample: arn:aws:acm:us-east-1:123456789012:certificate/abcd1234-1234-1234-abcd-123456abcdef - certificate_source: - description: Where certificate comes from - returned: always - type: string - sample: acm - minimum_protocol_version: - description: Minimum SSL/TLS protocol supported by this distribution - returned: always - type: string - sample: TLSv1 - ssl_support_method: - description: Support for pre-SNI browsers or not - returned: always - type: string - sample: sni-only -web_acl_id: - description: ID of Web Access Control List (from WAF service) - returned: always - type: string - sample: abcd1234-1234-abcd-abcd-abcd12345678 -''' - -from ansible.module_utils._text import to_text, to_native -from ansible.module_utils.aws.core import AnsibleAWSModule -from ansible.module_utils.aws.cloudfront_facts import CloudFrontFactsServiceManager -from ansible.module_utils.ec2 import get_aws_connection_info -from ansible.module_utils.ec2 import ec2_argument_spec, boto3_conn, compare_aws_tags -from ansible.module_utils.ec2 import camel_dict_to_snake_dict, ansible_dict_to_boto3_tag_list -from ansible.module_utils.ec2 import snake_dict_to_camel_dict, boto3_tag_list_to_ansible_dict -import datetime - -try: - from collections import OrderedDict -except ImportError: - try: - from ordereddict import OrderedDict - except ImportError: - pass # caught by AnsibleAWSModule (as python 2.6 + boto3 => ordereddict is installed) - -try: - import botocore -except ImportError: - pass - - -def change_dict_key_name(dictionary, old_key, new_key): - if old_key in dictionary: - dictionary[new_key] = dictionary.get(old_key) - dictionary.pop(old_key, None) - return dictionary - - -def merge_validation_into_config(config, validated_node, node_name): - if validated_node is not None: - if isinstance(validated_node, dict): - config_node = config.get(node_name) - if config_node is not None: - config_node_items = list(config_node.items()) - else: - config_node_items = [] - config[node_name] = dict(config_node_items + list(validated_node.items())) - if isinstance(validated_node, list): - config[node_name] = list(set(config.get(node_name) + validated_node)) - return config - - -def ansible_list_to_cloudfront_list(list_items=None, include_quantity=True): - if list_items is None: - list_items = [] - if not isinstance(list_items, list): - raise ValueError('Expected a list, got a {0} with value {1}'.format(type(list_items).__name__, str(list_items))) - result = {} - if include_quantity: - result['quantity'] = len(list_items) - if len(list_items) > 0: - result['items'] = list_items - return result - - -def recursive_diff(dict1, dict2): - left = dict((k, v) for (k, v) in dict1.items() if k not in dict2) - right = dict((k, v) for (k, v) in dict2.items() if k not in dict1) - for k in (set(dict1.keys()) & set(dict2.keys())): - if isinstance(dict1[k], dict) and isinstance(dict2[k], dict): - result = recursive_diff(dict1[k], dict2[k]) - if result: - left[k] = result[0] - right[k] = result[1] - elif dict1[k] != dict2[k]: - left[k] = dict1[k] - right[k] = dict2[k] - if left or right: - return left, right - else: - return None - - -def create_distribution(client, module, config, tags): - try: - if not tags: - return client.create_distribution(DistributionConfig=config)['Distribution'] - else: - distribution_config_with_tags = { - 'DistributionConfig': config, - 'Tags': { - 'Items': tags - } - } - return client.create_distribution_with_tags(DistributionConfigWithTags=distribution_config_with_tags)['Distribution'] - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg="Error creating distribution") - - -def delete_distribution(client, module, distribution): - try: - return client.delete_distribution(Id=distribution['Distribution']['Id'], IfMatch=distribution['ETag']) - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg="Error deleting distribution %s" % to_native(distribution['Distribution'])) - - -def update_distribution(client, module, config, distribution_id, e_tag): - try: - return client.update_distribution(DistributionConfig=config, Id=distribution_id, IfMatch=e_tag)['Distribution'] - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg="Error updating distribution to %s" % to_native(config)) - - -def tag_resource(client, module, arn, tags): - try: - return client.tag_resource(Resource=arn, Tags=dict(Items=tags)) - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg="Error tagging resource") - - -def untag_resource(client, module, arn, tag_keys): - try: - return client.untag_resource(Resource=arn, TagKeys=dict(Items=tag_keys)) - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg="Error untagging resource") - - -def list_tags_for_resource(client, module, arn): - try: - response = client.list_tags_for_resource(Resource=arn) - return boto3_tag_list_to_ansible_dict(response.get('Tags').get('Items')) - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg="Error listing tags for resource") - - -def update_tags(client, module, existing_tags, valid_tags, purge_tags, arn): - changed = False - to_add, to_remove = compare_aws_tags(existing_tags, valid_tags, purge_tags) - if to_remove: - untag_resource(client, module, arn, to_remove) - changed = True - if to_add: - tag_resource(client, module, arn, ansible_dict_to_boto3_tag_list(to_add)) - changed = True - return changed - - -class CloudFrontValidationManager(object): - """ - Manages Cloudfront validations - """ - - def __init__(self, module): - self.__cloudfront_facts_mgr = CloudFrontFactsServiceManager(module) - self.module = module - self.__default_distribution_enabled = True - self.__default_http_port = 80 - self.__default_https_port = 443 - self.__default_ipv6_enabled = False - self.__default_origin_ssl_protocols = [ - 'TLSv1', - 'TLSv1.1', - 'TLSv1.2' - ] - self.__default_custom_origin_protocol_policy = 'match-viewer' - self.__default_custom_origin_read_timeout = 30 - self.__default_custom_origin_keepalive_timeout = 5 - self.__default_datetime_string = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f') - self.__default_cache_behavior_min_ttl = 0 - self.__default_cache_behavior_max_ttl = 31536000 - self.__default_cache_behavior_default_ttl = 86400 - self.__default_cache_behavior_compress = False - self.__default_cache_behavior_viewer_protocol_policy = 'allow-all' - self.__default_cache_behavior_smooth_streaming = False - self.__default_cache_behavior_forwarded_values_forward_cookies = 'none' - self.__default_cache_behavior_forwarded_values_query_string = True - self.__default_trusted_signers_enabled = False - self.__valid_price_classes = set([ - 'PriceClass_100', - 'PriceClass_200', - 'PriceClass_All' - ]) - self.__valid_origin_protocol_policies = set([ - 'http-only', - 'match-viewer', - 'https-only' - ]) - self.__valid_origin_ssl_protocols = set([ - 'SSLv3', - 'TLSv1', - 'TLSv1.1', - 'TLSv1.2' - ]) - self.__valid_cookie_forwarding = set([ - 'none', - 'whitelist', - 'all' - ]) - self.__valid_viewer_protocol_policies = set([ - 'allow-all', - 'https-only', - 'redirect-to-https' - ]) - self.__valid_methods = set([ - 'GET', - 'HEAD', - 'POST', - 'PUT', - 'PATCH', - 'OPTIONS', - 'DELETE' - ]) - self.__valid_methods_cached_methods = [ - set([ - 'GET', - 'HEAD' - ]), - set([ - 'GET', - 'HEAD', - 'OPTIONS' - ]) - ] - self.__valid_methods_allowed_methods = [ - self.__valid_methods_cached_methods[0], - self.__valid_methods_cached_methods[1], - self.__valid_methods - ] - self.__valid_lambda_function_association_event_types = set([ - 'viewer-request', - 'viewer-response', - 'origin-request', - 'origin-response' - ]) - self.__valid_viewer_certificate_ssl_support_methods = set([ - 'sni-only', - 'vip' - ]) - self.__valid_viewer_certificate_minimum_protocol_versions = set([ - 'SSLv3', - 'TLSv1', - 'TLSv1_2016', - 'TLSv1.1_2016', - 'TLSv1.2_2018' - ]) - self.__valid_viewer_certificate_certificate_sources = set([ - 'cloudfront', - 'iam', - 'acm' - ]) - self.__valid_http_versions = set([ - 'http1.1', - 'http2' - ]) - self.__s3_bucket_domain_identifier = '.s3.amazonaws.com' - - def add_missing_key(self, dict_object, key_to_set, value_to_set): - if key_to_set not in dict_object and value_to_set is not None: - dict_object[key_to_set] = value_to_set - return dict_object - - def add_key_else_change_dict_key(self, dict_object, old_key, new_key, value_to_set): - if old_key not in dict_object and value_to_set is not None: - dict_object[new_key] = value_to_set - else: - dict_object = change_dict_key_name(dict_object, old_key, new_key) - return dict_object - - def add_key_else_validate(self, dict_object, key_name, attribute_name, value_to_set, valid_values, to_aws_list=False): - if key_name in dict_object: - self.validate_attribute_with_allowed_values(value_to_set, attribute_name, valid_values) - else: - if to_aws_list: - dict_object[key_name] = ansible_list_to_cloudfront_list(value_to_set) - elif value_to_set is not None: - dict_object[key_name] = value_to_set - return dict_object - - def validate_logging(self, logging): - try: - if logging is None: - return None - valid_logging = {} - if logging and not set(['enabled', 'include_cookies', 'bucket', 'prefix']).issubset(logging): - self.module.fail_json(msg="The logging parameters enabled, include_cookies, bucket and prefix must be specified.") - valid_logging['include_cookies'] = logging.get('include_cookies') - valid_logging['enabled'] = logging.get('enabled') - valid_logging['bucket'] = logging.get('bucket') - valid_logging['prefix'] = logging.get('prefix') - return valid_logging - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating distribution logging") - - def validate_is_list(self, list_to_validate, list_name): - if not isinstance(list_to_validate, list): - self.module.fail_json(msg='%s is of type %s. Must be a list.' % (list_name, type(list_to_validate).__name__)) - - def validate_required_key(self, key_name, full_key_name, dict_object): - if key_name not in dict_object: - self.module.fail_json(msg="%s must be specified." % full_key_name) - - def validate_origins(self, client, config, origins, default_origin_domain_name, - default_origin_path, create_distribution, purge_origins=False): - try: - if origins is None: - if default_origin_domain_name is None and not create_distribution: - if purge_origins: - return None - else: - return ansible_list_to_cloudfront_list(config) - if default_origin_domain_name is not None: - origins = [{ - 'domain_name': default_origin_domain_name, - 'origin_path': default_origin_path or '' - }] - else: - origins = [] - self.validate_is_list(origins, 'origins') - if not origins and default_origin_domain_name is None and create_distribution: - self.module.fail_json(msg="Both origins[] and default_origin_domain_name have not been specified. Please specify at least one.") - all_origins = OrderedDict() - get_origin_key = lambda origin: origin.get('id') if 'id' in origin else origin.get('domain_name') + (origin.get('origin_path') or '') - new_origins = list() - for origin in config: - all_origins[get_origin_key(origin)] = origin - for origin in origins: - origin = self.validate_origin(client, all_origins.get(get_origin_key(origin), {}), origin, default_origin_path) - all_origins[get_origin_key(origin)] = origin - new_origins.append(get_origin_key(origin)) - if purge_origins: - for origin_key in list(all_origins.keys()): - if origin_key not in new_origins: - del(all_origins[origin_key]) - return ansible_list_to_cloudfront_list(list(all_origins.values())) - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating distribution origins") - - def validate_s3_origin_configuration(self, client, existing_config, origin): - if origin['s3_origin_access_identity_enabled'] and existing_config.get('s3_origin_config', {}).get('origin_access_identity'): - return existing_config['s3_origin_config']['origin_access_identity'] - if not origin['s3_origin_access_identity_enabled']: - return None - try: - comment = "access-identity-by-ansible-%s-%s" % (origin.get('domain_name'), self.__default_datetime_string) - cfoai_config = dict(CloudFrontOriginAccessIdentityConfig=dict(CallerReference=self.__default_datetime_string, - Comment=comment)) - oai = client.create_cloud_front_origin_access_identity(**cfoai_config)['CloudFrontOriginAccessIdentity']['Id'] - except Exception as e: - self.module.fail_json_aws(e, msg="Couldn't create Origin Access Identity for id %s" % origin['id']) - return "origin-access-identity/cloudfront/%s" % oai - - def validate_origin(self, client, existing_config, origin, default_origin_path): - try: - origin = self.add_missing_key(origin, 'origin_path', existing_config.get('origin_path', default_origin_path or '')) - self.validate_required_key('origin_path', 'origins[].origin_path', origin) - origin = self.add_missing_key(origin, 'id', existing_config.get('id', self.__default_datetime_string)) - if 'custom_headers' in origin and len(origin.get('custom_headers')) > 0: - for custom_header in origin.get('custom_headers'): - if 'header_name' not in custom_header or 'header_value' not in custom_header: - self.module.fail_json(msg="Both origins[].custom_headers.header_name and origins[].custom_headers.header_value must be specified.") - origin['custom_headers'] = ansible_list_to_cloudfront_list(origin.get('custom_headers')) - else: - origin['custom_headers'] = ansible_list_to_cloudfront_list() - if self.__s3_bucket_domain_identifier in origin.get('domain_name').lower(): - if origin.get("s3_origin_access_identity_enabled") is not None: - s3_origin_config = self.validate_s3_origin_configuration(client, existing_config, origin) - if s3_origin_config: - oai = s3_origin_config - else: - oai = "" - origin["s3_origin_config"] = dict(origin_access_identity=oai) - del(origin["s3_origin_access_identity_enabled"]) - if 'custom_origin_config' in origin: - self.module.fail_json(msg="s3_origin_access_identity_enabled and custom_origin_config are mutually exclusive") - else: - origin = self.add_missing_key(origin, 'custom_origin_config', existing_config.get('custom_origin_config', {})) - custom_origin_config = origin.get('custom_origin_config') - custom_origin_config = self.add_key_else_validate(custom_origin_config, 'origin_protocol_policy', - 'origins[].custom_origin_config.origin_protocol_policy', - self.__default_custom_origin_protocol_policy, self.__valid_origin_protocol_policies) - custom_origin_config = self.add_missing_key(custom_origin_config, 'origin_read_timeout', self.__default_custom_origin_read_timeout) - custom_origin_config = self.add_missing_key(custom_origin_config, 'origin_keepalive_timeout', self.__default_custom_origin_keepalive_timeout) - custom_origin_config = self.add_key_else_change_dict_key(custom_origin_config, 'http_port', 'h_t_t_p_port', self.__default_http_port) - custom_origin_config = self.add_key_else_change_dict_key(custom_origin_config, 'https_port', 'h_t_t_p_s_port', self.__default_https_port) - if custom_origin_config.get('origin_ssl_protocols', {}).get('items'): - custom_origin_config['origin_ssl_protocols'] = custom_origin_config['origin_ssl_protocols']['items'] - if custom_origin_config.get('origin_ssl_protocols'): - self.validate_attribute_list_with_allowed_list(custom_origin_config['origin_ssl_protocols'], 'origins[].origin_ssl_protocols', - self.__valid_origin_ssl_protocols) - else: - custom_origin_config['origin_ssl_protocols'] = self.__default_origin_ssl_protocols - custom_origin_config['origin_ssl_protocols'] = ansible_list_to_cloudfront_list(custom_origin_config['origin_ssl_protocols']) - return origin - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - self.module.fail_json_aws(e, msg="Error validating distribution origin") - - def validate_cache_behaviors(self, config, cache_behaviors, valid_origins, purge_cache_behaviors=False): - try: - if cache_behaviors is None and valid_origins is not None and purge_cache_behaviors is False: - return ansible_list_to_cloudfront_list(config) - all_cache_behaviors = OrderedDict() - # cache behaviors are order dependent so we don't preserve the existing ordering when purge_cache_behaviors - # is true (if purge_cache_behaviors is not true, we can't really know the full new order) - if not purge_cache_behaviors: - for behavior in config: - all_cache_behaviors[behavior['path_pattern']] = behavior - for cache_behavior in cache_behaviors: - valid_cache_behavior = self.validate_cache_behavior(all_cache_behaviors.get(cache_behavior.get('path_pattern'), {}), - cache_behavior, valid_origins) - all_cache_behaviors[cache_behavior['path_pattern']] = valid_cache_behavior - if purge_cache_behaviors: - for target_origin_id in set(all_cache_behaviors.keys()) - set([cb['path_pattern'] for cb in cache_behaviors]): - del(all_cache_behaviors[target_origin_id]) - return ansible_list_to_cloudfront_list(list(all_cache_behaviors.values())) - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating distribution cache behaviors") - - def validate_cache_behavior(self, config, cache_behavior, valid_origins, is_default_cache=False): - if is_default_cache and cache_behavior is None: - cache_behavior = {} - if cache_behavior is None and valid_origins is not None: - return config - cache_behavior = self.validate_cache_behavior_first_level_keys(config, cache_behavior, valid_origins, is_default_cache) - cache_behavior = self.validate_forwarded_values(config, cache_behavior.get('forwarded_values'), cache_behavior) - cache_behavior = self.validate_allowed_methods(config, cache_behavior.get('allowed_methods'), cache_behavior) - cache_behavior = self.validate_lambda_function_associations(config, cache_behavior.get('lambda_function_associations'), cache_behavior) - cache_behavior = self.validate_trusted_signers(config, cache_behavior.get('trusted_signers'), cache_behavior) - cache_behavior = self.validate_field_level_encryption_id(config, cache_behavior.get('field_level_encryption_id'), cache_behavior) - return cache_behavior - - def validate_cache_behavior_first_level_keys(self, config, cache_behavior, valid_origins, is_default_cache): - try: - cache_behavior = self.add_key_else_change_dict_key(cache_behavior, 'min_ttl', 'min_t_t_l', - config.get('min_t_t_l', self.__default_cache_behavior_min_ttl)) - cache_behavior = self.add_key_else_change_dict_key(cache_behavior, 'max_ttl', 'max_t_t_l', - config.get('max_t_t_l', self.__default_cache_behavior_max_ttl)) - cache_behavior = self.add_key_else_change_dict_key(cache_behavior, 'default_ttl', 'default_t_t_l', - config.get('default_t_t_l', self.__default_cache_behavior_default_ttl)) - cache_behavior = self.add_missing_key(cache_behavior, 'compress', config.get('compress', self.__default_cache_behavior_compress)) - target_origin_id = cache_behavior.get('target_origin_id', config.get('target_origin_id')) - if not target_origin_id: - target_origin_id = self.get_first_origin_id_for_default_cache_behavior(valid_origins) - if target_origin_id not in [origin['id'] for origin in valid_origins.get('items', [])]: - if is_default_cache: - cache_behavior_name = 'Default cache behavior' - else: - cache_behavior_name = 'Cache behavior for path %s' % cache_behavior['path_pattern'] - self.module.fail_json(msg="%s has target_origin_id pointing to an origin that does not exist." % - cache_behavior_name) - cache_behavior['target_origin_id'] = target_origin_id - cache_behavior = self.add_key_else_validate(cache_behavior, 'viewer_protocol_policy', 'cache_behavior.viewer_protocol_policy', - config.get('viewer_protocol_policy', - self.__default_cache_behavior_viewer_protocol_policy), - self.__valid_viewer_protocol_policies) - cache_behavior = self.add_missing_key(cache_behavior, 'smooth_streaming', - config.get('smooth_streaming', self.__default_cache_behavior_smooth_streaming)) - return cache_behavior - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating distribution cache behavior first level keys") - - def validate_forwarded_values(self, config, forwarded_values, cache_behavior): - try: - if not forwarded_values: - forwarded_values = dict() - existing_config = config.get('forwarded_values', {}) - headers = forwarded_values.get('headers', existing_config.get('headers', {}).get('items')) - if headers: - headers.sort() - forwarded_values['headers'] = ansible_list_to_cloudfront_list(headers) - if 'cookies' not in forwarded_values: - forward = existing_config.get('cookies', {}).get('forward', self.__default_cache_behavior_forwarded_values_forward_cookies) - forwarded_values['cookies'] = {'forward': forward} - else: - existing_whitelist = existing_config.get('cookies', {}).get('whitelisted_names', {}).get('items') - whitelist = forwarded_values.get('cookies').get('whitelisted_names', existing_whitelist) - if whitelist: - self.validate_is_list(whitelist, 'forwarded_values.whitelisted_names') - forwarded_values['cookies']['whitelisted_names'] = ansible_list_to_cloudfront_list(whitelist) - cookie_forwarding = forwarded_values.get('cookies').get('forward', existing_config.get('cookies', {}).get('forward')) - self.validate_attribute_with_allowed_values(cookie_forwarding, 'cache_behavior.forwarded_values.cookies.forward', - self.__valid_cookie_forwarding) - forwarded_values['cookies']['forward'] = cookie_forwarding - query_string_cache_keys = forwarded_values.get('query_string_cache_keys', existing_config.get('query_string_cache_keys', {}).get('items', [])) - self.validate_is_list(query_string_cache_keys, 'forwarded_values.query_string_cache_keys') - forwarded_values['query_string_cache_keys'] = ansible_list_to_cloudfront_list(query_string_cache_keys) - forwarded_values = self.add_missing_key(forwarded_values, 'query_string', - existing_config.get('query_string', self.__default_cache_behavior_forwarded_values_query_string)) - cache_behavior['forwarded_values'] = forwarded_values - return cache_behavior - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating forwarded values") - - def validate_lambda_function_associations(self, config, lambda_function_associations, cache_behavior): - try: - if lambda_function_associations is not None: - self.validate_is_list(lambda_function_associations, 'lambda_function_associations') - for association in lambda_function_associations: - association = change_dict_key_name(association, 'lambda_function_arn', 'lambda_function_a_r_n') - self.validate_attribute_with_allowed_values(association.get('event_type'), 'cache_behaviors[].lambda_function_associations.event_type', - self.__valid_lambda_function_association_event_types) - cache_behavior['lambda_function_associations'] = ansible_list_to_cloudfront_list(lambda_function_associations) - else: - if 'lambda_function_associations' in config: - cache_behavior['lambda_function_associations'] = config.get('lambda_function_associations') - else: - cache_behavior['lambda_function_associations'] = ansible_list_to_cloudfront_list([]) - return cache_behavior - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating lambda function associations") - - def validate_field_level_encryption_id(self, config, field_level_encryption_id, cache_behavior): - # only set field_level_encryption_id if it's already set or if it was passed - if field_level_encryption_id is not None: - cache_behavior['field_level_encryption_id'] = field_level_encryption_id - elif 'field_level_encryption_id' in config: - cache_behavior['field_level_encryption_id'] = config.get('field_level_encryption_id') - return cache_behavior - - def validate_allowed_methods(self, config, allowed_methods, cache_behavior): - try: - if allowed_methods is not None: - self.validate_required_key('items', 'cache_behavior.allowed_methods.items[]', allowed_methods) - temp_allowed_items = allowed_methods.get('items') - self.validate_is_list(temp_allowed_items, 'cache_behavior.allowed_methods.items') - self.validate_attribute_list_with_allowed_list(temp_allowed_items, 'cache_behavior.allowed_methods.items[]', - self.__valid_methods_allowed_methods) - cached_items = allowed_methods.get('cached_methods') - if 'cached_methods' in allowed_methods: - self.validate_is_list(cached_items, 'cache_behavior.allowed_methods.cached_methods') - self.validate_attribute_list_with_allowed_list(cached_items, 'cache_behavior.allowed_items.cached_methods[]', - self.__valid_methods_cached_methods) - # we don't care if the order of how cloudfront stores the methods differs - preserving existing - # order reduces likelihood of making unnecessary changes - if 'allowed_methods' in config and set(config['allowed_methods']['items']) == set(temp_allowed_items): - cache_behavior['allowed_methods'] = config['allowed_methods'] - else: - cache_behavior['allowed_methods'] = ansible_list_to_cloudfront_list(temp_allowed_items) - - if cached_items and set(cached_items) == set(config.get('allowed_methods', {}).get('cached_methods', {}).get('items', [])): - cache_behavior['allowed_methods']['cached_methods'] = config['allowed_methods']['cached_methods'] - else: - cache_behavior['allowed_methods']['cached_methods'] = ansible_list_to_cloudfront_list(cached_items) - else: - if 'allowed_methods' in config: - cache_behavior['allowed_methods'] = config.get('allowed_methods') - return cache_behavior - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating allowed methods") - - def validate_trusted_signers(self, config, trusted_signers, cache_behavior): - try: - if trusted_signers is None: - trusted_signers = {} - if 'items' in trusted_signers: - valid_trusted_signers = ansible_list_to_cloudfront_list(trusted_signers.get('items')) - else: - valid_trusted_signers = dict(quantity=config.get('quantity', 0)) - if 'items' in config: - valid_trusted_signers = dict(items=config['items']) - valid_trusted_signers['enabled'] = trusted_signers.get('enabled', config.get('enabled', self.__default_trusted_signers_enabled)) - cache_behavior['trusted_signers'] = valid_trusted_signers - return cache_behavior - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating trusted signers") - - def validate_viewer_certificate(self, viewer_certificate): - try: - if viewer_certificate is None: - return None - if viewer_certificate.get('cloudfront_default_certificate') and viewer_certificate.get('ssl_support_method') is not None: - self.module.fail_json(msg="viewer_certificate.ssl_support_method should not be specified with viewer_certificate_cloudfront_default" + - "_certificate set to true.") - self.validate_attribute_with_allowed_values(viewer_certificate.get('ssl_support_method'), 'viewer_certificate.ssl_support_method', - self.__valid_viewer_certificate_ssl_support_methods) - self.validate_attribute_with_allowed_values(viewer_certificate.get('minimum_protocol_version'), 'viewer_certificate.minimum_protocol_version', - self.__valid_viewer_certificate_minimum_protocol_versions) - self.validate_attribute_with_allowed_values(viewer_certificate.get('certificate_source'), 'viewer_certificate.certificate_source', - self.__valid_viewer_certificate_certificate_sources) - viewer_certificate = change_dict_key_name(viewer_certificate, 'cloudfront_default_certificate', 'cloud_front_default_certificate') - viewer_certificate = change_dict_key_name(viewer_certificate, 'ssl_support_method', 's_s_l_support_method') - viewer_certificate = change_dict_key_name(viewer_certificate, 'iam_certificate_id', 'i_a_m_certificate_id') - viewer_certificate = change_dict_key_name(viewer_certificate, 'acm_certificate_arn', 'a_c_m_certificate_arn') - return viewer_certificate - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating viewer certificate") - - def validate_custom_error_responses(self, config, custom_error_responses, purge_custom_error_responses): - try: - if custom_error_responses is None and not purge_custom_error_responses: - return ansible_list_to_cloudfront_list(config) - self.validate_is_list(custom_error_responses, 'custom_error_responses') - result = list() - existing_responses = dict((response['error_code'], response) for response in custom_error_responses) - for custom_error_response in custom_error_responses: - self.validate_required_key('error_code', 'custom_error_responses[].error_code', custom_error_response) - custom_error_response = change_dict_key_name(custom_error_response, 'error_caching_min_ttl', 'error_caching_min_t_t_l') - if 'response_code' in custom_error_response: - custom_error_response['response_code'] = str(custom_error_response['response_code']) - if custom_error_response['error_code'] in existing_responses: - del(existing_responses[custom_error_response['error_code']]) - result.append(custom_error_response) - if not purge_custom_error_responses: - result.extend(existing_responses.values()) - - return ansible_list_to_cloudfront_list(result) - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating custom error responses") - - def validate_restrictions(self, config, restrictions, purge_restrictions=False): - try: - if restrictions is None: - if purge_restrictions: - return None - else: - return config - self.validate_required_key('geo_restriction', 'restrictions.geo_restriction', restrictions) - geo_restriction = restrictions.get('geo_restriction') - self.validate_required_key('restriction_type', 'restrictions.geo_restriction.restriction_type', geo_restriction) - existing_restrictions = config.get('geo_restriction', {}).get(geo_restriction['restriction_type'], {}).get('items', []) - geo_restriction_items = geo_restriction.get('items') - if not purge_restrictions: - geo_restriction_items.extend([rest for rest in existing_restrictions if - rest not in geo_restriction_items]) - valid_restrictions = ansible_list_to_cloudfront_list(geo_restriction_items) - valid_restrictions['restriction_type'] = geo_restriction.get('restriction_type') - return {'geo_restriction': valid_restrictions} - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating restrictions") - - def validate_distribution_config_parameters(self, config, default_root_object, ipv6_enabled, http_version, web_acl_id): - try: - config['default_root_object'] = default_root_object or config.get('default_root_object', '') - config['is_i_p_v_6_enabled'] = ipv6_enabled or config.get('i_p_v_6_enabled', self.__default_ipv6_enabled) - if http_version is not None or config.get('http_version'): - self.validate_attribute_with_allowed_values(http_version, 'http_version', self.__valid_http_versions) - config['http_version'] = http_version or config.get('http_version') - if web_acl_id or config.get('web_a_c_l_id'): - config['web_a_c_l_id'] = web_acl_id or config.get('web_a_c_l_id') - return config - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating distribution config parameters") - - def validate_common_distribution_parameters(self, config, enabled, aliases, logging, price_class, purge_aliases=False): - try: - if config is None: - config = {} - if aliases is not None: - if not purge_aliases: - aliases.extend([alias for alias in config.get('aliases', {}).get('items', []) - if alias not in aliases]) - config['aliases'] = ansible_list_to_cloudfront_list(aliases) - if logging is not None: - config['logging'] = self.validate_logging(logging) - config['enabled'] = enabled or config.get('enabled', self.__default_distribution_enabled) - if price_class is not None: - self.validate_attribute_with_allowed_values(price_class, 'price_class', self.__valid_price_classes) - config['price_class'] = price_class - return config - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating common distribution parameters") - - def validate_comment(self, config, comment): - config['comment'] = comment or config.get('comment', "Distribution created by Ansible with datetime stamp " + self.__default_datetime_string) - return config - - def validate_caller_reference(self, caller_reference): - return caller_reference or self.__default_datetime_string - - def get_first_origin_id_for_default_cache_behavior(self, valid_origins): - try: - if valid_origins is not None: - valid_origins_list = valid_origins.get('items') - if valid_origins_list is not None and isinstance(valid_origins_list, list) and len(valid_origins_list) > 0: - return str(valid_origins_list[0].get('id')) - self.module.fail_json(msg="There are no valid origins from which to specify a target_origin_id for the default_cache_behavior configuration.") - except Exception as e: - self.module.fail_json_aws(e, msg="Error getting first origin_id for default cache behavior") - - def validate_attribute_list_with_allowed_list(self, attribute_list, attribute_list_name, allowed_list): - try: - self.validate_is_list(attribute_list, attribute_list_name) - if (isinstance(allowed_list, list) and set(attribute_list) not in allowed_list or - isinstance(allowed_list, set) and not set(allowed_list).issuperset(attribute_list)): - self.module.fail_json(msg='The attribute list {0} must be one of [{1}]'.format(attribute_list_name, ' '.join(str(a) for a in allowed_list))) - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating attribute list with allowed value list") - - def validate_attribute_with_allowed_values(self, attribute, attribute_name, allowed_list): - if attribute is not None and attribute not in allowed_list: - self.module.fail_json(msg='The attribute {0} must be one of [{1}]'.format(attribute_name, ' '.join(str(a) for a in allowed_list))) - - def validate_distribution_from_caller_reference(self, caller_reference): - try: - distributions = self.__cloudfront_facts_mgr.list_distributions(False) - distribution_name = 'Distribution' - distribution_config_name = 'DistributionConfig' - distribution_ids = [dist.get('Id') for dist in distributions] - for distribution_id in distribution_ids: - config = self.__cloudfront_facts_mgr.get_distribution(distribution_id) - distribution = config.get(distribution_name) - if distribution is not None: - distribution_config = distribution.get(distribution_config_name) - if distribution_config is not None and distribution_config.get('CallerReference') == caller_reference: - distribution['DistributionConfig'] = distribution_config - return distribution - - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating distribution from caller reference") - - def validate_distribution_from_aliases_caller_reference(self, distribution_id, aliases, caller_reference): - try: - if caller_reference is not None: - return self.validate_distribution_from_caller_reference(caller_reference) - else: - if aliases: - distribution_id = self.validate_distribution_id_from_alias(aliases) - if distribution_id: - return self.__cloudfront_facts_mgr.get_distribution(distribution_id) - return None - except Exception as e: - self.module.fail_json_aws(e, msg="Error validating distribution_id from alias, aliases and caller reference") - - def validate_distribution_id_from_alias(self, aliases): - distributions = self.__cloudfront_facts_mgr.list_distributions(False) - if distributions: - for distribution in distributions: - distribution_aliases = distribution.get('Aliases', {}).get('Items', []) - if set(aliases) & set(distribution_aliases): - return distribution['Id'] - return None - - def wait_until_processed(self, client, wait_timeout, distribution_id, caller_reference): - if distribution_id is None: - distribution_id = self.validate_distribution_from_caller_reference(caller_reference=caller_reference)['Id'] - - try: - waiter = client.get_waiter('distribution_deployed') - attempts = 1 + int(wait_timeout / 60) - waiter.wait(Id=distribution_id, WaiterConfig={'MaxAttempts': attempts}) - except botocore.exceptions.WaiterError as e: - self.module.fail_json(msg="Timeout waiting for cloudfront action. Waited for {0} seconds before timeout. " - "Error: {1}".format(to_text(wait_timeout), to_native(e))) - - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - self.module.fail_json_aws(e, msg="Error getting distribution {0}".format(distribution_id)) - - -def main(): - argument_spec = ec2_argument_spec() - - argument_spec.update(dict( - state=dict(choices=['present', 'absent'], default='present'), - caller_reference=dict(), - comment=dict(), - distribution_id=dict(), - e_tag=dict(), - tags=dict(type='dict', default={}), - purge_tags=dict(type='bool', default=False), - alias=dict(), - aliases=dict(type='list', default=[]), - purge_aliases=dict(type='bool', default=False), - default_root_object=dict(), - origins=dict(type='list'), - purge_origins=dict(type='bool', default=False), - default_cache_behavior=dict(type='dict'), - cache_behaviors=dict(type='list'), - purge_cache_behaviors=dict(type='bool', default=False), - custom_error_responses=dict(type='list'), - purge_custom_error_responses=dict(type='bool', default=False), - logging=dict(type='dict'), - price_class=dict(), - enabled=dict(type='bool'), - viewer_certificate=dict(type='dict'), - restrictions=dict(type='dict'), - web_acl_id=dict(), - http_version=dict(), - ipv6_enabled=dict(type='bool'), - default_origin_domain_name=dict(), - default_origin_path=dict(), - wait=dict(default=False, type='bool'), - wait_timeout=dict(default=1800, type='int') - )) - - result = {} - changed = True - - module = AnsibleAWSModule( - argument_spec=argument_spec, - supports_check_mode=False, - mutually_exclusive=[ - ['distribution_id', 'alias'], - ['default_origin_domain_name', 'distribution_id'], - ['default_origin_domain_name', 'alias'], - ] - ) - - region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True) - client = boto3_conn(module, conn_type='client', resource='cloudfront', region=region, endpoint=ec2_url, **aws_connect_kwargs) - - validation_mgr = CloudFrontValidationManager(module) - - state = module.params.get('state') - caller_reference = module.params.get('caller_reference') - comment = module.params.get('comment') - e_tag = module.params.get('e_tag') - tags = module.params.get('tags') - purge_tags = module.params.get('purge_tags') - distribution_id = module.params.get('distribution_id') - alias = module.params.get('alias') - aliases = module.params.get('aliases') - purge_aliases = module.params.get('purge_aliases') - default_root_object = module.params.get('default_root_object') - origins = module.params.get('origins') - purge_origins = module.params.get('purge_origins') - default_cache_behavior = module.params.get('default_cache_behavior') - cache_behaviors = module.params.get('cache_behaviors') - purge_cache_behaviors = module.params.get('purge_cache_behaviors') - custom_error_responses = module.params.get('custom_error_responses') - purge_custom_error_responses = module.params.get('purge_custom_error_responses') - logging = module.params.get('logging') - price_class = module.params.get('price_class') - enabled = module.params.get('enabled') - viewer_certificate = module.params.get('viewer_certificate') - restrictions = module.params.get('restrictions') - purge_restrictions = module.params.get('purge_restrictions') - web_acl_id = module.params.get('web_acl_id') - http_version = module.params.get('http_version') - ipv6_enabled = module.params.get('ipv6_enabled') - default_origin_domain_name = module.params.get('default_origin_domain_name') - default_origin_path = module.params.get('default_origin_path') - wait = module.params.get('wait') - wait_timeout = module.params.get('wait_timeout') - - if alias and alias not in aliases: - aliases.append(alias) - - distribution = validation_mgr.validate_distribution_from_aliases_caller_reference(distribution_id, aliases, caller_reference) - - update = state == 'present' and distribution - create = state == 'present' and not distribution - delete = state == 'absent' and distribution - - if not (update or create or delete): - module.exit_json(changed=False) - - if update or delete: - config = distribution['Distribution']['DistributionConfig'] - e_tag = distribution['ETag'] - distribution_id = distribution['Distribution']['Id'] - else: - config = dict() - if update: - config = camel_dict_to_snake_dict(config, reversible=True) - - if create or update: - config = validation_mgr.validate_common_distribution_parameters(config, enabled, aliases, logging, price_class, purge_aliases) - config = validation_mgr.validate_distribution_config_parameters(config, default_root_object, ipv6_enabled, http_version, web_acl_id) - config['origins'] = validation_mgr.validate_origins(client, config.get('origins', {}).get('items', []), origins, default_origin_domain_name, - default_origin_path, create, purge_origins) - config['cache_behaviors'] = validation_mgr.validate_cache_behaviors(config.get('cache_behaviors', {}).get('items', []), - cache_behaviors, config['origins'], purge_cache_behaviors) - config['default_cache_behavior'] = validation_mgr.validate_cache_behavior(config.get('default_cache_behavior', {}), - default_cache_behavior, config['origins'], True) - config['custom_error_responses'] = validation_mgr.validate_custom_error_responses(config.get('custom_error_responses', {}).get('items', []), - custom_error_responses, purge_custom_error_responses) - valid_restrictions = validation_mgr.validate_restrictions(config.get('restrictions', {}), restrictions, purge_restrictions) - if valid_restrictions: - config['restrictions'] = valid_restrictions - valid_viewer_certificate = validation_mgr.validate_viewer_certificate(viewer_certificate) - config = merge_validation_into_config(config, valid_viewer_certificate, 'viewer_certificate') - config = validation_mgr.validate_comment(config, comment) - config = snake_dict_to_camel_dict(config, capitalize_first=True) - - if create: - config['CallerReference'] = validation_mgr.validate_caller_reference(caller_reference) - result = create_distribution(client, module, config, ansible_dict_to_boto3_tag_list(tags)) - result = camel_dict_to_snake_dict(result) - result['tags'] = list_tags_for_resource(client, module, result['arn']) - - if delete: - if config['Enabled']: - config['Enabled'] = False - result = update_distribution(client, module, config, distribution_id, e_tag) - validation_mgr.wait_until_processed(client, wait_timeout, distribution_id, config.get('CallerReference')) - distribution = validation_mgr.validate_distribution_from_aliases_caller_reference(distribution_id, aliases, caller_reference) - # e_tag = distribution['ETag'] - result = delete_distribution(client, module, distribution) - - if update: - changed = config != distribution['Distribution']['DistributionConfig'] - if changed: - result = update_distribution(client, module, config, distribution_id, e_tag) - else: - result = distribution['Distribution'] - existing_tags = list_tags_for_resource(client, module, result['ARN']) - distribution['Distribution']['DistributionConfig']['tags'] = existing_tags - changed |= update_tags(client, module, existing_tags, tags, purge_tags, result['ARN']) - result = camel_dict_to_snake_dict(result) - result['distribution_config']['tags'] = config['tags'] = list_tags_for_resource(client, module, result['arn']) - result['diff'] = dict() - diff = recursive_diff(distribution['Distribution']['DistributionConfig'], config) - if diff: - result['diff']['before'] = diff[0] - result['diff']['after'] = diff[1] - - if wait and (create or update): - validation_mgr.wait_until_processed(client, wait_timeout, distribution_id, config.get('CallerReference')) - - if 'distribution_config' in result: - result.update(result['distribution_config']) - del(result['distribution_config']) - - module.exit_json(changed=changed, **result) - - -if __name__ == '__main__': - main() diff --git a/requirements-python.txt b/requirements-python.txt index 81eee6cdb..848407e7e 100644 --- a/requirements-python.txt +++ b/requirements-python.txt @@ -4,6 +4,10 @@ ansible>=6,<7 # make sure this is BEFORE boto3 and boto awscli +# some tasks call aws command on localhost +# make sure this is BEFORE boto3 and boto +awscli + # needed for inventory and aws modules boto3 From a48d3fd72fb588695e2252ec6be0c38ef771096b Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Wed, 20 Jul 2022 16:17:52 +0200 Subject: [PATCH 07/19] Restore cloudflare custom role --- library/cloudfront_distribution.py | 2008 ++++++++++++++++++++++++++++ 1 file changed, 2008 insertions(+) create mode 100644 library/cloudfront_distribution.py diff --git a/library/cloudfront_distribution.py b/library/cloudfront_distribution.py new file mode 100644 index 000000000..8e6907eaf --- /dev/null +++ b/library/cloudfront_distribution.py @@ -0,0 +1,2008 @@ +#!/usr/bin/python +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +### WARNING!!! PATCHED MODULE AHEAD! #### +# This module override an be removed once the PR is merged +# and we migrate to the appropriate version - 2.10.x at the soonest. +# Fixes: https://github.com/ansible/ansible/issues/68354 +# This file was taken from ansible 2.7.16 and patched with: +# https://github.com/ansible/ansible/pull/68355 +# /FS + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = ''' +--- + +module: cloudfront_distribution + +short_description: create, update and delete aws cloudfront distributions. + +description: + - Allows for easy creation, updating and deletion of CloudFront distributions. + +requirements: + - boto3 >= 1.0.0 + - python >= 2.6 + +version_added: "2.5" + +author: + - Willem van Ketwich (@wilvk) + - Will Thames (@willthames) + +extends_documentation_fragment: + - aws + - ec2 + +options: + + state: + description: + - The desired state of the distribution + present - creates a new distribution or updates an existing distribution. + absent - deletes an existing distribution. + choices: ['present', 'absent'] + default: 'present' + + distribution_id: + description: + - The id of the cloudfront distribution. This parameter can be exchanged with I(alias) or I(caller_reference) and is used in conjunction with I(e_tag). + + e_tag: + description: + - A unique identifier of a modified or existing distribution. Used in conjunction with I(distribution_id). + Is determined automatically if not specified. + + caller_reference: + description: + - A unique identifier for creating and updating cloudfront distributions. Each caller reference must be unique across all distributions. e.g. a caller + reference used in a web distribution cannot be reused in a streaming distribution. This parameter can be used instead of I(distribution_id) + to reference an existing distribution. If not specified, this defaults to a datetime stamp of the format + 'YYYY-MM-DDTHH:MM:SS.ffffff'. + + tags: + description: + - Should be input as a dict() of key-value pairs. + Note that numeric keys or values must be wrapped in quotes. e.g. "Priority:" '1' + + purge_tags: + description: + - Specifies whether existing tags will be removed before adding new tags. When I(purge_tags=yes), existing tags are removed and I(tags) are added, if + specified. If no tags are specified, it removes all existing tags for the distribution. When I(purge_tags=no), existing tags are kept and I(tags) + are added, if specified. + default: 'no' + type: bool + + alias: + description: + - The name of an alias (CNAME) that is used in a distribution. This is used to effectively reference a distribution by its alias as an alias can only + be used by one distribution per AWS account. This variable avoids having to provide the I(distribution_id) as well as + the I(e_tag), or I(caller_reference) of an existing distribution. + + aliases: + description: + - A I(list[]) of domain name aliases (CNAMEs) as strings to be used for the distribution. Each alias must be unique across all distribution for the AWS + account. + + purge_aliases: + description: + - Specifies whether existing aliases will be removed before adding new aliases. When I(purge_aliases=yes), existing aliases are removed and I(aliases) + are added. + default: 'no' + type: bool + + default_root_object: + description: + - A config element that specifies the path to request when the user requests the origin. e.g. if specified as 'index.html', this maps to + www.example.com/index.html when www.example.com is called by the user. This prevents the entire distribution origin from being exposed at the root. + + default_origin_domain_name: + description: + - The domain name to use for an origin if no I(origins) have been specified. Should only be used on a first run of generating a distribution and not on + subsequent runs. Should not be used in conjunction with I(distribution_id), I(caller_reference) or I(alias). + + default_origin_path: + description: + - The default origin path to specify for an origin if no I(origins) have been specified. Defaults to empty if not specified. + + origins: + description: + - A config element that is a I(list[]) of complex origin objects to be specified for the distribution. Used for creating and updating distributions. + Each origin item comprises the attributes + I(id) + I(domain_name) (defaults to default_origin_domain_name if not specified) + I(origin_path) (defaults to default_origin_path if not specified) + I(custom_headers[]) + I(header_name) + I(header_value) + I(s3_origin_access_identity_enabled) + I(custom_origin_config) + I(http_port) + I(https_port) + I(origin_protocol_policy) + I(origin_ssl_protocols[]) + I(origin_read_timeout) + I(origin_keepalive_timeout) + + purge_origins: + description: Whether to remove any origins that aren't listed in I(origins) + default: false + + default_cache_behavior: + description: + - A config element that is a complex object specifying the default cache behavior of the distribution. If not specified, the I(target_origin_id) is + defined as the I(target_origin_id) of the first valid I(cache_behavior) in I(cache_behaviors) with defaults. + The default cache behavior comprises the attributes + I(target_origin_id) + I(forwarded_values) + I(query_string) + I(cookies) + I(forward) + I(whitelisted_names) + I(headers[]) + I(query_string_cache_keys[]) + I(trusted_signers) + I(enabled) + I(items[]) + I(viewer_protocol_policy) + I(min_ttl) + I(allowed_methods) + I(items[]) + I(cached_methods[]) + I(smooth_streaming) + I(default_ttl) + I(max_ttl) + I(compress) + I(lambda_function_associations[]) + I(lambda_function_arn) + I(event_type) + I(field_level_encryption_id) + + cache_behaviors: + description: + - A config element that is a I(list[]) of complex cache behavior objects to be specified for the distribution. The order + of the list is preserved across runs unless C(purge_cache_behavior) is enabled. + Each cache behavior comprises the attributes + I(path_pattern) + I(target_origin_id) + I(forwarded_values) + I(query_string) + I(cookies) + I(forward) + I(whitelisted_names) + I(headers[]) + I(query_string_cache_keys[]) + I(trusted_signers) + I(enabled) + I(items[]) + I(viewer_protocol_policy) + I(min_ttl) + I(allowed_methods) + I(items[]) + I(cached_methods[]) + I(smooth_streaming) + I(default_ttl) + I(max_ttl) + I(compress) + I(lambda_function_associations[]) + I(field_level_encryption_id) + + purge_cache_behaviors: + description: Whether to remove any cache behaviors that aren't listed in I(cache_behaviors). This switch + also allows the reordering of cache_behaviors. + default: false + + custom_error_responses: + description: + - A config element that is a I(list[]) of complex custom error responses to be specified for the distribution. This attribute configures custom http + error messages returned to the user. + Each custom error response object comprises the attributes + I(error_code) + I(reponse_page_path) + I(response_code) + I(error_caching_min_ttl) + + purge_custom_error_responses: + description: Whether to remove any custom error responses that aren't listed in I(custom_error_responses) + default: false + + comment: + description: + - A comment that describes the cloudfront distribution. If not specified, it defaults to a + generic message that it has been created with Ansible, and a datetime stamp. + + logging: + description: + - A config element that is a complex object that defines logging for the distribution. + The logging object comprises the attributes + I(enabled) + I(include_cookies) + I(bucket) + I(prefix) + + price_class: + description: + - A string that specifies the pricing class of the distribution. As per + U(https://aws.amazon.com/cloudfront/pricing/) + I(price_class=PriceClass_100) consists of the areas + United States + Canada + Europe + I(price_class=PriceClass_200) consists of the areas + United States + Canada + Europe + Hong Kong, Philippines, S. Korea, Singapore & Taiwan + Japan + India + I(price_class=PriceClass_All) consists of the areas + United States + Canada + Europe + Hong Kong, Philippines, S. Korea, Singapore & Taiwan + Japan + India + South America + Australia + choices: ['PriceClass_100', 'PriceClass_200', 'PriceClass_All'] + default: aws defaults this to 'PriceClass_All' + + enabled: + description: + - A boolean value that specifies whether the distribution is enabled or disabled. + default: 'yes' + type: bool + + viewer_certificate: + description: + - A config element that is a complex object that specifies the encryption details of the distribution. + Comprises the following attributes + I(cloudfront_default_certificate) + I(iam_certificate_id) + I(acm_certificate_arn) + I(ssl_support_method) + I(minimum_protocol_version) + I(certificate) + I(certificate_source) + + restrictions: + description: + - A config element that is a complex object that describes how a distribution should restrict it's content. + The restriction object comprises the following attributes + I(geo_restriction) + I(restriction_type) + I(items[]) + + web_acl_id: + description: + - The id of a Web Application Firewall (WAF) Access Control List (ACL). + + http_version: + description: + - The version of the http protocol to use for the distribution. + choices: [ 'http1.1', 'http2' ] + default: aws defaults this to 'http2' + + ipv6_enabled: + description: + - Determines whether IPv6 support is enabled or not. + type: bool + default: 'no' + + wait: + description: + - Specifies whether the module waits until the distribution has completed processing the creation or update. + type: bool + default: 'no' + + wait_timeout: + description: + - Specifies the duration in seconds to wait for a timeout of a cloudfront create or update. Defaults to 1800 seconds (30 minutes). + default: 1800 + +''' + +EXAMPLES = ''' + +# create a basic distribution with defaults and tags + +- cloudfront_distribution: + state: present + default_origin_domain_name: www.my-cloudfront-origin.com + tags: + Name: example distribution + Project: example project + Priority: '1' + +# update a distribution comment by distribution_id + +- cloudfront_distribution: + state: present + distribution_id: E1RP5A2MJ8073O + comment: modified by ansible cloudfront.py + +# update a distribution comment by caller_reference + +- cloudfront_distribution: + state: present + caller_reference: my cloudfront distribution 001 + comment: modified by ansible cloudfront.py + +# update a distribution's aliases and comment using the distribution_id as a reference + +- cloudfront_distribution: + state: present + distribution_id: E1RP5A2MJ8073O + comment: modified by cloudfront.py again + aliases: [ 'www.my-distribution-source.com', 'zzz.aaa.io' ] + +# update a distribution's aliases and comment using an alias as a reference + +- cloudfront_distribution: + state: present + caller_reference: my test distribution + comment: modified by cloudfront.py again + aliases: + - www.my-distribution-source.com + - zzz.aaa.io + +# update a distribution's comment and aliases and tags and remove existing tags + +- cloudfront_distribution: + state: present + distribution_id: E15BU8SDCGSG57 + comment: modified by cloudfront.py again + aliases: + - tested.com + tags: + Project: distribution 1.2 + purge_tags: yes + +# create a distribution with an origin, logging and default cache behavior + +- cloudfront_distribution: + state: present + caller_reference: unique test distribution id + origins: + - id: 'my test origin-000111' + domain_name: www.example.com + origin_path: /production + custom_headers: + - header_name: MyCustomHeaderName + header_value: MyCustomHeaderValue + default_cache_behavior: + target_origin_id: 'my test origin-000111' + forwarded_values: + query_string: true + cookies: + forward: all + headers: + - '*' + viewer_protocol_policy: allow-all + smooth_streaming: true + compress: true + allowed_methods: + items: + - GET + - HEAD + cached_methods: + - GET + - HEAD + logging: + enabled: true + include_cookies: false + bucket: mylogbucket.s3.amazonaws.com + prefix: myprefix/ + enabled: false + comment: this is a cloudfront distribution with logging + +# delete a distribution + +- cloudfront_distribution: + state: absent + caller_reference: replaceable distribution +''' + +RETURN = ''' +active_trusted_signers: + description: Key pair IDs that CloudFront is aware of for each trusted signer + returned: always + type: complex + contains: + enabled: + description: Whether trusted signers are in use + returned: always + type: bool + sample: false + quantity: + description: Number of trusted signers + returned: always + type: int + sample: 1 + items: + description: Number of trusted signers + returned: when there are trusted signers + type: list + sample: + - key_pair_id +aliases: + description: Aliases that refer to the distribution + returned: always + type: complex + contains: + items: + description: List of aliases + returned: always + type: list + sample: + - test.example.com + quantity: + description: Number of aliases + returned: always + type: int + sample: 1 +arn: + description: Amazon Resource Name of the distribution + returned: always + type: string + sample: arn:aws:cloudfront::123456789012:distribution/E1234ABCDEFGHI +cache_behaviors: + description: Cloudfront cache behaviors + returned: always + type: complex + contains: + items: + description: List of cache behaviors + returned: always + type: complex + contains: + allowed_methods: + description: Methods allowed by the cache behavior + returned: always + type: complex + contains: + cached_methods: + description: Methods cached by the cache behavior + returned: always + type: complex + contains: + items: + description: List of cached methods + returned: always + type: list + sample: + - HEAD + - GET + quantity: + description: Count of cached methods + returned: always + type: int + sample: 2 + items: + description: List of methods allowed by the cache behavior + returned: always + type: list + sample: + - HEAD + - GET + quantity: + description: Count of methods allowed by the cache behavior + returned: always + type: int + sample: 2 + compress: + description: Whether compression is turned on for the cache behavior + returned: always + type: bool + sample: false + default_ttl: + description: Default Time to Live of the cache behavior + returned: always + type: int + sample: 86400 + forwarded_values: + description: Values forwarded to the origin for this cache behavior + returned: always + type: complex + contains: + cookies: + description: Cookies to forward to the origin + returned: always + type: complex + contains: + forward: + description: Which cookies to forward to the origin for this cache behavior + returned: always + type: string + sample: none + whitelisted_names: + description: The names of the cookies to forward to the origin for this cache behavior + returned: when I(forward) is C(whitelist) + type: complex + contains: + quantity: + description: Count of cookies to forward + returned: always + type: int + sample: 1 + items: + description: List of cookies to forward + returned: when list is not empty + type: list + sample: my_cookie + headers: + description: Which headers are used to vary on cache retrievals + returned: always + type: complex + contains: + quantity: + description: Count of headers to vary on + returned: always + type: int + sample: 1 + items: + description: List of headers to vary on + returned: when list is not empty + type: list + sample: + - Host + query_string: + description: Whether the query string is used in cache lookups + returned: always + type: bool + sample: false + query_string_cache_keys: + description: Which query string keys to use in cache lookups + returned: always + type: complex + contains: + quantity: + description: Count of query string cache keys to use in cache lookups + returned: always + type: int + sample: 1 + items: + description: List of query string cache keys to use in cache lookups + returned: when list is not empty + type: list + sample: + lambda_function_associations: + description: Lambda function associations for a cache behavior + returned: always + type: complex + contains: + quantity: + description: Count of lambda function associations + returned: always + type: int + sample: 1 + items: + description: List of lambda function associations + returned: when list is not empty + type: list + sample: + - lambda_function_arn: arn:aws:lambda:123456789012:us-east-1/lambda/lambda-function + event_type: viewer-response + max_ttl: + description: Maximum Time to Live + returned: always + type: int + sample: 31536000 + min_ttl: + description: Minimum Time to Live + returned: always + type: int + sample: 0 + path_pattern: + description: Path pattern that determines this cache behavior + returned: always + type: string + sample: /path/to/files/* + smooth_streaming: + description: Whether smooth streaming is enabled + returned: always + type: bool + sample: false + target_origin_id: + description: Id of origin reference by this cache behavior + returned: always + type: string + sample: origin_abcd + trusted_signers: + description: Trusted signers + returned: always + type: complex + contains: + enabled: + description: Whether trusted signers are enabled for this cache behavior + returned: always + type: bool + sample: false + quantity: + description: Count of trusted signers + returned: always + type: int + sample: 1 + viewer_protocol_policy: + description: Policy of how to handle http/https + returned: always + type: string + sample: redirect-to-https + quantity: + description: Count of cache behaviors + returned: always + type: int + sample: 1 + +caller_reference: + description: Idempotency reference given when creating cloudfront distribution + returned: always + type: string + sample: '1484796016700' +comment: + description: Any comments you want to include about the distribution + returned: always + type: string + sample: 'my first cloudfront distribution' +custom_error_responses: + description: Custom error responses to use for error handling + returned: always + type: complex + contains: + items: + description: List of custom error responses + returned: always + type: complex + contains: + error_caching_min_ttl: + description: Mininum time to cache this error response + returned: always + type: int + sample: 300 + error_code: + description: Origin response code that triggers this error response + returned: always + type: int + sample: 500 + response_code: + description: Response code to return to the requester + returned: always + type: string + sample: '500' + response_page_path: + description: Path that contains the error page to display + returned: always + type: string + sample: /errors/5xx.html + quantity: + description: Count of custom error response items + returned: always + type: int + sample: 1 +default_cache_behavior: + description: Default cache behavior + returned: always + type: complex + contains: + allowed_methods: + description: Methods allowed by the cache behavior + returned: always + type: complex + contains: + cached_methods: + description: Methods cached by the cache behavior + returned: always + type: complex + contains: + items: + description: List of cached methods + returned: always + type: list + sample: + - HEAD + - GET + quantity: + description: Count of cached methods + returned: always + type: int + sample: 2 + items: + description: List of methods allowed by the cache behavior + returned: always + type: list + sample: + - HEAD + - GET + quantity: + description: Count of methods allowed by the cache behavior + returned: always + type: int + sample: 2 + compress: + description: Whether compression is turned on for the cache behavior + returned: always + type: bool + sample: false + default_ttl: + description: Default Time to Live of the cache behavior + returned: always + type: int + sample: 86400 + forwarded_values: + description: Values forwarded to the origin for this cache behavior + returned: always + type: complex + contains: + cookies: + description: Cookies to forward to the origin + returned: always + type: complex + contains: + forward: + description: Which cookies to forward to the origin for this cache behavior + returned: always + type: string + sample: none + whitelisted_names: + description: The names of the cookies to forward to the origin for this cache behavior + returned: when I(forward) is C(whitelist) + type: complex + contains: + quantity: + description: Count of cookies to forward + returned: always + type: int + sample: 1 + items: + description: List of cookies to forward + returned: when list is not empty + type: list + sample: my_cookie + headers: + description: Which headers are used to vary on cache retrievals + returned: always + type: complex + contains: + quantity: + description: Count of headers to vary on + returned: always + type: int + sample: 1 + items: + description: List of headers to vary on + returned: when list is not empty + type: list + sample: + - Host + query_string: + description: Whether the query string is used in cache lookups + returned: always + type: bool + sample: false + query_string_cache_keys: + description: Which query string keys to use in cache lookups + returned: always + type: complex + contains: + quantity: + description: Count of query string cache keys to use in cache lookups + returned: always + type: int + sample: 1 + items: + description: List of query string cache keys to use in cache lookups + returned: when list is not empty + type: list + sample: + lambda_function_associations: + description: Lambda function associations for a cache behavior + returned: always + type: complex + contains: + quantity: + description: Count of lambda function associations + returned: always + type: int + sample: 1 + items: + description: List of lambda function associations + returned: when list is not empty + type: list + sample: + - lambda_function_arn: arn:aws:lambda:123456789012:us-east-1/lambda/lambda-function + event_type: viewer-response + max_ttl: + description: Maximum Time to Live + returned: always + type: int + sample: 31536000 + min_ttl: + description: Minimum Time to Live + returned: always + type: int + sample: 0 + path_pattern: + description: Path pattern that determines this cache behavior + returned: always + type: string + sample: /path/to/files/* + smooth_streaming: + description: Whether smooth streaming is enabled + returned: always + type: bool + sample: false + target_origin_id: + description: Id of origin reference by this cache behavior + returned: always + type: string + sample: origin_abcd + trusted_signers: + description: Trusted signers + returned: always + type: complex + contains: + enabled: + description: Whether trusted signers are enabled for this cache behavior + returned: always + type: bool + sample: false + quantity: + description: Count of trusted signers + returned: always + type: int + sample: 1 + viewer_protocol_policy: + description: Policy of how to handle http/https + returned: always + type: string + sample: redirect-to-https +default_root_object: + description: The object that you want CloudFront to request from your origin (for example, index.html) + when a viewer requests the root URL for your distribution + returned: always + type: string + sample: '' +diff: + description: Difference between previous configuration and new configuration + returned: always + type: dict + sample: {} +domain_name: + description: Domain name of cloudfront distribution + returned: always + type: string + sample: d1vz8pzgurxosf.cloudfront.net +enabled: + description: Whether the cloudfront distribution is enabled or not + returned: always + type: bool + sample: true +http_version: + description: Version of HTTP supported by the distribution + returned: always + type: string + sample: http2 +id: + description: Cloudfront distribution ID + returned: always + type: string + sample: E123456ABCDEFG +in_progress_invalidation_batches: + description: The number of invalidation batches currently in progress + returned: always + type: int + sample: 0 +is_ipv6_enabled: + description: Whether IPv6 is enabled + returned: always + type: bool + sample: true +last_modified_time: + description: Date and time distribution was last modified + returned: always + type: string + sample: '2017-10-13T01:51:12.656000+00:00' +logging: + description: Logging information + returned: always + type: complex + contains: + bucket: + description: S3 bucket logging destination + returned: always + type: string + sample: logs-example-com.s3.amazonaws.com + enabled: + description: Whether logging is enabled + returned: always + type: bool + sample: true + include_cookies: + description: Whether to log cookies + returned: always + type: bool + sample: false + prefix: + description: Prefix added to logging object names + returned: always + type: string + sample: cloudfront/test +origins: + description: Origins in the cloudfront distribution + returned: always + type: complex + contains: + items: + description: List of origins + returned: always + type: complex + contains: + custom_headers: + description: Custom headers passed to the origin + returned: always + type: complex + contains: + quantity: + description: Count of headers + returned: always + type: int + sample: 1 + custom_origin_config: + description: Configuration of the origin + returned: always + type: complex + contains: + http_port: + description: Port on which HTTP is listening + returned: always + type: int + sample: 80 + https_port: + description: Port on which HTTPS is listening + returned: always + type: int + sample: 443 + origin_keepalive_timeout: + description: Keep-alive timeout + returned: always + type: int + sample: 5 + origin_protocol_policy: + description: Policy of which protocols are supported + returned: always + type: string + sample: https-only + origin_read_timeout: + description: Timeout for reads to the origin + returned: always + type: int + sample: 30 + origin_ssl_protocols: + description: SSL protocols allowed by the origin + returned: always + type: complex + contains: + items: + description: List of SSL protocols + returned: always + type: list + sample: + - TLSv1 + - TLSv1.1 + - TLSv1.2 + quantity: + description: Count of SSL protocols + returned: always + type: int + sample: 3 + domain_name: + description: Domain name of the origin + returned: always + type: string + sample: test-origin.example.com + id: + description: ID of the origin + returned: always + type: string + sample: test-origin.example.com + origin_path: + description: Subdirectory to prefix the request from the S3 or HTTP origin + returned: always + type: string + sample: '' + quantity: + description: Count of origins + returned: always + type: int + sample: 1 +price_class: + description: Price class of cloudfront distribution + returned: always + type: string + sample: PriceClass_All +restrictions: + description: Restrictions in use by Cloudfront + returned: always + type: complex + contains: + geo_restriction: + description: Controls the countries in which your content is distributed. + returned: always + type: complex + contains: + quantity: + description: Count of restrictions + returned: always + type: int + sample: 1 + items: + description: List of country codes allowed or disallowed + returned: always + type: list + sample: xy + restriction_type: + description: Type of restriction + returned: always + type: string + sample: blacklist +status: + description: Status of the cloudfront distribution + returned: always + type: string + sample: InProgress +tags: + description: Distribution tags + returned: always + type: dict + sample: + Hello: World +viewer_certificate: + description: Certificate used by cloudfront distribution + returned: always + type: complex + contains: + acm_certificate_arn: + description: ARN of ACM certificate + returned: when certificate comes from ACM + type: string + sample: arn:aws:acm:us-east-1:123456789012:certificate/abcd1234-1234-1234-abcd-123456abcdef + certificate: + description: Reference to certificate + returned: always + type: string + sample: arn:aws:acm:us-east-1:123456789012:certificate/abcd1234-1234-1234-abcd-123456abcdef + certificate_source: + description: Where certificate comes from + returned: always + type: string + sample: acm + minimum_protocol_version: + description: Minimum SSL/TLS protocol supported by this distribution + returned: always + type: string + sample: TLSv1 + ssl_support_method: + description: Support for pre-SNI browsers or not + returned: always + type: string + sample: sni-only +web_acl_id: + description: ID of Web Access Control List (from WAF service) + returned: always + type: string + sample: abcd1234-1234-abcd-abcd-abcd12345678 +''' + +from ansible.module_utils._text import to_text, to_native +from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.aws.cloudfront_facts import CloudFrontFactsServiceManager +from ansible.module_utils.ec2 import get_aws_connection_info +from ansible.module_utils.ec2 import ec2_argument_spec, boto3_conn, compare_aws_tags +from ansible.module_utils.ec2 import camel_dict_to_snake_dict, ansible_dict_to_boto3_tag_list +from ansible.module_utils.ec2 import snake_dict_to_camel_dict, boto3_tag_list_to_ansible_dict +import datetime + +try: + from collections import OrderedDict +except ImportError: + try: + from ordereddict import OrderedDict + except ImportError: + pass # caught by AnsibleAWSModule (as python 2.6 + boto3 => ordereddict is installed) + +try: + import botocore +except ImportError: + pass + + +def change_dict_key_name(dictionary, old_key, new_key): + if old_key in dictionary: + dictionary[new_key] = dictionary.get(old_key) + dictionary.pop(old_key, None) + return dictionary + + +def merge_validation_into_config(config, validated_node, node_name): + if validated_node is not None: + if isinstance(validated_node, dict): + config_node = config.get(node_name) + if config_node is not None: + config_node_items = list(config_node.items()) + else: + config_node_items = [] + config[node_name] = dict(config_node_items + list(validated_node.items())) + if isinstance(validated_node, list): + config[node_name] = list(set(config.get(node_name) + validated_node)) + return config + + +def ansible_list_to_cloudfront_list(list_items=None, include_quantity=True): + if list_items is None: + list_items = [] + if not isinstance(list_items, list): + raise ValueError('Expected a list, got a {0} with value {1}'.format(type(list_items).__name__, str(list_items))) + result = {} + if include_quantity: + result['quantity'] = len(list_items) + if len(list_items) > 0: + result['items'] = list_items + return result + + +def recursive_diff(dict1, dict2): + left = dict((k, v) for (k, v) in dict1.items() if k not in dict2) + right = dict((k, v) for (k, v) in dict2.items() if k not in dict1) + for k in (set(dict1.keys()) & set(dict2.keys())): + if isinstance(dict1[k], dict) and isinstance(dict2[k], dict): + result = recursive_diff(dict1[k], dict2[k]) + if result: + left[k] = result[0] + right[k] = result[1] + elif dict1[k] != dict2[k]: + left[k] = dict1[k] + right[k] = dict2[k] + if left or right: + return left, right + else: + return None + + +def create_distribution(client, module, config, tags): + try: + if not tags: + return client.create_distribution(DistributionConfig=config)['Distribution'] + else: + distribution_config_with_tags = { + 'DistributionConfig': config, + 'Tags': { + 'Items': tags + } + } + return client.create_distribution_with_tags(DistributionConfigWithTags=distribution_config_with_tags)['Distribution'] + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Error creating distribution") + + +def delete_distribution(client, module, distribution): + try: + return client.delete_distribution(Id=distribution['Distribution']['Id'], IfMatch=distribution['ETag']) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Error deleting distribution %s" % to_native(distribution['Distribution'])) + + +def update_distribution(client, module, config, distribution_id, e_tag): + try: + return client.update_distribution(DistributionConfig=config, Id=distribution_id, IfMatch=e_tag)['Distribution'] + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Error updating distribution to %s" % to_native(config)) + + +def tag_resource(client, module, arn, tags): + try: + return client.tag_resource(Resource=arn, Tags=dict(Items=tags)) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Error tagging resource") + + +def untag_resource(client, module, arn, tag_keys): + try: + return client.untag_resource(Resource=arn, TagKeys=dict(Items=tag_keys)) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Error untagging resource") + + +def list_tags_for_resource(client, module, arn): + try: + response = client.list_tags_for_resource(Resource=arn) + return boto3_tag_list_to_ansible_dict(response.get('Tags').get('Items')) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Error listing tags for resource") + + +def update_tags(client, module, existing_tags, valid_tags, purge_tags, arn): + changed = False + to_add, to_remove = compare_aws_tags(existing_tags, valid_tags, purge_tags) + if to_remove: + untag_resource(client, module, arn, to_remove) + changed = True + if to_add: + tag_resource(client, module, arn, ansible_dict_to_boto3_tag_list(to_add)) + changed = True + return changed + + +class CloudFrontValidationManager(object): + """ + Manages Cloudfront validations + """ + + def __init__(self, module): + self.__cloudfront_facts_mgr = CloudFrontFactsServiceManager(module) + self.module = module + self.__default_distribution_enabled = True + self.__default_http_port = 80 + self.__default_https_port = 443 + self.__default_ipv6_enabled = False + self.__default_origin_ssl_protocols = [ + 'TLSv1', + 'TLSv1.1', + 'TLSv1.2' + ] + self.__default_custom_origin_protocol_policy = 'match-viewer' + self.__default_custom_origin_read_timeout = 30 + self.__default_custom_origin_keepalive_timeout = 5 + self.__default_datetime_string = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f') + self.__default_cache_behavior_min_ttl = 0 + self.__default_cache_behavior_max_ttl = 31536000 + self.__default_cache_behavior_default_ttl = 86400 + self.__default_cache_behavior_compress = False + self.__default_cache_behavior_viewer_protocol_policy = 'allow-all' + self.__default_cache_behavior_smooth_streaming = False + self.__default_cache_behavior_forwarded_values_forward_cookies = 'none' + self.__default_cache_behavior_forwarded_values_query_string = True + self.__default_trusted_signers_enabled = False + self.__valid_price_classes = set([ + 'PriceClass_100', + 'PriceClass_200', + 'PriceClass_All' + ]) + self.__valid_origin_protocol_policies = set([ + 'http-only', + 'match-viewer', + 'https-only' + ]) + self.__valid_origin_ssl_protocols = set([ + 'SSLv3', + 'TLSv1', + 'TLSv1.1', + 'TLSv1.2' + ]) + self.__valid_cookie_forwarding = set([ + 'none', + 'whitelist', + 'all' + ]) + self.__valid_viewer_protocol_policies = set([ + 'allow-all', + 'https-only', + 'redirect-to-https' + ]) + self.__valid_methods = set([ + 'GET', + 'HEAD', + 'POST', + 'PUT', + 'PATCH', + 'OPTIONS', + 'DELETE' + ]) + self.__valid_methods_cached_methods = [ + set([ + 'GET', + 'HEAD' + ]), + set([ + 'GET', + 'HEAD', + 'OPTIONS' + ]) + ] + self.__valid_methods_allowed_methods = [ + self.__valid_methods_cached_methods[0], + self.__valid_methods_cached_methods[1], + self.__valid_methods + ] + self.__valid_lambda_function_association_event_types = set([ + 'viewer-request', + 'viewer-response', + 'origin-request', + 'origin-response' + ]) + self.__valid_viewer_certificate_ssl_support_methods = set([ + 'sni-only', + 'vip' + ]) + self.__valid_viewer_certificate_minimum_protocol_versions = set([ + 'SSLv3', + 'TLSv1', + 'TLSv1_2016', + 'TLSv1.1_2016', + 'TLSv1.2_2018' + ]) + self.__valid_viewer_certificate_certificate_sources = set([ + 'cloudfront', + 'iam', + 'acm' + ]) + self.__valid_http_versions = set([ + 'http1.1', + 'http2' + ]) + self.__s3_bucket_domain_identifier = '.s3.amazonaws.com' + + def add_missing_key(self, dict_object, key_to_set, value_to_set): + if key_to_set not in dict_object and value_to_set is not None: + dict_object[key_to_set] = value_to_set + return dict_object + + def add_key_else_change_dict_key(self, dict_object, old_key, new_key, value_to_set): + if old_key not in dict_object and value_to_set is not None: + dict_object[new_key] = value_to_set + else: + dict_object = change_dict_key_name(dict_object, old_key, new_key) + return dict_object + + def add_key_else_validate(self, dict_object, key_name, attribute_name, value_to_set, valid_values, to_aws_list=False): + if key_name in dict_object: + self.validate_attribute_with_allowed_values(value_to_set, attribute_name, valid_values) + else: + if to_aws_list: + dict_object[key_name] = ansible_list_to_cloudfront_list(value_to_set) + elif value_to_set is not None: + dict_object[key_name] = value_to_set + return dict_object + + def validate_logging(self, logging): + try: + if logging is None: + return None + valid_logging = {} + if logging and not set(['enabled', 'include_cookies', 'bucket', 'prefix']).issubset(logging): + self.module.fail_json(msg="The logging parameters enabled, include_cookies, bucket and prefix must be specified.") + valid_logging['include_cookies'] = logging.get('include_cookies') + valid_logging['enabled'] = logging.get('enabled') + valid_logging['bucket'] = logging.get('bucket') + valid_logging['prefix'] = logging.get('prefix') + return valid_logging + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating distribution logging") + + def validate_is_list(self, list_to_validate, list_name): + if not isinstance(list_to_validate, list): + self.module.fail_json(msg='%s is of type %s. Must be a list.' % (list_name, type(list_to_validate).__name__)) + + def validate_required_key(self, key_name, full_key_name, dict_object): + if key_name not in dict_object: + self.module.fail_json(msg="%s must be specified." % full_key_name) + + def validate_origins(self, client, config, origins, default_origin_domain_name, + default_origin_path, create_distribution, purge_origins=False): + try: + if origins is None: + if default_origin_domain_name is None and not create_distribution: + if purge_origins: + return None + else: + return ansible_list_to_cloudfront_list(config) + if default_origin_domain_name is not None: + origins = [{ + 'domain_name': default_origin_domain_name, + 'origin_path': default_origin_path or '' + }] + else: + origins = [] + self.validate_is_list(origins, 'origins') + if not origins and default_origin_domain_name is None and create_distribution: + self.module.fail_json(msg="Both origins[] and default_origin_domain_name have not been specified. Please specify at least one.") + all_origins = OrderedDict() + get_origin_key = lambda origin: origin.get('id') if 'id' in origin else origin.get('domain_name') + (origin.get('origin_path') or '') + new_origins = list() + for origin in config: + all_origins[get_origin_key(origin)] = origin + for origin in origins: + origin = self.validate_origin(client, all_origins.get(get_origin_key(origin), {}), origin, default_origin_path) + all_origins[get_origin_key(origin)] = origin + new_origins.append(get_origin_key(origin)) + if purge_origins: + for origin_key in list(all_origins.keys()): + if origin_key not in new_origins: + del(all_origins[origin_key]) + return ansible_list_to_cloudfront_list(list(all_origins.values())) + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating distribution origins") + + def validate_s3_origin_configuration(self, client, existing_config, origin): + if origin['s3_origin_access_identity_enabled'] and existing_config.get('s3_origin_config', {}).get('origin_access_identity'): + return existing_config['s3_origin_config']['origin_access_identity'] + if not origin['s3_origin_access_identity_enabled']: + return None + try: + comment = "access-identity-by-ansible-%s-%s" % (origin.get('domain_name'), self.__default_datetime_string) + cfoai_config = dict(CloudFrontOriginAccessIdentityConfig=dict(CallerReference=self.__default_datetime_string, + Comment=comment)) + oai = client.create_cloud_front_origin_access_identity(**cfoai_config)['CloudFrontOriginAccessIdentity']['Id'] + except Exception as e: + self.module.fail_json_aws(e, msg="Couldn't create Origin Access Identity for id %s" % origin['id']) + return "origin-access-identity/cloudfront/%s" % oai + + def validate_origin(self, client, existing_config, origin, default_origin_path): + try: + origin = self.add_missing_key(origin, 'origin_path', existing_config.get('origin_path', default_origin_path or '')) + self.validate_required_key('origin_path', 'origins[].origin_path', origin) + origin = self.add_missing_key(origin, 'id', existing_config.get('id', self.__default_datetime_string)) + if 'custom_headers' in origin and len(origin.get('custom_headers')) > 0: + for custom_header in origin.get('custom_headers'): + if 'header_name' not in custom_header or 'header_value' not in custom_header: + self.module.fail_json(msg="Both origins[].custom_headers.header_name and origins[].custom_headers.header_value must be specified.") + origin['custom_headers'] = ansible_list_to_cloudfront_list(origin.get('custom_headers')) + else: + origin['custom_headers'] = ansible_list_to_cloudfront_list() + if self.__s3_bucket_domain_identifier in origin.get('domain_name').lower(): + if origin.get("s3_origin_access_identity_enabled") is not None: + s3_origin_config = self.validate_s3_origin_configuration(client, existing_config, origin) + if s3_origin_config: + oai = s3_origin_config + else: + oai = "" + origin["s3_origin_config"] = dict(origin_access_identity=oai) + del(origin["s3_origin_access_identity_enabled"]) + if 'custom_origin_config' in origin: + self.module.fail_json(msg="s3_origin_access_identity_enabled and custom_origin_config are mutually exclusive") + else: + origin = self.add_missing_key(origin, 'custom_origin_config', existing_config.get('custom_origin_config', {})) + custom_origin_config = origin.get('custom_origin_config') + custom_origin_config = self.add_key_else_validate(custom_origin_config, 'origin_protocol_policy', + 'origins[].custom_origin_config.origin_protocol_policy', + self.__default_custom_origin_protocol_policy, self.__valid_origin_protocol_policies) + custom_origin_config = self.add_missing_key(custom_origin_config, 'origin_read_timeout', self.__default_custom_origin_read_timeout) + custom_origin_config = self.add_missing_key(custom_origin_config, 'origin_keepalive_timeout', self.__default_custom_origin_keepalive_timeout) + custom_origin_config = self.add_key_else_change_dict_key(custom_origin_config, 'http_port', 'h_t_t_p_port', self.__default_http_port) + custom_origin_config = self.add_key_else_change_dict_key(custom_origin_config, 'https_port', 'h_t_t_p_s_port', self.__default_https_port) + if custom_origin_config.get('origin_ssl_protocols', {}).get('items'): + custom_origin_config['origin_ssl_protocols'] = custom_origin_config['origin_ssl_protocols']['items'] + if custom_origin_config.get('origin_ssl_protocols'): + self.validate_attribute_list_with_allowed_list(custom_origin_config['origin_ssl_protocols'], 'origins[].origin_ssl_protocols', + self.__valid_origin_ssl_protocols) + else: + custom_origin_config['origin_ssl_protocols'] = self.__default_origin_ssl_protocols + custom_origin_config['origin_ssl_protocols'] = ansible_list_to_cloudfront_list(custom_origin_config['origin_ssl_protocols']) + return origin + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + self.module.fail_json_aws(e, msg="Error validating distribution origin") + + def validate_cache_behaviors(self, config, cache_behaviors, valid_origins, purge_cache_behaviors=False): + try: + if cache_behaviors is None and valid_origins is not None and purge_cache_behaviors is False: + return ansible_list_to_cloudfront_list(config) + all_cache_behaviors = OrderedDict() + # cache behaviors are order dependent so we don't preserve the existing ordering when purge_cache_behaviors + # is true (if purge_cache_behaviors is not true, we can't really know the full new order) + if not purge_cache_behaviors: + for behavior in config: + all_cache_behaviors[behavior['path_pattern']] = behavior + for cache_behavior in cache_behaviors: + valid_cache_behavior = self.validate_cache_behavior(all_cache_behaviors.get(cache_behavior.get('path_pattern'), {}), + cache_behavior, valid_origins) + all_cache_behaviors[cache_behavior['path_pattern']] = valid_cache_behavior + if purge_cache_behaviors: + for target_origin_id in set(all_cache_behaviors.keys()) - set([cb['path_pattern'] for cb in cache_behaviors]): + del(all_cache_behaviors[target_origin_id]) + return ansible_list_to_cloudfront_list(list(all_cache_behaviors.values())) + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating distribution cache behaviors") + + def validate_cache_behavior(self, config, cache_behavior, valid_origins, is_default_cache=False): + if is_default_cache and cache_behavior is None: + cache_behavior = {} + if cache_behavior is None and valid_origins is not None: + return config + cache_behavior = self.validate_cache_behavior_first_level_keys(config, cache_behavior, valid_origins, is_default_cache) + cache_behavior = self.validate_forwarded_values(config, cache_behavior.get('forwarded_values'), cache_behavior) + cache_behavior = self.validate_allowed_methods(config, cache_behavior.get('allowed_methods'), cache_behavior) + cache_behavior = self.validate_lambda_function_associations(config, cache_behavior.get('lambda_function_associations'), cache_behavior) + cache_behavior = self.validate_trusted_signers(config, cache_behavior.get('trusted_signers'), cache_behavior) + cache_behavior = self.validate_field_level_encryption_id(config, cache_behavior.get('field_level_encryption_id'), cache_behavior) + return cache_behavior + + def validate_cache_behavior_first_level_keys(self, config, cache_behavior, valid_origins, is_default_cache): + try: + cache_behavior = self.add_key_else_change_dict_key(cache_behavior, 'min_ttl', 'min_t_t_l', + config.get('min_t_t_l', self.__default_cache_behavior_min_ttl)) + cache_behavior = self.add_key_else_change_dict_key(cache_behavior, 'max_ttl', 'max_t_t_l', + config.get('max_t_t_l', self.__default_cache_behavior_max_ttl)) + cache_behavior = self.add_key_else_change_dict_key(cache_behavior, 'default_ttl', 'default_t_t_l', + config.get('default_t_t_l', self.__default_cache_behavior_default_ttl)) + cache_behavior = self.add_missing_key(cache_behavior, 'compress', config.get('compress', self.__default_cache_behavior_compress)) + target_origin_id = cache_behavior.get('target_origin_id', config.get('target_origin_id')) + if not target_origin_id: + target_origin_id = self.get_first_origin_id_for_default_cache_behavior(valid_origins) + if target_origin_id not in [origin['id'] for origin in valid_origins.get('items', [])]: + if is_default_cache: + cache_behavior_name = 'Default cache behavior' + else: + cache_behavior_name = 'Cache behavior for path %s' % cache_behavior['path_pattern'] + self.module.fail_json(msg="%s has target_origin_id pointing to an origin that does not exist." % + cache_behavior_name) + cache_behavior['target_origin_id'] = target_origin_id + cache_behavior = self.add_key_else_validate(cache_behavior, 'viewer_protocol_policy', 'cache_behavior.viewer_protocol_policy', + config.get('viewer_protocol_policy', + self.__default_cache_behavior_viewer_protocol_policy), + self.__valid_viewer_protocol_policies) + cache_behavior = self.add_missing_key(cache_behavior, 'smooth_streaming', + config.get('smooth_streaming', self.__default_cache_behavior_smooth_streaming)) + return cache_behavior + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating distribution cache behavior first level keys") + + def validate_forwarded_values(self, config, forwarded_values, cache_behavior): + try: + if not forwarded_values: + forwarded_values = dict() + existing_config = config.get('forwarded_values', {}) + headers = forwarded_values.get('headers', existing_config.get('headers', {}).get('items')) + if headers: + headers.sort() + forwarded_values['headers'] = ansible_list_to_cloudfront_list(headers) + if 'cookies' not in forwarded_values: + forward = existing_config.get('cookies', {}).get('forward', self.__default_cache_behavior_forwarded_values_forward_cookies) + forwarded_values['cookies'] = {'forward': forward} + else: + existing_whitelist = existing_config.get('cookies', {}).get('whitelisted_names', {}).get('items') + whitelist = forwarded_values.get('cookies').get('whitelisted_names', existing_whitelist) + if whitelist: + self.validate_is_list(whitelist, 'forwarded_values.whitelisted_names') + forwarded_values['cookies']['whitelisted_names'] = ansible_list_to_cloudfront_list(whitelist) + cookie_forwarding = forwarded_values.get('cookies').get('forward', existing_config.get('cookies', {}).get('forward')) + self.validate_attribute_with_allowed_values(cookie_forwarding, 'cache_behavior.forwarded_values.cookies.forward', + self.__valid_cookie_forwarding) + forwarded_values['cookies']['forward'] = cookie_forwarding + query_string_cache_keys = forwarded_values.get('query_string_cache_keys', existing_config.get('query_string_cache_keys', {}).get('items', [])) + self.validate_is_list(query_string_cache_keys, 'forwarded_values.query_string_cache_keys') + forwarded_values['query_string_cache_keys'] = ansible_list_to_cloudfront_list(query_string_cache_keys) + forwarded_values = self.add_missing_key(forwarded_values, 'query_string', + existing_config.get('query_string', self.__default_cache_behavior_forwarded_values_query_string)) + cache_behavior['forwarded_values'] = forwarded_values + return cache_behavior + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating forwarded values") + + def validate_lambda_function_associations(self, config, lambda_function_associations, cache_behavior): + try: + if lambda_function_associations is not None: + self.validate_is_list(lambda_function_associations, 'lambda_function_associations') + for association in lambda_function_associations: + association = change_dict_key_name(association, 'lambda_function_arn', 'lambda_function_a_r_n') + self.validate_attribute_with_allowed_values(association.get('event_type'), 'cache_behaviors[].lambda_function_associations.event_type', + self.__valid_lambda_function_association_event_types) + cache_behavior['lambda_function_associations'] = ansible_list_to_cloudfront_list(lambda_function_associations) + else: + if 'lambda_function_associations' in config: + cache_behavior['lambda_function_associations'] = config.get('lambda_function_associations') + else: + cache_behavior['lambda_function_associations'] = ansible_list_to_cloudfront_list([]) + return cache_behavior + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating lambda function associations") + + def validate_field_level_encryption_id(self, config, field_level_encryption_id, cache_behavior): + # only set field_level_encryption_id if it's already set or if it was passed + if field_level_encryption_id is not None: + cache_behavior['field_level_encryption_id'] = field_level_encryption_id + elif 'field_level_encryption_id' in config: + cache_behavior['field_level_encryption_id'] = config.get('field_level_encryption_id') + return cache_behavior + + def validate_allowed_methods(self, config, allowed_methods, cache_behavior): + try: + if allowed_methods is not None: + self.validate_required_key('items', 'cache_behavior.allowed_methods.items[]', allowed_methods) + temp_allowed_items = allowed_methods.get('items') + self.validate_is_list(temp_allowed_items, 'cache_behavior.allowed_methods.items') + self.validate_attribute_list_with_allowed_list(temp_allowed_items, 'cache_behavior.allowed_methods.items[]', + self.__valid_methods_allowed_methods) + cached_items = allowed_methods.get('cached_methods') + if 'cached_methods' in allowed_methods: + self.validate_is_list(cached_items, 'cache_behavior.allowed_methods.cached_methods') + self.validate_attribute_list_with_allowed_list(cached_items, 'cache_behavior.allowed_items.cached_methods[]', + self.__valid_methods_cached_methods) + # we don't care if the order of how cloudfront stores the methods differs - preserving existing + # order reduces likelihood of making unnecessary changes + if 'allowed_methods' in config and set(config['allowed_methods']['items']) == set(temp_allowed_items): + cache_behavior['allowed_methods'] = config['allowed_methods'] + else: + cache_behavior['allowed_methods'] = ansible_list_to_cloudfront_list(temp_allowed_items) + + if cached_items and set(cached_items) == set(config.get('allowed_methods', {}).get('cached_methods', {}).get('items', [])): + cache_behavior['allowed_methods']['cached_methods'] = config['allowed_methods']['cached_methods'] + else: + cache_behavior['allowed_methods']['cached_methods'] = ansible_list_to_cloudfront_list(cached_items) + else: + if 'allowed_methods' in config: + cache_behavior['allowed_methods'] = config.get('allowed_methods') + return cache_behavior + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating allowed methods") + + def validate_trusted_signers(self, config, trusted_signers, cache_behavior): + try: + if trusted_signers is None: + trusted_signers = {} + if 'items' in trusted_signers: + valid_trusted_signers = ansible_list_to_cloudfront_list(trusted_signers.get('items')) + else: + valid_trusted_signers = dict(quantity=config.get('quantity', 0)) + if 'items' in config: + valid_trusted_signers = dict(items=config['items']) + valid_trusted_signers['enabled'] = trusted_signers.get('enabled', config.get('enabled', self.__default_trusted_signers_enabled)) + cache_behavior['trusted_signers'] = valid_trusted_signers + return cache_behavior + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating trusted signers") + + def validate_viewer_certificate(self, viewer_certificate): + try: + if viewer_certificate is None: + return None + if viewer_certificate.get('cloudfront_default_certificate') and viewer_certificate.get('ssl_support_method') is not None: + self.module.fail_json(msg="viewer_certificate.ssl_support_method should not be specified with viewer_certificate_cloudfront_default" + + "_certificate set to true.") + self.validate_attribute_with_allowed_values(viewer_certificate.get('ssl_support_method'), 'viewer_certificate.ssl_support_method', + self.__valid_viewer_certificate_ssl_support_methods) + self.validate_attribute_with_allowed_values(viewer_certificate.get('minimum_protocol_version'), 'viewer_certificate.minimum_protocol_version', + self.__valid_viewer_certificate_minimum_protocol_versions) + self.validate_attribute_with_allowed_values(viewer_certificate.get('certificate_source'), 'viewer_certificate.certificate_source', + self.__valid_viewer_certificate_certificate_sources) + viewer_certificate = change_dict_key_name(viewer_certificate, 'cloudfront_default_certificate', 'cloud_front_default_certificate') + viewer_certificate = change_dict_key_name(viewer_certificate, 'ssl_support_method', 's_s_l_support_method') + viewer_certificate = change_dict_key_name(viewer_certificate, 'iam_certificate_id', 'i_a_m_certificate_id') + viewer_certificate = change_dict_key_name(viewer_certificate, 'acm_certificate_arn', 'a_c_m_certificate_arn') + return viewer_certificate + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating viewer certificate") + + def validate_custom_error_responses(self, config, custom_error_responses, purge_custom_error_responses): + try: + if custom_error_responses is None and not purge_custom_error_responses: + return ansible_list_to_cloudfront_list(config) + self.validate_is_list(custom_error_responses, 'custom_error_responses') + result = list() + existing_responses = dict((response['error_code'], response) for response in custom_error_responses) + for custom_error_response in custom_error_responses: + self.validate_required_key('error_code', 'custom_error_responses[].error_code', custom_error_response) + custom_error_response = change_dict_key_name(custom_error_response, 'error_caching_min_ttl', 'error_caching_min_t_t_l') + if 'response_code' in custom_error_response: + custom_error_response['response_code'] = str(custom_error_response['response_code']) + if custom_error_response['error_code'] in existing_responses: + del(existing_responses[custom_error_response['error_code']]) + result.append(custom_error_response) + if not purge_custom_error_responses: + result.extend(existing_responses.values()) + + return ansible_list_to_cloudfront_list(result) + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating custom error responses") + + def validate_restrictions(self, config, restrictions, purge_restrictions=False): + try: + if restrictions is None: + if purge_restrictions: + return None + else: + return config + self.validate_required_key('geo_restriction', 'restrictions.geo_restriction', restrictions) + geo_restriction = restrictions.get('geo_restriction') + self.validate_required_key('restriction_type', 'restrictions.geo_restriction.restriction_type', geo_restriction) + existing_restrictions = config.get('geo_restriction', {}).get(geo_restriction['restriction_type'], {}).get('items', []) + geo_restriction_items = geo_restriction.get('items') + if not purge_restrictions: + geo_restriction_items.extend([rest for rest in existing_restrictions if + rest not in geo_restriction_items]) + valid_restrictions = ansible_list_to_cloudfront_list(geo_restriction_items) + valid_restrictions['restriction_type'] = geo_restriction.get('restriction_type') + return {'geo_restriction': valid_restrictions} + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating restrictions") + + def validate_distribution_config_parameters(self, config, default_root_object, ipv6_enabled, http_version, web_acl_id): + try: + config['default_root_object'] = default_root_object or config.get('default_root_object', '') + config['is_i_p_v_6_enabled'] = ipv6_enabled or config.get('i_p_v_6_enabled', self.__default_ipv6_enabled) + if http_version is not None or config.get('http_version'): + self.validate_attribute_with_allowed_values(http_version, 'http_version', self.__valid_http_versions) + config['http_version'] = http_version or config.get('http_version') + if web_acl_id or config.get('web_a_c_l_id'): + config['web_a_c_l_id'] = web_acl_id or config.get('web_a_c_l_id') + return config + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating distribution config parameters") + + def validate_common_distribution_parameters(self, config, enabled, aliases, logging, price_class, purge_aliases=False): + try: + if config is None: + config = {} + if aliases is not None: + if not purge_aliases: + aliases.extend([alias for alias in config.get('aliases', {}).get('items', []) + if alias not in aliases]) + config['aliases'] = ansible_list_to_cloudfront_list(aliases) + if logging is not None: + config['logging'] = self.validate_logging(logging) + config['enabled'] = enabled or config.get('enabled', self.__default_distribution_enabled) + if price_class is not None: + self.validate_attribute_with_allowed_values(price_class, 'price_class', self.__valid_price_classes) + config['price_class'] = price_class + return config + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating common distribution parameters") + + def validate_comment(self, config, comment): + config['comment'] = comment or config.get('comment', "Distribution created by Ansible with datetime stamp " + self.__default_datetime_string) + return config + + def validate_caller_reference(self, caller_reference): + return caller_reference or self.__default_datetime_string + + def get_first_origin_id_for_default_cache_behavior(self, valid_origins): + try: + if valid_origins is not None: + valid_origins_list = valid_origins.get('items') + if valid_origins_list is not None and isinstance(valid_origins_list, list) and len(valid_origins_list) > 0: + return str(valid_origins_list[0].get('id')) + self.module.fail_json(msg="There are no valid origins from which to specify a target_origin_id for the default_cache_behavior configuration.") + except Exception as e: + self.module.fail_json_aws(e, msg="Error getting first origin_id for default cache behavior") + + def validate_attribute_list_with_allowed_list(self, attribute_list, attribute_list_name, allowed_list): + try: + self.validate_is_list(attribute_list, attribute_list_name) + if (isinstance(allowed_list, list) and set(attribute_list) not in allowed_list or + isinstance(allowed_list, set) and not set(allowed_list).issuperset(attribute_list)): + self.module.fail_json(msg='The attribute list {0} must be one of [{1}]'.format(attribute_list_name, ' '.join(str(a) for a in allowed_list))) + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating attribute list with allowed value list") + + def validate_attribute_with_allowed_values(self, attribute, attribute_name, allowed_list): + if attribute is not None and attribute not in allowed_list: + self.module.fail_json(msg='The attribute {0} must be one of [{1}]'.format(attribute_name, ' '.join(str(a) for a in allowed_list))) + + def validate_distribution_from_caller_reference(self, caller_reference): + try: + distributions = self.__cloudfront_facts_mgr.list_distributions(False) + distribution_name = 'Distribution' + distribution_config_name = 'DistributionConfig' + distribution_ids = [dist.get('Id') for dist in distributions] + for distribution_id in distribution_ids: + config = self.__cloudfront_facts_mgr.get_distribution(distribution_id) + distribution = config.get(distribution_name) + if distribution is not None: + distribution_config = distribution.get(distribution_config_name) + if distribution_config is not None and distribution_config.get('CallerReference') == caller_reference: + distribution['DistributionConfig'] = distribution_config + return distribution + + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating distribution from caller reference") + + def validate_distribution_from_aliases_caller_reference(self, distribution_id, aliases, caller_reference): + try: + if caller_reference is not None: + return self.validate_distribution_from_caller_reference(caller_reference) + else: + if aliases: + distribution_id = self.validate_distribution_id_from_alias(aliases) + if distribution_id: + return self.__cloudfront_facts_mgr.get_distribution(distribution_id) + return None + except Exception as e: + self.module.fail_json_aws(e, msg="Error validating distribution_id from alias, aliases and caller reference") + + def validate_distribution_id_from_alias(self, aliases): + distributions = self.__cloudfront_facts_mgr.list_distributions(False) + if distributions: + for distribution in distributions: + distribution_aliases = distribution.get('Aliases', {}).get('Items', []) + if set(aliases) & set(distribution_aliases): + return distribution['Id'] + return None + + def wait_until_processed(self, client, wait_timeout, distribution_id, caller_reference): + if distribution_id is None: + distribution_id = self.validate_distribution_from_caller_reference(caller_reference=caller_reference)['Id'] + + try: + waiter = client.get_waiter('distribution_deployed') + attempts = 1 + int(wait_timeout / 60) + waiter.wait(Id=distribution_id, WaiterConfig={'MaxAttempts': attempts}) + except botocore.exceptions.WaiterError as e: + self.module.fail_json(msg="Timeout waiting for cloudfront action. Waited for {0} seconds before timeout. " + "Error: {1}".format(to_text(wait_timeout), to_native(e))) + + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + self.module.fail_json_aws(e, msg="Error getting distribution {0}".format(distribution_id)) + + +def main(): + argument_spec = ec2_argument_spec() + + argument_spec.update(dict( + state=dict(choices=['present', 'absent'], default='present'), + caller_reference=dict(), + comment=dict(), + distribution_id=dict(), + e_tag=dict(), + tags=dict(type='dict', default={}), + purge_tags=dict(type='bool', default=False), + alias=dict(), + aliases=dict(type='list', default=[]), + purge_aliases=dict(type='bool', default=False), + default_root_object=dict(), + origins=dict(type='list'), + purge_origins=dict(type='bool', default=False), + default_cache_behavior=dict(type='dict'), + cache_behaviors=dict(type='list'), + purge_cache_behaviors=dict(type='bool', default=False), + custom_error_responses=dict(type='list'), + purge_custom_error_responses=dict(type='bool', default=False), + logging=dict(type='dict'), + price_class=dict(), + enabled=dict(type='bool'), + viewer_certificate=dict(type='dict'), + restrictions=dict(type='dict'), + web_acl_id=dict(), + http_version=dict(), + ipv6_enabled=dict(type='bool'), + default_origin_domain_name=dict(), + default_origin_path=dict(), + wait=dict(default=False, type='bool'), + wait_timeout=dict(default=1800, type='int') + )) + + result = {} + changed = True + + module = AnsibleAWSModule( + argument_spec=argument_spec, + supports_check_mode=False, + mutually_exclusive=[ + ['distribution_id', 'alias'], + ['default_origin_domain_name', 'distribution_id'], + ['default_origin_domain_name', 'alias'], + ] + ) + + region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True) + client = boto3_conn(module, conn_type='client', resource='cloudfront', region=region, endpoint=ec2_url, **aws_connect_kwargs) + + validation_mgr = CloudFrontValidationManager(module) + + state = module.params.get('state') + caller_reference = module.params.get('caller_reference') + comment = module.params.get('comment') + e_tag = module.params.get('e_tag') + tags = module.params.get('tags') + purge_tags = module.params.get('purge_tags') + distribution_id = module.params.get('distribution_id') + alias = module.params.get('alias') + aliases = module.params.get('aliases') + purge_aliases = module.params.get('purge_aliases') + default_root_object = module.params.get('default_root_object') + origins = module.params.get('origins') + purge_origins = module.params.get('purge_origins') + default_cache_behavior = module.params.get('default_cache_behavior') + cache_behaviors = module.params.get('cache_behaviors') + purge_cache_behaviors = module.params.get('purge_cache_behaviors') + custom_error_responses = module.params.get('custom_error_responses') + purge_custom_error_responses = module.params.get('purge_custom_error_responses') + logging = module.params.get('logging') + price_class = module.params.get('price_class') + enabled = module.params.get('enabled') + viewer_certificate = module.params.get('viewer_certificate') + restrictions = module.params.get('restrictions') + purge_restrictions = module.params.get('purge_restrictions') + web_acl_id = module.params.get('web_acl_id') + http_version = module.params.get('http_version') + ipv6_enabled = module.params.get('ipv6_enabled') + default_origin_domain_name = module.params.get('default_origin_domain_name') + default_origin_path = module.params.get('default_origin_path') + wait = module.params.get('wait') + wait_timeout = module.params.get('wait_timeout') + + if alias and alias not in aliases: + aliases.append(alias) + + distribution = validation_mgr.validate_distribution_from_aliases_caller_reference(distribution_id, aliases, caller_reference) + + update = state == 'present' and distribution + create = state == 'present' and not distribution + delete = state == 'absent' and distribution + + if not (update or create or delete): + module.exit_json(changed=False) + + if update or delete: + config = distribution['Distribution']['DistributionConfig'] + e_tag = distribution['ETag'] + distribution_id = distribution['Distribution']['Id'] + else: + config = dict() + if update: + config = camel_dict_to_snake_dict(config, reversible=True) + + if create or update: + config = validation_mgr.validate_common_distribution_parameters(config, enabled, aliases, logging, price_class, purge_aliases) + config = validation_mgr.validate_distribution_config_parameters(config, default_root_object, ipv6_enabled, http_version, web_acl_id) + config['origins'] = validation_mgr.validate_origins(client, config.get('origins', {}).get('items', []), origins, default_origin_domain_name, + default_origin_path, create, purge_origins) + config['cache_behaviors'] = validation_mgr.validate_cache_behaviors(config.get('cache_behaviors', {}).get('items', []), + cache_behaviors, config['origins'], purge_cache_behaviors) + config['default_cache_behavior'] = validation_mgr.validate_cache_behavior(config.get('default_cache_behavior', {}), + default_cache_behavior, config['origins'], True) + config['custom_error_responses'] = validation_mgr.validate_custom_error_responses(config.get('custom_error_responses', {}).get('items', []), + custom_error_responses, purge_custom_error_responses) + valid_restrictions = validation_mgr.validate_restrictions(config.get('restrictions', {}), restrictions, purge_restrictions) + if valid_restrictions: + config['restrictions'] = valid_restrictions + valid_viewer_certificate = validation_mgr.validate_viewer_certificate(viewer_certificate) + config = merge_validation_into_config(config, valid_viewer_certificate, 'viewer_certificate') + config = validation_mgr.validate_comment(config, comment) + config = snake_dict_to_camel_dict(config, capitalize_first=True) + + if create: + config['CallerReference'] = validation_mgr.validate_caller_reference(caller_reference) + result = create_distribution(client, module, config, ansible_dict_to_boto3_tag_list(tags)) + result = camel_dict_to_snake_dict(result) + result['tags'] = list_tags_for_resource(client, module, result['arn']) + + if delete: + if config['Enabled']: + config['Enabled'] = False + result = update_distribution(client, module, config, distribution_id, e_tag) + validation_mgr.wait_until_processed(client, wait_timeout, distribution_id, config.get('CallerReference')) + distribution = validation_mgr.validate_distribution_from_aliases_caller_reference(distribution_id, aliases, caller_reference) + # e_tag = distribution['ETag'] + result = delete_distribution(client, module, distribution) + + if update: + changed = config != distribution['Distribution']['DistributionConfig'] + if changed: + result = update_distribution(client, module, config, distribution_id, e_tag) + else: + result = distribution['Distribution'] + existing_tags = list_tags_for_resource(client, module, result['ARN']) + distribution['Distribution']['DistributionConfig']['tags'] = existing_tags + changed |= update_tags(client, module, existing_tags, tags, purge_tags, result['ARN']) + result = camel_dict_to_snake_dict(result) + result['distribution_config']['tags'] = config['tags'] = list_tags_for_resource(client, module, result['arn']) + result['diff'] = dict() + diff = recursive_diff(distribution['Distribution']['DistributionConfig'], config) + if diff: + result['diff']['before'] = diff[0] + result['diff']['after'] = diff[1] + + if wait and (create or update): + validation_mgr.wait_until_processed(client, wait_timeout, distribution_id, config.get('CallerReference')) + + if 'distribution_config' in result: + result.update(result['distribution_config']) + del(result['distribution_config']) + + module.exit_json(changed=changed, **result) + + +if __name__ == '__main__': + main() From 1d6f8198edaed96aeb7fa37c03b00e1ebe9861e5 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Mon, 29 Jan 2024 10:36:43 +0100 Subject: [PATCH 08/19] Update centos7 ami --- .../templates/vcl/subroutines/deliver.vcl.j2 | 89 ------------------- 1 file changed, 89 deletions(-) diff --git a/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 b/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 index 0f6a2600c..e69de29bb 100644 --- a/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 +++ b/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 @@ -1,89 +0,0 @@ -{% if varnish_do_not_expose_caching %} -if ( client.ip ~ trusted ) { -{% endif %} -if (obj.hits > 0) { - set resp.http.X-Magento-Cache-Debug = "HIT"; -} else { - set resp.http.X-Magento-Cache-Debug = "MISS"; -} -{% if varnish_do_not_expose_caching %} -}else { - unset resp.http.X-Cached-At; -} -{% endif %} - -if (resp.http.X-Rewrite-HTML-Maxage) { - unset resp.http.X-Rewrite-HTML-Maxage; - - {% if varnish_html_cache_control_max_age != 0 %} - # Rewrite only max-age, get rid of s-max-age - set resp.http.Cache-Control = "public, max-age={{ varnish_html_cache_control_max_age }}"; - - # Just in case somebody is using HTTP/1.0 fix the Expires header too - set resp.http.Expires = "" + (now + {{ varnish_html_cache_control_max_age }}s); - {% else %} - set resp.http.Cache-Control = "no-cache, no-store, must-revalidate"; - set resp.http.Pragma = "no-cache"; - set resp.http.Expires = "0"; - {% endif %} -} - -if (req.http.{{ varnish_debug_request_info_header_name }}) { - set resp.http.{{ varnish_debug_request_info_header_name }} = - req.http.{{ varnish_debug_request_info_header_name }} + "; Backend-Response-Delivered"; -} - -if (req.http.{{ varnish_bypass_request_info_header_name }}) { - set resp.http.{{ varnish_bypass_request_info_header_name }} = - req.http.{{ varnish_bypass_request_info_header_name }} + "; Backend-Response-Delivered"; - set resp.http.Set-Cookie = "{{ varnish_bypass_request_cookie_name}} = {{ varnish_bypass_request_token }}; Secure"; - set resp.http.Cache-Control = "no-cache"; -} - -if (req.http.{{ varnish_bypass_request_info_header_name }}) { - set resp.http.{{ varnish_bypass_request_info_header_name }} = - req.http.{{ varnish_bypass_request_info_header_name }} + "; Backend-Response-Delivered"; - set resp.http.Set-Cookie = "{{ varnish_bypass_request_cookie_name}} = {{ varnish_bypass_request_token }}; Secure"; - set resp.http.Cache-Control = "no-cache"; -} - -if (resp.status == 502 || resp.status == 503 || resp.status == 504) { - return(synth(resp.status)); -} - -if (req.http.X-Warmup) { - set resp.http.X-Warmup = "skipped-body"; - set resp.status = 204; -} - -{% if varnish_throttling_enabled %} - {% include "vcl/subroutines/deliver_throttling.vcl.j2" %} -{% endif %} - -{% if varnish_media_cors_enabled %} -if (req.url ~ "^/media/" && req.http.Origin ~ "^https?:\/\/({{ varnish_media_cors_allowed_origins | join("|") | replace(".", "\.") | replace("-", "\-") }})$") { - set resp.http.Access-Control-Max-Age = "{{ varnish_media_cors_max_age }}"; - set resp.http.Access-Control-Allow-Methods = "HEAD, GET, OPTIONS"; - set resp.http.Access-Control-Allow-Origin = req.http.Origin; - set resp.http.Access-Control-Allow-Headers = "*"; -} - -# https://bugs.chromium.org/p/chromium/issues/detail?id=260239 -# Chrome aggressively cache requests and won't try CORS request if there is already response in cache -# without Vary: Origin header. -if (resp.http.Vary) { - set resp.http.Vary = resp.http.Vary + ", Origin"; -}else { - set resp.http.Vary = "Origin"; -} - -{% endif %} - -unset resp.http.Age; -unset resp.http.X-Magento-Debug; -unset resp.http.X-Magento-Tags; -unset resp.http.X-Powered-By; -unset resp.http.Server; -unset resp.http.X-Varnish; -unset resp.http.Via; -unset resp.http.Link; From da12c1db761b4e5991c4c2cc434741776b383a6a Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Mon, 5 Feb 2024 10:59:07 +0100 Subject: [PATCH 09/19] Place cloudflare exclusive traffic into correct location Current location is not included in nginx configuration therefore it had no effect --- roles/cs.cloudflare/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cs.cloudflare/defaults/main.yml b/roles/cs.cloudflare/defaults/main.yml index dcf4357a9..a20e2291b 100644 --- a/roles/cs.cloudflare/defaults/main.yml +++ b/roles/cs.cloudflare/defaults/main.yml @@ -1,5 +1,5 @@ cloudflare_enabled: no -# Accept only traffic comming from Cloudflare +# Accept only traffic coming from Cloudflare cloudflare_exclusive_traffic: yes # configuration file paths From 89ab044b2cd0b6c4c12abeb8eda270c03568e403 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Fri, 2 Feb 2024 11:24:16 +0100 Subject: [PATCH 10/19] Expose pio quality target spread option as pio_quality_target_spread --- roles/cs.pio/defaults/main.yml | 1 + roles/cs.pio/templates/pio-config.conf.j2 | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/cs.pio/defaults/main.yml b/roles/cs.pio/defaults/main.yml index 5b8369b2c..6ed280e61 100644 --- a/roles/cs.pio/defaults/main.yml +++ b/roles/cs.pio/defaults/main.yml @@ -18,6 +18,7 @@ pio_media_fast_quality: 95 pio_target_directory: pio_cache pio_media_target_directory: pio_cache_cms pio_optimize_media: yes +pio_quality_target_spread: 80 # We cannot use magento_media_dir here as it might not be defined at this point pio_filesystem_local_dir: "{{ mageops_app_web_dir }}/shared/pub/media/" diff --git a/roles/cs.pio/templates/pio-config.conf.j2 b/roles/cs.pio/templates/pio-config.conf.j2 index 3ac3a2a4f..8817827c0 100644 --- a/roles/cs.pio/templates/pio-config.conf.j2 +++ b/roles/cs.pio/templates/pio-config.conf.j2 @@ -17,7 +17,7 @@ http_server: ( kind: "LazyResizeHttpServer", addr: "127.0.0.1:8441", - quality_target_spread: 100, + quality_target_spread: {{ pio_quality_target_spread }}, secret: "{{ lazy_resize_secret }}", source_directory: "catalog/product", target_directory: "{{ pio_target_directory }}", @@ -28,7 +28,7 @@ url: "/media", quality: ( quality: {{ pio_media_fast_quality }}, - spread: 100 + spread: {{ pio_quality_target_spread }} ), target_ssim: {{ pio_media_target_ssim }}, source_directory: "", From 1a56feac4e276cc67e103371f7788618cbb62c73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3=C5=BAwiak?= Date: Wed, 17 Jan 2024 10:17:08 +0100 Subject: [PATCH 11/19] QA/Testing Team request to skip varnish cache. For this purpose QA can add ?__mgo_bypass_token=token to URL. This action set MageOpsBypassToken cookie and works utill this cookie will be deleted. When the cookie is set Varnish will deliver content without using cache. --- group_vars/all.yml | 2 + .../templates/vcl/subroutines/deliver.vcl.j2 | 92 +++++++++++++++++++ site.step-40-app-node.yml | 1 - 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/group_vars/all.yml b/group_vars/all.yml index cc1a9c204..ebc05ef63 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1673,6 +1673,8 @@ varnish_strip_params: - "{{ https_termination_redirect_source_domain_param }}" varnish_debug_request_info_header_name: "{{ mageops_debug_http_header_prefix }}-Info-Varnish" +varnish_bypass_request_info_header_name: "{{ mageops_bypass_http_header_prefix }}-Info-Varnish" + # ---------------------------------------------------------- # -------- Varnish Language Detection & Redirects -------- diff --git a/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 b/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 index e69de29bb..9dc907c95 100644 --- a/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 +++ b/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 @@ -0,0 +1,92 @@ +<<<<<<< HEAD +======= +{% if varnish_do_not_expose_caching %} +if ( client.ip ~ trusted ) { +{% endif %} +if (obj.hits > 0) { + set resp.http.X-Magento-Cache-Debug = "HIT"; +} else { + set resp.http.X-Magento-Cache-Debug = "MISS"; +} +{% if varnish_do_not_expose_caching %} +}else { + unset resp.http.X-Cached-At; +} +{% endif %} + +if (resp.http.X-Rewrite-HTML-Maxage) { + unset resp.http.X-Rewrite-HTML-Maxage; + + {% if varnish_html_cache_control_max_age != 0 %} + # Rewrite only max-age, get rid of s-max-age + set resp.http.Cache-Control = "public, max-age={{ varnish_html_cache_control_max_age }}"; + + # Just in case somebody is using HTTP/1.0 fix the Expires header too + set resp.http.Expires = "" + (now + {{ varnish_html_cache_control_max_age }}s); + {% else %} + set resp.http.Cache-Control = "no-cache, no-store, must-revalidate"; + set resp.http.Pragma = "no-cache"; + set resp.http.Expires = "0"; + {% endif %} +} + +if (req.http.{{ varnish_debug_request_info_header_name }}) { + set resp.http.{{ varnish_debug_request_info_header_name }} = + req.http.{{ varnish_debug_request_info_header_name }} + "; Backend-Response-Delivered"; +} + +if (req.http.{{ varnish_bypass_request_info_header_name }}) { + set resp.http.{{ varnish_bypass_request_info_header_name }} = + req.http.{{ varnish_bypass_request_info_header_name }} + "; Backend-Response-Delivered"; + set resp.http.Set-Cookie = "{{ varnish_bypass_request_cookie_name}} = {{ varnish_bypass_request_token }}; Secure"; + set resp.http.Cache-Control = "no-cache"; +} + +if (req.http.{{ varnish_bypass_request_info_header_name }}) { + set resp.http.{{ varnish_bypass_request_info_header_name }} = + req.http.{{ varnish_bypass_request_info_header_name }} + "; Backend-Response-Delivered"; + set resp.http.Set-Cookie = "{{ varnish_bypass_request_cookie_name}} = {{ varnish_bypass_request_token }}; Secure"; + set resp.http.Cache-Control = "no-cache"; +} + +if (resp.status == 502 || resp.status == 504) { + return(synth(resp.status)); +} + +if (req.http.X-Warmup) { + set resp.http.X-Warmup = "skipped-body"; + set resp.status = 204; +} + +{% if varnish_throttling_enabled %} + {% include "vcl/subroutines/deliver_throttling.vcl.j2" %} +{% endif %} + +{% if varnish_media_cors_enabled %} +if (req.url ~ "^/media/" && req.http.Origin ~ "^https?:\/\/({{ varnish_media_cors_allowed_origins | join("|") | replace(".", "\.") | replace("-", "\-") }})$") { + set resp.http.Access-Control-Max-Age = "{{ varnish_media_cors_max_age }}"; + set resp.http.Access-Control-Allow-Methods = "HEAD, GET, OPTIONS"; + set resp.http.Access-Control-Allow-Origin = req.http.Origin; + set resp.http.Access-Control-Allow-Headers = "*"; +} + +# https://bugs.chromium.org/p/chromium/issues/detail?id=260239 +# Chrome aggressively cache requests and won't try CORS request if there is already response in cache +# without Vary: Origin header. +if (resp.http.Vary) { + set resp.http.Vary = resp.http.Vary + ", Origin"; +}else { + set resp.http.Vary = "Origin"; +} + +{% endif %} + +unset resp.http.Age; +unset resp.http.X-Magento-Debug; +unset resp.http.X-Magento-Tags; +unset resp.http.X-Powered-By; +unset resp.http.Server; +unset resp.http.X-Varnish; +unset resp.http.Via; +unset resp.http.Link; +>>>>>>> 599e99a (QA/Testing Team request to skip varnish cache.) diff --git a/site.step-40-app-node.yml b/site.step-40-app-node.yml index 45fb64b8f..4199c0597 100644 --- a/site.step-40-app-node.yml +++ b/site.step-40-app-node.yml @@ -198,6 +198,5 @@ nginx_debug_request_header_name: "{{ mageops_debug_token_http_header }}" nginx_debug_request_query_param_name: "{{ mageops_debug_token_query_param }}" nginx_debug_request_cookie_name: "{{ mageops_debug_token_request_cookie }}" - # On app node we might not have yet magento code deployed magento_facts_detect_from_artifacts: yes From f3cb2a2e55bf3ef241ed44d99b61db4e84af5529 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Thu, 1 Feb 2024 13:03:21 +0100 Subject: [PATCH 12/19] Allow dynamic setting of new_relic license --- group_vars/all.yml | 3 +-- roles/cs.new-relic/defaults/main.yml | 4 +-- .../cs.new-relic/files/newrelic_feature.bash | 26 ++++++++++++++++--- roles/cs.new-relic/tasks/main.yml | 4 +++ 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/group_vars/all.yml b/group_vars/all.yml index ebc05ef63..8d94dd14f 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -2055,5 +2055,4 @@ aws_pio_ebs_volume_size: "{{ aws_app_node_ebs_volume_size }}" # ----- New Relic ----- # --------------------- new_relic_app_name: "{{ mageops_app_name }}" -mageops_new_relic_enabled: no -# new_relic_license need to be set up +mageops_new_relic_enabled: yes diff --git a/roles/cs.new-relic/defaults/main.yml b/roles/cs.new-relic/defaults/main.yml index 32d3a9e27..702e8eee7 100644 --- a/roles/cs.new-relic/defaults/main.yml +++ b/roles/cs.new-relic/defaults/main.yml @@ -1,7 +1,7 @@ new_relic_repo_url: http://yum.newrelic.com/pub/newrelic/el5/x86_64/newrelic-repo-5-3.noarch.rpm new_relic_packages: - newrelic-php5 -# new_relic_license: +new_relic_license: new_relic_app_name: "New relic app name" new_relic_collector_enabled: yes new_relic_ignore_user_exception_handler: no @@ -15,7 +15,7 @@ new_relic_stact_trace_threshold: "3s" new_relic_explain_enabled: yes new_relic_explain_threshold: "500ms" new_relic_framework: magento2 -new_relic_enabled: yes +new_relic_enabled: "{{ new_relic_license != '' }}" new_relic_cron_enabled: no new_relic_cron_start: "0 7 * * *" # From 7:00 diff --git a/roles/cs.new-relic/files/newrelic_feature.bash b/roles/cs.new-relic/files/newrelic_feature.bash index de780e0ab..2edb8fb19 100644 --- a/roles/cs.new-relic/files/newrelic_feature.bash +++ b/roles/cs.new-relic/files/newrelic_feature.bash @@ -1,6 +1,7 @@ #!/usr/bin/env bash feature__flag_name="newrelic_apm" +feature__license_key="newrelic_license_key" feature::apply() { local expected @@ -9,9 +10,15 @@ feature::apply() { expected="$(feature::expected_value)" expected="$(feature::normalize_expected "$expected")" current="$(feature::current_value)" + current_license="$(feature::current_license_value)" + expected_license="$(feature::expected_license_value)" - if [ "$expected" != "$current" ];then - feature::update "$expected" + if [ -z "$expected_license" ];then + # We cannot enable the feature without a license key + expected="false" + fi + if [ "$expected" != "$current" ] || [ "$expected_license" != "$current_license" ];then + feature::update "$expected" "$expected_license" fi } @@ -34,6 +41,10 @@ feature::expected_value() { features::read_feature_flag "$feature__flag_name" "false" } +feature::expected_license_value() { + features::read_feature_flag "$feature__license_key" "" +} + feature::current_value() { local current @@ -46,12 +57,21 @@ feature::current_value() { echo "$current" } +feature::current_license_value() { + local current + + current="$(grep '^newrelic.license' /etc/php.d/newrelic.ini | sed 's/.*=\s*"\(.*\)"\s*$/\1/')" + + echo "$current" +} + feature::update() { local value=$1 - local config + local license=$2 echo "Setting newrelic apm to $value" sed -i -e "s/newrelic.enabled[[:space:]]=[[:space:]].*/newrelic.enabled = ${value}/" /etc/php.d/newrelic.ini + sed -i -e "s/newrelic.license[[:space:]]=[[:space:]].*/newrelic.license = \"${license}\"/" /etc/php.d/newrelic.ini echo "Reloading php-fpm" systemctl reload php-fpm } diff --git a/roles/cs.new-relic/tasks/main.yml b/roles/cs.new-relic/tasks/main.yml index 99f4f086a..e8fa5d1b3 100644 --- a/roles/cs.new-relic/tasks/main.yml +++ b/roles/cs.new-relic/tasks/main.yml @@ -32,6 +32,10 @@ shell: "mageopscli set_feature_flag newrelic_apm {{ new_relic_enabled | ternary('true', 'false') }}" when: aws_use +- name: Set license key feature flag + shell: "mageopscli set_feature_flag newrelic_license_key \"{{ new_relic_license }}\"" + when: aws_use + - name: Setup cron template: src: cron.j2 From 1c88493a620db5ffa2d9c67e27bccc9ab4a097d0 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Mon, 26 Feb 2024 16:01:54 +0100 Subject: [PATCH 13/19] Allow access to Data Lifecycle Manager for full access policy --- .../templates/employee_full_access.policy.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/cs.aws-iam-employee-access/templates/employee_full_access.policy.json b/roles/cs.aws-iam-employee-access/templates/employee_full_access.policy.json index 1d65901eb..f38ed3e80 100644 --- a/roles/cs.aws-iam-employee-access/templates/employee_full_access.policy.json +++ b/roles/cs.aws-iam-employee-access/templates/employee_full_access.policy.json @@ -29,7 +29,6 @@ "Effect": "Allow", "Resource": "arn:aws:pi:*:*:metrics/rds/*" }, - { "Action": "ec2:*", "Effect": "Allow", @@ -307,6 +306,11 @@ "freetier:Get*" ], "Resource": "*" + }, + { + "Action": "dlm:*", + "Effect": "Allow", + "Resource": "*" } ] } From 366a50e9a1bf294350550ebfbecec382ce757520 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Fri, 2 Feb 2024 17:09:17 +0100 Subject: [PATCH 14/19] Add missing help entry to mageopscli --- roles/cs.mageops-cli/files/mageopscli | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/cs.mageops-cli/files/mageopscli b/roles/cs.mageops-cli/files/mageopscli index c12ed0f5a..343e03d8c 100755 --- a/roles/cs.mageops-cli/files/mageopscli +++ b/roles/cs.mageops-cli/files/mageopscli @@ -151,6 +151,7 @@ main::help() { main::eprintln " is_feature_flag_set Checks if there is any value set for feature flag" main::eprintln " status code 0 means flag is set, 1 otherwise" main::eprintln " apply_features Apply feature updates to this host" + main::eprintln " clear_opcache Clears opcache for php and php-fpm" main::eprintln "" main::eprintln " Mageops cli tools" main::eprintln " (c) Creativestyle 2020" From 0b5f9cb9c0f80b91b6b72fe0d5bf84bf10e6c78f Mon Sep 17 00:00:00 2001 From: Szpadel Date: Mon, 11 Mar 2024 13:32:36 +0100 Subject: [PATCH 15/19] Create symlink to new geoip configuration location (#382) --- roles/cs.geolite2/tasks/main.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/roles/cs.geolite2/tasks/main.yml b/roles/cs.geolite2/tasks/main.yml index 1f8632e91..e158c82f6 100644 --- a/roles/cs.geolite2/tasks/main.yml +++ b/roles/cs.geolite2/tasks/main.yml @@ -10,7 +10,13 @@ - name: Install geoupdate configuration template: src: GeoIP.conf.j2 - dest: /etc/GeoIP.conf + dest: /usr/local/etc/GeoIP.conf + +- name: Symlink geoupdate configuration to new location + file: + dest: /usr/local/etc/GeoIP.conf + src: /etc/GeoIP.conf + state: link - name: Update geolite2 databases shell: geoipupdate From 33783c0c6ce7a2f189bea573bd535ff91380167f Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Mon, 11 Mar 2024 12:19:37 +0100 Subject: [PATCH 16/19] Generate new configuration for geoupdate 6.x --- roles/cs.geolite2/tasks/main.yml | 7 +++---- roles/cs.geolite2/templates/GeoIP6.conf.j2 | 3 +++ 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 roles/cs.geolite2/templates/GeoIP6.conf.j2 diff --git a/roles/cs.geolite2/tasks/main.yml b/roles/cs.geolite2/tasks/main.yml index e158c82f6..8610f44c9 100644 --- a/roles/cs.geolite2/tasks/main.yml +++ b/roles/cs.geolite2/tasks/main.yml @@ -12,11 +12,10 @@ src: GeoIP.conf.j2 dest: /usr/local/etc/GeoIP.conf -- name: Symlink geoupdate configuration to new location - file: +- name: Install geoupdate 6.x configuration + template: + src: GeoIP6.conf.j2 dest: /usr/local/etc/GeoIP.conf - src: /etc/GeoIP.conf - state: link - name: Update geolite2 databases shell: geoipupdate diff --git a/roles/cs.geolite2/templates/GeoIP6.conf.j2 b/roles/cs.geolite2/templates/GeoIP6.conf.j2 new file mode 100644 index 000000000..665729dd8 --- /dev/null +++ b/roles/cs.geolite2/templates/GeoIP6.conf.j2 @@ -0,0 +1,3 @@ +AccountID {{ geolite2_userid }} +LicenseKey {{ geolite2_license_key }} +EditionIDs {{ geolite2_product_ids }} From 060e4ea94a7c46c618a61a2ee940336c17a731be Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Wed, 13 Mar 2024 13:28:27 +0100 Subject: [PATCH 17/19] Add permissions required for aws dlm --- roles/cs.aws-iam/defaults/main.yml | 5 ++ roles/cs.aws-iam/tasks/dlm-roles.yaml | 23 ++++++++++ roles/cs.aws-iam/tasks/main.yml | 1 + .../templates/aws_dlm_access.policy.json | 46 +++++++++++++++++++ .../templates/aws_dlm_sts_access.policy.json | 12 +++++ 5 files changed, 87 insertions(+) create mode 100644 roles/cs.aws-iam/tasks/dlm-roles.yaml create mode 100644 roles/cs.aws-iam/templates/aws_dlm_access.policy.json create mode 100644 roles/cs.aws-iam/templates/aws_dlm_sts_access.policy.json diff --git a/roles/cs.aws-iam/defaults/main.yml b/roles/cs.aws-iam/defaults/main.yml index 6ed67a79c..aba263e8c 100644 --- a/roles/cs.aws-iam/defaults/main.yml +++ b/roles/cs.aws-iam/defaults/main.yml @@ -12,6 +12,9 @@ aws_iam_policy_cloudwatch_metrics_access: "{{ aws_iam_name_prefix }}CloudWatchMe aws_iam_policy_lambda_access: "{{ aws_iam_name_prefix }}LambdaAccess" aws_iam_policy_kms_access: "{{ aws_iam_name_prefix }}KmsAccess" +aws_iam_policy_dlm_sts_access: "{{ aws_iam_name_prefix }}DLMAllowSTSAccess" +aws_iam_policy_dlm_aws_access: "{{ aws_iam_name_prefix }}DLMAccess" + aws_iam_group_custom_policies: "{{ aws_iam_name_prefix }}CustomPolicies" aws_iam_group_standard_policies: "{{ aws_iam_name_prefix }}StandardPolicies" @@ -29,3 +32,5 @@ aws_iam_role_node_coordinator_lambda_execution: "{{ aws_iam_name_prefix }}Handle aws_iam_role_app_node: "{{ aws_iam_name_prefix }}AppNode" aws_iam_role_varnish: "{{ aws_iam_role_app_node }}" aws_iam_role_persistent_node: "{{ aws_iam_name_prefix }}PersistentNode" + +aws_iam_role_dlm: "{{ aws_iam_name_prefix }}DLM" diff --git a/roles/cs.aws-iam/tasks/dlm-roles.yaml b/roles/cs.aws-iam/tasks/dlm-roles.yaml new file mode 100644 index 000000000..be883a3ad --- /dev/null +++ b/roles/cs.aws-iam/tasks/dlm-roles.yaml @@ -0,0 +1,23 @@ +- name: Create DLM STS access policy + iam_managed_policy: + policy_name: "{{ aws_iam_policy_dlm_sts_access }}" + policy_description: Allow DLM to assume STS role + policy: "{{ lookup('template', 'dlm_sts_access.policy.json') }}" + state: present + register: iam_dlm_sts_access + +- name: Create DLM AWS access policy + iam_managed_policy: + policy_name: "{{ aws_iam_policy_dlm_aws_access }}" + policy_description: Allow DLM to access AWS resources + policy: "{{ lookup('template', 'dlm_aws_access.policy.json') }}" + state: present + register: iam_dlm_aws_access + +- name: Create role for DLM + iam_role: + assume_role_policy_document: "{{ lookup('template', 'dlm_sts_access.policy.json') }}" + name: "{{ aws_iam_role_dlm }}" + state: present + managed_policy: + - "{{ iam_dlm_aws_access.policy.arn }}" diff --git a/roles/cs.aws-iam/tasks/main.yml b/roles/cs.aws-iam/tasks/main.yml index 2db9ff45b..54a3057ba 100644 --- a/roles/cs.aws-iam/tasks/main.yml +++ b/roles/cs.aws-iam/tasks/main.yml @@ -3,3 +3,4 @@ - import_tasks: lambda-roles.yml - import_tasks: kms-roles.yml - import_tasks: provisioning-groups.yml +- import_tasks: dlm-roles.yml diff --git a/roles/cs.aws-iam/templates/aws_dlm_access.policy.json b/roles/cs.aws-iam/templates/aws_dlm_access.policy.json new file mode 100644 index 000000000..c868a4c47 --- /dev/null +++ b/roles/cs.aws-iam/templates/aws_dlm_access.policy.json @@ -0,0 +1,46 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:CreateSnapshot", + "ec2:CreateSnapshots", + "ec2:DeleteSnapshot", + "ec2:DescribeInstances", + "ec2:DescribeVolumes", + "ec2:DescribeSnapshots", + "ec2:EnableFastSnapshotRestores", + "ec2:DescribeFastSnapshotRestores", + "ec2:DisableFastSnapshotRestores", + "ec2:CopySnapshot", + "ec2:ModifySnapshotAttribute", + "ec2:DescribeSnapshotAttribute", + "ec2:DescribeSnapshotTierStatus", + "ec2:ModifySnapshotTier" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateTags" + ], + "Resource": "arn:aws:ec2:*::snapshot/*" + }, + { + "Effect": "Allow", + "Action": [ + "events:PutRule", + "events:DeleteRule", + "events:DescribeRule", + "events:EnableRule", + "events:DisableRule", + "events:ListTargetsByRule", + "events:PutTargets", + "events:RemoveTargets" + ], + "Resource": "arn:aws:events:*:*:rule/AwsDataLifecycleRule.managed-cwe.*" + } + ] +} diff --git a/roles/cs.aws-iam/templates/aws_dlm_sts_access.policy.json b/roles/cs.aws-iam/templates/aws_dlm_sts_access.policy.json new file mode 100644 index 000000000..85ec67594 --- /dev/null +++ b/roles/cs.aws-iam/templates/aws_dlm_sts_access.policy.json @@ -0,0 +1,12 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "dlm.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} From d4cff8c56dd6a4dae666b301f9cf03f3dab227c0 Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Tue, 19 Mar 2024 12:35:26 +0100 Subject: [PATCH 18/19] Fix typos in dlm roles profile names --- roles/cs.aws-iam/tasks/dlm-roles.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/cs.aws-iam/tasks/dlm-roles.yaml b/roles/cs.aws-iam/tasks/dlm-roles.yaml index be883a3ad..e17adf085 100644 --- a/roles/cs.aws-iam/tasks/dlm-roles.yaml +++ b/roles/cs.aws-iam/tasks/dlm-roles.yaml @@ -2,7 +2,7 @@ iam_managed_policy: policy_name: "{{ aws_iam_policy_dlm_sts_access }}" policy_description: Allow DLM to assume STS role - policy: "{{ lookup('template', 'dlm_sts_access.policy.json') }}" + policy: "{{ lookup('template', 'aws_dlm_sts_access.policy.json') }}" state: present register: iam_dlm_sts_access @@ -10,13 +10,13 @@ iam_managed_policy: policy_name: "{{ aws_iam_policy_dlm_aws_access }}" policy_description: Allow DLM to access AWS resources - policy: "{{ lookup('template', 'dlm_aws_access.policy.json') }}" + policy: "{{ lookup('template', 'aws_dlm_access.policy.json') }}" state: present register: iam_dlm_aws_access - name: Create role for DLM iam_role: - assume_role_policy_document: "{{ lookup('template', 'dlm_sts_access.policy.json') }}" + assume_role_policy_document: "{{ lookup('template', 'aws_dlm_sts_access.policy.json') }}" name: "{{ aws_iam_role_dlm }}" state: present managed_policy: From 17cb5b0037657a31282c955969f28c94cf9686bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3=C5=BAwiak?= Date: Tue, 9 Apr 2024 10:55:38 +0200 Subject: [PATCH 19/19] init --- group_vars/all.yml | 11 +---- requirements-python.txt | 4 -- .../employee_full_access.policy.json | 6 +-- roles/cs.aws-iam/defaults/main.yml | 5 -- roles/cs.aws-iam/tasks/dlm-roles.yaml | 23 ---------- roles/cs.aws-iam/tasks/main.yml | 1 - .../templates/aws_dlm_access.policy.json | 46 ------------------- .../templates/aws_dlm_sts_access.policy.json | 12 ----- roles/cs.cloudflare/defaults/main.yml | 2 +- roles/cs.geolite2/tasks/main.yml | 7 +-- roles/cs.geolite2/templates/GeoIP6.conf.j2 | 3 -- .../defaults/main/app-etc.yml | 3 -- .../tasks/000-prepare-runtime-config.yml | 8 +++- .../tasks/080-core-config.yml | 11 +++++ roles/cs.mageops-cli/files/mageopscli | 1 - roles/cs.mysql-configure/tasks/create-db.yml | 9 ++++ roles/cs.new-relic/defaults/main.yml | 4 +- .../cs.new-relic/files/newrelic_feature.bash | 26 ++--------- roles/cs.new-relic/meta/main.yml | 3 -- roles/cs.new-relic/tasks/main.yml | 4 -- roles/cs.pio/defaults/main.yml | 1 - roles/cs.pio/templates/pio-config.conf.j2 | 4 +- roles/cs.switch-pkg-mgr/defaults/main.yml | 1 - roles/cs.switch-pkg-mgr/files/yum.conf | 7 --- roles/cs.switch-pkg-mgr/tasks/main.yml | 12 ----- roles/cs.switch-pkg-mgr/tasks/yum.yml | 9 ---- .../files/dnf.conf | 0 .../tasks/main.yml} | 0 .../templates/vcl/subroutines/deliver.vcl.j2 | 10 ---- roles/cs.wait-for-warmup/defaults/main.yml | 2 - roles/cs.wait-for-warmup/tasks/main.yml | 10 ---- site.step-15-varnish.yml | 3 +- site.step-20-persistent.yml | 3 +- site.step-40-app-node.yml | 4 +- site.step-45-app-deploy.yml | 3 +- site.step-80-monitoring.yml | 6 --- 36 files changed, 44 insertions(+), 220 deletions(-) delete mode 100644 roles/cs.aws-iam/tasks/dlm-roles.yaml delete mode 100644 roles/cs.aws-iam/templates/aws_dlm_access.policy.json delete mode 100644 roles/cs.aws-iam/templates/aws_dlm_sts_access.policy.json delete mode 100644 roles/cs.geolite2/templates/GeoIP6.conf.j2 delete mode 100644 roles/cs.new-relic/meta/main.yml delete mode 100644 roles/cs.switch-pkg-mgr/defaults/main.yml delete mode 100644 roles/cs.switch-pkg-mgr/files/yum.conf delete mode 100644 roles/cs.switch-pkg-mgr/tasks/main.yml delete mode 100644 roles/cs.switch-pkg-mgr/tasks/yum.yml rename roles/{cs.switch-pkg-mgr => cs.switch-to-dnf}/files/dnf.conf (100%) rename roles/{cs.switch-pkg-mgr/tasks/dnf.yml => cs.switch-to-dnf/tasks/main.yml} (100%) delete mode 100644 roles/cs.wait-for-warmup/defaults/main.yml delete mode 100644 roles/cs.wait-for-warmup/tasks/main.yml diff --git a/group_vars/all.yml b/group_vars/all.yml index 8d94dd14f..80dce4642 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -955,7 +955,6 @@ magento_admin_user_lastname: Suite # Path to node warmup script executed at instance start relative to magento app root dir - never need to override... magento_node_warmup_script_path: /bin/node-warmup.sh -mageops_wait_for_warmup_secs: 600 # ------------------------------ # -------- Magento SCD -------- @@ -1673,8 +1672,6 @@ varnish_strip_params: - "{{ https_termination_redirect_source_domain_param }}" varnish_debug_request_info_header_name: "{{ mageops_debug_http_header_prefix }}-Info-Varnish" -varnish_bypass_request_info_header_name: "{{ mageops_bypass_http_header_prefix }}-Info-Varnish" - # ---------------------------------------------------------- # -------- Varnish Language Detection & Redirects -------- @@ -1823,11 +1820,6 @@ mageops_cli_features_dir: /usr/local/lib/mageops/features # Whether to perform full update mageops_packages_full_update: yes -# Package manager to use -mageops_pkg_mgr: - # Supported options: dnf, yum - centos7: dnf - # Packages that are ensured to be absent on all nodes mageops_packages_mirrorlist_countrycode: "de" @@ -2055,4 +2047,5 @@ aws_pio_ebs_volume_size: "{{ aws_app_node_ebs_volume_size }}" # ----- New Relic ----- # --------------------- new_relic_app_name: "{{ mageops_app_name }}" -mageops_new_relic_enabled: yes +mageops_new_relic_enabled: no +# new_relic_license need to be set up diff --git a/requirements-python.txt b/requirements-python.txt index 848407e7e..81eee6cdb 100644 --- a/requirements-python.txt +++ b/requirements-python.txt @@ -4,10 +4,6 @@ ansible>=6,<7 # make sure this is BEFORE boto3 and boto awscli -# some tasks call aws command on localhost -# make sure this is BEFORE boto3 and boto -awscli - # needed for inventory and aws modules boto3 diff --git a/roles/cs.aws-iam-employee-access/templates/employee_full_access.policy.json b/roles/cs.aws-iam-employee-access/templates/employee_full_access.policy.json index f38ed3e80..1d65901eb 100644 --- a/roles/cs.aws-iam-employee-access/templates/employee_full_access.policy.json +++ b/roles/cs.aws-iam-employee-access/templates/employee_full_access.policy.json @@ -29,6 +29,7 @@ "Effect": "Allow", "Resource": "arn:aws:pi:*:*:metrics/rds/*" }, + { "Action": "ec2:*", "Effect": "Allow", @@ -306,11 +307,6 @@ "freetier:Get*" ], "Resource": "*" - }, - { - "Action": "dlm:*", - "Effect": "Allow", - "Resource": "*" } ] } diff --git a/roles/cs.aws-iam/defaults/main.yml b/roles/cs.aws-iam/defaults/main.yml index aba263e8c..6ed67a79c 100644 --- a/roles/cs.aws-iam/defaults/main.yml +++ b/roles/cs.aws-iam/defaults/main.yml @@ -12,9 +12,6 @@ aws_iam_policy_cloudwatch_metrics_access: "{{ aws_iam_name_prefix }}CloudWatchMe aws_iam_policy_lambda_access: "{{ aws_iam_name_prefix }}LambdaAccess" aws_iam_policy_kms_access: "{{ aws_iam_name_prefix }}KmsAccess" -aws_iam_policy_dlm_sts_access: "{{ aws_iam_name_prefix }}DLMAllowSTSAccess" -aws_iam_policy_dlm_aws_access: "{{ aws_iam_name_prefix }}DLMAccess" - aws_iam_group_custom_policies: "{{ aws_iam_name_prefix }}CustomPolicies" aws_iam_group_standard_policies: "{{ aws_iam_name_prefix }}StandardPolicies" @@ -32,5 +29,3 @@ aws_iam_role_node_coordinator_lambda_execution: "{{ aws_iam_name_prefix }}Handle aws_iam_role_app_node: "{{ aws_iam_name_prefix }}AppNode" aws_iam_role_varnish: "{{ aws_iam_role_app_node }}" aws_iam_role_persistent_node: "{{ aws_iam_name_prefix }}PersistentNode" - -aws_iam_role_dlm: "{{ aws_iam_name_prefix }}DLM" diff --git a/roles/cs.aws-iam/tasks/dlm-roles.yaml b/roles/cs.aws-iam/tasks/dlm-roles.yaml deleted file mode 100644 index e17adf085..000000000 --- a/roles/cs.aws-iam/tasks/dlm-roles.yaml +++ /dev/null @@ -1,23 +0,0 @@ -- name: Create DLM STS access policy - iam_managed_policy: - policy_name: "{{ aws_iam_policy_dlm_sts_access }}" - policy_description: Allow DLM to assume STS role - policy: "{{ lookup('template', 'aws_dlm_sts_access.policy.json') }}" - state: present - register: iam_dlm_sts_access - -- name: Create DLM AWS access policy - iam_managed_policy: - policy_name: "{{ aws_iam_policy_dlm_aws_access }}" - policy_description: Allow DLM to access AWS resources - policy: "{{ lookup('template', 'aws_dlm_access.policy.json') }}" - state: present - register: iam_dlm_aws_access - -- name: Create role for DLM - iam_role: - assume_role_policy_document: "{{ lookup('template', 'aws_dlm_sts_access.policy.json') }}" - name: "{{ aws_iam_role_dlm }}" - state: present - managed_policy: - - "{{ iam_dlm_aws_access.policy.arn }}" diff --git a/roles/cs.aws-iam/tasks/main.yml b/roles/cs.aws-iam/tasks/main.yml index 54a3057ba..2db9ff45b 100644 --- a/roles/cs.aws-iam/tasks/main.yml +++ b/roles/cs.aws-iam/tasks/main.yml @@ -3,4 +3,3 @@ - import_tasks: lambda-roles.yml - import_tasks: kms-roles.yml - import_tasks: provisioning-groups.yml -- import_tasks: dlm-roles.yml diff --git a/roles/cs.aws-iam/templates/aws_dlm_access.policy.json b/roles/cs.aws-iam/templates/aws_dlm_access.policy.json deleted file mode 100644 index c868a4c47..000000000 --- a/roles/cs.aws-iam/templates/aws_dlm_access.policy.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:CreateSnapshot", - "ec2:CreateSnapshots", - "ec2:DeleteSnapshot", - "ec2:DescribeInstances", - "ec2:DescribeVolumes", - "ec2:DescribeSnapshots", - "ec2:EnableFastSnapshotRestores", - "ec2:DescribeFastSnapshotRestores", - "ec2:DisableFastSnapshotRestores", - "ec2:CopySnapshot", - "ec2:ModifySnapshotAttribute", - "ec2:DescribeSnapshotAttribute", - "ec2:DescribeSnapshotTierStatus", - "ec2:ModifySnapshotTier" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": [ - "ec2:CreateTags" - ], - "Resource": "arn:aws:ec2:*::snapshot/*" - }, - { - "Effect": "Allow", - "Action": [ - "events:PutRule", - "events:DeleteRule", - "events:DescribeRule", - "events:EnableRule", - "events:DisableRule", - "events:ListTargetsByRule", - "events:PutTargets", - "events:RemoveTargets" - ], - "Resource": "arn:aws:events:*:*:rule/AwsDataLifecycleRule.managed-cwe.*" - } - ] -} diff --git a/roles/cs.aws-iam/templates/aws_dlm_sts_access.policy.json b/roles/cs.aws-iam/templates/aws_dlm_sts_access.policy.json deleted file mode 100644 index 85ec67594..000000000 --- a/roles/cs.aws-iam/templates/aws_dlm_sts_access.policy.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "dlm.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} diff --git a/roles/cs.cloudflare/defaults/main.yml b/roles/cs.cloudflare/defaults/main.yml index a20e2291b..dcf4357a9 100644 --- a/roles/cs.cloudflare/defaults/main.yml +++ b/roles/cs.cloudflare/defaults/main.yml @@ -1,5 +1,5 @@ cloudflare_enabled: no -# Accept only traffic coming from Cloudflare +# Accept only traffic comming from Cloudflare cloudflare_exclusive_traffic: yes # configuration file paths diff --git a/roles/cs.geolite2/tasks/main.yml b/roles/cs.geolite2/tasks/main.yml index 8610f44c9..1f8632e91 100644 --- a/roles/cs.geolite2/tasks/main.yml +++ b/roles/cs.geolite2/tasks/main.yml @@ -10,12 +10,7 @@ - name: Install geoupdate configuration template: src: GeoIP.conf.j2 - dest: /usr/local/etc/GeoIP.conf - -- name: Install geoupdate 6.x configuration - template: - src: GeoIP6.conf.j2 - dest: /usr/local/etc/GeoIP.conf + dest: /etc/GeoIP.conf - name: Update geolite2 databases shell: geoipupdate diff --git a/roles/cs.geolite2/templates/GeoIP6.conf.j2 b/roles/cs.geolite2/templates/GeoIP6.conf.j2 deleted file mode 100644 index 665729dd8..000000000 --- a/roles/cs.geolite2/templates/GeoIP6.conf.j2 +++ /dev/null @@ -1,3 +0,0 @@ -AccountID {{ geolite2_userid }} -LicenseKey {{ geolite2_license_key }} -EditionIDs {{ geolite2_product_ids }} diff --git a/roles/cs.magento-configure/defaults/main/app-etc.yml b/roles/cs.magento-configure/defaults/main/app-etc.yml index 7c4680958..a130a3b1c 100644 --- a/roles/cs.magento-configure/defaults/main/app-etc.yml +++ b/roles/cs.magento-configure/defaults/main/app-etc.yml @@ -165,9 +165,6 @@ magento_app_etc_config_consumer_workers: cron_consumers_runner: cron_run: false max_messages: "{{ magento_consumer_workers_max_messages | default(500) }}" - queue: - consumers_wait_for_messages: 0 - only_spawn_when_message_available: 1 magento_app_etc_config_cron_consumers: cron_consumers_runner: diff --git a/roles/cs.magento-configure/tasks/000-prepare-runtime-config.yml b/roles/cs.magento-configure/tasks/000-prepare-runtime-config.yml index 52feb9bd2..2610f2254 100644 --- a/roles/cs.magento-configure/tasks/000-prepare-runtime-config.yml +++ b/roles/cs.magento-configure/tasks/000-prepare-runtime-config.yml @@ -63,8 +63,14 @@ magento_core_config_settings: "{{ magento_core_config_settings + magento_baler_js_bundling_core_config }}" when: magento_scd_advanced_js_bundling and magento_scd_advanced_js_bundling_strategy == 'baler' +- name: Download CA RDS + ansible.builtin.get_url: + url: https://truststore.pki.rds.amazonaws.com/eu-central-1/eu-central-1-bundle.pem + dest: /tmp/eu-central-1-bundle.pem + mode: '0666' + - name: Check if database is initialized - command: mysql -N --batch -u {{ mageops_app_mysql_user|quote }} -p{{ mageops_app_mysql_pass|quote }} -h {{ mageops_mysql_host|quote }} -e "SHOW TABLES FROM `{{ mageops_app_mysql_db }}` LIKE 'admin_user';" + command: mysql --ssl-ca=/tmp/eu-central-1-bundle.pem -N --batch -u {{ mageops_app_mysql_user|quote }} -p{{ mageops_app_mysql_pass|quote }} -h {{ mageops_mysql_host|quote }} -e "SHOW TABLES FROM `{{ mageops_app_mysql_db }}` LIKE 'admin_user';" changed_when: false register: admins diff --git a/roles/cs.magento-configure/tasks/080-core-config.yml b/roles/cs.magento-configure/tasks/080-core-config.yml index e3ec18022..db87a1ae1 100644 --- a/roles/cs.magento-configure/tasks/080-core-config.yml +++ b/roles/cs.magento-configure/tasks/080-core-config.yml @@ -9,12 +9,20 @@ magento_core_config_settings: "{{ magento_core_config_settings + _extra_items }}" when: magento_varnish_host | default(false, true) +- name: Download CA RDS + ansible.builtin.get_url: + url: https://truststore.pki.rds.amazonaws.com/eu-central-1/eu-central-1-bundle.pem + dest: /tmp/eu-central-1-bundle.pem + mode: '0666' + + - name: Ensure core config database settings' values community.mysql.mysql_query: login_db: "{{ mageops_app_mysql_db }}" login_host: "{{ mageops_mysql_host }}" login_user: "{{ mageops_app_mysql_user }}" login_password: "{{ mageops_app_mysql_pass }}" + ca_cert: /tmp/eu-central-1-bundle.pem query: | INSERT INTO `core_config_data` SET @@ -35,6 +43,7 @@ login_host: "{{ mageops_mysql_host }}" login_user: "{{ mageops_app_mysql_user }}" login_password: "{{ mageops_app_mysql_pass }}" + ca_cert: /tmp/eu-central-1-bundle.pem query: | INSERT IGNORE INTO `core_config_data` SET @@ -47,12 +56,14 @@ loop_control: loop_var: magento_db_setting + - name: Ensure core config database settings are absent (defaults are used) community.mysql.mysql_query: login_db: "{{ mageops_app_mysql_db }}" login_host: "{{ mageops_mysql_host }}" login_user: "{{ mageops_app_mysql_user }}" login_password: "{{ mageops_app_mysql_pass }}" + ca_cert: /tmp/eu-central-1-bundle.pem query: | DELETE FROM core_config_data WHERE diff --git a/roles/cs.mageops-cli/files/mageopscli b/roles/cs.mageops-cli/files/mageopscli index 343e03d8c..c12ed0f5a 100755 --- a/roles/cs.mageops-cli/files/mageopscli +++ b/roles/cs.mageops-cli/files/mageopscli @@ -151,7 +151,6 @@ main::help() { main::eprintln " is_feature_flag_set Checks if there is any value set for feature flag" main::eprintln " status code 0 means flag is set, 1 otherwise" main::eprintln " apply_features Apply feature updates to this host" - main::eprintln " clear_opcache Clears opcache for php and php-fpm" main::eprintln "" main::eprintln " Mageops cli tools" main::eprintln " (c) Creativestyle 2020" diff --git a/roles/cs.mysql-configure/tasks/create-db.yml b/roles/cs.mysql-configure/tasks/create-db.yml index 008dc5cd4..a0133473d 100644 --- a/roles/cs.mysql-configure/tasks/create-db.yml +++ b/roles/cs.mysql-configure/tasks/create-db.yml @@ -1,3 +1,9 @@ +- name: Download CA RDS + ansible.builtin.get_url: + url: https://truststore.pki.rds.amazonaws.com/eu-central-1/eu-central-1-bundle.pem + dest: /tmp/eu-central-1-bundle.pem + mode: '0666' + - name: Ensure project database exists mysql_db: login_host: "{{ mageops_mysql_host }}" @@ -5,6 +11,7 @@ login_password: "{{ mageops_mysql_root_pass }}" name: "{{ mageops_app_mysql_db }}" state: present + ca_cert: /tmp/eu-central-1-bundle.pem - name: Ensure project db user for external connections exists mysql_user: @@ -15,6 +22,7 @@ password: "{{ mageops_app_mysql_pass }}" host: "%" state: present + ca_cert: /tmp/eu-central-1-bundle.pem priv: "{{ mageops_app_mysql_db }}.*:{{ mysql_configure_all_db_permissions }}" - name: Ensure project db user for localhost exists @@ -26,5 +34,6 @@ password: "{{ mageops_app_mysql_pass }}" host: "localhost" state: present + ca_cert: /tmp/eu-central-1-bundle.pem priv: "{{ mageops_app_mysql_db }}.*:{{ mysql_configure_all_db_permissions }}" when: mysql_user_localhost_access diff --git a/roles/cs.new-relic/defaults/main.yml b/roles/cs.new-relic/defaults/main.yml index 702e8eee7..32d3a9e27 100644 --- a/roles/cs.new-relic/defaults/main.yml +++ b/roles/cs.new-relic/defaults/main.yml @@ -1,7 +1,7 @@ new_relic_repo_url: http://yum.newrelic.com/pub/newrelic/el5/x86_64/newrelic-repo-5-3.noarch.rpm new_relic_packages: - newrelic-php5 -new_relic_license: +# new_relic_license: new_relic_app_name: "New relic app name" new_relic_collector_enabled: yes new_relic_ignore_user_exception_handler: no @@ -15,7 +15,7 @@ new_relic_stact_trace_threshold: "3s" new_relic_explain_enabled: yes new_relic_explain_threshold: "500ms" new_relic_framework: magento2 -new_relic_enabled: "{{ new_relic_license != '' }}" +new_relic_enabled: yes new_relic_cron_enabled: no new_relic_cron_start: "0 7 * * *" # From 7:00 diff --git a/roles/cs.new-relic/files/newrelic_feature.bash b/roles/cs.new-relic/files/newrelic_feature.bash index 2edb8fb19..de780e0ab 100644 --- a/roles/cs.new-relic/files/newrelic_feature.bash +++ b/roles/cs.new-relic/files/newrelic_feature.bash @@ -1,7 +1,6 @@ #!/usr/bin/env bash feature__flag_name="newrelic_apm" -feature__license_key="newrelic_license_key" feature::apply() { local expected @@ -10,15 +9,9 @@ feature::apply() { expected="$(feature::expected_value)" expected="$(feature::normalize_expected "$expected")" current="$(feature::current_value)" - current_license="$(feature::current_license_value)" - expected_license="$(feature::expected_license_value)" - if [ -z "$expected_license" ];then - # We cannot enable the feature without a license key - expected="false" - fi - if [ "$expected" != "$current" ] || [ "$expected_license" != "$current_license" ];then - feature::update "$expected" "$expected_license" + if [ "$expected" != "$current" ];then + feature::update "$expected" fi } @@ -41,10 +34,6 @@ feature::expected_value() { features::read_feature_flag "$feature__flag_name" "false" } -feature::expected_license_value() { - features::read_feature_flag "$feature__license_key" "" -} - feature::current_value() { local current @@ -57,21 +46,12 @@ feature::current_value() { echo "$current" } -feature::current_license_value() { - local current - - current="$(grep '^newrelic.license' /etc/php.d/newrelic.ini | sed 's/.*=\s*"\(.*\)"\s*$/\1/')" - - echo "$current" -} - feature::update() { local value=$1 - local license=$2 + local config echo "Setting newrelic apm to $value" sed -i -e "s/newrelic.enabled[[:space:]]=[[:space:]].*/newrelic.enabled = ${value}/" /etc/php.d/newrelic.ini - sed -i -e "s/newrelic.license[[:space:]]=[[:space:]].*/newrelic.license = \"${license}\"/" /etc/php.d/newrelic.ini echo "Reloading php-fpm" systemctl reload php-fpm } diff --git a/roles/cs.new-relic/meta/main.yml b/roles/cs.new-relic/meta/main.yml deleted file mode 100644 index 68605ed6d..000000000 --- a/roles/cs.new-relic/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ -dependencies: - - role: cs.mageops-cli - when: aws_use diff --git a/roles/cs.new-relic/tasks/main.yml b/roles/cs.new-relic/tasks/main.yml index e8fa5d1b3..99f4f086a 100644 --- a/roles/cs.new-relic/tasks/main.yml +++ b/roles/cs.new-relic/tasks/main.yml @@ -32,10 +32,6 @@ shell: "mageopscli set_feature_flag newrelic_apm {{ new_relic_enabled | ternary('true', 'false') }}" when: aws_use -- name: Set license key feature flag - shell: "mageopscli set_feature_flag newrelic_license_key \"{{ new_relic_license }}\"" - when: aws_use - - name: Setup cron template: src: cron.j2 diff --git a/roles/cs.pio/defaults/main.yml b/roles/cs.pio/defaults/main.yml index 6ed280e61..5b8369b2c 100644 --- a/roles/cs.pio/defaults/main.yml +++ b/roles/cs.pio/defaults/main.yml @@ -18,7 +18,6 @@ pio_media_fast_quality: 95 pio_target_directory: pio_cache pio_media_target_directory: pio_cache_cms pio_optimize_media: yes -pio_quality_target_spread: 80 # We cannot use magento_media_dir here as it might not be defined at this point pio_filesystem_local_dir: "{{ mageops_app_web_dir }}/shared/pub/media/" diff --git a/roles/cs.pio/templates/pio-config.conf.j2 b/roles/cs.pio/templates/pio-config.conf.j2 index 8817827c0..3ac3a2a4f 100644 --- a/roles/cs.pio/templates/pio-config.conf.j2 +++ b/roles/cs.pio/templates/pio-config.conf.j2 @@ -17,7 +17,7 @@ http_server: ( kind: "LazyResizeHttpServer", addr: "127.0.0.1:8441", - quality_target_spread: {{ pio_quality_target_spread }}, + quality_target_spread: 100, secret: "{{ lazy_resize_secret }}", source_directory: "catalog/product", target_directory: "{{ pio_target_directory }}", @@ -28,7 +28,7 @@ url: "/media", quality: ( quality: {{ pio_media_fast_quality }}, - spread: {{ pio_quality_target_spread }} + spread: 100 ), target_ssim: {{ pio_media_target_ssim }}, source_directory: "", diff --git a/roles/cs.switch-pkg-mgr/defaults/main.yml b/roles/cs.switch-pkg-mgr/defaults/main.yml deleted file mode 100644 index e1274760e..000000000 --- a/roles/cs.switch-pkg-mgr/defaults/main.yml +++ /dev/null @@ -1 +0,0 @@ -switch_pkg_mgr_option: "" diff --git a/roles/cs.switch-pkg-mgr/files/yum.conf b/roles/cs.switch-pkg-mgr/files/yum.conf deleted file mode 100644 index d04e79d4d..000000000 --- a/roles/cs.switch-pkg-mgr/files/yum.conf +++ /dev/null @@ -1,7 +0,0 @@ -[main] -gpgcheck=True -installonly_limit=3 -clean_requirements_on_remove=True -best=False -fastestmirror=True -max_parallel_downloads=20 diff --git a/roles/cs.switch-pkg-mgr/tasks/main.yml b/roles/cs.switch-pkg-mgr/tasks/main.yml deleted file mode 100644 index b72899007..000000000 --- a/roles/cs.switch-pkg-mgr/tasks/main.yml +++ /dev/null @@ -1,12 +0,0 @@ -- name: Switch to dnf - include_tasks: dnf.yml - when: switch_pkg_mgr_option == 'dnf' - -- name: Switch to yum - include_tasks: yum.yml - when: switch_pkg_mgr_option == 'yum' - -- name: Fail if option is invalid - fail: - msg: "Invalid switch_pkg_mgr_option value: {{ switch_pkg_mgr_option }}" - when: switch_pkg_mgr_option != 'dnf' and switch_pkg_mgr_option != 'yum' and switch_pkg_mgr_option is not empty diff --git a/roles/cs.switch-pkg-mgr/tasks/yum.yml b/roles/cs.switch-pkg-mgr/tasks/yum.yml deleted file mode 100644 index 97b0b493c..000000000 --- a/roles/cs.switch-pkg-mgr/tasks/yum.yml +++ /dev/null @@ -1,9 +0,0 @@ -- name: Set yum configuration - copy: - src: yum.conf - dest: /etc/yum.conf - -- name: Switch package manager to yum - set_fact: - ansible_facts: - pkg_mgr: yum diff --git a/roles/cs.switch-pkg-mgr/files/dnf.conf b/roles/cs.switch-to-dnf/files/dnf.conf similarity index 100% rename from roles/cs.switch-pkg-mgr/files/dnf.conf rename to roles/cs.switch-to-dnf/files/dnf.conf diff --git a/roles/cs.switch-pkg-mgr/tasks/dnf.yml b/roles/cs.switch-to-dnf/tasks/main.yml similarity index 100% rename from roles/cs.switch-pkg-mgr/tasks/dnf.yml rename to roles/cs.switch-to-dnf/tasks/main.yml diff --git a/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 b/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 index 9dc907c95..252040bce 100644 --- a/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 +++ b/roles/cs.varnish/templates/vcl/subroutines/deliver.vcl.j2 @@ -1,5 +1,3 @@ -<<<<<<< HEAD -======= {% if varnish_do_not_expose_caching %} if ( client.ip ~ trusted ) { {% endif %} @@ -42,13 +40,6 @@ if (req.http.{{ varnish_bypass_request_info_header_name }}) { set resp.http.Cache-Control = "no-cache"; } -if (req.http.{{ varnish_bypass_request_info_header_name }}) { - set resp.http.{{ varnish_bypass_request_info_header_name }} = - req.http.{{ varnish_bypass_request_info_header_name }} + "; Backend-Response-Delivered"; - set resp.http.Set-Cookie = "{{ varnish_bypass_request_cookie_name}} = {{ varnish_bypass_request_token }}; Secure"; - set resp.http.Cache-Control = "no-cache"; -} - if (resp.status == 502 || resp.status == 504) { return(synth(resp.status)); } @@ -89,4 +80,3 @@ unset resp.http.Server; unset resp.http.X-Varnish; unset resp.http.Via; unset resp.http.Link; ->>>>>>> 599e99a (QA/Testing Team request to skip varnish cache.) diff --git a/roles/cs.wait-for-warmup/defaults/main.yml b/roles/cs.wait-for-warmup/defaults/main.yml deleted file mode 100644 index c270a16a7..000000000 --- a/roles/cs.wait-for-warmup/defaults/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -wait_for_warmup_retries: "{{ (mageops_wait_for_warmup_secs / wait_for_warmup_delay)|int }}" -wait_for_warmup_delay: 5 diff --git a/roles/cs.wait-for-warmup/tasks/main.yml b/roles/cs.wait-for-warmup/tasks/main.yml deleted file mode 100644 index 43e2a261f..000000000 --- a/roles/cs.wait-for-warmup/tasks/main.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: Deployment finished, waiting for magento warmup. - uri: - method: HEAD - url: "http://localhost/WARMUP" - status_code: 200 - register: _warmup_result - until: _warmup_result is not failed - retries: "{{ wait_for_warmup_retries }}" - delay: "{{ wait_for_warmup_delay }}" - changed_when: false diff --git a/site.step-15-varnish.yml b/site.step-15-varnish.yml index a4b8084ea..2ca4a8ac3 100644 --- a/site.step-15-varnish.yml +++ b/site.step-15-varnish.yml @@ -6,8 +6,7 @@ delegate_to: localhost become: no when: aws_use - - role: cs.switch-pkg-mgr - switch_pkg_mgr_option: "{{ mageops_pkg_mgr.centos7 }}" + - role: cs.switch-to-dnf - role: cs.versionlock node_name: varnish versionlock_packages: "{{ versionlock_varnish_packages + versionlock_varnish_packages_base + versionlock_varnish_packages_extra }}" diff --git a/site.step-20-persistent.yml b/site.step-20-persistent.yml index 30e74c5ad..608d69830 100644 --- a/site.step-20-persistent.yml +++ b/site.step-20-persistent.yml @@ -16,8 +16,7 @@ delegate_facts: no become: no when: aws_use - - role: cs.switch-pkg-mgr - switch_pkg_mgr_option: "{{ mageops_pkg_mgr.centos7 }}" + - role: cs.switch-to-dnf - role: cs.versionlock node_name: persistent versionlock_packages: "{{ versionlock_persistent_packages + versionlock_persistent_packages_base + versionlock_persistent_packages_extra }}" diff --git a/site.step-40-app-node.yml b/site.step-40-app-node.yml index 4199c0597..4905cb169 100644 --- a/site.step-40-app-node.yml +++ b/site.step-40-app-node.yml @@ -22,8 +22,7 @@ become: no when: aws_use - - role: cs.switch-pkg-mgr - switch_pkg_mgr_option: "{{ mageops_pkg_mgr.centos7 }}" + - role: cs.switch-to-dnf - role: cs.mageops-cli-user mageops_cli_user: root @@ -198,5 +197,6 @@ nginx_debug_request_header_name: "{{ mageops_debug_token_http_header }}" nginx_debug_request_query_param_name: "{{ mageops_debug_token_query_param }}" nginx_debug_request_cookie_name: "{{ mageops_debug_token_request_cookie }}" + # On app node we might not have yet magento code deployed magento_facts_detect_from_artifacts: yes diff --git a/site.step-45-app-deploy.yml b/site.step-45-app-deploy.yml index 97ef4edc3..de473c551 100644 --- a/site.step-45-app-deploy.yml +++ b/site.step-45-app-deploy.yml @@ -87,8 +87,7 @@ become: no when: aws_use - - role: cs.switch-pkg-mgr - switch_pkg_mgr_option: "{{ mageops_pkg_mgr.centos7 }}" + - role: cs.switch-to-dnf - role: cs.deploy deploy_install_actions: - name: Configure Magento diff --git a/site.step-80-monitoring.yml b/site.step-80-monitoring.yml index 4bae517c2..aa5ab4024 100644 --- a/site.step-80-monitoring.yml +++ b/site.step-80-monitoring.yml @@ -1,5 +1,3 @@ -- import_playbook: site.common.group-current-hosts.yml - - hosts: localhost connection: local roles: @@ -7,7 +5,3 @@ - role: cs.aws-logs-slack when: aws_logs_slack_notifications - role: cs.aws-logs-retention -- hosts: app:¤t - roles: - - role: cs.wait-for-warmup - run_once: true