diff --git a/generic_config_updater/gu_common.py b/generic_config_updater/gu_common.py index 7821557e715..c5bf097a244 100644 --- a/generic_config_updater/gu_common.py +++ b/generic_config_updater/gu_common.py @@ -9,6 +9,7 @@ import copy import re import os +import hashlib from sonic_py_common import logger, multi_asic from enum import Enum @@ -80,6 +81,9 @@ def __init__(self, yang_dir=YANG_DIR, scope=multi_asic.DEFAULT_NAMESPACE): self.scope = scope self.yang_dir = YANG_DIR self.sonic_yang_with_loaded_models = None + self._validate_config_cache = {} + self._currently_loaded_hash = None + self._loaded_sy = None def get_config_db_as_json(self): return get_config_db_as_json(self.scope) @@ -138,6 +142,12 @@ def validate_sonic_yang_config(self, sonic_yang_as_json): return False, ex def validate_config_db_config(self, config_db_as_json): + _cache_key = hashlib.md5( + json.dumps(config_db_as_json, sort_keys=True).encode() + ).hexdigest() + if _cache_key in self._validate_config_cache: + return self._validate_config_cache[_cache_key] + sy = self.create_sonic_yang_with_loaded_models() # TODO: Move these validators to YANG models @@ -145,20 +155,28 @@ def validate_config_db_config(self, config_db_as_json): self.validate_lanes] try: - tmp_config_db_as_json = copy.deepcopy(config_db_as_json) - - sy.loadData(tmp_config_db_as_json) + sy.loadData(config_db_as_json) + self._currently_loaded_hash = _cache_key + self._loaded_sy = sy sy.validate_data_tree() for supplemental_yang_validator in supplemental_yang_validators: success, error = supplemental_yang_validator(config_db_as_json) if not success: - return success, error + result = (success, error) + self._validate_config_cache[_cache_key] = result + return result except sonic_yang.SonicYangException as ex: - return False, ex + self._currently_loaded_hash = None + self._loaded_sy = None + result = (False, ex) + self._validate_config_cache[_cache_key] = result + return result - return True, None + result = (True, None) + self._validate_config_cache[_cache_key] = result + return result def validate_field_operation(self, old_config, target_config): """ @@ -522,7 +540,7 @@ def create_xpath(self, tokens): def _create_sonic_yang_with_loaded_models(self): return self.config_wrapper.create_sonic_yang_with_loaded_models() - def find_ref_paths(self, path, config): + def find_ref_paths(self, path, config, reload_config=True): """ Finds the paths referencing any line under the given 'path' within the given 'config'. Example: @@ -560,22 +578,39 @@ def find_ref_paths(self, path, config): /ACL_TABLE/EVERFLOW6/ports/1 """ # TODO: Also fetch references by must statement (check similar statements) - return self._find_leafref_paths(path, config) - - def _find_leafref_paths(self, path, config): - sy = self._create_sonic_yang_with_loaded_models() - - tmp_config = copy.deepcopy(config) - - sy.loadData(tmp_config) - - xpath = self.convert_path_to_xpath(path, config, sy) + return self._find_leafref_paths(path, config, reload_config=reload_config) + + def _find_leafref_paths(self, path, config, reload_config=True): + if reload_config: + _config_hash = hashlib.md5( + json.dumps(config, sort_keys=True).encode() + ).hexdigest() + already_loaded = ( + self.config_wrapper is not None and + self.config_wrapper._currently_loaded_hash == _config_hash and + self.config_wrapper._loaded_sy is not None + ) + if not already_loaded: + sy = self._create_sonic_yang_with_loaded_models() + sy.loadData(config) + if self.config_wrapper is not None: + self.config_wrapper._currently_loaded_hash = _config_hash + self.config_wrapper._loaded_sy = sy + else: + sy = self.config_wrapper._loaded_sy + else: + sy = self._create_sonic_yang_with_loaded_models() + sy.loadData(config) - leaf_xpaths = self._get_inner_leaf_xpaths(xpath, sy) + if not isinstance(path, list): + path = [path] ref_xpaths = [] - for xpath in leaf_xpaths: - ref_xpaths.extend(sy.find_data_dependencies(xpath)) + for inner_path in path: + xpath = self.convert_path_to_xpath(inner_path, config, sy) + leaf_xpaths = self._get_inner_leaf_xpaths(xpath, sy) + for leaf_xpath in leaf_xpaths: + ref_xpaths.extend(sy.find_data_dependencies(leaf_xpath)) ref_paths = [] ref_paths_set = set() diff --git a/generic_config_updater/patch_sorter.py b/generic_config_updater/patch_sorter.py index 829c3c2b684..07dc7d861f3 100644 --- a/generic_config_updater/patch_sorter.py +++ b/generic_config_updater/patch_sorter.py @@ -850,14 +850,14 @@ def _validate_replace(self, move, diff): simulated_config = move.apply(diff.current_config) deleted_paths, added_paths = self._get_paths(diff.current_config, simulated_config, []) - # For deleted paths, we check the current config has no dependencies between nodes under the removed path - if not self._validate_paths_config(deleted_paths, diff.current_config): - return False - # For added paths, we check the simulated config has no dependencies between nodes under the added path if not self._validate_paths_config(added_paths, simulated_config): return False + # For deleted paths, we check the current config has no dependencies between nodes under the removed path + if not self._validate_paths_config(deleted_paths, diff.current_config): + return False + return True def _get_paths(self, current_ptr, target_ptr, tokens): @@ -935,10 +935,7 @@ def _validate_paths_config(self, paths, config): return True def _find_ref_paths(self, paths, config): - refs = [] - for path in paths: - refs.extend(self.path_addressing.find_ref_paths(path, config)) - return refs + return self.path_addressing.find_ref_paths(paths, config) class NoEmptyTableMoveValidator: """ @@ -1055,6 +1052,43 @@ def _get_non_existing_tables_tokens(self, config1, config2): if not(table in config2): yield [table] + +class BulkLeafListMoveGenerator: + """ + A class that generates bulk REPLACE moves for leaf-lists (lists of primitive + values) that differ between current and target configs. + """ + def generate(self, diff): + for move in self._traverse(diff, diff.current_config, diff.target_config, []): + yield move + + def _traverse(self, diff, current_ptr, target_ptr, tokens): + if not isinstance(current_ptr, dict) or not isinstance(target_ptr, dict): + return + + for key in current_ptr: + if key not in target_ptr: + continue + + current_val = current_ptr[key] + target_val = target_ptr[key] + tokens.append(key) + + if isinstance(current_val, list) and isinstance(target_val, list): + if (current_val != target_val and + self._is_leaf_list(current_val) and + self._is_leaf_list(target_val)): + yield JsonMove(diff, OperationType.REPLACE, list(tokens), list(tokens)) + elif isinstance(current_val, dict) and isinstance(target_val, dict): + for move in self._traverse(diff, current_val, target_val, tokens): + yield move + + tokens.pop() + + @staticmethod + def _is_leaf_list(lst): + return all(isinstance(item, (str, int, float, bool)) for item in lst) + class KeyLevelMoveGenerator: """ A class that key level moves. The item name at the root level of ConfigDB is called 'Table', the item @@ -1639,7 +1673,8 @@ def create(self, algorithm=Algorithm.DFS): move_generators = [RemoveCreateOnlyDependencyMoveGenerator(self.path_addressing), LowLevelMoveGenerator(self.path_addressing)] # TODO: Enable TableLevelMoveGenerator once it is confirmed whole table can be updated at the same time - move_non_extendable_generators = [KeyLevelMoveGenerator()] + move_non_extendable_generators = [BulkLeafListMoveGenerator(), + KeyLevelMoveGenerator()] move_extenders = [RequiredValueMoveExtender(self.path_addressing, self.operation_wrapper), UpperLevelMoveExtender(), DeleteInsteadOfReplaceMoveExtender(),