From d03f82f66246e6466484aaba1440f7f8e9bec291 Mon Sep 17 00:00:00 2001 From: "Vinu.Kaveesha" Date: Wed, 26 Mar 2025 15:15:28 +0530 Subject: [PATCH] initial --- apply_secret_mappings.py | 41 ++++++++++++++ extract_common_secret_mappings.py | 94 +++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 apply_secret_mappings.py create mode 100644 extract_common_secret_mappings.py diff --git a/apply_secret_mappings.py b/apply_secret_mappings.py new file mode 100644 index 0000000..05c41c9 --- /dev/null +++ b/apply_secret_mappings.py @@ -0,0 +1,41 @@ +import os +import yaml + +# Directory containing the YAML files +SECRET_DIR = "secrets" +MAPPING_YML = "secret_mappings.yml" + +# Load the placeholder mappings +with open(MAPPING_YML, "r") as yamlfile: + placeholder_mappings = yaml.safe_load(yamlfile) + +# Function to replace placeholders with original values +def replace_placeholders(obj): + if isinstance(obj, dict): + for k, v in obj.items(): + if isinstance(v, str) and v in placeholder_mappings: + obj[k] = placeholder_mappings[v] + else: + replace_placeholders(v) + elif isinstance(obj, list): + for i, item in enumerate(obj): + if isinstance(item, str) and item in placeholder_mappings: + obj[i] = placeholder_mappings[item] + else: + replace_placeholders(item) + +# Process all YAML files and replace placeholders +for filename in os.listdir(SECRET_DIR): + if filename.endswith(".yml") or filename.endswith(".yaml"): + file_path = os.path.join(SECRET_DIR, filename) + + with open(file_path, "r") as file: + data = yaml.safe_load(file) + + if data: + replace_placeholders(data) + + with open(file_path, "w") as file: + yaml.safe_dump(data, file, default_flow_style=False,sort_keys=False) + +print("Placeholders replaced with original values in all files.") diff --git a/extract_common_secret_mappings.py b/extract_common_secret_mappings.py new file mode 100644 index 0000000..d720566 --- /dev/null +++ b/extract_common_secret_mappings.py @@ -0,0 +1,94 @@ +import os +import yaml +from collections import defaultdict + +SECRET_DIR = "secrets" +OUTPUT_YML = "secret_mappings.yml" +PLACEHOLDER_PREFIX = "PH_" + +secret_mappings = {} +common_values = defaultdict(int) +value_to_placeholder = {} +value_to_first_key = {} +value_to_first_file = {} + +def find_common_values(obj, parent_key=None, filename=None): + if isinstance(obj, dict): + for k, v in obj.items(): + find_common_values(v, k, filename) + elif isinstance(obj, list): + for item in obj: + find_common_values(item, parent_key, filename) + elif isinstance(obj, str): + common_values[obj] += 1 + if obj not in value_to_first_key: + value_to_first_key[obj] = parent_key + value_to_first_file[obj] = filename + +def replace_common_values(obj): + global modified + if isinstance(obj, dict): + for k, v in obj.items(): + if isinstance(v, str) and common_values[v] > 1: + if v not in value_to_placeholder: + placeholder = ( + f"{PLACEHOLDER_PREFIX}{value_to_first_key[v].replace('-', '_').upper()}_" + f"{value_to_first_file[v].replace('.', '_').upper()}" + ) + value_to_placeholder[v] = placeholder + obj[k] = value_to_placeholder[v] + secret_mappings[value_to_placeholder[v]] = v + modified = True + elif isinstance(v, str) and "\n" in v: # Handle multi-line strings + lines = v.split("\n") + for i, line in enumerate(lines): + if line.strip() in common_values and common_values[line.strip()] > 1: + if line.strip() not in value_to_placeholder: + placeholder = ( + f"{PLACEHOLDER_PREFIX}{value_to_first_key[line.strip()].replace('-', '_').upper()}_" + f"{value_to_first_file[line.strip()].replace('.', '_').upper()}" + ) + value_to_placeholder[line.strip()] = placeholder + lines[i] = value_to_placeholder[line.strip()] + secret_mappings[value_to_placeholder[line.strip()]] = line.strip() + modified = True + obj[k] = "\n".join(lines) # Preserve formatting + else: + replace_common_values(v) + elif isinstance(obj, list): + for i, item in enumerate(obj): + if isinstance(item, str) and common_values[item] > 1: + if item not in value_to_placeholder: + placeholder = f"{PLACEHOLDER_PREFIX}{i}_{value_to_first_file[item].replace('.', '_').upper()}" + value_to_placeholder[item] = placeholder + obj[i] = value_to_placeholder[item] + secret_mappings[value_to_placeholder[item]] = item + modified = True + else: + replace_common_values(item) + +# First pass: Identify common values +for filename in os.listdir(SECRET_DIR): + if filename.endswith(".yml") or filename.endswith(".yaml"): + file_path = os.path.join(SECRET_DIR, filename) + with open(file_path, "r") as file: + data = yaml.safe_load(file) + find_common_values(data, filename=filename) + +# Second pass: Replace common values +for filename in os.listdir(SECRET_DIR): + if filename.endswith(".yml") or filename.endswith(".yaml"): + file_path = os.path.join(SECRET_DIR, filename) + with open(file_path, "r") as file: + data = yaml.safe_load(file) + modified = False + replace_common_values(data) + if modified: + with open(file_path, "w") as file: + yaml.safe_dump(data, file, default_flow_style=False) + +# Save mappings +with open(OUTPUT_YML, "w") as yamlfile: + yaml.safe_dump(secret_mappings, yamlfile, default_flow_style=False) + +print(f"Processing completed. Placeholder mappings saved in {OUTPUT_YML}.")