From d03f82f66246e6466484aaba1440f7f8e9bec291 Mon Sep 17 00:00:00 2001 From: "Vinu.Kaveesha" Date: Wed, 26 Mar 2025 15:15:28 +0530 Subject: [PATCH 1/4] initial --- apply_secret_mappings.py | 41 ++++++++++++++ extract_common_secret_mappings.py | 94 +++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 apply_secret_mappings.py create mode 100644 extract_common_secret_mappings.py diff --git a/apply_secret_mappings.py b/apply_secret_mappings.py new file mode 100644 index 0000000..05c41c9 --- /dev/null +++ b/apply_secret_mappings.py @@ -0,0 +1,41 @@ +import os +import yaml + +# Directory containing the YAML files +SECRET_DIR = "secrets" +MAPPING_YML = "secret_mappings.yml" + +# Load the placeholder mappings +with open(MAPPING_YML, "r") as yamlfile: + placeholder_mappings = yaml.safe_load(yamlfile) + +# Function to replace placeholders with original values +def replace_placeholders(obj): + if isinstance(obj, dict): + for k, v in obj.items(): + if isinstance(v, str) and v in placeholder_mappings: + obj[k] = placeholder_mappings[v] + else: + replace_placeholders(v) + elif isinstance(obj, list): + for i, item in enumerate(obj): + if isinstance(item, str) and item in placeholder_mappings: + obj[i] = placeholder_mappings[item] + else: + replace_placeholders(item) + +# Process all YAML files and replace placeholders +for filename in os.listdir(SECRET_DIR): + if filename.endswith(".yml") or filename.endswith(".yaml"): + file_path = os.path.join(SECRET_DIR, filename) + + with open(file_path, "r") as file: + data = yaml.safe_load(file) + + if data: + replace_placeholders(data) + + with open(file_path, "w") as file: + yaml.safe_dump(data, file, default_flow_style=False,sort_keys=False) + +print("Placeholders replaced with original values in all files.") diff --git a/extract_common_secret_mappings.py b/extract_common_secret_mappings.py new file mode 100644 index 0000000..d720566 --- /dev/null +++ b/extract_common_secret_mappings.py @@ -0,0 +1,94 @@ +import os +import yaml +from collections import defaultdict + +SECRET_DIR = "secrets" +OUTPUT_YML = "secret_mappings.yml" +PLACEHOLDER_PREFIX = "PH_" + +secret_mappings = {} +common_values = defaultdict(int) +value_to_placeholder = {} +value_to_first_key = {} +value_to_first_file = {} + +def find_common_values(obj, parent_key=None, filename=None): + if isinstance(obj, dict): + for k, v in obj.items(): + find_common_values(v, k, filename) + elif isinstance(obj, list): + for item in obj: + find_common_values(item, parent_key, filename) + elif isinstance(obj, str): + common_values[obj] += 1 + if obj not in value_to_first_key: + value_to_first_key[obj] = parent_key + value_to_first_file[obj] = filename + +def replace_common_values(obj): + global modified + if isinstance(obj, dict): + for k, v in obj.items(): + if isinstance(v, str) and common_values[v] > 1: + if v not in value_to_placeholder: + placeholder = ( + f"{PLACEHOLDER_PREFIX}{value_to_first_key[v].replace('-', '_').upper()}_" + f"{value_to_first_file[v].replace('.', '_').upper()}" + ) + value_to_placeholder[v] = placeholder + obj[k] = value_to_placeholder[v] + secret_mappings[value_to_placeholder[v]] = v + modified = True + elif isinstance(v, str) and "\n" in v: # Handle multi-line strings + lines = v.split("\n") + for i, line in enumerate(lines): + if line.strip() in common_values and common_values[line.strip()] > 1: + if line.strip() not in value_to_placeholder: + placeholder = ( + f"{PLACEHOLDER_PREFIX}{value_to_first_key[line.strip()].replace('-', '_').upper()}_" + f"{value_to_first_file[line.strip()].replace('.', '_').upper()}" + ) + value_to_placeholder[line.strip()] = placeholder + lines[i] = value_to_placeholder[line.strip()] + secret_mappings[value_to_placeholder[line.strip()]] = line.strip() + modified = True + obj[k] = "\n".join(lines) # Preserve formatting + else: + replace_common_values(v) + elif isinstance(obj, list): + for i, item in enumerate(obj): + if isinstance(item, str) and common_values[item] > 1: + if item not in value_to_placeholder: + placeholder = f"{PLACEHOLDER_PREFIX}{i}_{value_to_first_file[item].replace('.', '_').upper()}" + value_to_placeholder[item] = placeholder + obj[i] = value_to_placeholder[item] + secret_mappings[value_to_placeholder[item]] = item + modified = True + else: + replace_common_values(item) + +# First pass: Identify common values +for filename in os.listdir(SECRET_DIR): + if filename.endswith(".yml") or filename.endswith(".yaml"): + file_path = os.path.join(SECRET_DIR, filename) + with open(file_path, "r") as file: + data = yaml.safe_load(file) + find_common_values(data, filename=filename) + +# Second pass: Replace common values +for filename in os.listdir(SECRET_DIR): + if filename.endswith(".yml") or filename.endswith(".yaml"): + file_path = os.path.join(SECRET_DIR, filename) + with open(file_path, "r") as file: + data = yaml.safe_load(file) + modified = False + replace_common_values(data) + if modified: + with open(file_path, "w") as file: + yaml.safe_dump(data, file, default_flow_style=False) + +# Save mappings +with open(OUTPUT_YML, "w") as yamlfile: + yaml.safe_dump(secret_mappings, yamlfile, default_flow_style=False) + +print(f"Processing completed. Placeholder mappings saved in {OUTPUT_YML}.") From 2181683927526512b005ab083dbcd42438d07653 Mon Sep 17 00:00:00 2001 From: "Vinu.Kaveesha" Date: Wed, 26 Mar 2025 15:31:53 +0530 Subject: [PATCH 2/4] secrets and config mappings added --- apply_config_mappings.py | 41 +++++++++++ extract_common_secret_mappings copy.py | 94 ++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 apply_config_mappings.py create mode 100644 extract_common_secret_mappings copy.py diff --git a/apply_config_mappings.py b/apply_config_mappings.py new file mode 100644 index 0000000..9748a7e --- /dev/null +++ b/apply_config_mappings.py @@ -0,0 +1,41 @@ +import os +import yaml + +# Directory containing the YAML files +SECRET_DIR = "configmaps" +MAPPING_YML = "config_mappings.yml" + +# Load the placeholder mappings +with open(MAPPING_YML, "r") as yamlfile: + placeholder_mappings = yaml.safe_load(yamlfile) + +# Function to replace placeholders with original values +def replace_placeholders(obj): + if isinstance(obj, dict): + for k, v in obj.items(): + if isinstance(v, str) and v in placeholder_mappings: + obj[k] = placeholder_mappings[v] + else: + replace_placeholders(v) + elif isinstance(obj, list): + for i, item in enumerate(obj): + if isinstance(item, str) and item in placeholder_mappings: + obj[i] = placeholder_mappings[item] + else: + replace_placeholders(item) + +# Process all YAML files and replace placeholders +for filename in os.listdir(SECRET_DIR): + if filename.endswith(".yml") or filename.endswith(".yaml"): + file_path = os.path.join(SECRET_DIR, filename) + + with open(file_path, "r") as file: + data = yaml.safe_load(file) + + if data: + replace_placeholders(data) + + with open(file_path, "w") as file: + yaml.safe_dump(data, file, default_flow_style=False,sort_keys=False) + +print("Placeholders replaced with original values in all files.") diff --git a/extract_common_secret_mappings copy.py b/extract_common_secret_mappings copy.py new file mode 100644 index 0000000..c2343c0 --- /dev/null +++ b/extract_common_secret_mappings copy.py @@ -0,0 +1,94 @@ +import os +import yaml +from collections import defaultdict + +SECRET_DIR = "configmaps" +OUTPUT_YML = "config_mappings.yml" +PLACEHOLDER_PREFIX = "PH_" + +secret_mappings = {} +common_values = defaultdict(int) +value_to_placeholder = {} +value_to_first_key = {} +value_to_first_file = {} + +def find_common_values(obj, parent_key=None, filename=None): + if isinstance(obj, dict): + for k, v in obj.items(): + find_common_values(v, k, filename) + elif isinstance(obj, list): + for item in obj: + find_common_values(item, parent_key, filename) + elif isinstance(obj, str): + common_values[obj] += 1 + if obj not in value_to_first_key: + value_to_first_key[obj] = parent_key + value_to_first_file[obj] = filename + +def replace_common_values(obj): + global modified + if isinstance(obj, dict): + for k, v in obj.items(): + if isinstance(v, str) and common_values[v] > 1: + if v not in value_to_placeholder: + placeholder = ( + f"{PLACEHOLDER_PREFIX}{value_to_first_key[v].replace('-', '_').upper()}_" + f"{value_to_first_file[v].replace('.', '_').upper()}" + ) + value_to_placeholder[v] = placeholder + obj[k] = value_to_placeholder[v] + secret_mappings[value_to_placeholder[v]] = v + modified = True + elif isinstance(v, str) and "\n" in v: # Handle multi-line strings + lines = v.split("\n") + for i, line in enumerate(lines): + if line.strip() in common_values and common_values[line.strip()] > 1: + if line.strip() not in value_to_placeholder: + placeholder = ( + f"{PLACEHOLDER_PREFIX}{value_to_first_key[line.strip()].replace('-', '_').upper()}_" + f"{value_to_first_file[line.strip()].replace('.', '_').upper()}" + ) + value_to_placeholder[line.strip()] = placeholder + lines[i] = value_to_placeholder[line.strip()] + secret_mappings[value_to_placeholder[line.strip()]] = line.strip() + modified = True + obj[k] = "\n".join(lines) # Preserve formatting + else: + replace_common_values(v) + elif isinstance(obj, list): + for i, item in enumerate(obj): + if isinstance(item, str) and common_values[item] > 1: + if item not in value_to_placeholder: + placeholder = f"{PLACEHOLDER_PREFIX}{i}_{value_to_first_file[item].replace('.', '_').upper()}" + value_to_placeholder[item] = placeholder + obj[i] = value_to_placeholder[item] + secret_mappings[value_to_placeholder[item]] = item + modified = True + else: + replace_common_values(item) + +# First pass: Identify common values +for filename in os.listdir(SECRET_DIR): + if filename.endswith(".yml") or filename.endswith(".yaml"): + file_path = os.path.join(SECRET_DIR, filename) + with open(file_path, "r") as file: + data = yaml.safe_load(file) + find_common_values(data, filename=filename) + +# Second pass: Replace common values +for filename in os.listdir(SECRET_DIR): + if filename.endswith(".yml") or filename.endswith(".yaml"): + file_path = os.path.join(SECRET_DIR, filename) + with open(file_path, "r") as file: + data = yaml.safe_load(file) + modified = False + replace_common_values(data) + if modified: + with open(file_path, "w") as file: + yaml.safe_dump(data, file, default_flow_style=False) + +# Save mappings +with open(OUTPUT_YML, "w") as yamlfile: + yaml.safe_dump(secret_mappings, yamlfile, default_flow_style=False) + +print(f"Processing completed. Placeholder mappings saved in {OUTPUT_YML}.") From 2a968238e29c39dc63814bb11bcedef0b8dea133 Mon Sep 17 00:00:00 2001 From: "Vinu.Kaveesha" Date: Wed, 26 Mar 2025 15:53:31 +0530 Subject: [PATCH 3/4] few chnages --- extract_common_secret_mappings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extract_common_secret_mappings.py b/extract_common_secret_mappings.py index d720566..c2343c0 100644 --- a/extract_common_secret_mappings.py +++ b/extract_common_secret_mappings.py @@ -2,8 +2,8 @@ import os import yaml from collections import defaultdict -SECRET_DIR = "secrets" -OUTPUT_YML = "secret_mappings.yml" +SECRET_DIR = "configmaps" +OUTPUT_YML = "config_mappings.yml" PLACEHOLDER_PREFIX = "PH_" secret_mappings = {} From b3ca1d5a8af9d00a9c2c68060854edc2137b1a1d Mon Sep 17 00:00:00 2001 From: "Vinu.Kaveesha" Date: Wed, 26 Mar 2025 15:56:07 +0530 Subject: [PATCH 4/4] ok --- ...common_secret_mappings.py => extract_common_config_mappings.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename extract_common_secret_mappings.py => extract_common_config_mappings.py (100%) diff --git a/extract_common_secret_mappings.py b/extract_common_config_mappings.py similarity index 100% rename from extract_common_secret_mappings.py rename to extract_common_config_mappings.py