initial

apply_secret_mappings.py

@@ -0,0 +1,41 @@
+import os
+import yaml
+
+# Directory containing the YAML files
+SECRET_DIR = "secrets"
+MAPPING_YML = "secret_mappings.yml"
+
+# Load the placeholder mappings
+with open(MAPPING_YML, "r") as yamlfile:
+    placeholder_mappings = yaml.safe_load(yamlfile)
+
+# Function to replace placeholders with original values
+def replace_placeholders(obj):
+    if isinstance(obj, dict):
+        for k, v in obj.items():
+            if isinstance(v, str) and v in placeholder_mappings:
+                obj[k] = placeholder_mappings[v]
+            else:
+                replace_placeholders(v)
+    elif isinstance(obj, list):
+        for i, item in enumerate(obj):
+            if isinstance(item, str) and item in placeholder_mappings:
+                obj[i] = placeholder_mappings[item]
+            else:
+                replace_placeholders(item)
+
+# Process all YAML files and replace placeholders
+for filename in os.listdir(SECRET_DIR):
+    if filename.endswith(".yml") or filename.endswith(".yaml"):
+        file_path = os.path.join(SECRET_DIR, filename)
+        
+        with open(file_path, "r") as file:
+            data = yaml.safe_load(file)
+        
+        if data:
+            replace_placeholders(data)
+            
+            with open(file_path, "w") as file:
+                yaml.safe_dump(data, file, default_flow_style=False,sort_keys=False)
+
+print("Placeholders replaced with original values in all files.")

extract_common_secret_mappings.py

@@ -0,0 +1,94 @@
+import os
+import yaml
+from collections import defaultdict
+
+SECRET_DIR = "secrets"
+OUTPUT_YML = "secret_mappings.yml"
+PLACEHOLDER_PREFIX = "PH_"
+
+secret_mappings = {}
+common_values = defaultdict(int)
+value_to_placeholder = {}
+value_to_first_key = {}
+value_to_first_file = {}
+
+def find_common_values(obj, parent_key=None, filename=None):
+    if isinstance(obj, dict):
+        for k, v in obj.items():
+            find_common_values(v, k, filename)
+    elif isinstance(obj, list):
+        for item in obj:
+            find_common_values(item, parent_key, filename)
+    elif isinstance(obj, str):
+        common_values[obj] += 1
+        if obj not in value_to_first_key:
+            value_to_first_key[obj] = parent_key
+            value_to_first_file[obj] = filename
+
+def replace_common_values(obj):
+    global modified
+    if isinstance(obj, dict):
+        for k, v in obj.items():
+            if isinstance(v, str) and common_values[v] > 1:
+                if v not in value_to_placeholder:
+                    placeholder = (
+                        f"{PLACEHOLDER_PREFIX}{value_to_first_key[v].replace('-', '_').upper()}_"
+                        f"{value_to_first_file[v].replace('.', '_').upper()}"
+                    )
+                    value_to_placeholder[v] = placeholder
+                obj[k] = value_to_placeholder[v]
+                secret_mappings[value_to_placeholder[v]] = v
+                modified = True
+            elif isinstance(v, str) and "\n" in v:  # Handle multi-line strings
+                lines = v.split("\n")
+                for i, line in enumerate(lines):
+                    if line.strip() in common_values and common_values[line.strip()] > 1:
+                        if line.strip() not in value_to_placeholder:
+                            placeholder = (
+                                f"{PLACEHOLDER_PREFIX}{value_to_first_key[line.strip()].replace('-', '_').upper()}_"
+                                f"{value_to_first_file[line.strip()].replace('.', '_').upper()}"
+                            )
+                            value_to_placeholder[line.strip()] = placeholder
+                        lines[i] = value_to_placeholder[line.strip()]
+                        secret_mappings[value_to_placeholder[line.strip()]] = line.strip()
+                        modified = True
+                obj[k] = "\n".join(lines)  # Preserve formatting
+            else:
+                replace_common_values(v)
+    elif isinstance(obj, list):
+        for i, item in enumerate(obj):
+            if isinstance(item, str) and common_values[item] > 1:
+                if item not in value_to_placeholder:
+                    placeholder = f"{PLACEHOLDER_PREFIX}{i}_{value_to_first_file[item].replace('.', '_').upper()}"
+                    value_to_placeholder[item] = placeholder
+                obj[i] = value_to_placeholder[item]
+                secret_mappings[value_to_placeholder[item]] = item
+                modified = True
+            else:
+                replace_common_values(item)
+
+# First pass: Identify common values
+for filename in os.listdir(SECRET_DIR):
+    if filename.endswith(".yml") or filename.endswith(".yaml"):
+        file_path = os.path.join(SECRET_DIR, filename)
+        with open(file_path, "r") as file:
+            data = yaml.safe_load(file)
+        find_common_values(data, filename=filename)
+
+# Second pass: Replace common values
+for filename in os.listdir(SECRET_DIR):
+    if filename.endswith(".yml") or filename.endswith(".yaml"):
+        file_path = os.path.join(SECRET_DIR, filename)
+        with open(file_path, "r") as file:
+            data = yaml.safe_load(file)
+        modified = False
+        replace_common_values(data)
+        if modified:
+            with open(file_path, "w") as file:
+                yaml.safe_dump(data, file, default_flow_style=False)
+
+# Save mappings
+with open(OUTPUT_YML, "w") as yamlfile:
+    yaml.safe_dump(secret_mappings, yamlfile, default_flow_style=False)
+
+print(f"Processing completed. Placeholder mappings saved in {OUTPUT_YML}.")