Warn if api_key in Config

garak/_config.py

@@ -7,11 +7,13 @@
 
 # logging should be set up before config is loaded
 
+import platform
 from collections import defaultdict
 from dataclasses import dataclass
 import importlib
 import logging
 import os
+import stat
 import pathlib
 from typing import List
 import yaml
@@ -116,13 +118,25 @@ def _nested_dict():
 # generator, probe, detector, buff = {}, {}, {}, {}
 
 
+def _key_exists(d: dict, key: str) -> bool:
+    # Check for the presence of a key in a nested dict.
+    if not isinstance(d, dict) and not isinstance(d, list):
+        return False
+    if isinstance(d, list):
+        return any([_key_exists(item, key) for item in d])
+    if isinstance(d, dict) and key in d.keys():
+        return True
+    else:
+        return any([_key_exists(val, key) for val in d.values()])
+
+
 def _set_settings(config_obj, settings_obj: dict):
     for k, v in settings_obj.items():
         setattr(config_obj, k, v)
     return config_obj
 
 
-def _combine_into(d: dict, combined: dict) -> None:
+def _combine_into(d: dict, combined: dict) -> dict:
     if d is None:
         return combined
     for k, v in d.items():
@@ -141,6 +155,22 @@ def _load_yaml_config(settings_filenames) -> dict:
         with open(settings_filename, encoding="utf-8") as settings_file:
             settings = yaml.safe_load(settings_file)
             if settings is not None:
+                if _key_exists(settings, "api_key"):
+                    if platform.system() == "Windows":
+                        msg = (f"A possibly secret value (`api_key`) was detected in {settings_filename}. "
+                               f"We recommend removing potentially sensitive values from config files or "
+                               f"ensuring the file is readable only by you.")
+                        logging.warning(msg)
+                        print(f"⚠️  {msg}")
+                    else:
+                        logging.info(f"API key found in {settings_filename}. Checking readability...")
+                        res = os.stat(settings_filename)
+                        if res.st_mode & stat.S_IROTH or res.st_mode & stat.S_IRGRP:
+                            msg = (f"A possibly secret value (`api_key`) was detected in {settings_filename}, "
+                                   f"which is readable by users other than yourself. "
+                                   f"Consider changing permissions on this file to only be readable by you.")
+                            logging.warning(msg)
+                            print(f"⚠️  {msg}")
                 config = _combine_into(settings, config)
     return config
 

tests/test_config.py

@@ -826,3 +826,10 @@ async def main():
         "/", headers={"User-Agent": AGENT_TEST}
     ).respond_with_data("")
     asyncio.run(main())
+
+
+def test_api_key_in_config():
+    importlib.reload(_config)
+
+    _config.plugins.generators["a"]["b"]["c"]["api_key"] = "something"
+    assert _config._key_exists(_config.plugins.generators, "api_key")