Add augment_from setting

This replaces (and removes) the [Server] section take_from setting.

doc/configuration.md

@@ -177,7 +177,8 @@ will be downloaded automatically.
   file containing the token. This can be useful to share the
   configuration file with others, without accidentally sharing a
   personal GitHub token. The path may be specified absolute, or relative
-  to the configuration file.
+  to the configuration file. Note that setting `augment_from` in section
+  `[Local]` can also be used to specify an external token.
 
 ## Section Directory
 
@@ -211,11 +212,6 @@ Configuration options are:
 * **https**: whether to use HTTPS to access the server; boolean, default
   is `false`.
 
-* **take_from**: file path (absolute or relative to the configuration
-  file) containing (part of) the server details. Anything in this file
-  will override the settings specified above. Not all settings need be
-  present; the `[Server]` section header is not needed but allowed.
-
 ## Section Local
 
 This section specifies where to create the export file, and what and
@@ -272,3 +268,8 @@ where to log.
   storage definitions.
 
   For XML sources, this setting is usually best left empty.
+
+* **augment_from** (filename) specifies an external toml file to merge
+  into the current one. It can be used to specify settings used in
+  multiple toml files once. The toml file must have the same structure
+  as the regular one, but does not have to be complete.

src/config.py

@@ -43,6 +43,9 @@ def get_config() -> ns.Namespace:
     levels = 'debug,info,warning,error,critical'.split(',')
     ns.check_oneof(local, 'loglevel', levels, 'info')
 
+    # Merge-in setting from the file specified in augment_from, if any
+    merge_augmented_settings(config)
+
     # Do final setup of logging
     setup_logging(config)
 
@@ -78,6 +81,27 @@ def get_config() -> ns.Namespace:
 
 # =====
 
+def merge_augmented_settings(config:ns.Namespace):
+    """ Merges settings from file in setting augment_from, if any """
+
+    local = ns.get_section(config, 'Local')
+    if local is None:
+        return
+    fname = local._get('augment_from')
+    if fname is None:
+        return
+    if not isabs(fname):
+        fname = join(config.cfgdir, fname)
+    if not exists(fname):
+        raise ConfigurationError(f"augment_from file {local._get('augment_from')} not found")
+    cs = ns.dict2ns(toml.load(fname))
+    if (aug_local := ns.get_section(cs, 'Local')) and 'augment_from' in aug_local:
+        raise ConfigurationError("Recursive augment_from not supported")
+    # Add/override each key/value in augment_from
+    for k, v in cs._flattened():
+        ns.set_in_path(config, k, v)
+
+
 def merge_overrides(args:argparse.Namespace, config:ns.Namespace):
     """Merge command line overrides into configuration"""
 
@@ -162,7 +186,7 @@ def check(config:ns.Namespace):
     # Check server configuration
     if src.srctype == 'udl':
         check_server(config)
-        
+
 
 def check_server(config):
     """ Check UDL -> XML server configuration """
@@ -180,28 +204,11 @@ def check_server(config):
     ns.check_default(svr, 'password', 'SYS')
     ns.check_default(svr, 'namespace', 'USER')
     ns.check_default(svr, 'https', False)
-
-    # Is an external definition file specified?
-    if not (take_from := svr._get('take_from')):
-        return
-
-    # Make file path absolute, if not already so
-    if not isabs(take_from):
-        take_from = join(config.cfgdir, take_from)
 
-    # Load the server specification from the file
-    ext_svr = ns.dict2ns(toml.load(take_from))
+    if 'take_from' in svr:
+        raise ConfigurationError("Setting take_from no longer supported." \
+            " Use augment_from in section Local instead.")
 
-    # Allow, but don't require, a [Server] section header
-    if 'Server' in ext_svr and isinstance(ext_svr.Server, ns.Namespace):
-        ext_svr = ext_svr.Server
-
-    # Overwrite default settings with what's in the external file
-    for prop in 'host,port,user,password,namespace,https'.split(','):
-        if not prop in ext_svr:
-            continue
-        svr[prop] = ext_svr[prop]
-
 
 def check_csp(config:ns.Namespace):
     """ Checks the CSP configuration """

template.toml

@@ -81,10 +81,6 @@ path = ''
 ##### Server connection details: the server to use to convert UDL to XML.
 [Server]
 
-# Filename to take the settings from. Merges with (and overrides) settings
-# defined here. Does not need, but may use, [Server] section header.
-#take_from = 'server.toml'
-
 # Hostname or IP address, default 'localhost'
 #host = ''
 # Caché/IRIS internal web server port, default '52773'
@@ -122,3 +118,5 @@ timestamps = 'leave'
 # Force override of export version. Either 25 (Cache) or 26 (IRIS).
 export_version = ''
 
+# Augment/override settings in this file with the one specified here.
+augment_from = ''

tests/test_augment_from.py

@@ -0,0 +1,86 @@
+"""
+Tests the [Local] augment_from setting.
+"""
+
+import pytest
+
+import namespace as ns
+
+
+# Base configuration
+CFG = """
+[Source]
+type = "directory"
+[Directory]
+path = '{path}'
+[Local]
+outfile = 'out.xml'
+augment_from = 'ovr.toml'
+"""
+
+# Some overrides to merge-in
+OVR = """
+[Source]
+type = "github"
+[GitHub]
+user = 'a'
+repo = 'b'
+tag = 'main'
+[Local]
+outfile = 'out2.xml'
+"""
+
+
+@pytest.mark.usefixtures("reload_modules")
+def test_augment(tmp_path, get_config):
+    """ Tests overriding directory settings using augment_from setting """
+
+    # Write override toml
+    ovr = tmp_path / 'ovr.toml'
+    with open(ovr, 'wt', encoding="UTF-8") as f:
+        f.write(OVR)
+
+    # Get the parsed and augmented configuration settings
+    cfg = get_config(CFG, tmp_path)
+
+    assert cfg.Source.type == 'github', "Override not applied to Source.type"
+    assert cfg.GitHub.user == 'a', "Setting GitHub.user not correct"
+    assert cfg.GitHub.repo == 'b', "Setting GitHub.repo not correct"
+    assert cfg.GitHub.tag == 'main', "Setting GitHub.tag not correct"
+    assert cfg.Local.outfile == 'out2.xml', "Setting Local.outfile not updated"
+
+
+@pytest.mark.usefixtures("reload_modules")
+def test_augment_ovr(tmp_path, get_config):
+    """ Tests recursive override error """
+
+    # Write override toml with another augment_from setting
+    ovr = tmp_path / 'ovr.toml'
+    with open(ovr, 'wt', encoding="UTF-8") as f:
+        f.write(OVR+"augment_from='abcde'")
+
+    # Make sure this raises a configuration error
+    with pytest.raises(ns.ConfigurationError) as e:
+        get_config(CFG, tmp_path)
+
+    # Check the error message
+    msg:str = e.value.args[0]
+    assert msg == "Recursive augment_from not supported", f"Unexpected message {msg}"
+
+
+@pytest.mark.usefixtures("reload_modules")
+def test_take_from_error(tmp_path, get_config):
+    """ Tests error message on removed setting """
+
+    # Write (empty) override toml
+    ovr = tmp_path / 'ovr.toml'
+    with open(ovr, 'wt', encoding="UTF-8") as f:
+        f.write("")
+
+    # Make sure this raises a configuration error
+    with pytest.raises(ns.ConfigurationError) as e:
+        get_config(CFG+"[Server]\ntake_from='xxx'\n", tmp_path)
+
+    # Check the error message
+    msg:str = e.value.args[0]
+    assert "Setting take_from no longer supported" in msg, f"Unexpected message {msg}"

tests/test_server_config.py

@@ -61,7 +61,8 @@ def test_external_def_all(tmp_path:Path, get_config):
     """ Tests overriding server with external file """
 
     extdef = tmp_path / 'svr.toml'
-    settings = "host = '127.0.0.1'\n" \
+    settings = "[Server]\n" \
+        "host = '127.0.0.1'\n" \
         "port = '12345'\n" \
         "user = 'Asterix'\n" \
         "password = 'Obelix'\n" \
@@ -71,9 +72,7 @@ def test_external_def_all(tmp_path:Path, get_config):
     with open(extdef, 'wt', encoding='utf8') as f:
         f.write(settings)
 
-    override = f"[Server]\ntake_from = '{extdef}'"
-
-    cfg = get_config(f"{CFG}\n{override}", tmp_path) # type: ns.Namespace
+    cfg = get_config(f"{CFG}\naugment_from='{extdef}'", tmp_path)
     assert 'Server' in cfg, "No Server section"
     svr = cfg.Server
     assert isinstance(svr, ns.Namespace), "Server not a section"
@@ -94,37 +93,16 @@ def test_external_relative_location(tmp_path:Path, get_config):
     dir.mkdir(parents=True)
 
     extdef = dir / 'svr.toml'
-    settings = "host = '127.0.0.1'\n"
-    with open(extdef, 'wt', encoding='utf8') as f:
-        f.write(settings)
-
-    relpath = extdef.relative_to(tmp_path)
-    override = f"[Server]\ntake_from = '{relpath}'"
-
-    cfg = get_config(f"{CFG}\n{override}", tmp_path) # type: ns.Namespace
-    assert 'Server' in cfg, "No Server section"
-    svr = cfg.Server
-    assert isinstance(svr, ns.Namespace), "Server not a section"
-
-    assert svr.host == '127.0.0.1', f"Unexpected value for host: {svr.host}"
-
-
-@pytest.mark.usefixtures("reload_modules")
-def test_external_with_header(tmp_path:Path, get_config):
-    """ Tests external server definition with a section header """
-
-    extdef = tmp_path / 'svr.toml'
     settings = "[Server]\nhost = '127.0.0.1'\n"
     with open(extdef, 'wt', encoding='utf8') as f:
         f.write(settings)
 
-    override = "[Server]\ntake_from = 'svr.toml'"
+    relpath = extdef.relative_to(tmp_path)
+    override = f"augment_from = '{relpath}'"
 
-    cfg = get_config(f"{CFG}\n{override}", tmp_path) # type: ns.Namespace
+    cfg = get_config(f"{CFG}\n{override}", tmp_path)
     assert 'Server' in cfg, "No Server section"
     svr = cfg.Server
     assert isinstance(svr, ns.Namespace), "Server not a section"
 
     assert svr.host == '127.0.0.1', f"Unexpected value for host: {svr.host}"
-
-