From 5e63e7e68d599b0b370897c78186bfc2fae5c332 Mon Sep 17 00:00:00 2001 From: Gertjan Klein Date: Thu, 30 May 2024 16:51:09 +0200 Subject: [PATCH] Implement a 'take' option. --- doc/configuration.md | 8 ++++ src/config.py | 22 ++++++----- src/fsrepo.py | 7 +++- src/ziprepo.py | 11 ++++-- template.toml | 11 +++++- tests/test_take.py | 91 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 134 insertions(+), 16 deletions(-) create mode 100644 tests/test_take.py diff --git a/doc/configuration.md b/doc/configuration.md index a210365..918f274 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -51,6 +51,14 @@ There are several configuration sections in the file: slash) from the sources root. Supports asterisk-style wildcards. Example: `['/src/tests/*','/data/Test_*.lut']` +* **take**: a list (enclosed in square brackets) of items to include in + the export. Items are specified as a path (including leading slash) + from the sources root. Supports asterisk-style wildcards. Example: + `['/src/tests/*','/data/Test_*.lut']`. If this list is empty, all + items will be included. If an item is present in both the `skip` and + `take` lists (or the latter is empty), the `skip` list takes + precedence. + ## Section CSP (This section and the CSP.parsers sections are only needed if CSP files diff --git a/src/config.py b/src/config.py index 4c4fb4d..5afb6b7 100644 --- a/src/config.py +++ b/src/config.py @@ -55,16 +55,17 @@ def get_config() -> ns.Namespace: # Make sure configuration is complete check(config) - # Converts specification(s) of files to skip to regexes - config.skip_regexes = [] - for spec in config.Source.skip: - spec = spec.replace('\\', '\\\\') - spec = spec.replace('.', '\\.') - # Create valid regex for star - spec = spec.replace('*', '.*') - regex = re.compile(spec, re.I) - config.skip_regexes.append(regex) - + # # Converts specification(s) of files to skip/take to regexes + for src, dst in (('skip', 'skip_regexes'), ('take', 'take_regexes')): + config[dst] = [] + for spec in config.Source[src]: + spec = spec.replace('\\', '\\\\') + spec = spec.replace('.', '\\.') + # Create valid regex for star + spec = spec.replace('*', '.*') + regex = re.compile(spec, re.I) + config[dst].append(regex) + # Load token contents from file, if specified if config.Source.type == 'github' and config.GitHub.token and config.GitHub.token[0] == '@': path = config.GitHub.token[1:] @@ -131,6 +132,7 @@ def check(config:ns.Namespace): ns.check_default(src, 'datadir', '') ns.check_default(src, 'cspdir', '') ns.check_default(src, 'skip', []) + ns.check_default(src, 'take', []) # Strip leading slash if present, we don't need it if src.srcdir == '/': diff --git a/src/fsrepo.py b/src/fsrepo.py index 3bd592d..e606edd 100644 --- a/src/fsrepo.py +++ b/src/fsrepo.py @@ -44,9 +44,14 @@ def get_names(self): norm = '/' + (name.replace(os.sep, '/') if os.sep != '/' else name) skip = any(rx.match(norm) for rx in self.config.skip_regexes) if skip: - logging.debug('Skipping %s because config requested so', name) + logging.debug('Skipping %s because item in "skip" list', name) continue + if self.config.take_regexes: + if not any(rx.match(norm) for rx in self.config.take_regexes): + logging.debug('Skipping %s because not in "take" list', name) + continue + parts = name.split(os.sep) if path_matches(cspdir, parts): diff --git a/src/ziprepo.py b/src/ziprepo.py index 8eda5f5..368f221 100644 --- a/src/ziprepo.py +++ b/src/ziprepo.py @@ -51,13 +51,18 @@ def get_names(self): if parts[-1][0] == '.': continue - # Check for items we should skip (remove base directory name first) + # Check for items we should skip/take (remove base directory name first) tmp = '/' + name.split('/', 1)[1] skip = any(rx.match(tmp) for rx in self.config.skip_regexes) if skip: - logging.debug('Skipping %s because config requested so', tmp) + logging.debug('Skipping %s because item in "skip" list', name) continue - + + if self.config.take_regexes: + if not any(rx.match(tmp) for rx in self.config.take_regexes): + logging.debug('Skipping %s because not in "take" list', name) + continue + # Configure separately for CSP and source? encoding = self.config.Source.encoding diff --git a/template.toml b/template.toml index daf8865..1e9e79f 100644 --- a/template.toml +++ b/template.toml @@ -23,11 +23,18 @@ cspdir = 'csp' # Directory to search for data files to include, if any. datadir = 'data' -# List of items to skip when building the export. Supports *-style wildcards. -# The items are matched against their full path from the root of the repository. +# List of items to skip when building the export. Supports *-style +# wildcards. The items are matched against their full path in the +# repository, e.g. '/src/Strix/Lib/String.cls'. skip = [ ] +# List of items to include in the export. Supports *-style wildcards. If +# this list is empty, all items will be included. Items both in the skip +# and take lists will be excluded. Items are matched against their full +# path in the repository, e.g. '/src/Strix/Lib/String.cls'. +take = [ +] # Details on what to do with CSP items [CSP] diff --git a/tests/test_take.py b/tests/test_take.py new file mode 100644 index 0000000..8cfefd2 --- /dev/null +++ b/tests/test_take.py @@ -0,0 +1,91 @@ +from importlib import import_module +from io import BytesIO + +from lxml import etree + +import pytest + +builder = import_module("build-export") + + +CFG = """ +[Source] +type = "directory" +srctype = "xml" +srcdir = 'src' +cspdir = 'csp' +datadir = 'data' +skip = [{skip}] +take = [{take}] +[CSP] +export = 'embed' +[Data] +export = 'embed' +[[CSP.parsers]] +regex = '((/csp)?/[^/]+)/(.+)' +app = '\\1' +item = '\\3' +nomatch = 'error' +[Directory] +path = '{path}' +[Local] +outfile = 'out.xml' +loglevel = 'debug' +""" + + +@pytest.mark.usefixtures("reload_modules") +def test_take_basic(src_tree, tmp_path, get_build, validate_schema): + """Basic test for take""" + + # Get parsed configuration + paths = '/csp/app/hello.csp', '/data/Test*', '/src/a.cls.xml' + takes = ','.join((f"'{p}'" for p in paths)) + + cfg = CFG.format(path=src_tree, skip='', take=takes) + xml = get_build(cfg, tmp_path) + + # Parse with ElementTree + tree = etree.parse(BytesIO(xml)) + assert tree.docinfo.root_name == 'Export' + + export = tree.getroot() + assert len(export) == 3, "Expect 3 items in export" + + # Make sure the things we skipped are not in the export + assert tree.find('./CSP[@name="hello.csp"]') is not None, "hello.csp in export" + assert tree.find('./Document[@name="Test.LUT"]') is not None, "Test.LUT in export" + assert tree.find('./Class[@name="tmp.a"]') is not None, "tmp.a in export" + + validate_schema(xml) + + +@pytest.mark.usefixtures("reload_modules") +def test_skip_overrides_take(src_tree, tmp_path, get_build, validate_schema): + """Tests that skip has priority over take""" + + # Get parsed configuration + paths = '/csp/app/hello.csp', '/data/Test*', '/src/a.cls.xml' + takes = ','.join((f"'{p}'" for p in paths)) + skips = "'/csp/app/hello.csp'" + + cfg = CFG.format(path=src_tree, skip=skips, take=takes) + xml = get_build(cfg, tmp_path) + + # Parse with ElementTree + tree = etree.parse(BytesIO(xml)) + assert tree.docinfo.root_name == 'Export' + + export = tree.getroot() + assert len(export) == 2, "Expect 2 items in export" + + # Make sure the things we skipped are not in the export + assert tree.find('./CSP[@name="hello.csp"]') is None, "hello.csp must not in export" + assert tree.find('./Document[@name="Test.LUT"]') is not None, "Test.LUT in export" + assert tree.find('./Class[@name="tmp.a"]') is not None, "tmp.a in export" + + validate_schema(xml) + + + +