Improve handling of class timestamps

doc/configuration.md

@@ -248,3 +248,17 @@ where to log.
 * **loglevel**: the level of logging. Possible values are `'debug'`,
   `'info'`, `'warning'`, `'error'`, and `'critical'`. Note that most
   logging is done as `'info'` or `'debug'`.
+
+* **timestamps**: how to handle timestamps (TimeCreated and TimeChanged
+  elements) in classes. XML sources may or may not have them. Sources on
+  the filesystem can use the file timestamp to set these properties.
+  Possible values are:
+
+  * `'clear'`: remove timestamp elements from class exports, if present.
+  * `'update'`: if timestamp information can be determined, as on a
+    local filesystem, add or update the elements, otherwise remove them.
+  * `'leave'`: don't add or remove timestamps, leave them as they are.
+    This is the default.
+
+  UDL sources from GitHub don't have timestamps, but they get one from
+  the UDL to XML conversion. These are always stripped.

src/config.py

@@ -120,6 +120,7 @@ def check(config:ns.Namespace):
     ns.check_default(local, 'logdir', '')
     ns.check_default(local, 'loglevel', '')
     ns.check_default(local, 'threads', 4)
+    ns.check_oneof(local, 'timestamps', ('clear', 'update', 'leave'), 'leave')
 
     # Check data configuration
     if src.datadir:

src/split_export.py

@@ -11,13 +11,17 @@
 
 from lxml import etree
 
+import namespace as ns
 from config import ConfigurationError
 from repo import RepositoryItem, RepositoryCspItem
 
 
 class ExportFile:
     """Represents a file with items to export"""
 
+    # The configuration object
+    config:ns.Namespace
+
     # The filename to export to
     filename:str
 
@@ -28,7 +32,8 @@ class ExportFile:
     root:Optional[etree.Element]
 
 
-    def __init__(self, filename, items=None):
+    def __init__(self, config, filename, items=None):
+        self.config = config
         self.filename = filename
         self.items = items or []
         self.root = None
@@ -76,6 +81,13 @@ def create_export(self) -> etree.Element:
         minver = maxver = 0
         parser = etree.XMLParser(strip_cdata=False)
 
+        # How to handle timestamps ('clear', 'update', 'leave')
+        timestamps = self.config.Local.timestamps
+        # If we get data from GitHub, we get nonsense timestamps and
+        # have no better info, so clear them regardless of setting
+        if self.config.Source.srctype == 'udl' and self.config.Source.type == 'github':
+            timestamps = 'clear'
+
         for item in self.items:
             # Get the lxml element for this item
             item_root = item.get_xml_element()
@@ -96,10 +108,13 @@ def create_export(self) -> etree.Element:
                 if version > maxver: maxver = version
 
             # Handle item timestamps
-            if ts := item.horolog:
-                self.update_timestamp(item_root, ts)
-            else:
+            if timestamps == 'clear':
                 self.remove_timestamps(item_root)
+            elif timestamps == 'update':
+                if ts := item.horolog:
+                    self.update_timestamp(item_root, ts)
+                else:
+                    self.remove_timestamps(item_root)
 
             for el in item_root:
                 el.tail = '\n\n'
@@ -127,9 +142,13 @@ def update_timestamp(self, item_root:etree.Element, horolog):
         for el in item_root:
             if el.tag != "Class":
                 continue
-            for subel in el:
-                if subel.tag not in ("TimeChanged", "TimeCreated"):
-                    continue
+            for tag in ("TimeChanged", "TimeCreated"):
+                subel = el.find(tag)
+                if subel is None:
+                    # Add the element at the first position of the parent
+                    subel = etree.Element(tag)
+                    subel.tail = '\n'
+                    el.insert(0, subel)
                 subel.text = horolog
 
 
@@ -139,7 +158,7 @@ def get_files(config, repo):
 
     export_files = []
     main_name = get_export_name(config, repo.name)
-    main_export = ExportFile(main_name)
+    main_export = ExportFile(config, main_name)
     export_files.append(main_export)
 
     # Export of code items
@@ -154,7 +173,7 @@ def get_files(config, repo):
         else:
             name, ext = splitext(main_name)
             name = f'{name}_data{ext}'
-            data_export = ExportFile(name, repo.data_items)
+            data_export = ExportFile(config, name, repo.data_items)
             export_files.append(data_export)
 
     # Export of CSP items
@@ -164,7 +183,7 @@ def get_files(config, repo):
         else:
             name, ext = splitext(main_name)
             name = f'{name}_csp{ext}'
-            csp_export = ExportFile(name)
+            csp_export = ExportFile(config, name)
             export_files.append(csp_export)
 
         for item in repo.csp_items:

template.toml

@@ -116,3 +116,6 @@ logdir = ''
 # Logging level. One of 'debug', 'info', 'warning', 'error', or 'critical'.
 loglevel = 'info'
 
+# Timestamp handling. One of 'clear', 'update', or 'leave' (default).
+timestamps = 'leave'
+

tests/conftest.py

@@ -216,7 +216,12 @@ def src_tree(tmp_path_factory):
     dir = base / 'src' / 'c'
     dir.mkdir(parents=True)
     file = dir / "cc.cls.xml"
-    file.write_text(CLS_TPL.format(name="tmp.c.cc"), encoding='UTF-8')
+    contents = CLS_TPL.format(name="tmp.c.cc")
+    timestamps = '<TimeCreated>65958,79762.139</TimeCreated>\n' \
+        '<TimeChanged>65958,79762.139</TimeChanged>\n'
+    idx = contents.find('<Desc')
+    contents = contents[:idx] + timestamps + contents[idx:]
+    file.write_text(contents, encoding='UTF-8')
 
     # An include file
     dir = base / 'src'

tests/test_fs_timestamps.py

@@ -0,0 +1,117 @@
+import sys
+from importlib import reload, import_module
+from typing import Any
+from io import BytesIO
+import re
+
+from lxml import etree
+
+import pytest
+
+builder = import_module("build-export") # type: Any
+
+
+CFG = """
+[Source]
+type = "directory"
+srctype = "xml"
+srcdir = 'src'
+cspdir = ''
+datadir = ''
+skip = ['/csp/*', '/data/*']
+[Directory]
+path = '{path}'
+[Local]
+outfile = 'out.xml'
+"""
+
+
+@pytest.mark.usefixtures("reload_modules")
+def test_has_timestamps(src_tree, tmp_path, get_build):
+    """ Tests creating an export with updated timestamps. """
+
+    cfg = CFG.format(path=src_tree) + "\ntimestamps='update'"
+    export = get_build(cfg, tmp_path)
+
+    tree = etree.parse(BytesIO(export))
+    assert tree.docinfo.root_name == 'Export'
+
+    # Regex matching $horolog
+    dh = re.compile(r'\d+,\d+(\.\d+)?')
+    for el in tree.getroot():
+        if el.tag != 'Class':
+            continue
+
+        # Make sure the TimeCreated element is present and resembles a $Horolog value
+        assert el.find('TimeCreated') is not None, "No TimeCreated element in class"
+        text = el.find('TimeCreated').text
+        assert dh.match(text), f"TimeCreated not in $Horolog format: '{text}'." 
+
+        assert el.find('TimeChanged') is not None, "No TimeChanged element in class"
+        text = el.find('TimeChanged').text
+        assert dh.match(text), f"TimeChanged not in $Horolog format: '{text}'." 
+
+
+@pytest.mark.usefixtures("reload_modules")
+def test_clear_timestamps(src_tree, tmp_path, get_build):
+    """ Tests creating an export with removed timestamps. """
+
+    cfg = CFG.format(path=src_tree) + "\ntimestamps='clear'"
+    export = get_build(cfg, tmp_path)
+
+    tree = etree.parse(BytesIO(export))
+    assert tree.docinfo.root_name == 'Export'
+
+    for el in tree.getroot():
+        if el.tag != 'Class':
+            continue
+        assert el.find('TimeCreated') is None, "TimeCreated element in class"
+        assert el.find('TimeChanged') is None, "TimeChanged element in class"
+
+
+@pytest.mark.usefixtures("reload_modules")
+def test_leave_timestamps(src_tree, tmp_path, get_build):
+    """ Tests creating an export with unchanged timestamps. """
+
+    cfg = CFG.format(path=src_tree) + "\ntimestamps='leave'"
+    export = get_build(cfg, tmp_path)
+
+    tree = etree.parse(BytesIO(export))
+    assert tree.docinfo.root_name == 'Export'
+
+    for el in tree.getroot():
+        if el.tag != 'Class':
+            continue
+
+        # Classes a and b had no timestamp and still shouldn't have
+        if el.attrib['name'] in ("tmp.a", "tmp.b"):
+            assert el.find('TimeCreated') is None, "TimeCreated element in class"
+            assert el.find('TimeChanged') is None, "TimeChanged element in class"
+
+        # Class c.cc had a timestamp and still should have
+        elif el.attrib['name'] in ("tmp.c.cc",):
+            assert el.find('TimeCreated') is not None, "No TimeCreated element in class"
+            text = el.find('TimeCreated').text
+            assert text == '65958,79762.139', f"Unexpected value of TimeCreated: '{text}'." 
+
+            assert el.find('TimeChanged') is not None, "No TimeChanged element in class"
+            text = el.find('TimeChanged').text
+            assert text == '65958,79762.139', f"Unexpected value of TimeCreated: '{text}'." 
+
+
+@pytest.mark.usefixtures("reload_modules")
+def test_empty_is_leave(src_tree, tmp_path, get_build):
+    """ Tests creating an export with default timestamps setting. """
+
+    cfg = CFG.format(path=src_tree) + "\ntimestamps='leave'"
+    export1 = get_build(cfg, tmp_path)
+
+    reload(sys.modules['logging'])
+    reload(sys.modules['config'])
+    reload(sys.modules['build-export'])
+
+    cfg = CFG.format(path=src_tree)
+    export2 = get_build(cfg, tmp_path)
+
+    assert export1 == export2, "Export without setting not same as setting 'leave'"
+