feat: Use fonts from xml2rfc-fonts (#1124)

* ci: Use xml2rfc-fonts for tests

* ci: Bump up setup-python action to v5

* ci: Bump up checout action to v4

* ci: Restrict macOS testing to intel processor

See #1129

* fix: Fix text extraction issues with updated fonts

* build: Use xml2rfc-fonts in docker image

* fix: Use xml2rfc-fonts xml2rfc compatible version

* docs: Update documentation to use xml2rfc-fonts

* docs: Update font install instructions

.github/workflows/checks.yml

@@ -18,7 +18,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v1
@@ -39,10 +39,10 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2.3.1
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -56,30 +56,23 @@ jobs:
     - name: Download Fonts
       if: steps.cache-fonts-linux.outputs.cache-hit != 'true'
       run: |
-        echo "Downloading Noto Font..."
-        mkdir -p ~/.fonts/opentype/noto
-        wget -q https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip
-        unzip -q Noto-unhinted.zip -d ~/.fonts/opentype/noto/
-        mkdir -p /usr/share/fonts/truetype/noto/
-        ln -sf ~/.fonts/opentype/noto/*.[to]tf /usr/share/fonts/truetype/noto/
-        echo "Downloading Roboto Mono Font..."
-        mkdir -p ~/.fonts/opentype/roboto
-        wget -q https://fonts.google.com/download?family=Roboto%20Mono -O roboto-mono.zip
-        unzip -q roboto-mono.zip -d ~/.fonts/opentype/roboto/
-        mkdir -p /usr/share/fonts/truetype/roboto/
-        ln -sf ~/.fonts/opentype/roboto/*.[to]tf /usr/share/fonts/truetype/roboto/
+        echo "Downloading xml2rfc-fonts"
+        mkdir -p ~/.fonts/opentype ~/fonts
+        wget -q -O fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.22.0.tar.gz
+        tar zxf fonts.tar.gz -C ~/fonts
+        mv ~/fonts/*/noto/* ~/.fonts/opentype/
+        mv ~/fonts/*/roboto_mono/* ~/.fonts/opentype/
+        mkdir -p /usr/share/fonts/truetype
+        ln -sf ~/.fonts/opentype/*.[to]tf /usr/share/fonts/truetype/
         echo "Reloading Font Cache..."
         fc-cache -f -v
         
     - name: Link Fonts
       if: steps.cache-fonts-linux.outputs.cache-hit == 'true'
       run: |
-        echo "Linking Noto Font..."
-        mkdir -p /usr/share/fonts/truetype/noto/
-        ln -sf ~/.fonts/opentype/noto/*.[to]tf /usr/share/fonts/truetype/noto/
-        echo "Linking Roboto Mono Font..."
-        mkdir -p /usr/share/fonts/truetype/roboto/
-        ln -sf ~/.fonts/opentype/roboto/*.[to]tf /usr/share/fonts/truetype/roboto/
+        echo "Linking Fonts..."
+        mkdir -p /usr/share/fonts/truetype/
+        ln -sf ~/.fonts/opentype/*.[to]tf /usr/share/fonts/truetype/
         echo "Reloading Font Cache..."
         fc-cache -f -v
         
@@ -104,7 +97,7 @@ jobs:
 
   tests-macos:
     name: Unit Tests (macOS)
-    runs-on: macos-latest
+    runs-on: macos-12
     if: github.event_name == 'pull_request'
 
     strategy:
@@ -113,41 +106,32 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2.3.1
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Fonts Cache
       id: cache-fonts-mac
       uses: pat-s/[email protected]
       with:
-        path: ~/new-fonts
+        path: ~/fonts
         key: fonts-macos
 
     - name: Download Fonts
-      if: steps.cache-fonts-mac.outputs.cache-hit != 'true'
       run: |
-        mkdir -p ~/new-fonts
-        cd ~/new-fonts
-        echo "Downloading Noto Font..."
-        wget -q https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip
-        unzip -o -q Noto-unhinted.zip -d /Library/Fonts/
-        echo "Downloading Roboto Mono Font..."
-        wget -q https://fonts.google.com/download?family=Roboto%20Mono -O roboto-mono.zip
-        unzip -o -q roboto-mono.zip -d /Library/Fonts/
-        
-    - name: Extract Fonts
-      if: steps.cache-fonts-mac.outputs.cache-hit == 'true'
-      run: |
-        unzip -o -q ~/new-fonts/Noto-unhinted.zip -d /Library/Fonts/
-        unzip -o -q ~/new-fonts/roboto-mono.zip -d /Library/Fonts/
+        echo "Downloading xml2rfc-fonts"
+        mkdir -p ~/fonts /Library/Fonts
+        wget -q -O fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.22.0.tar.gz
+        tar zxf fonts.tar.gz -C ~/fonts
+        mv ~/fonts/*/noto/* /Library/Fonts/
+        mv ~/fonts/*/roboto_mono/* /Library/Fonts/
         
     - name: Install dependencies
       run: |
-        brew install pango libffi groff make diffutils
+        brew install pango pillow libffi groff make diffutils
         echo "Installing pip + wheel..."
         python -m pip install --upgrade pip wheel
         echo "Installing requirements.txt + test dependencies..."
@@ -177,10 +161,10 @@ jobs:
 
 #     steps:
 #     - name: Checkout repository
-#       uses: actions/checkout@v2
+#       uses: actions/checkout@v4
 
 #     - name: Setup Python ${{ matrix.python-version }}
-#       uses: actions/setup-python@v2.3.1
+#       uses: actions/setup-python@v5
 #       with:
 #         python-version: ${{ matrix.python-version }}
 

README.md

@@ -44,16 +44,11 @@ In order to generate PDFs, **xml2rfc** uses the [WeasyPrint] module, which depen
 ```sh
 pip install 'weasyprint>=53.0,!=57.0,!=60.0'
 ```
-3. Finally, install the full **Noto Font** and **Roboto Mono** packages:
-  * Download the full font file from:
-    https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip
-  * Follow the installation instructions at
-    https://www.google.com/get/noto/help/install/
-  * Go to https://fonts.google.com/specimen/Roboto+Mono, and download the
-    font. Follow the installation instructions above, as applied to this download.
-  * Go to https://fonts.google.com/noto/specimen/Noto+Sans+Math, and
-    download the font. Follow the installation instructions above, as
-    applied to this download.
+3. Finally, install the required fonts:
+  * Download latest fonts from [xml2rfc-fonts](https://github.com/ietf-tools/xml2rfc-fonts/releases/latest).
+  * In the **Assets** section, download either the `tar.gz` or the `zip` archive.
+  * Extract the contents of the downloaded `xml2rfc-fonts` archive.
+  * Install the fonts found in the `noto` and `roboto_mono` directories to your operating system.
 
 With these installed and available to **xml2rfc**, the `--pdf` switch will be enabled.
 

docker/base.Dockerfile

@@ -34,17 +34,12 @@ RUN apt-get update --fix-missing && \
     apt-get clean -y
 
 # Install required fonts
-RUN mkdir -p ~/.fonts/opentype && \
-    wget -q https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip && \
-    unzip -q Noto-unhinted.zip -d ~/.fonts/opentype/ && \
-    rm Noto-unhinted.zip && \
-    wget -q https://fonts.google.com/download?family=Roboto%20Mono -O roboto-mono.zip && \
-    unzip -q roboto-mono.zip -d ~/.fonts/opentype/ && \
-    rm roboto-mono.zip && \
-    wget -q https://fonts.google.com/download?family=Noto%20Sans%20Math -O noto-sans-math.zip && \
-    unzip -q noto-sans-math.zip -d ~/.fonts/opentype/ && \
-    rm noto-sans-math.zip && \
-    ln -sf ~/.fonts/opentype/*.[to]tf /usr/share/fonts/truetype/ && \
+RUN mkdir -p ~/.fonts/opentype /tmp/fonts && \
+    wget -q -O /tmp/fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.22.0.tar.gz && \
+    tar zxf /tmp/fonts.tar.gz -C /tmp/fonts && \
+    mv /tmp/fonts/*/noto/* ~/.fonts/opentype/ && \
+    mv /tmp/fonts/*/roboto_mono/* ~/.fonts/opentype/ && \
+    rm -rf /tmp/fonts.tar.gz /tmp/fonts/ && \
     fc-cache -f
 
 # Copy everything required to build xml2rfc

xml2rfc/run.py

@@ -89,25 +89,18 @@ def get_pdf_help(missing_libs=""):
 
         pip install 'weasyprint>=53.0,!=57.0,!=60.0'
 
+    3. Finally, install the required fonts:
+       * Download latest fonts from xml2rfc-fonts.
+         https://github.com/ietf-tools/xml2rfc-fonts/releases/latest
 
-    3. Finally, install the full Noto Font and Roboto Mono packages:
+       * In the "Assets" section, download either the tar.gz or
+         the zip archive.
 
-       * Download the full font file from:
-         https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip
-         or follow the 'DOWNLOAD ALL FONTS' link on this page:
-         https://www.google.com/get/noto/
+       * Extract the contents of the downloaded xml2rfc-fonts archive.
 
-       * Follow the installation instructions at
-         https://www.google.com/get/noto/help/install/
+       * Install the fonts found in the noto and roboto_mono directories
+         to your operating system.
 
-       * Go to https://fonts.google.com/specimen/Roboto+Mono, and download the
-         font. Follow the installation instructions above, as applied to this
-         download.
-
-       * Go to https://fonts.google.com/noto/specimen/Noto+Sans+Math, and
-         download the font. Follow the installation instructions above, as
-         applied to this download.
-    
     With these libraries, modules, and fonts installed and available to
     xml2rfc, the --pdf switch will be enabled.
     """

xml2rfc/walkpdf.py

@@ -22,7 +22,7 @@ def walk(obj, seen):
             dobj[k] = d
             iobj += i
         if hasattr(obj, 'extract_text'):
-            dobj['text'] = obj.extract_text()
+            dobj['text'] = obj.extract_text(extraction_mode="layout")
     elif isinstance(obj, pypdf.generic.ArrayObject):
         dobj = []
         for o in obj: