fix crash on aarch64 linux wheels #735
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "**" | |
| pull_request: {} | |
| jobs: | |
| resolve: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| MSRV: ${{ steps.resolve-msrv.outputs.MSRV }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - name: resolve MSRV | |
| id: resolve-msrv | |
| run: echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["workspace"]["package"]["rust-version"])'` >> $GITHUB_OUTPUT | |
| test: | |
| needs: [resolve] | |
| name: test rust-${{ matrix.rust-version }} on ${{ matrix.runs-on }}, Python ${{ matrix.python-version }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| rust-version: [stable] | |
| runs-on: [ubuntu, macos] | |
| python-version: ["3.13"] | |
| include: | |
| - rust-version: ${{ needs.resolve.outputs.MSRV }} | |
| runs-on: ubuntu | |
| - rust-version: stable | |
| runs-on: ubuntu | |
| python-version: "3.13t" | |
| - rust-version: nightly | |
| runs-on: ubuntu | |
| runs-on: ${{ matrix.runs-on }}-latest | |
| env: | |
| RUNS_ON: ${{ matrix.runs-on }}-latest | |
| RUST_VERSION: ${{ matrix.rust-version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: quansight-labs/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: ${{ matrix.rust-version }} | |
| - uses: Swatinem/rust-cache@v2 | |
| - uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: install cargo-careful | |
| if: matrix.rust-version == 'nightly' | |
| uses: taiki-e/install-action@cargo-careful | |
| - if: matrix.rust-version == 'nightly' | |
| run: cargo careful t -F python | |
| - run: cargo llvm-cov -F python --codecov --output-path codecov.json | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - uses: codecov/codecov-action@v4 | |
| with: | |
| env_vars: RUNS_ON,RUST_VERSION | |
| files: codecov.json | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| test-python: | |
| name: test jiter-python ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.13", "3.13t"] | |
| env: | |
| RUNS_ON: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: quansight-labs/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Install python dependencies | |
| run: pip install -r crates/jiter-python/tests/requirements.txt | |
| - name: Build jiter-python | |
| run: | | |
| source <(cargo llvm-cov show-env --export-prefix) | |
| cargo llvm-cov clean --workspace --profraw-only | |
| pip install -e crates/jiter-python --config-settings=build-args='--profile dev' | |
| - name: Run tests | |
| run: | | |
| source <(cargo llvm-cov show-env --export-prefix) | |
| pytest crates/jiter-python/tests | |
| python crates/jiter-python/bench.py jiter jiter-cache --fast | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - name: Coverage report | |
| run: | | |
| source <(cargo llvm-cov show-env --export-prefix) | |
| cargo llvm-cov report --codecov --output-path codecov.json | |
| - uses: codecov/codecov-action@v4 | |
| with: | |
| files: codecov.json | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| bench: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - uses: moonrepo/setup-rust@v1 | |
| with: | |
| channel: stable | |
| cache-target: release | |
| bins: cargo-codspeed | |
| - run: python crates/jiter/benches/generate_big.py | |
| - run: cargo codspeed build -F python -p jiter | |
| - uses: CodSpeedHQ/action@v3 | |
| with: | |
| run: cargo codspeed run | |
| token: ${{ secrets.CODSPEED_TOKEN }} | |
| fuzz: | |
| name: fuzz on ${{ matrix.runs-on }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runs-on: [ubuntu, macos] | |
| runs-on: ${{ matrix.runs-on }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@nightly | |
| - id: cache-rust | |
| uses: Swatinem/rust-cache@v2 | |
| - if: steps.cache-rust.outputs.cache-hit != 'true' | |
| run: cargo install cargo-fuzz | |
| - run: | | |
| # cargo fuzz defaults to musl targets, which is seeming incomatible with sanitizers according to CI failures | |
| RUST_TARGET=$(rustc -Vv | grep host | cut -d ' ' -f 2) | |
| cargo fuzz run --target=$RUST_TARGET --fuzz-dir crates/fuzz compare_to_serde --release -- -max_total_time=300s | |
| fuzz-skip: | |
| name: fuzz skip | |
| # we only run this on ubuntu since architecture should make no difference | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@nightly | |
| - id: cache-rust | |
| uses: Swatinem/rust-cache@v2 | |
| - if: steps.cache-rust.outputs.cache-hit != 'true' | |
| run: cargo install cargo-fuzz | |
| - run: | | |
| # cargo fuzz defaults to musl targets, which is seeming incomatible with sanitizers according to CI failures | |
| RUST_TARGET=$(rustc -Vv | grep host | cut -d ' ' -f 2) | |
| cargo fuzz run --target=$RUST_TARGET --fuzz-dir crates/fuzz compare_skip --release -- -max_total_time=300s | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt,clippy | |
| - uses: Swatinem/rust-cache@v2 | |
| - uses: pre-commit/[email protected] | |
| with: | |
| extra_args: --all-files --verbose | |
| env: | |
| PRE_COMMIT_COLOR: always | |
| SKIP: test | |
| - run: cargo doc -F python | |
| build-sdist: | |
| name: build sdist | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - uses: PyO3/maturin-action@v1 | |
| with: | |
| command: sdist | |
| args: --out dist | |
| rust-toolchain: stable | |
| working-directory: crates/jiter-python | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: pypi_files_sdist | |
| path: crates/jiter-python/dist | |
| build: | |
| name: build on ${{ matrix.os }} (${{ matrix.target }} - ${{ matrix.interpreter || 'all' }}${{ matrix.os == 'linux' && format(' - {0}', matrix.manylinux == 'auto' && 'manylinux' || matrix.manylinux) || '' }}) | |
| # only run on push to main and on release | |
| if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [linux] # windows and macos builds added manually | |
| target: [x86_64, aarch64] | |
| manylinux: [auto] | |
| include: | |
| # manylinux for various platforms | |
| - os: linux | |
| manylinux: auto | |
| target: i686 | |
| - os: linux | |
| manylinux: auto | |
| target: aarch64 | |
| - os: linux | |
| manylinux: auto | |
| target: armv7 | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| - os: linux | |
| manylinux: auto | |
| target: ppc64le | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| - os: linux | |
| manylinux: auto | |
| target: s390x | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| # musllinux | |
| - os: linux | |
| manylinux: musllinux_1_1 | |
| target: x86_64 | |
| - os: linux | |
| manylinux: musllinux_1_1 | |
| target: aarch64 | |
| # macos; | |
| # all versions x86_64 | |
| # older pythons which can't be run on the arm hardware for PGO | |
| - os: macos | |
| target: x86_64 | |
| - os: macos | |
| target: aarch64 | |
| interpreter: 3.8 3.9 | |
| # windows; | |
| # aarch64 only 3.11 and up, also not PGO optimized | |
| - os: windows | |
| target: i686 | |
| python-architecture: x86 | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| # FIXME had link failures, needs investigation | |
| # - os: windows | |
| # target: aarch64 | |
| # interpreter: 3.11 3.12 | |
| exclude: | |
| # PGO optimized below | |
| - os: linux | |
| manylinux: auto | |
| target: x86_64 | |
| runs-on: ${{ (matrix.os == 'linux' && 'ubuntu') || matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| architecture: ${{ matrix.python-architecture || 'x64' }} | |
| - name: build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.target }} | |
| manylinux: ${{ matrix.manylinux }} | |
| args: --release --out dist --interpreter ${{ matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13' }} | |
| rust-toolchain: stable | |
| working-directory: crates/jiter-python | |
| - run: ${{ (matrix.os == 'windows' && 'dir') || 'ls -lh' }} crates/jiter-python/dist/ | |
| - run: | | |
| pip install -U twine | |
| # FIXME: restore `twine` check when metadata version 2.4 (used by `maturin`) supported by `twine`, | |
| # see https://github.com/pypa/twine/pull/1180 | |
| # twine check --strict crates/jiter-python/dist/* | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: pypi_files_${{ matrix.os }}_${{ matrix.target }}_${{ matrix.interpreter || 'all' }}_${{ matrix.manylinux }} | |
| path: crates/jiter-python/dist | |
| build-pgo: | |
| name: build pgo-optimized on ${{ matrix.os }} / ${{ matrix.interpreter }} | |
| # only run on push to main and on release | |
| if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [linux, windows, macos] | |
| interpreter: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] | |
| include: | |
| # standard runners with override for macos arm | |
| - os: linux | |
| runs-on: ubuntu-latest | |
| - os: windows | |
| ls: dir | |
| runs-on: windows-latest | |
| - os: macos | |
| runs-on: macos-latest | |
| exclude: | |
| # macos arm only supported from 3.10 and up | |
| - os: macos | |
| interpreter: "3.8" | |
| - os: macos | |
| interpreter: "3.9" | |
| # windows free-threaded build has a problematic wheel name, | |
| # needs something like https://github.com/PyO3/maturin/pull/2325 to fix | |
| - os: windows | |
| interpreter: "3.13t" | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: setup-python | |
| name: set up python | |
| uses: quansight-labs/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.interpreter }} | |
| allow-prereleases: true | |
| - name: install rust stable | |
| id: rust-toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: llvm-tools | |
| # linux: builds are done inside manylinux docker images, need to help maturin-action | |
| # find the right path | |
| # | |
| # TODO, would be nice to remove, see | |
| # https://github.com/PyO3/maturin-action/issues/300 | |
| - if: ${{ matrix.os == 'linux' }} | |
| id: resolve-interpreter | |
| name: resolve interpreter | |
| run: | | |
| echo python-path=/opt/python/$(python -c "print({ | |
| '3.8': 'cp38-cp38', | |
| '3.9': 'cp39-cp39', | |
| '3.10': 'cp310-cp310', | |
| '3.11': 'cp311-cp311', | |
| '3.12': 'cp312-cp312', | |
| '3.13': 'cp313-cp313', | |
| '3.13t': 'cp313-cp313t', | |
| }['${{ matrix.interpreter }}'])")/bin/python >> $GITHUB_OUTPUT | |
| - name: build pgo wheel | |
| uses: ./.github/actions/build-pgo-wheel | |
| with: | |
| interpreter: ${{ matrix.os == 'linux' && steps.resolve-interpreter.outputs.python-path || steps.setup-python.outputs.python-path }} | |
| rust-toolchain: ${{ steps.rust-toolchain.outputs.name }} | |
| - run: ${{ matrix.ls || 'ls -lh' }} crates/jiter-python/dist/ | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: pypi_files_${{ matrix.os }}_${{ matrix.interpreter }} | |
| path: crates/jiter-python/dist | |
| inspect-pypi-assets: | |
| needs: [build, build-sdist, build-pgo] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_* | |
| merge-multiple: true | |
| path: dist | |
| - name: list dist files | |
| run: | | |
| ls -lh dist/ | |
| ls -l dist/ | |
| echo "`ls dist | wc -l` files" | |
| - name: extract and list sdist file | |
| run: | | |
| mkdir sdist-files | |
| tar -xvf dist/*.tar.gz -C sdist-files | |
| tree -a sdist-files | |
| - name: extract and list wheel file | |
| run: | | |
| ls dist/*cp310-manylinux*x86_64.whl | head -n 1 | |
| python -m zipfile --list `ls dist/*cp310-manylinux*x86_64.whl | head -n 1` | |
| test-builds-arch: | |
| name: test build on ${{ matrix.target }}-${{ matrix.distro }} | |
| needs: [build] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: [aarch64, armv7, s390x, ppc64le] | |
| distro: ["ubuntu22.04"] | |
| include: | |
| - target: aarch64 | |
| distro: alpine_latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_linux_* | |
| merge-multiple: true | |
| path: crates/jiter-python/dist | |
| - uses: uraimo/run-on-arch-action@v2 | |
| name: install & test | |
| with: | |
| arch: ${{ matrix.target }} | |
| distro: ${{ matrix.distro }} | |
| githubToken: ${{ github.token }} | |
| install: | | |
| set -x | |
| if command -v apt-get &> /dev/null; then | |
| echo "installing python & pip with apt-get..." | |
| apt-get update | |
| apt-get install -y --no-install-recommends python3 python3-pip python3-venv git | |
| else | |
| echo "installing python & pip with apk..." | |
| apk update | |
| apk add python3 py3-pip git | |
| fi | |
| run: | | |
| cd crates/jiter-python | |
| set -x | |
| # typing-extensions isn't automatically installed because of `--no-index --no-deps` | |
| python3 -m venv venv | |
| source venv/bin/activate | |
| python3 -m pip install -U pip -r tests/requirements.txt | |
| python3 -m pip install jiter --no-index --no-deps --find-links dist --force-reinstall | |
| python3 -m pytest | |
| python3 -c 'import jiter; print(jiter.__version__)' | |
| test-builds-os: | |
| name: test build on ${{ matrix.os }} | |
| needs: [build, build-pgo] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu, macos, windows] | |
| runs-on: ${{ matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_* | |
| merge-multiple: true | |
| path: crates/jiter-python/dist | |
| - name: run tests | |
| run: | | |
| python3 -m pip install -U pip -r tests/requirements.txt | |
| python3 -m pip install jiter --no-index --no-deps --find-links dist --force-reinstall | |
| python3 -m pytest | |
| python3 -c 'import jiter; print(jiter.__version__)' | |
| working-directory: crates/jiter-python | |
| # https://github.com/marketplace/actions/alls-green#why used for branch protection checks | |
| check: | |
| if: always() | |
| needs: [test, test-python, bench, fuzz, fuzz-skip, lint] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Decide whether the needed jobs succeeded or failed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| jobs: ${{ toJSON(needs) }} | |
| release: | |
| needs: [check] | |
| if: "success() && startsWith(github.ref, 'refs/tags/')" | |
| runs-on: ubuntu-latest | |
| environment: release | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - run: cargo publish -p jiter | |
| env: | |
| CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} | |
| release-python: | |
| needs: [test-builds-arch, test-builds-os, build-sdist, check] | |
| if: "success() && startsWith(github.ref, 'refs/tags/')" | |
| runs-on: ubuntu-latest | |
| environment: release-python | |
| permissions: | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_* | |
| merge-multiple: true | |
| path: dist | |
| - run: pip install -U twine | |
| - run: ls -l dist/ | |
| # FIXME: restore `twine` check when metadata version 2.4 (used by `maturin`) supported by `twine` | |
| # see https://github.com/pypa/twine/pull/1180 | |
| # - run: twine check --strict dist/* | |
| - name: upload to pypi | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| packages-dir: dist/ |