Add Spark 4.0 support #2611
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Code analysis | |
| on: | |
| push: | |
| branches: | |
| - develop | |
| pull_request: | |
| branches-ignore: | |
| - master | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # flake8-commas is failing on Python 3.12 | |
| # as well as bandit https://github.com/PyCQA/bandit/issues/1077 | |
| DEFAULT_PYTHON: '3.11' | |
| jobs: | |
| linters: | |
| name: Linters | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ env.DEFAULT_PYTHON }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.DEFAULT_PYTHON }} | |
| - name: Install Kerberos headers | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install --no-install-recommends libkrb5-dev | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('requirements*.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql-${{ hashFiles('requirements*.txt') }} | |
| ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-codeql- | |
| - name: Upgrade pip | |
| run: python -m pip install --upgrade pip setuptools wheel | |
| - name: Install dependencies | |
| run: | | |
| pip install -I \ | |
| -r requirements/core.txt \ | |
| -r requirements/dev.txt \ | |
| -r requirements/ftp.txt \ | |
| -r requirements/hdfs.txt \ | |
| -r requirements/kerberos.txt \ | |
| -r requirements/s3.txt \ | |
| -r requirements/sftp.txt \ | |
| -r requirements/spark.txt \ | |
| -r requirements/webdav.txt | |
| # Set the `CODEQL-PYTHON` environment variable to the Python executable | |
| # that includes the dependencies | |
| echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV | |
| - name: Run flake8 | |
| run: python3 -m flake8 --config setup.cfg . | |
| - name: Run mypy | |
| run: python3 -m mypy --config-file setup.cfg onetl | |
| codeql: | |
| name: CodeQL | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ env.DEFAULT_PYTHON }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.DEFAULT_PYTHON }} | |
| # Initializes the CodeQL tools for scanning. | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@v3 | |
| with: | |
| languages: python | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v3 | |
| with: | |
| category: /language:python |