Eval Assist integration #916
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Performance | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| performance: | |
| runs-on: ubuntu-latest | |
| env: | |
| OS: ubuntu-latest | |
| UNITXT_DEFAULT_VERBOSITY: error | |
| DATASETS_VERBOSITY: error | |
| HF_HUB_VERBOSITY: error | |
| HF_DATASETS_DISABLE_PROGRESS_BARS: "True" | |
| TQDM_DISABLE: "True" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Install Requirements | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| uv pip install --system ".[tests]" | |
| - name: Prepare the dirs for performance evaluation in main | |
| run: | | |
| mkdir -p performance_action | |
| mkdir -p performance_action/logs | |
| echo "" > performance_action/__init__.py | |
| echo " " > performance_action/logs/cards_benchmark.prof | |
| echo " " > performance_action/logs/cards_benchmark.json | |
| cp performance/card_profiler.py performance_action/card_profiler.py | |
| cp performance/compare_performance_results.py performance_action/compare_performance_results.py | |
| - name: Checkout main branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| clean: false | |
| - name: Run performance on main branch | |
| run: | | |
| python performance_action/card_profiler.py --output_file performance_action/main_results.json | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| clean: false | |
| - name: Run performance on PR branch | |
| run: | | |
| python performance_action/card_profiler.py --output_file performance_action/pr_results.json | |
| - name: Compare main and PR performance results | |
| run: | | |
| python performance_action/compare_performance_results.py performance_action/main_results.json performance_action/pr_results.json >> $GITHUB_STEP_SUMMARY |