[pull] master from ClickHouse:master #14859
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # yamllint disable rule:comments-indentation | |
| name: PullRequestCI | |
| env: | |
| # Force the stdout and stderr streams to be unbuffered | |
| PYTHONUNBUFFERED: 1 | |
| on: # yamllint disable-line rule:truthy | |
| pull_request: | |
| types: | |
| - synchronize | |
| - reopened | |
| - opened | |
| branches: | |
| - master | |
| # Cancel the previous wf run in PRs. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| RunConfig: | |
| runs-on: [self-hosted, style-checker-aarch64] | |
| outputs: | |
| data: ${{ steps.runconfig.outputs.CI_DATA }} | |
| steps: | |
| - name: Check out repository code | |
| uses: ClickHouse/checkout@v1 | |
| with: | |
| clear-repository: true # to ensure correct digests | |
| fetch-depth: 0 # to get a version | |
| filter: tree:0 | |
| - name: Debug Info | |
| uses: ./.github/actions/debug | |
| - name: Set pending Sync status | |
| run: | | |
| python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --set-pending-status | |
| - name: Labels check | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| python3 run_check.py | |
| - name: Python unit tests | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| echo "Testing the main ci directory" | |
| python3 -m unittest discover -s . -p 'test_*.py' | |
| - name: PrepareRunConfig | |
| id: runconfig | |
| run: | | |
| python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json | |
| echo "::group::CI configuration" | |
| python3 -m json.tool ${{ runner.temp }}/ci_run_data.json | |
| echo "::endgroup::" | |
| { | |
| echo 'CI_DATA<<EOF' | |
| cat ${{ runner.temp }}/ci_run_data.json | |
| echo 'EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Re-create GH statuses for skipped jobs if any | |
| run: | | |
| python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses | |
| BuildDockers: | |
| needs: [RunConfig] | |
| if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }} | |
| uses: ./.github/workflows/docker_test_images.yml | |
| with: | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| StyleCheck: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Style check')}} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Style check | |
| runner_type: style-checker-aarch64 | |
| run_command: | | |
| python3 style_check.py | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| secrets: | |
| secret_envs: | | |
| ROBOT_CLICKHOUSE_SSH_KEY<<RCSK | |
| ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}} | |
| RCSK | |
| FastTest: | |
| needs: [RunConfig, BuildDockers, StyleCheck] | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Fast test | |
| runner_type: builder | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| run_command: | | |
| python3 fast_test_check.py | |
| ################################# Main stages ################################# | |
| # for main CI chain | |
| # | |
| Builds_1: | |
| needs: [RunConfig, StyleCheck, FastTest] | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }} | |
| # using callable wf (reusable_stage.yml) allows grouping all nested jobs under a tab | |
| uses: ./.github/workflows/reusable_build_stage.yml | |
| with: | |
| stage: Builds_1 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| Tests_1: | |
| needs: [RunConfig, Builds_1] | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }} | |
| uses: ./.github/workflows/reusable_test_stage.yml | |
| with: | |
| stage: Tests_1 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| Builds_2: | |
| needs: [RunConfig, Builds_1] | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_2') }} | |
| uses: ./.github/workflows/reusable_build_stage.yml | |
| with: | |
| stage: Builds_2 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| # stage for running non-required checks without being blocked by required checks (Test_1) if corresponding settings is selected | |
| Tests_2_ww: | |
| needs: [RunConfig, Builds_1] | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2_ww') }} | |
| uses: ./.github/workflows/reusable_test_stage.yml | |
| with: | |
| stage: Tests_2_ww | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| Tests_2: | |
| needs: [RunConfig, Builds_1, Tests_1] | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }} | |
| uses: ./.github/workflows/reusable_test_stage.yml | |
| with: | |
| stage: Tests_2 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ################################# Reports ################################# | |
| # Reports should run even if Builds_1/2 fail - run them separately (not in Tests_1/2/3) | |
| Builds_Report: | |
| # run report check for failed builds to indicate the CI error | |
| if: ${{ !cancelled() | |
| && needs.RunConfig.result == 'success' | |
| && needs.StyleCheck.result != 'failure' | |
| && needs.FastTest.result != 'failure' | |
| && needs.BuildDockers.result != 'failure' | |
| && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }} | |
| needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2] | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Builds | |
| runner_type: style-checker-aarch64 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| CheckReadyForMerge: | |
| if: ${{ !cancelled() }} | |
| # Test_2 or Test_3 do not have the jobs required for Mergeable check, | |
| # however, set them as "needs" to get all checks results before the automatic merge occurs. | |
| needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2_ww, Tests_2] | |
| runs-on: [self-hosted, style-checker-aarch64] | |
| steps: | |
| - name: Check out repository code | |
| uses: ClickHouse/checkout@v1 | |
| with: | |
| filter: tree:0 | |
| - name: Check and set merge status | |
| if: ${{ needs.StyleCheck.result == 'success' }} | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json" | |
| cat > "$WORKFLOW_RESULT_FILE" << 'EOF' | |
| ${{ toJson(needs) }} | |
| EOF | |
| python3 merge_pr.py --set-ci-status | |
| - name: Check Workflow results | |
| uses: ./.github/actions/check_workflow | |
| with: | |
| needs: ${{ toJson(needs) }} | |
| ################################# Stage Final ################################# | |
| # | |
| FinishCheck: | |
| if: ${{ !failure() && !cancelled() }} | |
| needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2_ww, Tests_2] | |
| runs-on: [self-hosted, style-checker-aarch64] | |
| steps: | |
| - name: Check out repository code | |
| uses: ClickHouse/checkout@v1 | |
| with: | |
| filter: tree:0 | |
| - name: Finish label | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }} | |
| ############################################################################################# | |
| ###################################### JEPSEN TESTS ######################################### | |
| ############################################################################################# | |
| # This is special test NOT INCLUDED in FinishCheck | |
| # When it's skipped, all dependent tasks will be skipped too. | |
| # DO NOT add it there | |
| Jepsen: | |
| # we need concurrency as the job uses dedicated instances in the cloud | |
| concurrency: | |
| group: jepsen | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse Keeper Jepsen') }} | |
| needs: [RunConfig, Builds_1] | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: ClickHouse Keeper Jepsen | |
| runner_type: style-checker-aarch64 | |
| data: ${{ needs.RunConfig.outputs.data }} |