diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f87a1f7..a5d007d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -216,7 +216,10 @@ jobs: sca: environment: ci - runs-on: ${{ vars.SCA_RUNNER_LABEL || 'arm64-mo-shanghai-8c16g' }} + # SCA is CPU-bound but does not run the database or BVT; validate it on + # the smaller standard ARM runner. A repository variable remains an + # explicit escape hatch for a larger runner. + runs-on: ${{ vars.SCA_RUNNER_LABEL || 'arm64-mo-shanghai-4c8g' }} name: SCA Test on Linux/arm64 steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 diff --git a/.github/workflows/coverage-merge.yaml b/.github/workflows/coverage-merge.yaml new file mode 100644 index 0000000..61cc8d3 --- /dev/null +++ b/.github/workflows/coverage-merge.yaml @@ -0,0 +1,270 @@ +name: MatrixOne Coverage Merge + +on: + workflow_call: + inputs: + prerequisites_ready: + description: "Whether every coverage producer completed successfully" + required: false + type: boolean + default: true + expected_bvt_generation: + description: "Assignment generation expected in both BVT manifests" + required: false + type: string + default: "" + ci_ref: + description: "CI ref containing the coverage parser" + required: false + type: string + default: "main" + secrets: + TOKEN_ACTION: + description: "Token for checkout and pull-request metadata" + required: false + +jobs: + coverage_merge: + name: Coverage + environment: ci + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + timeout-minutes: 20 + steps: + - name: Verify coverage prerequisites + env: + PREREQUISITES_READY: ${{ inputs.prerequisites_ready }} + run: | + if [ "${PREREQUISITES_READY}" != 'true' ]; then + echo '::error::coverage merge skipped because at least one required producer was skipped or failed' + exit 1 + fi + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "1" + path: ./matrixone + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + - name: Checkout coverage parser + uses: actions/checkout@v6 + with: + token: ${{ secrets.TOKEN_ACTION }} + repository: matrixorigin/CI + ref: ${{ inputs.ci_ref }} + fetch-depth: "1" + path: ./CI + - name: Set up Go + uses: matrixorigin/CI/actions/setup-env@main + with: + setup-java: false + go-version-file: "${{ github.workspace }}/matrixone/go.mod" + - name: Generate diff.patch + env: + GH_TOKEN: ${{ secrets.TOKEN_ACTION }} + IS_PUB_REPO: ${{ vars.IS_PUB_REPO }} + EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + cd "$GITHUB_WORKSPACE/matrixone" + pr_repo='${{ github.event.pull_request.base.repo.full_name }}' + pr_number='${{ github.event.pull_request.number }}' + current_head_sha=$(curl -fsSL --retry 5 --retry-delay 3 \ + -H 'Accept: application/vnd.github+json' \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${pr_repo}/pulls/${pr_number}" \ + | jq -r '.head.sha') + if [ "${current_head_sha}" != "${EXPECTED_HEAD_SHA}" ]; then + echo "::error::PR head moved from ${EXPECTED_HEAD_SHA} to ${current_head_sha}; refusing to merge coverage against a different diff" + exit 1 + fi + if [ -n "${IS_PUB_REPO:-}" ] && [ "${IS_PUB_REPO}" != "0" ] && [ "${IS_PUB_REPO}" != "false" ]; then + curl -fL --retry 5 --retry-delay 3 "https://github.com/${pr_repo}/pull/${pr_number}.diff" -o diff.patch + else + curl -fL --retry 5 --retry-delay 3 \ + -H 'Accept: application/vnd.github.v3.diff' \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + "https://api.github.com/repos/${pr_repo}/pulls/${pr_number}" -o diff.patch + fi + - name: Download test coverage artifacts + uses: actions/download-artifact@v7 + with: + # A full rerun rotates the shared generation. Filter out artifacts + # left by previous attempts while retaining successful producers + # from an earlier attempt during "Re-run failed jobs". + pattern: ${{ inputs.expected_bvt_generation != '' && format('*coverage*generation-{0}-*', inputs.expected_bvt_generation) || '*coverage*' }} + merge-multiple: true + path: ${{ github.workspace }}/coverage-artifacts + - name: Merge profiles and check coverage rate + id: merge_coverage + run: | + set -euo pipefail + artifact_dir="$GITHUB_WORKSPACE/coverage-artifacts" + processed_dir="$GITHUB_WORKSPACE/coverage-processed" + mkdir -p "${processed_dir}" + find "${artifact_dir}" -type f -printf '%P %s bytes\n' | sort \ + | tee "${RUNNER_TEMP}/coverage-inputs.txt" + + mapfile -t ut_profiles < <(find "${artifact_dir}" -type f -name 'ut-coverage.out' | sort) + mapfile -t bvt_profiles < <(find "${artifact_dir}" -type f -name 'bvt-*.out' | sort) + if [ "${#ut_profiles[@]}" -ne 1 ] || [ "${#bvt_profiles[@]}" -ne 2 ]; then + echo '::error::expected one UT profile and two BVT profiles' + find "${artifact_dir}" -type f -print || true + exit 1 + fi + + mapfile -t manifests < <(find "${artifact_dir}" -type f -name 'bvt-*-manifest.json' | sort) + if [ "${#manifests[@]}" -ne 2 ]; then + echo "::error::expected exactly two BVT manifests, found ${#manifests[@]}" + exit 1 + fi + jq -s '.' "${manifests[@]}" > "${RUNNER_TEMP}/bvt-group-manifests.json" + if ! jq -e \ + --arg expected_generation '${{ inputs.expected_bvt_generation }}' \ + --arg expected_sha '${{ github.event.pull_request.head.sha }}' \ + ' + length == 2 + and (map(.schema_version) == [1, 1]) + and ((map(.deployment) | sort) == ["compose-proxy", "launch-pessimistic"]) + and ((map(.group) | sort) == [0, 1]) + and ((map(.generation) | unique | length) == 1) + and ($expected_generation == "" or all(.[]; .generation == $expected_generation)) + and all(.[]; .head_sha == $expected_sha) + ' "${RUNNER_TEMP}/bvt-group-manifests.json" >/dev/null; then + echo '::error::BVT manifests do not describe complementary groups from the expected generation and PR head' + jq . "${RUNNER_TEMP}/bvt-group-manifests.json" || true + exit 1 + fi + + coverage_files=() + for profile in "${ut_profiles[@]}"; do + output="${processed_dir}/$(basename "${profile}")" + grep -Ev 'pkg/pb|pkg/sql/parsers/goyacc|yaccpar' "${profile}" > "${output}" || true + test -s "${output}" + coverage_files+=("${output}") + done + for profile in "${bvt_profiles[@]}"; do + output="${processed_dir}/$(basename "${profile}")" + grep -Ev 'pkg/pb|yaccpar' "${profile}" > "${output}" || true + test -s "${output}" + coverage_files+=("${output}") + done + cd "$GITHUB_WORKSPACE" + cp "$GITHUB_WORKSPACE/CI/scripts/.ignore" "$GITHUB_WORKSPACE/.ignore" + set +e + python CI/scripts/parse_coverage.py \ + -coverage_files "${coverage_files[@]}" \ + -diff_path "$GITHUB_WORKSPACE/matrixone/diff.patch" \ + -minimal_coverage 0.75 \ + -summary_path "$GITHUB_WORKSPACE/coverage-summary.json" \ + 2>&1 | tee "${RUNNER_TEMP}/coverage-parser.log" + parser_status=${PIPESTATUS[0]} + set -e + + has_pr_coverage=false + if [ -s "$GITHUB_WORKSPACE/coverage-summary.json" ]; then + has_pr_coverage=$(jq -r '.has_pr_coverage' "$GITHUB_WORKSPACE/coverage-summary.json") + echo "overall_coverage=$(jq -r '.overall_coverage' "$GITHUB_WORKSPACE/coverage-summary.json")" >> "$GITHUB_OUTPUT" + echo "pr_coverage=$(jq -r '.pr_coverage' "$GITHUB_WORKSPACE/coverage-summary.json")" >> "$GITHUB_OUTPUT" + echo "coverage_approved=$(jq -r '.approved' "$GITHUB_WORKSPACE/coverage-summary.json")" >> "$GITHUB_OUTPUT" + fi + echo "has_pr_coverage=${has_pr_coverage}" >> "$GITHUB_OUTPUT" + + # Preserve the legacy PR-focused HTML semantics and generate it even + # when the coverage threshold rejects the PR. + if [ "${has_pr_coverage}" = 'true' ]; then + test -s "$GITHUB_WORKSPACE/pr_coverage.out" + cd "$GITHUB_WORKSPACE/matrixone" + go tool cover \ + -o "$GITHUB_WORKSPACE/matrixone/pr_coverage.html" \ + -html="$GITHUB_WORKSPACE/pr_coverage.out" + fi + exit "${parser_status}" + - name: Summarize coverage merge result + if: ${{ always() && !cancelled() }} + run: | + set -euo pipefail + echo '### Coverage merge result' >> "$GITHUB_STEP_SUMMARY" + echo "- Merge step: \`${{ steps.merge_coverage.outcome }}\`" >> "$GITHUB_STEP_SUMMARY" + echo '- Expected inputs: one UT profile and two BVT profiles.' >> "$GITHUB_STEP_SUMMARY" + summary="$GITHUB_WORKSPACE/coverage-summary.json" + if [ -s "${summary}" ]; then + overall=$(jq -r '.overall_coverage' "${summary}") + pr_rate=$(jq -r '.pr_coverage' "${summary}") + threshold=$(jq -r '.minimal_coverage' "${summary}") + covered=$(jq -r '.covered_modified_lines' "${summary}") + total=$(jq -r '.total_modified_lines' "${summary}") + has_changes=$(jq -r '.has_go_changes' "${summary}") + has_pr_coverage=$(jq -r '.has_pr_coverage' "${summary}") + approved=$(jq -r '.approved' "${summary}") + overall_pct=$(awk -v rate="${overall}" 'BEGIN { printf "%.2f%%", rate * 100 }') + pr_pct=$(awk -v rate="${pr_rate}" 'BEGIN { printf "%.2f%%", rate * 100 }') + threshold_pct=$(awk -v rate="${threshold}" 'BEGIN { printf "%.2f%%", rate * 100 }') + echo "- Overall coverage: \`${overall_pct}\`" >> "$GITHUB_STEP_SUMMARY" + if [ "${has_pr_coverage}" = 'true' ]; then + if [ "${approved}" = 'true' ]; then + gate='PASS' + else + gate='FAIL' + fi + echo "- Changed-code coverage: \`${covered}/${total} (${pr_pct})\`" >> "$GITHUB_STEP_SUMMARY" + echo "- Required changed-code coverage: \`>${threshold_pct}\`" >> "$GITHUB_STEP_SUMMARY" + echo "- Coverage gate: \`${gate}\`" >> "$GITHUB_STEP_SUMMARY" + elif [ "${has_changes}" = 'true' ]; then + if [ "${approved}" = 'true' ]; then + gate='PASS' + else + gate='FAIL' + fi + echo '- Changed-code coverage: Go changes were found, but no instrumented blocks matched the coverage profiles.' >> "$GITHUB_STEP_SUMMARY" + echo "- Coverage gate: \`${gate}\`" >> "$GITHUB_STEP_SUMMARY" + else + echo '- Changed-code coverage: no modified Go blocks; gate passes.' >> "$GITHUB_STEP_SUMMARY" + fi + else + echo '- Coverage rates unavailable because profile processing failed before the summary was written.' >> "$GITHUB_STEP_SUMMARY" + fi + if [ -s "${RUNNER_TEMP}/bvt-group-manifests.json" ]; then + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '#### BVT group manifests' >> "$GITHUB_STEP_SUMMARY" + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '```json' >> "$GITHUB_STEP_SUMMARY" + jq . "${RUNNER_TEMP}/bvt-group-manifests.json" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + fi + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '#### Downloaded profiles' >> "$GITHUB_STEP_SUMMARY" + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '```text' >> "$GITHUB_STEP_SUMMARY" + cat "${RUNNER_TEMP}/coverage-inputs.txt" >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + echo '```' >> "$GITHUB_STEP_SUMMARY" + if [ "${{ steps.merge_coverage.outcome }}" = 'success' ]; then + echo '- Result: `final-result-files` contains the changed-code profile, merged profile, summary, and PR coverage HTML.' >> "$GITHUB_STEP_SUMMARY" + else + echo '- Diagnostics: `coverage-merge-debug` contains the input inventory and PR diff; `final-result-files` retains any merged outputs.' >> "$GITHUB_STEP_SUMMARY" + fi + - name: Upload coverage result + if: ${{ always() && !cancelled() && steps.merge_coverage.outputs.has_pr_coverage == 'true' }} + uses: actions/upload-artifact@v7 + with: + name: final-result-files + path: | + ${{ github.workspace }}/matrixone/pr_coverage.html + ${{ github.workspace }}/pr_coverage.out + ${{ github.workspace }}/merged_coverage.out + ${{ github.workspace }}/coverage-summary.json + if-no-files-found: error + retention-days: 7 + - name: Upload coverage merge diagnostics + if: ${{ failure() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-merge-debug + path: | + ${{ runner.temp }}/coverage-inputs.txt + ${{ runner.temp }}/coverage-parser.log + ${{ runner.temp }}/bvt-group-manifests.json + ${{ github.workspace }}/matrixone/diff.patch + ${{ github.workspace }}/coverage-summary.json + if-no-files-found: warn + retention-days: 1 diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml new file mode 100644 index 0000000..c49927d --- /dev/null +++ b/.github/workflows/coverage-ut.yaml @@ -0,0 +1,359 @@ +name: MatrixOne UT Coverage + +on: + workflow_call: + inputs: + bvt_generation: + description: "Shared CI generation used to select this run's coverage artifacts" + required: false + type: string + default: "" + outputs: + coverage_ready: + description: "Whether the UT coverage profile was eligible to run" + value: ${{ jobs.check_coverage_eligibility.outputs.allowed }} + secrets: + TOKEN_ACTION: + description: "Token for checkout (e.g. pull from fork/private)" + required: false + TEST_S3FS_ALIYUN: + description: "Object storage configuration for UT coverage" + required: false + TEST_S3FS_QCLOUD: + description: "Object storage configuration for UT coverage" + required: false + S3ENDPOINT: + description: "S3ENDPOINT For Test" + required: false + S3REGION: + description: "S3REGION For Test" + required: false + S3APIKEY: + description: "S3APIKEY For Test" + required: false + S3APISECRET: + description: "S3APISECRET For Test" + required: false + S3BUCKET: + description: "S3BUCKET For Test" + required: false + +jobs: + check_coverage_eligibility: + name: Check coverage eligibility + environment: ci + runs-on: ubuntu-latest + outputs: + allowed: ${{ steps.eligibility.outputs.allowed }} + steps: + - id: check_in_org + name: Check organization membership + env: + GH_TOKEN: ${{ secrets.TOKEN_ACTION }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + set -euo pipefail + page=1 + user_in_org=0 + while :; do + users=$(curl -fsSL \ + -H 'Accept: application/vnd.github+json' \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/orgs/matrixorigin/members?page=${page}&per_page=100") + if jq -e --arg user "${PR_AUTHOR}" '.[] | select(.login == $user)' <<<"${users}" >/dev/null; then + user_in_org=1 + break + fi + [ "$(jq length <<<"${users}")" -eq 100 ] || break + page=$((page + 1)) + done + echo "in_org=${user_in_org}" >> "$GITHUB_OUTPUT" + - id: check_safe_label + name: Check pull request label + env: + GH_TOKEN: ${{ secrets.TOKEN_ACTION }} + PR_URL: ${{ github.event.pull_request.url }} + run: | + set -euo pipefail + labels=$(curl -fsSL \ + -H 'Accept: application/vnd.github+json' \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "${PR_URL}" | jq -r '.labels[].name') + if grep -Fxq 'safe-to-test' <<<"${labels}"; then + echo 'safe_label=1' >> "$GITHUB_OUTPUT" + else + echo 'safe_label=0' >> "$GITHUB_OUTPUT" + fi + - id: check_coverage_secrets + name: Check coverage secrets + env: + TEST_S3FS_ALIYUN: ${{ secrets.TEST_S3FS_ALIYUN }} + TEST_S3FS_QCLOUD: ${{ secrets.TEST_S3FS_QCLOUD }} + S3ENDPOINT: ${{ secrets.S3ENDPOINT }} + S3REGION: ${{ secrets.S3REGION }} + S3APIKEY: ${{ secrets.S3APIKEY }} + S3APISECRET: ${{ secrets.S3APISECRET }} + S3BUCKET: ${{ secrets.S3BUCKET }} + run: | + set -euo pipefail + missing=() + for name in TEST_S3FS_ALIYUN TEST_S3FS_QCLOUD S3ENDPOINT S3REGION S3APIKEY S3APISECRET S3BUCKET; do + if [ -z "${!name:-}" ]; then + missing+=("${name}") + fi + done + if [ "${#missing[@]}" -eq 0 ]; then + echo 'ready=true' >> "$GITHUB_OUTPUT" + else + echo 'ready=false' >> "$GITHUB_OUTPUT" + echo "::notice::UT coverage skipped because required secrets are unavailable: ${missing[*]}" + fi + - id: eligibility + name: Decide coverage eligibility + env: + IN_ORG: ${{ steps.check_in_org.outputs.in_org }} + SAFE_LABEL: ${{ steps.check_safe_label.outputs.safe_label }} + SECRETS_READY: ${{ steps.check_coverage_secrets.outputs.ready }} + run: | + set -euo pipefail + if { [ "${IN_ORG}" = '1' ] || [ "${SAFE_LABEL}" = '1' ]; } && [ "${SECRETS_READY}" = 'true' ]; then + echo 'allowed=true' >> "$GITHUB_OUTPUT" + else + echo 'allowed=false' >> "$GITHUB_OUTPUT" + if [ "${IN_ORG}" != '1' ] && [ "${SAFE_LABEL}" != '1' ]; then + echo '::notice::UT coverage skipped for an untrusted pull request' + fi + fi + + ut-coverage-linux-x86: + name: UT Coverage on Ubuntu/x86 + needs: [check_coverage_eligibility] + if: ${{ needs.check_coverage_eligibility.outputs.allowed == 'true' }} + environment: ci + # Preserve the Guangzhou network path and capacity used by main's + # external Aliyun/QCloud coverage UT. This is intentionally larger than + # the UT-only minimum while we validate the external-storage timeout fix. + runs-on: amd64-mo-guangzhou-2xlarge16 + timeout-minutes: 60 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "3" + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Go + uses: matrixorigin/CI/actions/setup-env@main + with: + setup-java: true + go-version-file: "${{ github.workspace }}/go.mod" + - id: coverage_ut + name: Run coverage unit tests + timeout-minutes: 45 + continue-on-error: true + run: | + set -euo pipefail + cd "$GITHUB_WORKSPACE" + export endpoint='${{ secrets.S3ENDPOINT }}' + export region='${{ secrets.S3REGION }}' + export apikey='${{ secrets.S3APIKEY }}' + export apisecret='${{ secrets.S3APISECRET }}' + export bucket='${{ secrets.S3BUCKET }}' + export TEST_S3FS_ALIYUN='${{ secrets.TEST_S3FS_ALIYUN }}' + export TEST_S3FS_QCLOUD='${{ secrets.TEST_S3FS_QCLOUD }}' + + test_scope=$(go list -mod=readonly ./... | grep -v 'driver\|engine/aoe\|engine/memEngine\|pkg/catalog') + cover_pkgs=$(printf '%s\n' "${test_scope}" | paste -sd, -) + test -n "${cover_pkgs}" + + make clean && make config && make cgo && make thirdparties + thirdparties_install_dir="$GITHUB_WORKSPACE/thirdparties/install" + cgo_cflags="-I${GITHUB_WORKSPACE}/cgo -I${thirdparties_install_dir}/include" + cgo_ldflags="-L${GITHUB_WORKSPACE}/cgo -lmo -L${thirdparties_install_dir}/lib -lusearch_c -Wl,-rpath,${thirdparties_install_dir}/lib -lm" + coverage_profile="${RUNNER_TEMP}/ut-coverage.out" + coverage_report="${RUNNER_TEMP}/ut-coverage-report.json" + + # Keep the same all-package coverage scope and external storage + # coverage as main. Save the verbose JSON report as an artifact + # rather than flooding the Actions console; the next step prints a + # concise parsed failure summary when this command fails. + set +e + CGO_CFLAGS="${cgo_cflags}" CGO_LDFLAGS="${cgo_ldflags}" \ + go test -mod=readonly -json -short -v -tags matrixone_test -p 6 \ + -covermode=set -coverprofile="${coverage_profile}" -coverpkg="${cover_pkgs}" \ + ${test_scope} \ + > "${coverage_report}" 2>&1 + test_status=$? + set -e + echo "test_status=${test_status}" >> "$GITHUB_OUTPUT" + if (( test_status != 0 )); then + exit "${test_status}" + fi + test -s "${coverage_profile}" + raw_coverage_bytes=$(wc -c < "${coverage_profile}") + # -coverpkg emits the same instrumented block once per tested + # package. Collapse those duplicate records before crossing the + # artifact boundary; the coverage gate only needs hit/not-hit + # semantics and parse_coverage.py applies the same max(hit) merge. + LC_ALL=C awk ' + NR == 1 { next } + $1 ~ /pkg\/pb|pkg\/sql\/parsers\/goyacc|yaccpar/ { next } + NF >= 3 { + key = $1 " " $2 + hit = ($3 + 0 > 0) ? 1 : 0 + if (!(key in coverage) || hit > coverage[key]) { + coverage[key] = hit + } + } + END { + print "mode: set" + for (key in coverage) { + print key, coverage[key] + } + } + ' "${coverage_profile}" > "${coverage_profile}.compacted" + test "$(wc -l < "${coverage_profile}.compacted")" -gt 1 + mv "${coverage_profile}.compacted" "${coverage_profile}" + compacted_coverage_bytes=$(wc -c < "${coverage_profile}") + echo "UT coverage profile compacted from ${raw_coverage_bytes} to ${compacted_coverage_bytes} bytes" + - name: Summarize 30 slowest coverage UT tests + if: ${{ always() && !cancelled() }} + run: | + set -euo pipefail + report="${RUNNER_TEMP}/ut-coverage-report.json" + timing="${RUNNER_TEMP}/ut-coverage-top30.txt" + { + echo '### Coverage UT timing' + if [ "${{ steps.coverage_ut.outcome }}" = 'failure' ]; then + echo '- Partial result: the failed run did not complete every test.' + fi + if [ ! -s "${report}" ]; then + echo '- No Go JSON report was produced.' + exit 0 + fi + if ! jq -r 'select(.Action == "pass" and (.Test | type == "string") and .Elapsed != null) | [(.Elapsed | tonumber), .Package, .Test] | @tsv' "${report}" \ + | sort -nr -k1,1 | awk 'NR <= 30 { print }' > "${timing}"; then + echo '::warning::failed to generate the UT timing summary' + exit 0 + fi + echo '' + echo '#### Top 30 slowest completed tests' + echo '' + echo '```text' + cat "${timing}" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + - name: Print failed coverage UT packages and tests + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} + run: | + set -euo pipefail + report="${RUNNER_TEMP}/ut-coverage-report.json" + echo '::group::Failed coverage UT packages' + if [ -s "${report}" ]; then + jq -r 'select(.Action == "fail" and .Package != null) | "FAIL \(.Package) (\(.Elapsed // 0)s)"' "${report}" | sort -u || true + echo '::endgroup::' + echo '::group::Failure details' + jq -r 'select(.Action == "output" and (.Output | test("panic: test timed out|--- FAIL:|^FAIL\\t"))) | "\(.Package // "unknown") / \(.Test // "package"): \(.Output | gsub("\\n$"; ""))"' "${report}" \ + | tail -n 100 | cut -c 1-1000 || true + else + echo 'No JSON test report was produced; inspect the failed test step.' + fi + echo '::endgroup::' + # Match the legacy coverage job's actionable failure report while keeping + # the large JSON stream out of successful coverage artifacts. + - name: Checkout CI repository for UT failure summary + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} + uses: actions/checkout@v6 + with: + token: ${{ secrets.TOKEN_ACTION }} + repository: matrixorigin/CI + fetch-depth: "1" + path: CI + - name: Analyze and summarize failed coverage UT + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} + continue-on-error: true + run: | + set -uo pipefail + report="${RUNNER_TEMP}/ut-coverage-report.json" + summary="${RUNNER_TEMP}/ut-failure-summary.md" + : > "${summary}" + if [ ! -s "${report}" ]; then + echo 'UT JSON report was not produced; inspect the failed test step.' | tee -a "${summary}" + else + go install github.com/matrixorigin/go-ut-analysis@latest || true + if command -v go-ut-analysis >/dev/null 2>&1; then + go-ut-analysis test -f "${report}" --first 10 --report-path "${RUNNER_TEMP}/ut-report" --stdout=false || true + else + echo 'go-ut-analysis is unavailable; the raw JSON report is attached.' | tee -a "${summary}" + fi + python "$GITHUB_WORKSPACE/CI/scripts/summarize_ut_report.py" "${report}" | tee -a "${summary}" || true + fi + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + cat "${summary}" >> "$GITHUB_STEP_SUMMARY" + fi + - name: Collect coverage UT runner diagnostics + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} + continue-on-error: true + run: | + set +e + diagnostics="${RUNNER_TEMP}/ut-runner-diagnostics.txt" + { + echo '## Memory' + free -h + echo + echo '## Disk' + df -h + echo + echo '## Limits' + ulimit -a + echo + echo '## Largest processes by RSS' + ps -eo pid,ppid,rss,vsz,stat,etime,cmd --sort=-rss | head -n 50 + echo + echo '## Cgroup memory' + for file in /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory.events; do + echo "### ${file}" + if [ -r "${file}" ]; then + cat "${file}" + else + echo unavailable + fi + done + echo + echo '## Recent kernel messages' + sudo dmesg -T 2>&1 | tail -n 1000 + } | tee "${diagnostics}" + { + echo '### Coverage UT runner diagnostics' + echo "- Full diagnostics: \`ut-coverage-diagnostics-attempt-${{ github.run_attempt }}/ut-runner-diagnostics.txt\`" + if grep -Eqi 'out of memory|oom-kill|killed process' "${diagnostics}"; then + echo '- OOM signal: detected in runner diagnostics.' + else + echo '- OOM signal: not detected in available runner diagnostics.' + fi + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload UT coverage profile + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'success' }} + uses: actions/upload-artifact@v7 + with: + name: ut-coverage-generation-${{ inputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/ut-coverage.out + if-no-files-found: warn + retention-days: 7 + - name: Upload failed UT diagnostics + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} + uses: actions/upload-artifact@v7 + with: + name: ut-coverage-diagnostics-attempt-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/ut-coverage-report.json + ${{ runner.temp }}/ut-failure-summary.md + ${{ runner.temp }}/ut-report + ${{ runner.temp }}/ut-runner-diagnostics.txt + if-no-files-found: warn + retention-days: 7 + - name: Fail on coverage UT failure + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} + run: exit 1 diff --git a/.github/workflows/e2e-compose-parallel.yaml b/.github/workflows/e2e-compose-parallel.yaml new file mode 100644 index 0000000..5139d1a --- /dev/null +++ b/.github/workflows/e2e-compose-parallel.yaml @@ -0,0 +1,453 @@ +name: MatrixOne Compose Test + +on: + workflow_call: + inputs: + bvt_group: + description: "BVT group assigned by the caller (0 or 1)" + required: false + type: string + default: "" + bvt_generation: + description: "Shared assignment generation for complementary BVT jobs" + required: false + type: string + default: "" + secrets: + TOKEN_ACTION: + description: "Token for checkout (e.g. pull from fork/private)" + required: false + +jobs: + bvt-docker-compose-push: + if: false + environment: ci + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + name: multi cn e2e bvt test docker compose(Optimistic/PUSH) + timeout-minutes: 75 + + steps: + - name: Pre Clean Some Unneeded Images + run: | + sudo df -h /* + echo "=========================================" + sudo docker builder prune -f -a; + sudo docker images | grep -v REPOSITORY | awk '{system("sudo docker rmi "$1":"$2)}'; + sudo docker volume prune -f; + echo "=========================================" + sudo df -h /* + - name: checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "3" + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Go And Java + uses: matrixorigin/CI/actions/setup-env@main + with: + go-version-file: "${{ github.workspace }}/go.mod" + + - name: Print run attempt + run: echo "run attempt is ${{ github.run_attempt }}" + + - name: Print Disk Usage Before Container Start + if: ${{ always() }} + run: | + set +e; + sudo df -h /*; + echo "==========================================="; + sudo du -hs /var/lib/docker; + echo "==========================================="; + sudo docker system df -v; + + - name: docker compose launch-multi-cn + timeout-minutes: 10 + run: | + echo "" >> ./etc/launch-tae-compose/config/cn-0.toml + echo '[cn.txn]' >> ./etc/launch-tae-compose/config/cn-0.toml + echo 'mode = "Optimistic"' >> ./etc/launch-tae-compose/config/cn-0.toml + echo "" >> ./etc/launch-tae-compose/config/cn-1.toml + echo '[cn.txn]' >> ./etc/launch-tae-compose/config/cn-1.toml + echo 'mode = "Optimistic"' >> ./etc/launch-tae-compose/config/cn-1.toml + echo "" >> ./etc/launch-tae-compose/config/tn.toml + echo '[tn.txn]' >> ./etc/launch-tae-compose/config/tn.toml + echo 'mode = "Optimistic"' >> ./etc/launch-tae-compose/config/tn.toml + cat ./etc/launch-tae-compose/config/cn-0.toml + cat ./etc/launch-tae-compose/config/cn-1.toml + cat ./etc/launch-tae-compose/config/tn.toml + + # Update compose.yaml to mount test directory to $GITHUB_WORKSPACE/test instead of /test + # Replace container mount point from /test to $GITHUB_WORKSPACE/test + # Handle various docker-compose volume formats: /test, /test:options, /test followed by space + sed -i 's|:/test\b|:'"$GITHUB_WORKSPACE"'/test|g' ./etc/launch-tae-compose/compose.yaml + + # build matrixone docker image first + docker build -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile + + mkdir -p ${{ github.workspace }}/docker-compose-log + docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn up -d + + # wait for ready + i=1 + while [ $(mysql -h 127.0.0.1 -P 6001 -u dump -p111 --execute 'create database if not exists compose_test;use compose_test; create table if not exists compose_test_table(col1 int auto_increment primary key);show tables;' 2>&1 | tee /dev/stderr | grep 'compose_test_table' | wc -l) -lt 1 ]; do + echo "wait mo init finished...$i" + if [ $i -ge 300 ]; then + echo "wait for $i seconds, mo init not finish, so exit 1" + docker ps + exit 1; + fi + i=$(($i+1)) + sleep 1 + done + docker ps + + - name: Print Disk Usage After Container Start + if: ${{ always() }} + run: | + set +e; + sudo df -h /*; + echo "==========================================="; + sudo du -hs /var/lib/docker; + echo "==========================================="; + sudo docker system df -v; + + - name: Clean Docker Image Build Cache + if: ${{ always() }} + run: | + sudo docker builder prune -f -a; + echo "==========================================="; + sudo docker system df -v; + + - name: Clone test-tool repository + # Node.js 12 actions are deprecated. + # For more information see: + # - https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/. + # Please update the following actions to use Node.js 16: actions/checkout@v3 + uses: actions/checkout@v6 + with: + repository: matrixorigin/mo-tester + path: ./mo-tester + ref: main + + - name: Update BVT SQL Timeout + run: | + cd $GITHUB_WORKSPACE/mo-tester + sed -i "s/socketTimeout:.*/socketTimeout: 300000/g" mo.yml + - name: Start BVT Test + id: bvt_on_pr_version + run: | + export LC_ALL="C.UTF-8" + locale + cd $GITHUB_WORKSPACE/mo-tester + sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml + cat mo.yml + echo "=============================" + + ./run.sh -n -g -o -p $GITHUB_WORKSPACE/test/distributed/cases -e pessimistic_transaction 2>&1 + + - name: Print Docker Info Before Container Shutdown + if: ${{ always() }} + run: | + set +e; + sudo docker ps; + echo "==========================================="; + sudo df -h /*; + echo "==========================================="; + sudo du -hs /var/lib/docker; + echo "==========================================="; + sudo docker system df -v; + - name: Print System Dmesg + if: ${{ always() }} + continue-on-error: true + run: | + sudo dmesg -T + + - name: export log + if: ${{ failure() || cancelled()}} + run: | + mkdir -p ${{ github.workspace }}/mo-tester/report + mv ${{ github.workspace }}/mo-tester/report ${{ github.workspace }}/docker-compose-log + curl http://localhost:12345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/docker-compose-log/cn-0-dump-stacks.log + curl http://localhost:22345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/docker-compose-log/cn-1-dump-stacks.log + + - name: Check Log Messages Count per second + if: ${{ always() && !cancelled() }} + run: | + ./optools/check_log_count.sh 1000 60 # {count threshold} {metric collected interval} + + - name: shutdown containers + if: ${{ always() }} + run: | + docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn down --remove-orphans + docker volume rm launch-tae-compose_minio_storage + + - uses: actions/upload-artifact@v7 + if: ${{ failure() || cancelled()}} + continue-on-error: true + with: + name: Compose-multi-cn-e2e-bvt-test-docker-log(Optimistic,PUSH) + path: | + ${{ github.workspace }}/docker-compose-log + retention-days: 7 + + multi-CN-bvt-docker-compose-proxy: + environment: ci + # Match the existing PR BVT runner policy. + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + name: multi cn e2e bvt test docker compose(PROXY) + timeout-minutes: 75 + steps: + - name: Pre Clean Some Unneeded Images + run: | + sudo df -h /* + echo "=========================================" + sudo docker builder prune -f -a; + sudo docker images | grep -v REPOSITORY | awk '{system("sudo docker rmi "$1":"$2)}'; + sudo docker volume prune -f; + echo "=========================================" + sudo df -h /* + - name: checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "3" + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Go And Java + uses: matrixorigin/CI/actions/setup-env@main + with: + go-version-file: "${{ github.workspace }}/go.mod" + + - name: Print run attempt + run: echo "run attempt is ${{ github.run_attempt }}" + + - name: Print Disk Usage Before Container Start + if: ${{ always() }} + run: | + set +e; + sudo df -h /*; + echo "==========================================="; + sudo du -hs /var/lib/docker; + echo "==========================================="; + sudo docker system df -v; + + - name: docker compose launch-multi-cn + timeout-minutes: 10 + run: | + cat ./etc/launch-tae-compose/config/cn-0.toml + cat ./etc/launch-tae-compose/config/cn-1.toml + cat ./etc/launch-tae-compose/config/tn.toml + + # Update compose.yaml to mount test directory to $GITHUB_WORKSPACE/test instead of /test + # Replace container mount point from /test to $GITHUB_WORKSPACE/test + # Handle various docker-compose volume formats: /test, /test:options, /test followed by space + sed -i 's|:/test\b|:'"$GITHUB_WORKSPACE"'/test|g' ./etc/launch-tae-compose/compose.yaml + + mkdir -p "$GITHUB_WORKSPACE/coverage" + docker build --build-arg GOBUILD_OPT=-cover \ + -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile + + mkdir -p ${{ github.workspace }}/docker-compose-log + docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn up -d + + # wait for ready + i=1 + while [ $(mysql -h 127.0.0.1 -P 6001 -u dump -p111 --execute 'create database if not exists compose_test;use compose_test; create table if not exists compose_test_table(col1 int auto_increment primary key);show tables;' 2>&1 | tee /dev/stderr | grep 'compose_test_table' | wc -l) -lt 1 ]; do + echo "wait mo init finished...$i" + if [ $i -ge 300 ]; then + echo "wait for $i seconds, mo init not finish, so exit 1" + docker ps + exit 1; + fi + i=$(($i+1)) + sleep 1 + done + docker ps + + - name: Print Disk Usage After Container Start + if: ${{ always() }} + run: | + set +e; + sudo df -h /*; + echo "==========================================="; + sudo du -hs /var/lib/docker; + echo "==========================================="; + sudo docker system df -v; + + - name: Clean Docker Image Build Cache + if: ${{ always() }} + run: | + sudo docker builder prune -f -a; + echo "==========================================="; + sudo docker system df -v; + + - name: Clone test-tool repository + uses: actions/checkout@v6 + with: + repository: matrixorigin/mo-tester + path: ./mo-tester + ref: main + + - name: Update BVT SQL Timeout + run: | + cd $GITHUB_WORKSPACE/mo-tester + sed -i "s/socketTimeout:.*/socketTimeout: 300000/g" mo.yml + + - name: Start BVT Test + id: bvt_on_pr_version + run: | + export LC_ALL="C.UTF-8" + locale + cd $GITHUB_WORKSPACE/mo-tester + sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml + cat mo.yml + echo "=============================" + + bvt_group='${{ inputs.bvt_group }}' + bvt_generation='${{ inputs.bvt_generation }}' + if [ -z "${bvt_group}" ]; then + bvt_group=$(( (${{ github.run_id }} + ${{ github.run_attempt }}) % 2 )) + fi + if [ -z "${bvt_generation}" ]; then + bvt_generation='${{ github.run_id }}-${{ github.run_attempt }}' + fi + if [[ ! "${bvt_group}" =~ ^[01]$ ]] || [ -z "${bvt_generation}" ]; then + echo "::error::invalid BVT assignment: group=${bvt_group}, generation=${bvt_generation}" + exit 1 + fi + echo "bvt_group=${bvt_group}" >> "$GITHUB_OUTPUT" + echo "bvt_generation=${bvt_generation}" >> "$GITHUB_OUTPUT" + jq -n \ + --arg deployment 'compose-proxy' \ + --argjson group "${bvt_group}" \ + --arg generation "${bvt_generation}" \ + --arg head_sha '${{ github.event.pull_request.head.sha }}' \ + '{schema_version: 1, deployment: $deployment, group: $group, generation: $generation, head_sha: $head_sha}' \ + > "${RUNNER_TEMP}/bvt-compose-manifest.json" + echo "Compose + Proxy BVT: generation ${bvt_generation}, group ${bvt_group}" + { + echo "### BVT group assignment" + echo "- Deployment: Compose Multi-CN + Proxy" + echo "- Group: ${bvt_group}" + echo "- Generation: ${bvt_generation}" + } >> "$GITHUB_STEP_SUMMARY" + cd "$GITHUB_WORKSPACE" + set -o pipefail + bash ./optools/run_bvt_group.sh \ + "$GITHUB_WORKSPACE/mo-tester" \ + "$GITHUB_WORKSPACE/test/distributed/cases" \ + "${bvt_group}" 2>&1 | tee "${RUNNER_TEMP}/bvt-compose.log" + + - name: Summarize Compose BVT result + if: ${{ always() && !cancelled() }} + run: | + set -euo pipefail + report="${RUNNER_TEMP}/bvt-compose.log" + timing="${RUNNER_TEMP}/bvt-compose-slowest.tsv" + if [ ! -s "${report}" ]; then + echo '### Compose + Proxy BVT result' | tee -a "$GITHUB_STEP_SUMMARY" + echo '- No BVT execution log was produced.' | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + { + echo '### Compose + Proxy BVT result' + grep '^Run BVT group' "${report}" | sed 's/^/- /' || true + echo + echo '#### Top 10 slow BVT scripts' + echo + } | tee -a "$GITHUB_STEP_SUMMARY" + + sed -nE 's#.*script file\[(.*)\] has been executed, and cost: ([0-9.]+)s.*#\2s\t\1#p' "${report}" \ + | sort -nr -k1,1 | awk 'NR <= 10 { print }' > "${timing}" + if [ ! -s "${timing}" ]; then + echo 'No per-script timing records found in BVT output.' | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + { + echo '```text' + cat "${timing}" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload Compose BVT execution log + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-execution-compose + path: ${{ runner.temp }}/bvt-compose.log + if-no-files-found: warn + retention-days: 7 + + - name: Print Docker Info Before Container Shutdown + if: ${{ always() }} + run: | + set +e; + sudo docker ps; + echo "==========================================="; + sudo df -h /*; + echo "==========================================="; + sudo du -hs /var/lib/docker; + echo "==========================================="; + sudo docker system df -v; + + - name: Print System Dmesg + if: ${{ always() }} + continue-on-error: true + run: | + sudo dmesg -T + + - name: export log + if: ${{ failure() || cancelled()}} + run: | + mkdir -p ${{ github.workspace }}/mo-tester/report + mv ${{ github.workspace }}/mo-tester/report ${{ github.workspace }}/docker-compose-log + curl http://localhost:12345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/docker-compose-log/cn-0-dump-stacks.log + curl http://localhost:22345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/docker-compose-log/cn-1-dump-stacks.log + + - name: Check Log Messages Count per second + if: ${{ always() && !cancelled() }} + run: | + ./optools/check_log_count.sh 1000 60 # {count threshold} {metric collected interval} + + - name: shutdown containers + if: ${{ always() }} + run: | + docker ps + docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn down --remove-orphans + docker volume rm launch-tae-compose_minio_storage || true + + - name: Generate Compose BVT coverage profile + if: ${{ always() && !cancelled() }} + run: | + set -uo pipefail + coverage_dir="$GITHUB_WORKSPACE/coverage" + coverage_profile="$RUNNER_TEMP/bvt-compose.out" + if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then + test -s "${coverage_profile}" + elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then + echo '::error::failed to generate Compose BVT coverage after a successful BVT' + exit 1 + else + echo '::warning::Compose BVT coverage unavailable because BVT did not finish successfully' + fi + + - name: Upload Compose BVT coverage + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-coverage-compose-generation-${{ steps.bvt_on_pr_version.outputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/bvt-compose.out + ${{ runner.temp }}/bvt-compose-manifest.json + if-no-files-found: warn + retention-days: 7 + + - uses: actions/upload-artifact@v7 + if: ${{ failure() || cancelled()}} + continue-on-error: true + with: + name: Compose-multi-cn-e2e-bvt-test-docker-log(PROXY) + path: | + ${{ github.workspace }}/docker-compose-log + retention-days: 7 diff --git a/.github/workflows/e2e-standalone-parallel.yaml b/.github/workflows/e2e-standalone-parallel.yaml new file mode 100644 index 0000000..74b4ff9 --- /dev/null +++ b/.github/workflows/e2e-standalone-parallel.yaml @@ -0,0 +1,464 @@ +name: MatrixOne e2e CI(Standalone) + +on: + workflow_call: + inputs: + bvt_group: + description: "BVT group assigned by the caller (0 or 1)" + required: false + type: string + default: "" + bvt_generation: + description: "Shared assignment generation for complementary BVT jobs" + required: false + type: string + default: "" + secrets: + TOKEN_ACTION: + description: "Token for checkout (e.g. pull from fork/private)" + required: false + +jobs: + bvt-linux-x86: + if: false + environment: ci + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + name: e2e BVT Test on Linux/x64(LAUNCH,Optimistic) + timeout-minutes: 90 + steps: + - name: checkout head + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "3" + path: ./head + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Go And Java + uses: matrixorigin/CI/actions/setup-env@main + with: + go-version-file: "${{ github.workspace }}/head/go.mod" + - name: Build MatrixOne + run: | + cd $GITHUB_WORKSPACE/head && make clean && make build + git rev-parse --short HEAD + - name: echo config + run: | + cd $GITHUB_WORKSPACE/head + echo "" >> ./etc/launch/cn.toml + echo '[cn.txn]' >> ./etc/launch/cn.toml + echo 'mode = "Optimistic"' >> ./etc/launch/cn.toml + echo " enable-leak-check = 1" >> ./etc/launch/cn.toml + echo ' max-active-ages = "2m"'>> ./etc/launch/cn.toml + echo "" >> ./etc/launch/tn.toml + echo '[tn.txn]' >> ./etc/launch/tn.toml + echo 'mode = "Optimistic"' >> ./etc/launch/tn.toml + cat ./etc/launch/cn.toml + cat ./etc/launch/tn.toml + - name: Start MO + run: | + cd $GITHUB_WORKSPACE/head + ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch + - name: Clone test-tool repository + uses: actions/checkout@v6 + with: + repository: matrixorigin/mo-tester + path: ./mo-tester + ref: main + - name: Update BVT SQL Timeout + run: | + cd $GITHUB_WORKSPACE/mo-tester + sed -i "s/socketTimeout:.*/socketTimeout: 300000/g" mo.yml + - name: Start BVT Test + id: bvt_on_pr_version + run: | + export LC_ALL="C.UTF-8" + locale + cd $GITHUB_WORKSPACE/mo-tester + sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml + cat mo.yml + echo "=============================" + + ./run.sh -n -g -o -p $GITHUB_WORKSPACE/head/test/distributed/cases -e pessimistic_transaction 2>&1 + - name: Dump mo-service goroutines + if: ${{ always() && !cancelled() }} + run: | + max_retry=5 + retry=0 + while [ "$(ps -ef | grep 'mo-service' | grep -v "grep" | wc -l)" -gt 0 ]; do + curl http://localhost:12345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/head/dump-stacks${retry}.log + pkill -9 mo-service + retry=$((retry+1)) + if [ ${retry} -ge ${max_retry} ]; then + echo 'after retry, still cannot shutdown mo-service' + exit 1 + fi + sleep 2 + done + - name: Restart the Version of Head of MO + run: | + cd $GITHUB_WORKSPACE/head + mv mo-service.log mo-service.r1.log + + # delete for clear the start finish status of mo + rm -rf mo-data/local/system_init_completed + + ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch + - name: Start BVT Test for MO with Version of Head Restarted + id: bvt_on_latest_head_version_run2 + run: | + export LC_ALL="C.UTF-8" + locale + cd $GITHUB_WORKSPACE/mo-tester + sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml + cat mo.yml + echo "=============================" + + ./run.sh -n -g -o -p $GITHUB_WORKSPACE/head/test/distributed/cases -e pessimistic_transaction 2>&1 + - name: Dump restarted mo-service goroutines + if: ${{ always() && !cancelled() }} + run: | + if [ "$(ps -ef | grep 'mo-service' | grep -v "grep" | wc -l)" -gt 0 ]; then curl http://localhost:12345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/head/restarted-dump-stacks.log; pkill -9 mo-service; else echo 'current mo-service has already crashed'; exit 1; fi + - name: Check Log Messages Count per second + if: ${{ always() && !cancelled() }} + run: | + cd $GITHUB_WORKSPACE/head + # 4 nodes in one Process + ./optools/check_log_count.sh 4000 60 # {count threshold} {metric collected interval} + - name: generate upload files + if: ${{ always() }} + continue-on-error: true + run: | + set +e + mkdir -p ${{ github.workspace }}/upload + rm -rf ./mo-tester/.git + rm -rf ./mo-tester/lib + mv ${{ github.workspace }}/head/mo-service.r1.log ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/head/mo-service.log ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/mo-tester ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/head/dump-stacks* ${{ github.workspace }}/upload/ + - uses: actions/upload-artifact@v7 + if: ${{ failure() || cancelled()}} + continue-on-error: true + with: + name: Standalone-e2e-BVT-Test-on-Linux-x64(LAUNCH,Optimistic)-reports + path: | + ${{ github.workspace }}/upload + retention-days: 7 + - uses: actions/upload-artifact@v7 + name: Upload MO DATA + if: ${{ failure() }} + continue-on-error: true + with: + name: Standalone-e2e-BVT-Test-on-Linux-x64(LAUNCH,Optimistic)-mo-data + path: | + ${{ github.workspace }}/head/mo-data + retention-days: 7 + + multi-cn-proxy-bvt-linux-x86: + # Compose + Proxy is the active proxy coverage deployment. Keep the + # native proxy launcher out of the PR matrix until it has a distinct role. + if: false + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + name: Legacy Launch + Proxy BVT (disabled in parallel workflow) + timeout-minutes: 75 + env: + mo_reuse_enable_checker: true + steps: + - name: checkout head + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + path: ./head + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Go And Java + uses: matrixorigin/CI/actions/setup-env@main + with: + go-version-file: "${{ github.workspace }}/head/go.mod" + - name: Build MatrixOne + run: | + cd $GITHUB_WORKSPACE/head && make clean && make build + git rev-parse --short HEAD + - name: Prepare Hosts + run: | + sudo bash -c 'echo -e "127.0.0.1\tcn0" >> /etc/hosts;'; + sudo bash -c 'echo -e "127.0.0.1\tcn1" >> /etc/hosts;'; + - name: Add cn.txn + run: | + cd $GITHUB_WORKSPACE/head + echo "" >> ./etc/launch-dynamic-with-proxy/cn.toml.base + echo "" >> ./etc/launch-dynamic-with-proxy/cn.toml.base + echo "[cn.txn]" >> ./etc/launch-dynamic-with-proxy/cn.toml.base + echo " enable-leak-check = 1" >> ./etc/launch-dynamic-with-proxy/cn.toml.base + echo ' max-active-ages = "2m"'>> ./etc/launch-dynamic-with-proxy/cn.toml.base + - name: Start MO + run: | + sudo cat /etc/hosts | grep '127.0.0.1'; + cd $GITHUB_WORKSPACE/head + ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch-dynamic-with-proxy -with-proxy + - name: Clone test-tool repository + uses: actions/checkout@v6 + with: + repository: matrixorigin/mo-tester + path: ./mo-tester + ref: main + - name: Update BVT SQL Timeout + run: | + cd $GITHUB_WORKSPACE/mo-tester + sed -i "s/socketTimeout:.*/socketTimeout: 300000/g" mo.yml + - name: Start BVT Test + id: bvt_on_pr_version + run: | + export LC_ALL="C.UTF-8" + locale + cd $GITHUB_WORKSPACE/mo-tester + sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml + cat mo.yml + echo "=============================" + + ./run.sh -n -g -o -p $GITHUB_WORKSPACE/head/test/distributed/cases -e optimistic 2>&1 + - name: Dump mo-service goroutines + if: ${{ always() && !cancelled() }} + run: | + if [ "$(ps -ef | grep 'mo-service' | grep -v "grep" | wc -l)" -gt 0 ]; then curl http://localhost:12345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/head/dump-stacks.log; else echo 'current mo-service has already crashed'; exit 1; fi + - name: Check Log Messages Count per second + if: ${{ always() && !cancelled() }} + run: | + cd $GITHUB_WORKSPACE/head + # one node in one Process + ./optools/check_log_count.sh 1000 60 # {count threshold} {metric collected interval} + - name: generate upload files + if: ${{ always() }} + continue-on-error: true + run: | + mkdir -p ${{ github.workspace }}/upload + rm -rf ./mo-tester/.git + rm -rf ./mo-tester/lib + mv ${{ github.workspace }}/head/mo-service.log ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/mo-tester ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/head/dump-stacks.log ${{ github.workspace }}/upload/ + - uses: actions/upload-artifact@v7 + if: ${{ failure() || cancelled()}} + continue-on-error: true + with: + name: Standalone-Multi-CN-e2e-BVT-Test-on-Linux-x64(LAUNCH,PROXY)-reports + path: | + ${{ github.workspace }}/upload + retention-days: 7 + + pessimistic-bvt-linux-x86: + # Match the existing PR BVT runner policy. + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + name: e2e BVT Test on Linux/x64(LAUNCH, PESSIMISTIC) + timeout-minutes: 75 + env: + mo_reuse_enable_checker: true + steps: + - name: checkout head + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "3" + path: ./head + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Go And Java + uses: matrixorigin/CI/actions/setup-env@main + with: + go-version-file: "${{ github.workspace }}/head/go.mod" + + - name: Build MatrixOne + run: | + cd "$GITHUB_WORKSPACE/head" + make clean && make GOBUILD_OPT=-cover build + git rev-parse --short HEAD + + - name: Add cn.txn + run: | + cd $GITHUB_WORKSPACE/head + echo "[cn.txn]" >> ./etc/launch/cn.toml + echo " enable-leak-check = 1" >> ./etc/launch/cn.toml + echo ' max-active-ages = "2m"'>> ./etc/launch/cn.toml + + - name: echo config + run: | + cd $GITHUB_WORKSPACE/head + cat ./etc/launch/cn.toml + cat ./etc/launch/tn.toml + + - name: Start MO + run: | + cd $GITHUB_WORKSPACE/head + export GOCOVERDIR="${RUNNER_TEMP}/bvt-coverage-pessimistic" + rm -rf "${GOCOVERDIR}" + mkdir -p "${GOCOVERDIR}" + ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch + + - name: Clone test-tool repository + uses: actions/checkout@v6 + with: + repository: matrixorigin/mo-tester + path: ./mo-tester + ref: main + - name: Update BVT SQL Timeout + run: | + cd $GITHUB_WORKSPACE/mo-tester + sed -i "s/socketTimeout:.*/socketTimeout: 300000/g" mo.yml + - name: Start BVT Test + id: bvt_on_pr_version + run: | + export LC_ALL="C.UTF-8" + locale + cd "$GITHUB_WORKSPACE/mo-tester" + sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml + cat mo.yml + echo "=============================" + + bvt_group='${{ inputs.bvt_group }}' + bvt_generation='${{ inputs.bvt_generation }}' + if [ -z "${bvt_group}" ]; then + bvt_group=$(( 1 - ((${{ github.run_id }} + ${{ github.run_attempt }}) % 2) )) + fi + if [ -z "${bvt_generation}" ]; then + bvt_generation='${{ github.run_id }}-${{ github.run_attempt }}' + fi + if [[ ! "${bvt_group}" =~ ^[01]$ ]] || [ -z "${bvt_generation}" ]; then + echo "::error::invalid BVT assignment: group=${bvt_group}, generation=${bvt_generation}" + exit 1 + fi + echo "bvt_group=${bvt_group}" >> "$GITHUB_OUTPUT" + echo "bvt_generation=${bvt_generation}" >> "$GITHUB_OUTPUT" + jq -n \ + --arg deployment 'launch-pessimistic' \ + --argjson group "${bvt_group}" \ + --arg generation "${bvt_generation}" \ + --arg head_sha '${{ github.event.pull_request.head.sha }}' \ + '{schema_version: 1, deployment: $deployment, group: $group, generation: $generation, head_sha: $head_sha}' \ + > "${RUNNER_TEMP}/bvt-pessimistic-manifest.json" + echo "Launch + Pessimistic BVT: generation ${bvt_generation}, group ${bvt_group}" + { + echo "### BVT group assignment" + echo "- Deployment: native launch (PESSIMISTIC)" + echo "- Group: ${bvt_group}" + echo "- Generation: ${bvt_generation}" + } >> "$GITHUB_STEP_SUMMARY" + cd "$GITHUB_WORKSPACE/head" + set -o pipefail + bash ./optools/run_bvt_group.sh \ + "$GITHUB_WORKSPACE/mo-tester" \ + "$GITHUB_WORKSPACE/head/test/distributed/cases" \ + "${bvt_group}" \ + "$GITHUB_WORKSPACE/head/test/distributed/resources" \ + 2>&1 | tee "${RUNNER_TEMP}/bvt-pessimistic.log" + + - name: Summarize Launch + Pessimistic BVT result + if: ${{ always() && !cancelled() }} + run: | + set -euo pipefail + report="${RUNNER_TEMP}/bvt-pessimistic.log" + timing="${RUNNER_TEMP}/bvt-pessimistic-slowest.tsv" + if [ ! -s "${report}" ]; then + echo '### Launch + Pessimistic BVT result' | tee -a "$GITHUB_STEP_SUMMARY" + echo '- No BVT execution log was produced.' | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + { + echo '### Launch + Pessimistic BVT result' + grep '^Run BVT group' "${report}" | sed 's/^/- /' || true + echo + echo '#### Top 10 slow BVT scripts' + echo + } | tee -a "$GITHUB_STEP_SUMMARY" + + sed -nE 's#.*script file\[(.*)\] has been executed, and cost: ([0-9.]+)s.*#\2s\t\1#p' "${report}" \ + | sort -nr -k1,1 | awk 'NR <= 10 { print }' > "${timing}" + if [ ! -s "${timing}" ]; then + echo 'No per-script timing records found in BVT output.' | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + { + echo '```text' + cat "${timing}" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload Launch + Pessimistic BVT execution log + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-execution-pessimistic + path: ${{ runner.temp }}/bvt-pessimistic.log + if-no-files-found: warn + retention-days: 7 + + - name: Dump mo-service goroutines + if: ${{ always() && !cancelled() }} + run: | + if [ "$(ps -ef | grep 'mo-service' | grep -v "grep" | wc -l)" -gt 0 ]; then curl http://localhost:12345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/head/dump-stacks.log; else echo 'current mo-service has already crashed'; exit 1; fi + + - name: Check Log Messages Count per second + if: ${{ always() && !cancelled() }} + run: | + cd $GITHUB_WORKSPACE/head + # 4 nodes in one Process + ./optools/check_log_count.sh 4000 60 # {count threshold} {metric collected interval} + + - name: Stop service and generate BVT coverage profile + if: ${{ always() && !cancelled() }} + run: | + set -uo pipefail + coverage_dir="${RUNNER_TEMP}/bvt-coverage-pessimistic" + coverage_profile="${RUNNER_TEMP}/bvt-pessimistic.out" + pids=$(pgrep -f 'mo-service' || true) + if [ -n "${pids}" ]; then + kill -TERM ${pids} || true + fi + for _ in $(seq 1 30); do + pgrep -f 'mo-service' >/dev/null || break + sleep 1 + done + pids=$(pgrep -f 'mo-service' || true) + if [ -n "${pids}" ]; then + kill -KILL ${pids} || true + fi + if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then + test -s "${coverage_profile}" + elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then + echo '::error::failed to generate Launch + Pessimistic BVT coverage after a successful BVT' + exit 1 + else + echo '::warning::Launch + Pessimistic BVT coverage unavailable because BVT did not finish successfully' + fi + + - name: Upload Launch + Pessimistic BVT coverage + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-coverage-pessimistic-generation-${{ steps.bvt_on_pr_version.outputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/bvt-pessimistic.out + ${{ runner.temp }}/bvt-pessimistic-manifest.json + if-no-files-found: warn + retention-days: 7 + + - name: generate upload files + if: ${{ always() }} + continue-on-error: true + run: | + mkdir -p ${{ github.workspace }}/upload + rm -rf ./mo-tester/.git + rm -rf ./mo-tester/lib + mv ${{ github.workspace }}/head/mo-service.log ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/mo-tester ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/head/dump-stacks.log ${{ github.workspace }}/upload/ + + - uses: actions/upload-artifact@v7 + if: ${{ failure() || cancelled()}} + continue-on-error: true + with: + name: Standalone-e2e-BVT-Test-on-Linux-x64(LAUNCH,PESSIMISTIC)-reports + path: | + ${{ github.workspace }}/upload + retention-days: 7 diff --git a/.github/workflows/utils-parallel.yaml b/.github/workflows/utils-parallel.yaml new file mode 100644 index 0000000..67446de --- /dev/null +++ b/.github/workflows/utils-parallel.yaml @@ -0,0 +1,29 @@ +# Utilities that remain outside the parallel coverage pipeline. +name: MatrixOne Utils CI + +on: + workflow_call: + secrets: + TOKEN_ACTION: + description: "A token passed from the caller workflow" + required: true + +jobs: + pr-size-label: + environment: ci + runs-on: arm64-mo-shanghai-4c8g + steps: + - name: size-label + uses: matrixorigin/CI/actions/label-size-action@main + with: + size_token: ${{ secrets.TOKEN_ACTION }} + ignore: ".md,.pb.go" + sizes: > + { + "XS":0, + "S":10, + "M":100, + "L":500, + "XL":1000, + "XXL":2000 + } diff --git a/scripts/parse_coverage.py b/scripts/parse_coverage.py index d43dbf2..eddab13 100644 --- a/scripts/parse_coverage.py +++ b/scripts/parse_coverage.py @@ -4,6 +4,7 @@ import logging import argparse import fnmatch +import json # 设置日志配置 logging.basicConfig( @@ -56,7 +57,7 @@ def merge_coverage_files(output_path, *coverage_files): # 计算覆盖率 coverage_percentage = (covered_blocks / total_blocks) if total_blocks > 0 else 0 - logging.info(f"Total code blocks: {total_blocks}, Covered blocks: {covered_blocks}, Coverage: {coverage_percentage}%") + logging.info(f"Total code blocks: {total_blocks}, Covered blocks: {covered_blocks}, Coverage: {coverage_percentage:.2%}") # 将合并后的结果写入输出文件 try: @@ -278,7 +279,7 @@ def diff_coverage(diff_path, coverage_path, output_path='pr_coverage.out', ignor except Exception as e: logging.error(f"An error occurred during the process: {e}") - return 0,0,0 + raise def is_valid_code_segment(segment): """判断一个代码片段是否包含有效的代码(忽略结构符号、关键字和空白)""" @@ -452,6 +453,39 @@ def _line_matches_block(line_num, mincol, maxcol, block): return False +def write_coverage_summary( + output_path, + total_blocks, + covered_blocks, + overall_coverage, + total_modified_lines, + covered_modified_lines, + pr_coverage, + minimal_coverage, + has_go_changes=None, +): + """Write the coverage gate result before the caller enforces the threshold.""" + if has_go_changes is None: + has_go_changes = total_modified_lines > 0 + summary = { + "schema_version": 1, + "total_blocks": total_blocks, + "covered_blocks": covered_blocks, + "overall_coverage": overall_coverage, + "total_modified_lines": total_modified_lines, + "covered_modified_lines": covered_modified_lines, + "pr_coverage": pr_coverage, + "minimal_coverage": minimal_coverage, + "has_go_changes": has_go_changes, + "has_pr_coverage": total_modified_lines > 0, + "approved": pr_coverage > minimal_coverage, + } + with open(output_path, "w") as summary_file: + json.dump(summary, summary_file, indent=2, sort_keys=True) + summary_file.write("\n") + return summary + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Merge coverage files and calculate coverage based on diff.") @@ -483,20 +517,39 @@ def _line_matches_block(line_num, mincol, maxcol, block): help='Count coverage by line instead of block. Default is False (count by block).' ) + parser.add_argument( + '-summary_path', + type=str, + default='coverage_summary.json', + help='Path for the machine-readable coverage summary.' + ) + args = parser.parse_args() - total_blocks, covered_blocks, coverage_percentage = merge_coverage_files('merged_coverage.out', *args.coverage_files) - logging.info(f"Total modified blocks: {total_blocks}, Covered blocks: {covered_blocks}, Coverage: {coverage_percentage:.2f}%") + total_blocks, covered_blocks, overall_coverage = merge_coverage_files('merged_coverage.out', *args.coverage_files) + logging.info(f"Total code blocks: {total_blocks}, Covered blocks: {covered_blocks}, Coverage: {overall_coverage:.2%}") # 调用主函数 diff_path = args.diff_path # 可以根据实际情况修改路径 coverage_path = 'merged_coverage.out' # 可以根据实际情况修改路径 - total_modified_lines, covered_modified_lines, coverage_percentage = diff_coverage(diff_path, coverage_path, count_by_line=args.count_by_line) - logging.info(f"total_modified_lines: {total_modified_lines}, covered_modified_lines: {covered_modified_lines}, coverage_percentage:{coverage_percentage}") + total_modified_lines, covered_modified_lines, pr_coverage = diff_coverage(diff_path, coverage_path, count_by_line=args.count_by_line) + logging.info(f"total_modified_lines: {total_modified_lines}, covered_modified_lines: {covered_modified_lines}, coverage_percentage:{pr_coverage:.2%}") + + summary = write_coverage_summary( + args.summary_path, + total_blocks, + covered_blocks, + overall_coverage, + total_modified_lines, + covered_modified_lines, + pr_coverage, + args.minimal_coverage, + has_go_changes=os.path.exists("pr_coverage.out"), + ) - if coverage_percentage <= args.minimal_coverage: + if not summary["approved"]: parse_file_coverage(args.minimal_coverage) - logging.warning(f"The code coverage:{coverage_percentage} is below or equal {args.minimal_coverage}, not approved.") + logging.warning(f"The code coverage:{pr_coverage:.2%} is below or equal {args.minimal_coverage:.2%}, not approved.") sys.exit(1) - logging.info(f"The code coverage:{coverage_percentage} is above {args.minimal_coverage}, pass.") \ No newline at end of file + logging.info(f"The code coverage:{pr_coverage:.2%} is above {args.minimal_coverage:.2%}, pass.") diff --git a/scripts/test_parse_coverage.py b/scripts/test_parse_coverage.py index 2708dde..761748e 100644 --- a/scripts/test_parse_coverage.py +++ b/scripts/test_parse_coverage.py @@ -1,11 +1,15 @@ import pytest import os +import json +import subprocess +import sys from parse_coverage import ( parse_diff, parse_coverage_and_generate_report, merge_coverage_files, diff_coverage, normalize_path, + write_coverage_summary, ) @@ -231,7 +235,6 @@ def test_multiple_modified_lines_one_block_by_line(self, tmp_path): assert total == 3 assert covered == 3 assert pct == 1.0 - def test_multiple_lines_partial_coverage(self, tmp_path): """按块计数(默认): 2块,1covered → 50%""" coverage_content = """\ @@ -309,6 +312,93 @@ def test_uncovered_block(self, tmp_path): assert pct == 0.0 +class TestCoverageSummary: + def test_writes_rates_counts_and_gate_result(self, tmp_path): + summary_path = tmp_path / "coverage-summary.json" + + summary = write_coverage_summary( + str(summary_path), + total_blocks=200, + covered_blocks=120, + overall_coverage=0.6, + total_modified_lines=4, + covered_modified_lines=3, + pr_coverage=0.75, + minimal_coverage=0.75, + ) + + assert summary["schema_version"] == 1 + assert summary["has_go_changes"] is True + assert summary["has_pr_coverage"] is True + assert summary["approved"] is False + assert '"overall_coverage": 0.6' in summary_path.read_text() + + def test_no_go_changes_is_explicit_and_passes(self, tmp_path): + summary = write_coverage_summary( + str(tmp_path / "coverage-summary.json"), + total_blocks=10, + covered_blocks=5, + overall_coverage=0.5, + total_modified_lines=0, + covered_modified_lines=0, + pr_coverage=1.0, + minimal_coverage=0.75, + ) + + assert summary["has_go_changes"] is False + assert summary["has_pr_coverage"] is False + assert summary["approved"] is True + + def test_diff_coverage_propagates_parser_errors(self, tmp_path): + with pytest.raises(FileNotFoundError): + diff_coverage( + str(tmp_path / "missing.diff"), + str(tmp_path / "missing-coverage.out"), + ignore_path=str(tmp_path / "missing.ignore"), + ) + + def test_cli_keeps_summary_and_pr_profile_when_gate_fails(self, tmp_path): + coverage_path = tmp_path / "coverage.out" + coverage_path.write_text( + "mode: set\n" + "github.com/matrixorigin/matrixone/pkg/example.go:1.1,2.2 1 0\n" + ) + diff_path = tmp_path / "diff.patch" + diff_path.write_text( + "diff --git a/pkg/example.go b/pkg/example.go\n" + "--- a/pkg/example.go\n" + "+++ b/pkg/example.go\n" + "@@ -1,1 +1,1 @@\n" + "-old\n" + "+new\n" + ) + (tmp_path / ".ignore").write_text("!*.go\n") + script_path = os.path.join(os.path.dirname(__file__), "parse_coverage.py") + + result = subprocess.run( + [ + sys.executable, + script_path, + "-coverage_files", + str(coverage_path), + "-diff_path", + str(diff_path), + "-summary_path", + "coverage-summary.json", + ], + cwd=tmp_path, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + summary = json.loads((tmp_path / "coverage-summary.json").read_text()) + assert summary["has_go_changes"] is True + assert summary["approved"] is False + assert (tmp_path / "pr_coverage.out").read_text().startswith("mode: set\n") + + class TestPathNormalization: """测试路径规范化"""