diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 8bdff7e4e..14d01b1d1 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -259,12 +259,6 @@ qualified = "RubyCode::get_op_type" metric = "halstead.effort" value = 101217.532578918 -[[entry]] -path = "big-code-analysis-ast/src/lang_helpers/doc_comment.rs" -qualified = "" -metric = "loc.ploc" -value = 642.0 - [[entry]] path = "big-code-analysis-ast/src/lang_helpers/doc_comment.rs" qualified = "bind_leading" @@ -775,12 +769,6 @@ qualified = "assemble_globals" metric = "nargs" value = 6.0 -[[entry]] -path = "big-code-analysis-cli/src/walk.rs" -qualified = "classify_dropped_child" -metric = "nargs" -value = 5.0 - [[entry]] path = "big-code-analysis-cli/src/walk.rs" qualified = "expand_seed_paths" @@ -1267,6 +1255,12 @@ qualified = "RustCode::compute" metric = "nargs" value = 5.0 +[[entry]] +path = "src/metrics/loc/shared.rs" +qualified = "credit_bare_continuation_rows" +metric = "nargs" +value = 5.0 + [[entry]] path = "src/metrics/loc/tcl.rs" qualified = "TclCode::compute" diff --git a/.claude/skills/add-lang/SKILL.md b/.claude/skills/add-lang/SKILL.md index 9ef3e015d..2f915dd79 100644 --- a/.claude/skills/add-lang/SKILL.md +++ b/.claude/skills/add-lang/SKILL.md @@ -531,7 +531,7 @@ Create the file, add `mod ;` to `checker.rs`, and write `holds_doc_comment_in_body`) — what the language's own documentation tool reads (#1597). They feed `dloc`; the defaults answer `false`, which is right for a language with no doc - convention. Where one exists, `lang_helpers/doc_comment.rs` has the + convention. Where one exists, `lang_helpers/doc_text.rs` has the shared opener tests. - `is_func_space` / `is_func_space_with_code`, `is_func`, `is_closure` — forward to the table: `crate::roles::::opens_space` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a6e459f0..84f799e11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -427,7 +427,18 @@ jobs: # runs doctests for correctness — only their coverage is omitted. - name: Clean coverage data run: cargo llvm-cov clean --workspace + # + # LLVM_PROFILE_FILE_NAME drops the `%p` cargo-llvm-cov puts in its + # default name. nextest runs every test in a process of its own, so + # one raw profile per process came to ~8,000 files and 15 GB, and + # the job ran the runner out of disk (PR #1804). Without `%p`, every + # process of one test binary merges on-line into `%4m`'s pool of + # four files: LLVM locks the file and adds the counters, so no count + # is lost. Measured on that PR: 101 files and 62 MB, with every + # file's covered lines and regions identical to the default naming. - name: Run tests with coverage instrumentation + env: + LLVM_PROFILE_FILE_NAME: big-code-analysis-%4m.profraw run: cargo llvm-cov --no-report nextest --all-features --workspace --locked # Codecov's native format preserves region coverage (lcov would drop to # line-only). @@ -739,7 +750,8 @@ jobs: # SHA256, mirroring the actionlint and checkmake patterns below. # Bump RUMDL_SHA256 in lockstep with `RUMDL_VERSION` (and with the # `rumdl = …` pin in `mise.toml` and the `Dockerfile`'s - # `RUMDL_VERSION` + per-arch SHA256 table). To regenerate the + # `RUMDL_VERSION` + per-arch SHA256 table); `make + # check-rumdl-lockstep` fails when they disagree. To regenerate the # SHA256 after a version bump (swap the triple for # `aarch64-unknown-linux-gnu` to get the Dockerfile's arm64 entry): # curl -fsSL https://github.com/rvben/rumdl/releases/download/v${RUMDL_VERSION}/rumdl-v${RUMDL_VERSION}-x86_64-unknown-linux-gnu.tar.gz.sha256 @@ -1013,6 +1025,15 @@ jobs: # gate without surfacing the regression in CI. - name: check-ruff-lockstep self-tests (explicit) run: python3 -m unittest -q utils/check-ruff-lockstep-test.py + # Defensive twin for the rumdl-lockstep gate (#1779): mise.toml, + # the `Install rumdl` step above and the Dockerfile must pin one + # rumdl version, and the Dockerfile's x86_64 hash must equal + # `RUMDL_SHA256`. Comments were the whole mechanism before, and + # #1653's bump nearly left the Dockerfile behind. + - name: check-rumdl-lockstep (explicit) + run: python3 utils/check-rumdl-lockstep.py + - name: check-rumdl-lockstep self-tests (explicit) + run: python3 -m unittest -q utils/check-rumdl-lockstep-test.py # Defensive twin for the enums-codegen-drift gate (#405): # running any grammar regen previously regenerated # `src/c_langs_macros/*.rs` to a pre-optimization form diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index adb3a1d09..f60aa4aad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -295,6 +295,27 @@ repos: entry: python3 -m unittest -q utils/check-ruff-lockstep-test.py pass_filenames: false + # rumdl version lockstep (#1779). mise.toml, the CI lint job and + # the Dockerfile each pin rumdl, and the Dockerfile's x86_64 hash + # must equal CI's. #1653's bump nearly left the Dockerfile behind. + # .rumdl-lockstep-baseline.txt records the last accepted version and + # hash, so a bump that keeps the old hash fails too. + - id: check-rumdl-lockstep + name: check-rumdl-lockstep + language: system + files: '^(mise\.toml|Dockerfile|\.github/workflows/ci\.yml|\.rumdl-lockstep-baseline\.txt|utils/check-rumdl-lockstep\.py)$' + entry: python3 utils/check-rumdl-lockstep.py + pass_filenames: false + + # Self-tests for the rumdl-lockstep gate. Runs on edits to the + # script or its test file. + - id: check-rumdl-lockstep-test + name: check-rumdl-lockstep-test + language: system + files: '^utils/check-rumdl-lockstep(-test)?\.py$' + entry: python3 -m unittest -q utils/check-rumdl-lockstep-test.py + pass_filenames: false + # Publish metadata for the crates.io uploads (#1224). The four # top-level crates pin internal deps at `=`, so none can # be `cargo publish --dry-run`-ed before the tag that publishes diff --git a/.rumdl-lockstep-baseline.txt b/.rumdl-lockstep-baseline.txt new file mode 100644 index 000000000..c9fef39d4 --- /dev/null +++ b/.rumdl-lockstep-baseline.txt @@ -0,0 +1,13 @@ +# rumdl lockstep baseline: the rumdl release and its x86_64 tarball +# SHA256 that check-rumdl-lockstep.py last accepted (#1779). +# Maintained by `python3 utils/check-rumdl-lockstep.py --update`; see the +# script's docstring. +# +# The gate fails when the version pinned in mise.toml, ci.yml and the +# Dockerfile has moved but the hash in ci.yml and the Dockerfile is still +# the one recorded here: the bump kept the old release's hash. A bump +# that pastes the new release's hash into both files fails too, until +# `--update` records the new pair; commit this file with that bump. +# +# One record: ` `. +0.3.2 f527d1374765d826c1627ceab3d1f46e548268731a495b5b12a3af529b3d79e8 diff --git a/AGENTS.md b/AGENTS.md index d1044632f..71408b46c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -97,6 +97,7 @@ and `cargo run -p big-code-analysis-web --`. `check-grammar-marker-sync.py`, `check-enums-codegen-drift.sh`, `check-grammar-crate.py`, `check-grammars-crates.sh`, `check-excluded-manifests.py`, `check-ruff-lockstep.py`, + `check-rumdl-lockstep.py`, `check-publish-metadata.py`, `check-test-lang-gates.py`, `check-role-census.py`, `check-role-coverage.py`, `check-role-bypass.py` (the three role-table gates, all reading diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a9d64099..fa7b41ae9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -217,6 +217,18 @@ for historical reference. (`EISDIR` on unix) rather than a skip. The Python `Ast.from_path` `ValueError` names the reason. +- **`make check-rumdl-lockstep`** (`utils/check-rumdl-lockstep.py`, + #1779) fails when the rumdl pin in `mise.toml`, the CI lint job's + `RUMDL_VERSION` / `RUMDL_SHA256` and the `Dockerfile` disagree on the + version, naming the file that differs, or when the `Dockerfile`'s + x86_64 SHA256 differs from CI's. `.rumdl-lockstep-baseline.txt` + records the version and hash it last accepted, so a bump that moves + the version but keeps the old hash fails by name, and an honest bump + fails until `python3 utils/check-rumdl-lockstep.py --update` records + the new pair in the same commit. It is wired into `make lint`, `make pre-commit`, `make ci`, + the pre-commit hooks (which also run on an edit to the baseline) and + the CI lint job. + ### Changed - **The role-bypass baseline is empty** (#1701, part of #1690). The @@ -901,6 +913,14 @@ for historical reference. whose predicates call it (metrics, `ops`, `find` and `count`) all lend the cache. (#1785) +- **`bca check` measures its ignored-file summary on the `--jobs` + thread budget** (#1307). Walked directories were re-listed serially + after the walk; they are now claimed per directory by scoped worker + threads, which cuts a 65,720-directory tree from 340 ms to 143 ms. + The `--report-skipped` listing is byte-identical, including which + spelling of a path reached by overlapping seeds is printed (the + first-named seed's). + ### Deprecated - `FromPathError::Unreadable`: superseded by `Skipped(SkipReason)`; @@ -1187,9 +1207,8 @@ for historical reference. than an identifier walk, because `[temp.pre]/5` allows a template declaration at most one declarator and a walk would read an array bound, an initialiser's callee or a function pointer's parameter - name. Cpp and Mozcpp alike. A pointer-to-member-function data member - (`static int (S::*fp)(T);`, and the plain `int (S::*pmf)(int);`) - still reads as a method — #1718. + name. Cpp and Mozcpp alike. A templated pointer-to-member-function + data member (`static int (S::*fp)(T);`) is left to #1718, below. - **Ruby's `npm` counts a `def` wherever the class body's statements reach it** (#1401). The member walk read the body's direct children @@ -1332,6 +1351,52 @@ for historical reference. stdin, so the after side applied no exclusions and reported the files they should have hidden as added. +- `utils/perturb.py` refuses a `cargo test` argv that can run more than + one test target without `--no-fail-fast`, instead of reporting the + guards in later targets as uncaught (#1787). It sees through a wrapper + in front of cargo (`timeout`, `env`, `nice`, `rustup run`), counts a + `-p` inside a short-option cluster (`-qpa`), and refuses a second + `--target`. + +- `npa` / `npm` (C++, Mozcpp): a templated pointer-to-member data + member such as `template static int (S::*fp)(T);` is counted + as an attribute instead of a method, and the methods + `template int (*S::*getPp())(T);` and + `template static int S::*getPd();` as methods instead of + attributes. A method returning a member-function pointer stays a + method, and the parenthesised non-templated `int (S::*pmf)(int);` was + already correct (#1718). A member function template whose every + parameter parses as an expression, such as + `template void connect(T* obj, void (T::*slot)());`, is + counted as a method instead of an attribute: both grammars read it as + a variable with a parenthesised initialiser, which a class member + cannot have. Still misread by the grammars' error recovery, as before: + the unparenthesised `int S::*getPd();` as data, and + `static constexpr int (S::*fp)(int) = nullptr;` as a method. + +- Ruby `npa` counts `attr_*` macros and `@x` / `@@x` class-scope + assignments wherever the class body runs them: under a conditional, + loop, `begin` / `rescue` / `ensure`, `case` arm, statement modifier, + parentheses, an assignment's right side, or a call's argument. A bare + `private` / `public` / `protected` shadowed by a local of the same + name — bound by an assignment, `for` or `rescue =>` the walk reads — + no longer changes attribute visibility, and a marker inside a + wrapper or in expression position now sets it, as it already did for + `npm`. The attribute and method walks share one traversal and agree + on a body's visibility (#1719). + +- `loc` (Bash, Tcl, iRules): a line-continuation row holding only the + `\` counts as code instead of blank, or as comment where it continues + a Tcl or iRules comment (`# note \`), in its enclosing function, proc + or event handler as well as the file (#1445). One that continues + nothing counts as the line it starts once joined to the next row: a + comment where that row is one (`\` + `# note`), blank where it is + empty, code otherwise. Where CRLF line endings + reach the parser unchanged (the library's `analyze`, Python's + `Ast.parse`; `bca` converts them first), the Tcl and iRules grammars + still lex such a row as a word (#1802). The C family, Ruby and Elixir + splice the row the same way and still count it as blank (#1801). + ## [2.4.0] - 2026-10-21 ### Added diff --git a/Dockerfile b/Dockerfile index 60cbc7f88..08158cfa8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -251,6 +251,8 @@ EOF # --------------------------------------------------------------------------- # rumdl (Markdown linter/formatter) — SHA256-verified release tarball. +# `make check-rumdl-lockstep` holds RUMDL_VERSION and the x86_64 hash to +# the CI lint job's pin, and the version to mise.toml (#1779). # --------------------------------------------------------------------------- RUN <<'EOF' set -eux diff --git a/Makefile b/Makefile index 4f0cc0b63..194d08714 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ find-by-ext = $(if $(FD),$(FD) --extension $(1) $(FD_EXCLUDE) $(2),find . -name NEXTEST := $(shell command -v cargo-nextest 2>/dev/null) TEST_CMD = $(if $(NEXTEST),$(NEXTEST) nextest run --workspace --all-features,cargo test --workspace --all-features --lib --bins --tests) -.PHONY: role-census role-census-test role-coverage role-coverage-test role-coverage-observe role-coverage-claimed role-parity-corpus role-corpus-refresh role-bypass role-bypass-test mutation-report-test mutation-workflow-test drift-common-test drift-oracle-test drift-compare-test recovery-oracle-test perturb-test coverage-diff-test self-scan-bca-test _pc-role-census _pc-role-census-test _pc-role-coverage _pc-role-coverage-test _pc-role-bypass _pc-role-bypass-test _ci-role-census _ci-role-census-test _ci-role-coverage _ci-role-coverage-test _ci-role-bypass _ci-role-bypass-test help check-tools worktree-setup worktree-setup-test build build-release check test test-doc chain-audit fmt fmt-check markdown-fmt markdown-lint shellcheck sh-fmt sh-fmt-check toml-fmt toml-fmt-check toml-lint makefile-check actionlint snapshot-anchors snapshot-anchors-test rustfmt-bail rustfmt-bail-test grammar-marker-sync grammar-marker-sync-test check-versions check-excluded-manifests check-excluded-manifests-test check-ruff-lockstep check-ruff-lockstep-test check-publish-metadata check-publish-metadata-test check-manpage-assets check-manpage-drift-test check-diagnostic-prefix check-diagnostic-prefix-test check-feature-gates check-feature-gates-test check-test-lang-gates check-test-lang-gates-compare check-test-lang-gates-test check-safety-doc-pin check-safety-doc-pin-test check-grammar-build-watch check-grammar-build-watch-test gate-status-test check-tools-test enums-check enums-codegen-drift enums-codegen-drift-test self-scan self-scan-headroom self-scan-write-baseline self-scan-write-baseline-headroom self-scan-baseline-fresh self-scan-profile-parity vcs lint clippy udeps insta-review insta-accept clean distclean install install-cli install-web doc doc-open doc-check doc-check-docsrs book book-serve book-pot book-po-update book-ja book-deploy all pre-commit ci release-check verify-changelog pkg-deb-local pkg-rpm-local dev-env-build dev-env-run dev-env-shell dev-env-rm py-bootstrap py-sync py-relock py-clean py-fmt py-fmt-check py-lint py-typecheck py-test py-stubtest smoke smoke-cli smoke-lib bench bench-scaling bench-walk fuzz-check fuzz-smoke fuzz-replay fuzz-run fuzz-tmin _check-find _pc-all _pc-fmt _pc-clippy _pc-test _pc-doc-check _pc-udeps _pc-shellcheck _pc-markdown-lint _pc-toml-lint _pc-makefile-check _pc-actionlint _pc-snapshot-anchors _pc-snapshot-anchors-test _pc-rustfmt-bail _pc-rustfmt-bail-test _pc-role-census _pc-role-census-test _pc-role-coverage _pc-role-coverage-test _pc-role-bypass _pc-role-bypass-test _pc-mutation-report-test _pc-mutation-workflow-test _pc-drift-common-test _pc-drift-oracle-test _pc-drift-compare-test _pc-recovery-oracle-test _pc-perturb-test _pc-coverage-diff-test _pc-self-scan-bca-test _pc-grammar-marker-sync _pc-grammar-marker-sync-test _pc-check-versions _pc-check-versions-test _pc-check-grammar-crate-test _pc-check-excluded-manifests _pc-check-excluded-manifests-test _pc-check-ruff-lockstep _pc-check-ruff-lockstep-test _pc-check-publish-metadata _pc-check-publish-metadata-test _pc-check-manpage-assets _pc-check-manpage-drift-test _pc-check-diagnostic-prefix _pc-check-diagnostic-prefix-test _pc-check-feature-gates-test _pc-check-test-lang-gates _pc-check-test-lang-gates-test _pc-check-safety-doc-pin _pc-check-safety-doc-pin-test _pc-check-grammar-build-watch _pc-check-grammar-build-watch-test _pc-worktree-setup-test _pc-gate-status-test _pc-check-tools-test _pc-enums-check _pc-enums-codegen-drift _pc-enums-codegen-drift-test _pc-self-scan _pc-self-scan-headroom _pc-py-fmt _pc-py-typecheck _pc-py-test _pc-py-stubtest _ci-all _ci-fmt-check _ci-clippy _ci-test _ci-doc-check _ci-build _ci-udeps _ci-shellcheck _ci-markdown-lint _ci-toml-lint _ci-makefile-check _ci-actionlint _ci-snapshot-anchors _ci-snapshot-anchors-test _ci-rustfmt-bail _ci-rustfmt-bail-test _ci-role-census _ci-role-census-test _ci-role-coverage _ci-role-coverage-test _ci-role-bypass _ci-role-bypass-test _ci-mutation-report-test _ci-mutation-workflow-test _ci-drift-common-test _ci-drift-oracle-test _ci-drift-compare-test _ci-recovery-oracle-test _ci-perturb-test _ci-coverage-diff-test _ci-self-scan-bca-test _ci-grammar-marker-sync _ci-grammar-marker-sync-test _ci-check-versions _ci-check-versions-test _ci-check-grammar-crate-test _ci-check-excluded-manifests _ci-check-excluded-manifests-test _ci-check-ruff-lockstep _ci-check-ruff-lockstep-test _ci-check-publish-metadata _ci-check-publish-metadata-test _ci-check-manpage-assets _ci-check-manpage-drift-test _ci-check-diagnostic-prefix _ci-check-diagnostic-prefix-test _ci-check-feature-gates-test _ci-check-test-lang-gates _ci-check-test-lang-gates-test _ci-check-safety-doc-pin _ci-check-safety-doc-pin-test _ci-check-grammar-build-watch _ci-check-grammar-build-watch-test _ci-worktree-setup-test _ci-gate-status-test _ci-check-tools-test _ci-enums-check _ci-enums-codegen-drift _ci-enums-codegen-drift-test _ci-enums-codegen-drift-test _ci-self-scan _ci-self-scan-headroom _ci-cargo-pipeline _ci-py-fmt-check _ci-py-lint _ci-py-typecheck _ci-py-test _ci-py-stubtest +.PHONY: role-census role-census-test role-coverage role-coverage-test role-coverage-observe role-coverage-claimed role-parity-corpus role-corpus-refresh role-bypass role-bypass-test mutation-report-test mutation-workflow-test drift-common-test drift-oracle-test drift-compare-test recovery-oracle-test perturb-test coverage-diff-test self-scan-bca-test _pc-role-census _pc-role-census-test _pc-role-coverage _pc-role-coverage-test _pc-role-bypass _pc-role-bypass-test _ci-role-census _ci-role-census-test _ci-role-coverage _ci-role-coverage-test _ci-role-bypass _ci-role-bypass-test help check-tools worktree-setup worktree-setup-test build build-release check test test-doc chain-audit fmt fmt-check markdown-fmt markdown-lint shellcheck sh-fmt sh-fmt-check toml-fmt toml-fmt-check toml-lint makefile-check actionlint snapshot-anchors snapshot-anchors-test rustfmt-bail rustfmt-bail-test grammar-marker-sync grammar-marker-sync-test check-versions check-excluded-manifests check-excluded-manifests-test check-ruff-lockstep check-ruff-lockstep-test check-rumdl-lockstep check-rumdl-lockstep-test check-publish-metadata check-publish-metadata-test check-manpage-assets check-manpage-drift-test check-diagnostic-prefix check-diagnostic-prefix-test check-feature-gates check-feature-gates-test check-test-lang-gates check-test-lang-gates-compare check-test-lang-gates-test check-safety-doc-pin check-safety-doc-pin-test check-grammar-build-watch check-grammar-build-watch-test gate-status-test check-tools-test enums-check enums-codegen-drift enums-codegen-drift-test self-scan self-scan-headroom self-scan-write-baseline self-scan-write-baseline-headroom self-scan-baseline-fresh self-scan-profile-parity vcs lint clippy udeps insta-review insta-accept clean distclean install install-cli install-web doc doc-open doc-check doc-check-docsrs book book-serve book-pot book-po-update book-ja book-deploy all pre-commit ci release-check verify-changelog pkg-deb-local pkg-rpm-local dev-env-build dev-env-run dev-env-shell dev-env-rm py-bootstrap py-sync py-relock py-clean py-fmt py-fmt-check py-lint py-typecheck py-test py-stubtest smoke smoke-cli smoke-lib bench bench-scaling bench-walk fuzz-check fuzz-smoke fuzz-replay fuzz-run fuzz-tmin _check-find _pc-all _pc-fmt _pc-clippy _pc-test _pc-doc-check _pc-udeps _pc-shellcheck _pc-markdown-lint _pc-toml-lint _pc-makefile-check _pc-actionlint _pc-snapshot-anchors _pc-snapshot-anchors-test _pc-rustfmt-bail _pc-rustfmt-bail-test _pc-role-census _pc-role-census-test _pc-role-coverage _pc-role-coverage-test _pc-role-bypass _pc-role-bypass-test _pc-mutation-report-test _pc-mutation-workflow-test _pc-drift-common-test _pc-drift-oracle-test _pc-drift-compare-test _pc-recovery-oracle-test _pc-perturb-test _pc-coverage-diff-test _pc-self-scan-bca-test _pc-grammar-marker-sync _pc-grammar-marker-sync-test _pc-check-versions _pc-check-versions-test _pc-check-grammar-crate-test _pc-check-excluded-manifests _pc-check-excluded-manifests-test _pc-check-ruff-lockstep _pc-check-ruff-lockstep-test _pc-check-rumdl-lockstep _pc-check-rumdl-lockstep-test _pc-check-publish-metadata _pc-check-publish-metadata-test _pc-check-manpage-assets _pc-check-manpage-drift-test _pc-check-diagnostic-prefix _pc-check-diagnostic-prefix-test _pc-check-feature-gates-test _pc-check-test-lang-gates _pc-check-test-lang-gates-test _pc-check-safety-doc-pin _pc-check-safety-doc-pin-test _pc-check-grammar-build-watch _pc-check-grammar-build-watch-test _pc-worktree-setup-test _pc-gate-status-test _pc-check-tools-test _pc-enums-check _pc-enums-codegen-drift _pc-enums-codegen-drift-test _pc-self-scan _pc-self-scan-headroom _pc-py-fmt _pc-py-typecheck _pc-py-test _pc-py-stubtest _ci-all _ci-fmt-check _ci-clippy _ci-test _ci-doc-check _ci-build _ci-udeps _ci-shellcheck _ci-markdown-lint _ci-toml-lint _ci-makefile-check _ci-actionlint _ci-snapshot-anchors _ci-snapshot-anchors-test _ci-rustfmt-bail _ci-rustfmt-bail-test _ci-role-census _ci-role-census-test _ci-role-coverage _ci-role-coverage-test _ci-role-bypass _ci-role-bypass-test _ci-mutation-report-test _ci-mutation-workflow-test _ci-drift-common-test _ci-drift-oracle-test _ci-drift-compare-test _ci-recovery-oracle-test _ci-perturb-test _ci-coverage-diff-test _ci-self-scan-bca-test _ci-grammar-marker-sync _ci-grammar-marker-sync-test _ci-check-versions _ci-check-versions-test _ci-check-grammar-crate-test _ci-check-excluded-manifests _ci-check-excluded-manifests-test _ci-check-ruff-lockstep _ci-check-ruff-lockstep-test _ci-check-rumdl-lockstep _ci-check-rumdl-lockstep-test _ci-check-publish-metadata _ci-check-publish-metadata-test _ci-check-manpage-assets _ci-check-manpage-drift-test _ci-check-diagnostic-prefix _ci-check-diagnostic-prefix-test _ci-check-feature-gates-test _ci-check-test-lang-gates _ci-check-test-lang-gates-test _ci-check-safety-doc-pin _ci-check-safety-doc-pin-test _ci-check-grammar-build-watch _ci-check-grammar-build-watch-test _ci-worktree-setup-test _ci-gate-status-test _ci-check-tools-test _ci-enums-check _ci-enums-codegen-drift _ci-enums-codegen-drift-test _ci-enums-codegen-drift-test _ci-self-scan _ci-self-scan-headroom _ci-cargo-pipeline _ci-py-fmt-check _ci-py-lint _ci-py-typecheck _ci-py-test _ci-py-stubtest .PHONY: plugin-test _pc-plugin-test _ci-plugin-test @@ -162,6 +162,8 @@ help: @echo " check-excluded-manifests-test Self-tests for the check-excluded-manifests gate" @echo " check-ruff-lockstep Assert the ruff-pre-commit rev, uv.lock, and the requirements export agree" @echo " check-ruff-lockstep-test Self-tests for the check-ruff-lockstep gate" + @echo " check-rumdl-lockstep Assert mise.toml, ci.yml and the Dockerfile pin one rumdl version and hash" + @echo " check-rumdl-lockstep-test Self-tests for the check-rumdl-lockstep gate" @echo " check-publish-metadata Assert every publishable crate carries the metadata crates.io needs" @echo " check-publish-metadata-test Self-tests for the check-publish-metadata gate" @echo " check-manpage-assets Assert every bca-*.1 man page is in deb+rpm asset lists" @@ -730,6 +732,24 @@ check-ruff-lockstep-test: @echo "Running check-ruff-lockstep self-tests..." @(cd $(BASE_DIR) && python3 -m unittest -q utils/check-ruff-lockstep-test.py) +# rumdl version lockstep (#1779). One rumdl release is pinned in three +# files that only comments kept in step: `rumdl` under [tools] in +# mise.toml, the CI lint job's RUMDL_VERSION / RUMDL_SHA256, and the +# Dockerfile's ENV RUMDL_VERSION plus per-arch SHA256 table. #1653's +# bump nearly left the Dockerfile behind, which would have linted the +# container on a different rumdl than CI. Also checks the Dockerfile's +# x86_64 hash equals CI's, since both verify the same tarball. Static +# lint — no network, no cargo. +check-rumdl-lockstep: + @echo "Checking rumdl version lockstep..." + @python3 $(BASE_DIR)utils/check-rumdl-lockstep.py + +# Self-tests for the rumdl-lockstep gate, kept separate for the same +# reason as the ruff-lockstep self-tests. +check-rumdl-lockstep-test: + @echo "Running check-rumdl-lockstep self-tests..." + @(cd $(BASE_DIR) && python3 -m unittest -q utils/check-rumdl-lockstep-test.py) + # Publish-metadata gate (#1224). `cargo publish --dry-run` is the # natural pre-tag check, but it cannot run for the four top-level # crates: each pins an internal dependency at `=`, and the @@ -1667,7 +1687,7 @@ lint: $(MAKE) -j --output-sync=target \ _ci-clippy _ci-plugin-test \ _ci-shellcheck _ci-markdown-lint _ci-toml-lint _ci-makefile-check \ - _ci-actionlint _ci-snapshot-anchors _ci-snapshot-anchors-test _ci-rustfmt-bail _ci-rustfmt-bail-test _ci-role-census _ci-role-census-test _ci-role-coverage _ci-role-coverage-test _ci-role-bypass _ci-role-bypass-test _ci-mutation-report-test _ci-mutation-workflow-test _ci-drift-common-test _ci-drift-oracle-test _ci-drift-compare-test _ci-recovery-oracle-test _ci-perturb-test _ci-coverage-diff-test _ci-self-scan-bca-test _ci-grammar-marker-sync _ci-grammar-marker-sync-test _ci-check-versions _ci-check-versions-test _ci-check-grammar-crate-test _ci-check-excluded-manifests _ci-check-excluded-manifests-test _ci-check-ruff-lockstep _ci-check-ruff-lockstep-test _ci-check-publish-metadata _ci-check-publish-metadata-test _ci-check-manpage-assets _ci-check-manpage-drift-test _ci-check-diagnostic-prefix _ci-check-diagnostic-prefix-test _ci-check-feature-gates-test _ci-check-test-lang-gates _ci-check-test-lang-gates-test _ci-check-safety-doc-pin _ci-check-safety-doc-pin-test _ci-check-grammar-build-watch _ci-check-grammar-build-watch-test _ci-worktree-setup-test _ci-gate-status-test _ci-check-tools-test _ci-enums-check _ci-enums-codegen-drift _ci-enums-codegen-drift-test + _ci-actionlint _ci-snapshot-anchors _ci-snapshot-anchors-test _ci-rustfmt-bail _ci-rustfmt-bail-test _ci-role-census _ci-role-census-test _ci-role-coverage _ci-role-coverage-test _ci-role-bypass _ci-role-bypass-test _ci-mutation-report-test _ci-mutation-workflow-test _ci-drift-common-test _ci-drift-oracle-test _ci-drift-compare-test _ci-recovery-oracle-test _ci-perturb-test _ci-coverage-diff-test _ci-self-scan-bca-test _ci-grammar-marker-sync _ci-grammar-marker-sync-test _ci-check-versions _ci-check-versions-test _ci-check-grammar-crate-test _ci-check-excluded-manifests _ci-check-excluded-manifests-test _ci-check-ruff-lockstep _ci-check-ruff-lockstep-test _ci-check-rumdl-lockstep _ci-check-rumdl-lockstep-test _ci-check-publish-metadata _ci-check-publish-metadata-test _ci-check-manpage-assets _ci-check-manpage-drift-test _ci-check-diagnostic-prefix _ci-check-diagnostic-prefix-test _ci-check-feature-gates-test _ci-check-test-lang-gates _ci-check-test-lang-gates-test _ci-check-safety-doc-pin _ci-check-safety-doc-pin-test _ci-check-grammar-build-watch _ci-check-grammar-build-watch-test _ci-worktree-setup-test _ci-gate-status-test _ci-check-tools-test _ci-enums-check _ci-enums-codegen-drift _ci-enums-codegen-drift-test # --------------------------------------------------------------------------- # Maintenance @@ -1854,7 +1874,7 @@ _pc-all: $(MAKE) -j --output-sync=target \ _pc-test _pc-plugin-test \ _pc-shellcheck _pc-markdown-lint _pc-toml-lint _pc-makefile-check \ - _pc-actionlint _pc-snapshot-anchors _pc-snapshot-anchors-test _pc-rustfmt-bail _pc-rustfmt-bail-test _pc-role-census _pc-role-census-test _pc-role-coverage _pc-role-coverage-test _pc-role-bypass _pc-role-bypass-test _pc-mutation-report-test _pc-mutation-workflow-test _pc-drift-common-test _pc-drift-oracle-test _pc-drift-compare-test _pc-recovery-oracle-test _pc-perturb-test _pc-coverage-diff-test _pc-self-scan-bca-test _pc-grammar-marker-sync _pc-grammar-marker-sync-test _pc-check-versions _pc-check-versions-test _pc-check-grammar-crate-test _pc-check-excluded-manifests _pc-check-excluded-manifests-test _pc-check-ruff-lockstep _pc-check-ruff-lockstep-test _pc-check-publish-metadata _pc-check-publish-metadata-test _pc-check-manpage-assets _pc-check-manpage-drift-test _pc-check-diagnostic-prefix _pc-check-diagnostic-prefix-test _pc-check-feature-gates-test _pc-check-test-lang-gates _pc-check-test-lang-gates-test _pc-check-safety-doc-pin _pc-check-safety-doc-pin-test _pc-check-grammar-build-watch _pc-check-grammar-build-watch-test _pc-worktree-setup-test _pc-gate-status-test _pc-check-tools-test _pc-enums-check _pc-enums-codegen-drift _pc-enums-codegen-drift-test \ + _pc-actionlint _pc-snapshot-anchors _pc-snapshot-anchors-test _pc-rustfmt-bail _pc-rustfmt-bail-test _pc-role-census _pc-role-census-test _pc-role-coverage _pc-role-coverage-test _pc-role-bypass _pc-role-bypass-test _pc-mutation-report-test _pc-mutation-workflow-test _pc-drift-common-test _pc-drift-oracle-test _pc-drift-compare-test _pc-recovery-oracle-test _pc-perturb-test _pc-coverage-diff-test _pc-self-scan-bca-test _pc-grammar-marker-sync _pc-grammar-marker-sync-test _pc-check-versions _pc-check-versions-test _pc-check-grammar-crate-test _pc-check-excluded-manifests _pc-check-excluded-manifests-test _pc-check-ruff-lockstep _pc-check-ruff-lockstep-test _pc-check-rumdl-lockstep _pc-check-rumdl-lockstep-test _pc-check-publish-metadata _pc-check-publish-metadata-test _pc-check-manpage-assets _pc-check-manpage-drift-test _pc-check-diagnostic-prefix _pc-check-diagnostic-prefix-test _pc-check-feature-gates-test _pc-check-test-lang-gates _pc-check-test-lang-gates-test _pc-check-safety-doc-pin _pc-check-safety-doc-pin-test _pc-check-grammar-build-watch _pc-check-grammar-build-watch-test _pc-worktree-setup-test _pc-gate-status-test _pc-check-tools-test _pc-enums-check _pc-enums-codegen-drift _pc-enums-codegen-drift-test \ _pc-manpages \ _pc-self-scan _pc-self-scan-headroom \ _pc-py-fmt _pc-py-typecheck _pc-py-test _pc-py-stubtest @@ -1864,7 +1884,7 @@ _ci-all: $(MAKE) -j --output-sync=target \ _ci-cargo-pipeline _ci-plugin-test \ _ci-shellcheck _ci-markdown-lint _ci-toml-lint _ci-makefile-check \ - _ci-actionlint _ci-snapshot-anchors _ci-snapshot-anchors-test _ci-rustfmt-bail _ci-rustfmt-bail-test _ci-role-census _ci-role-census-test _ci-role-coverage _ci-role-coverage-test _ci-role-bypass _ci-role-bypass-test _ci-mutation-report-test _ci-mutation-workflow-test _ci-drift-common-test _ci-drift-oracle-test _ci-drift-compare-test _ci-recovery-oracle-test _ci-perturb-test _ci-coverage-diff-test _ci-self-scan-bca-test _ci-grammar-marker-sync _ci-grammar-marker-sync-test _ci-check-versions _ci-check-versions-test _ci-check-grammar-crate-test _ci-check-excluded-manifests _ci-check-excluded-manifests-test _ci-check-ruff-lockstep _ci-check-ruff-lockstep-test _ci-check-publish-metadata _ci-check-publish-metadata-test _ci-check-manpage-assets _ci-check-manpage-drift-test _ci-check-diagnostic-prefix _ci-check-diagnostic-prefix-test _ci-check-feature-gates-test _ci-check-test-lang-gates _ci-check-test-lang-gates-test _ci-check-safety-doc-pin _ci-check-safety-doc-pin-test _ci-check-grammar-build-watch _ci-check-grammar-build-watch-test _ci-worktree-setup-test _ci-gate-status-test _ci-check-tools-test _ci-enums-check _ci-enums-codegen-drift _ci-enums-codegen-drift-test \ + _ci-actionlint _ci-snapshot-anchors _ci-snapshot-anchors-test _ci-rustfmt-bail _ci-rustfmt-bail-test _ci-role-census _ci-role-census-test _ci-role-coverage _ci-role-coverage-test _ci-role-bypass _ci-role-bypass-test _ci-mutation-report-test _ci-mutation-workflow-test _ci-drift-common-test _ci-drift-oracle-test _ci-drift-compare-test _ci-recovery-oracle-test _ci-perturb-test _ci-coverage-diff-test _ci-self-scan-bca-test _ci-grammar-marker-sync _ci-grammar-marker-sync-test _ci-check-versions _ci-check-versions-test _ci-check-grammar-crate-test _ci-check-excluded-manifests _ci-check-excluded-manifests-test _ci-check-ruff-lockstep _ci-check-ruff-lockstep-test _ci-check-rumdl-lockstep _ci-check-rumdl-lockstep-test _ci-check-publish-metadata _ci-check-publish-metadata-test _ci-check-manpage-assets _ci-check-manpage-drift-test _ci-check-diagnostic-prefix _ci-check-diagnostic-prefix-test _ci-check-feature-gates-test _ci-check-test-lang-gates _ci-check-test-lang-gates-test _ci-check-safety-doc-pin _ci-check-safety-doc-pin-test _ci-check-grammar-build-watch _ci-check-grammar-build-watch-test _ci-worktree-setup-test _ci-gate-status-test _ci-check-tools-test _ci-enums-check _ci-enums-codegen-drift _ci-enums-codegen-drift-test \ _ci-py-fmt-check _ci-py-lint _ci-py-typecheck _ci-py-test _ci-py-stubtest # --------------------------------------------------------------------------- @@ -1916,6 +1936,8 @@ _ci-all: # ├── _pc-check-versions # ├── _pc-check-ruff-lockstep # ├── _pc-check-ruff-lockstep-test +# ├── _pc-check-rumdl-lockstep +# ├── _pc-check-rumdl-lockstep-test # ├── _pc-check-publish-metadata # ├── _pc-check-publish-metadata-test # ├── _pc-check-diagnostic-prefix @@ -2088,6 +2110,12 @@ _pc-check-ruff-lockstep: _pc-fmt _pc-check-ruff-lockstep-test: _pc-fmt $(MAKE) check-ruff-lockstep-test +_pc-check-rumdl-lockstep: _pc-fmt + $(MAKE) check-rumdl-lockstep + +_pc-check-rumdl-lockstep-test: _pc-fmt + $(MAKE) check-rumdl-lockstep-test + _pc-check-publish-metadata: _pc-fmt $(MAKE) check-publish-metadata @@ -2356,6 +2384,12 @@ _ci-check-ruff-lockstep: _ci-check-ruff-lockstep-test: $(MAKE) check-ruff-lockstep-test +_ci-check-rumdl-lockstep: + $(MAKE) check-rumdl-lockstep + +_ci-check-rumdl-lockstep-test: + $(MAKE) check-rumdl-lockstep-test + _ci-check-publish-metadata: $(MAKE) check-publish-metadata diff --git a/big-code-analysis-ast/src/lang_helpers.rs b/big-code-analysis-ast/src/lang_helpers.rs index 40da8ae86..63f6cb1dc 100644 --- a/big-code-analysis-ast/src/lang_helpers.rs +++ b/big-code-analysis-ast/src/lang_helpers.rs @@ -22,6 +22,10 @@ pub(crate) mod case_arms; // `Checker` impls; only the run walk the LOC metric drives is // re-exported below. pub(crate) mod doc_comment; +// Crate-private: the per-tool doc-marker tests over a comment's bytes, +// split from the binding walk in `doc_comment` (#1649), which re-exports +// them so their callers keep one path. +pub(crate) mod doc_text; // Crate-private: Doxygen's attachment rule, read by the four C-family // `Checker` impls only (#1613). pub(crate) mod doxygen; diff --git a/big-code-analysis-ast/src/lang_helpers/doc_comment.rs b/big-code-analysis-ast/src/lang_helpers/doc_comment.rs index 414ed0458..957637a86 100644 --- a/big-code-analysis-ast/src/lang_helpers/doc_comment.rs +++ b/big-code-analysis-ast/src/lang_helpers/doc_comment.rs @@ -1,26 +1,14 @@ -//! Doc-comment recognition by bytes, for the grammars that emit one -//! comment kind for every comment form (#1597). -//! -//! Rust (a `doc_comment` child), Groovy (`groovydoc_comment`) and -//! Perl (`pod_statement`) mark documentation structurally. Every other -//! grammar parses `/** … */`, `///` or `---` to the same `comment` node -//! as a plain comment, so the opening marker is the only evidence, and -//! each language's own tool decides which markers count: Doxygen, javac, -//! Kotlin's lexer, JSDoc, PHP's scanner, Roslyn and LDoc disagree at the -//! edges (`/***`, `////`, `/**/`), and the predicates below follow each -//! tool rather than one rule for all. The slice they inspect is the -//! comment node's own span, which never reaches `to_string_lossy` -//! (grammar-dispatch rule 10): a comment's bytes may be any encoding, -//! and a marker is ASCII. -//! -//! Deliberate deviations, all on the strict side: `////` is never -//! documentation (tree-sitter-rust and Roslyn reject it; Doxygen's and -//! javac's lexers accept it, but a line of slashes is a banner, not -//! prose); the empty `/**/` is never documentation (javac lexes it as -//! an empty doc comment, which documents nothing); and a doc marker has -//! to open the comment — a `/**` found later in a comment's text is -//! text. - +//! Doc-comment binding: which comments document, by where they sit and +//! by the declaration after them (#1597, #1613). The per-tool marker +//! tests these rules consult live in [`super::doc_text`] and are +//! re-exported here, so a `Checker` or role row names one path for both +//! halves (#1649). + +pub(crate) use super::doc_text::{ + csharp_doc, doxygen, go_directive, java_doc, javadoc_block, jsdoc_virtual, kotlin_doc, + ldoc_opener, lua_long_comment, phpdoc, pod_comment_region, ruby_directive_opener, + ruby_magic_comment, tsdoc_block, tsdoc_module, +}; use crate::checker::Checker; use crate::node::{Ancestors, FoldLane, Node}; @@ -149,195 +137,6 @@ pub(crate) fn node_bytes<'a>(node: &Node, code: &'a [u8]) -> &'a [u8] { code.get(node.start_byte()..node.end_byte()).unwrap_or(&[]) } -/// `/**` opened by a byte that is neither `*` nor `/`: the Javadoc / -/// JSDoc / Doxygen block form. Rejects the empty `/**/` and the banner -/// `/***`, as JSDoc ("more than 3 stars will be ignored"), Doxygen -/// (`JAVADOC_BANNER` defaults to `NO`) and Roslyn do. -#[inline] -#[must_use] -pub(crate) fn javadoc_block(s: &[u8]) -> bool { - matches!(s, [b'/', b'*', b'*', b, ..] if *b != b'*' && *b != b'/') -} - -/// A JSDoc comment as tsc reads it (`isJSDocLikeText`): `/**` followed -/// by any byte but `/`. Unlike JSDoc itself it takes the banner `/***`; -/// only the empty `/**/` is rejected. TypeScript and TSX follow tsc, since -/// tsc is what reads their doc comments (#1597). -#[inline] -#[must_use] -pub(crate) fn tsdoc_block(s: &[u8]) -> bool { - matches!(s, [b'/', b'*', b'*', b, ..] if *b != b'/') -} - -/// `///` not followed by a fourth slash: the C# / Doxygen / Rust line -/// form, including the after-member `///<`. Rejects `////`. -#[inline] -#[must_use] -fn triple_slash(s: &[u8]) -> bool { - matches!(s, [b'/', b'/', b'/', rest @ ..] if rest.first() != Some(&b'/')) -} - -/// A Doxygen comment block in a C-family language: the Javadoc block, -/// the Qt `/*!` block, and the `///` / `//!` line forms, each also in -/// its after-member `<` spelling (Doxygen, "Comment blocks for C-like -/// languages"). `////` is rejected — stricter than Doxygen's own lexer, -/// see the module doc. -#[inline] -#[must_use] -pub(crate) fn doxygen(s: &[u8]) -> bool { - javadoc_block(s) || s.starts_with(b"/*!") || triple_slash(s) || s.starts_with(b"//!") -} - -/// A Javadoc comment: any `/**` block but the empty `/**/` (a third -/// `*` is just a leading asterisk the spec discards), plus the JEP 467 -/// Markdown form `///`. Javac's lexer also takes `/**/` and `////` as -/// doc comments; both are rejected here, as the module doc says. -#[inline] -#[must_use] -pub(crate) fn java_doc(s: &[u8]) -> bool { - kotlin_doc(s) || triple_slash(s) -} - -/// A KDoc block as Kotlin's lexer tokenises it: `/**` except the empty -/// `/**/`. `/***` is KDoc; line comments never are. -#[inline] -#[must_use] -pub(crate) fn kotlin_doc(s: &[u8]) -> bool { - s.starts_with(b"/**") && s != b"/**/" -} - -/// A PHPDoc block as the Zend scanner tokenises `T_DOC_COMMENT`: `/**` -/// followed by whitespace. `/**/`, `/***` and `/**x` are plain comments. -#[inline] -#[must_use] -pub(crate) fn phpdoc(s: &[u8]) -> bool { - s.starts_with(b"/**") && s.get(3).is_some_and(u8::is_ascii_whitespace) -} - -/// A C# XML documentation comment as Roslyn lexes it: `///` (not -/// `////`) or `/**` (not `/**/` nor `/***`). `//!` and `/*!` are the -/// Doxygen spellings and are plain comments here. -#[inline] -#[must_use] -pub(crate) fn csharp_doc(s: &[u8]) -> bool { - triple_slash(s) || javadoc_block(s) -} - -/// A doc comment's line with the delimiters JSDoc's `unwrap` strips: the -/// indent, one leading `*` and the space after it. -fn jsdoc_line(line: &[u8]) -> &[u8] { - let line = line.trim_ascii_start(); - let line = line.strip_prefix(b"*").unwrap_or(line); - line.strip_prefix(b" ").unwrap_or(line) -} - -/// The lines of a `/** … */` body, delimiters and the extra stars at -/// either end stripped (`unwrap`), each with its leading ` * ` stripped -/// ([`jsdoc_line`]). `Clone`, so a tag's text — every line up to the next -/// tag, as JSDoc's `toTags` reads it — can be read in place from the line -/// that opened it. -fn jsdoc_lines(s: &[u8]) -> impl Iterator + Clone { - let body = s.strip_prefix(b"/**").unwrap_or(s); - let body = body.strip_suffix(b"*/").unwrap_or(body); - let start = body.iter().position(|&b| b != b'*').unwrap_or(body.len()); - let end = body - .iter() - .rposition(|&b| b != b'*') - .map_or(start, |at| at + 1); - body[start..end].split(|&b| b == b'\n').map(jsdoc_line) -} - -/// The `(title, text)` a line opens when it starts a tag. -fn jsdoc_tag_open(line: &[u8]) -> Option<(&[u8], &[u8])> { - let tag = line.trim_ascii_start().strip_prefix(b"@")?; - let title = tag.iter().take_while(|b| !b.is_ascii_whitespace()).count(); - (title > 0).then(|| tag.split_at(title)) -} - -/// Whether a JSDoc tag's text names a symbol: anything left once the -/// `{type}` that may lead it is set aside. -fn jsdoc_names<'a>(lines: impl IntoIterator) -> bool { - // The lines joined, read once without joining them: a `{type}` is - // skipped by brace depth, and anything non-blank after it names. - let mut bytes = lines - .into_iter() - .flat_map(|line| line.iter().copied()) - .skip_while(u8::is_ascii_whitespace) - .peekable(); - if bytes.peek() == Some(&b'{') { - let mut depth = 0_usize; - for b in bytes.by_ref() { - match b { - b'{' => depth += 1, - b'}' => depth -= 1, - _ => {} - } - if depth == 0 { - break; - } - } - } - bytes.any(|b| !b.is_ascii_whitespace()) -} - -/// A JSDoc *virtual* comment, which stands alone: it documents the symbol -/// its tags name rather than the code after it, and the code after it gets -/// no doc from it (`createSymbolDoclet` drops a comment that set a name). -/// `@file`, `@fileoverview`, `@overview` and `@module` always set one; the -/// tags that take a name set one only when given it. `@license` sets none, -/// so a licence comment documents what follows like any other (#1613). -/// Titles match in any case, as the tag dictionary's `normalize` does. -#[must_use] -pub(crate) fn jsdoc_virtual(s: &[u8]) -> bool { - const ALWAYS: [&[u8]; 4] = [b"file", b"fileoverview", b"overview", b"module"]; - const NAMING: [&[u8]; 18] = [ - b"name", - b"function", - b"func", - b"method", - b"class", - b"constructor", - b"constant", - b"const", - b"member", - b"var", - b"namespace", - b"mixin", - b"interface", - b"event", - b"external", - b"host", - b"typedef", - b"callback", - ]; - let is = |title: &[u8], names: &[&[u8]]| names.iter().any(|n| title.eq_ignore_ascii_case(n)); - let mut lines = jsdoc_lines(s); - while let Some(line) = lines.next() { - let Some((title, text)) = jsdoc_tag_open(line) else { - continue; - }; - if is(title, &ALWAYS) { - return true; - } - // The tag's text runs over every line up to the next tag. - let rest = lines.clone().take_while(|l| jsdoc_tag_open(l).is_none()); - if is(title, &NAMING) && jsdoc_names(std::iter::once(text).chain(rest)) { - return true; - } - } - false -} - -/// A TypeDoc module comment: one carrying `@module` or -/// `@packageDocumentation`, which `getCommentImpl` withholds from any -/// declaration and reads as the file's documentation (#1613). -#[must_use] -pub(crate) fn tsdoc_module(s: &[u8]) -> bool { - jsdoc_lines(s) - .filter_map(jsdoc_tag_open) - .any(|(title, _)| title == b"module" || title == b"packageDocumentation") -} - /// Whether `comment` lies outside every function and closure body, by the /// same fold [`documents_where_it_sits`] runs, for a language whose /// in-body doc comments document wherever they sit and so must not bind. @@ -849,167 +648,6 @@ fn run_first_member<'a, T: Checker>( } } -/// A go/doc directive line, part of a lead comment group but not of -/// its text (go.dev/doc/comment, "Directives"): `//go:build`, -/// `//line `, `//extern `, `//export `, and the general -/// `//[a-z0-9]+:[a-z0-9]` form. A directive has no space after `//`. -#[must_use] -pub(crate) fn go_directive(s: &[u8]) -> bool { - let Some(rest) = s.strip_prefix(b"//") else { - return false; - }; - if rest.starts_with(b"line ") || rest.starts_with(b"extern ") || rest.starts_with(b"export ") { - return true; - } - let word = rest - .iter() - .take_while(|b| b.is_ascii_lowercase() || b.is_ascii_digit()) - .count(); - word > 0 - && rest.get(word) == Some(&b':') - && rest - .get(word.saturating_add(1)) - .is_some_and(|b| b.is_ascii_lowercase() || b.is_ascii_digit()) -} - -/// A Ruby comment that opens a documentation block whatever follows it: -/// RDoc's `##`, which introduces a metaprogrammed method's docs -/// (`RDoc::Parser::Ruby`), or a YARD `@!` directive (`# @!method`, -/// `# @!attribute`, `# @!macro`, `# @!parse`), which documents a DSL call -/// rather than a `def` (#1597). The `##` is exactly two `#`: YARD sets its -/// hash flag only for a `##` prefix and RDoc reads a lone `##` line, so a -/// `####…` banner or a `##### Helpers #####` divider opens nothing. -#[must_use] -pub(crate) fn ruby_directive_opener(s: &[u8]) -> bool { - if s.starts_with(b"##") && s.get(2) != Some(&b'#') { - return true; - } - s.strip_prefix(b"#").is_some_and(|rest| { - rest.iter() - .position(|b| *b != b' ' && *b != b'\t') - .is_some_and(|at| rest[at..].starts_with(b"@!")) - }) -} - -/// The comment lines YARD drops before documenting: a shebang, an -/// encoding line and the `frozen_string_literal` magic comment, as -/// YARD's `SHEBANG_LINE`, `ENCODING_LINE` and `FROZEN_STRING_LINE` name -/// them, case-insensitively. YARD applies them only before the first -/// token; here a line is excluded wherever it sits, so the match is on -/// the whole comment — `# coding: utf-8`, `# -*- coding: utf-8 -*-`, -/// `# vim: set fileencoding=utf-8 :`, `# frozen_string_literal: true` — -/// and a documentation row that only mentions an encoding (`# Encoding: -/// defaults to UTF-8.`) stays documentation. What this loses is a -/// comment that repeats a magic line mid-file. -#[must_use] -pub(crate) fn ruby_magic_comment(s: &[u8]) -> bool { - if s.starts_with(b"#!") { - return true; - } - s.strip_prefix(b"#") - .map(<[u8]>::trim_ascii) - .is_some_and(|body| ruby_modeline(body) || ruby_magic_setting(body)) -} - -fn contains_ci(hay: &[u8], needle: &[u8]) -> bool { - hay.windows(needle.len()) - .any(|window| window.eq_ignore_ascii_case(needle)) -} - -/// An Emacs (`-*- … -*-`) or Vim (`vim: …`) modeline that carries the -/// magic setting among others: an encoding, or for Emacs the -/// frozen-string flag. A modeline's key is never a magic one, so -/// `ruby_magic_setting` cannot also match it. -fn ruby_modeline(body: &[u8]) -> bool { - if body.len() >= 6 && body.starts_with(b"-*-") && body.ends_with(b"-*-") { - return contains_ci(body, b"coding") || contains_ci(body, b"frozen"); - } - body.get(..4) - .is_some_and(|head| head.eq_ignore_ascii_case(b"vim:")) - && contains_ci(body, b"coding") -} - -/// A comment that is one magic setting and nothing else: `key: value` -/// or `key=value`, the key a magic one and the value a single word. -fn ruby_magic_setting(body: &[u8]) -> bool { - let Some(separator) = body.iter().position(|b| matches!(b, b':' | b'=')) else { - return false; - }; - let (key, value) = body.split_at(separator); - let value = value.get(1..).unwrap_or_default().trim_ascii_start(); - [ - &b"coding"[..], - b"encoding", - b"frozen_string_literal", - b"frozen-string-literal", - ] - .iter() - .any(|known| key.trim_ascii_end().eq_ignore_ascii_case(known)) - && !value.is_empty() - && value - .iter() - .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) -} - -/// A POD block that is nothing but `comment` regions — `=begin comment` -/// … `=end comment` or a `=for comment` paragraph — which perlpod leaves -/// to no formatter and perlfaq7 recommends for commenting code out: POD -/// syntax, not documentation. A region covers only its own text, as -/// perlpod scopes it: a `=for comment` paragraph ends at the next blank -/// line and a `=begin comment` region at its `=end comment`, so a block -/// that goes on to document something (`=head1 NAME` …) is documentation. -#[must_use] -pub(crate) fn pod_comment_region(s: &[u8]) -> bool { - enum Region { - Outside, - Paragraph, - Begin, - } - let mut region = Region::Outside; - let mut regions = 0_usize; - for line in s.split(|&byte| byte == b'\n') { - let blank = line.trim_ascii().is_empty(); - match region { - Region::Paragraph if blank => region = Region::Outside, - Region::Begin if pod_command(line, b"=end").is_some_and(is_comment_format) => { - region = Region::Outside; - } - Region::Paragraph | Region::Begin => {} - Region::Outside if blank => {} - Region::Outside => { - if pod_command(line, b"=for").is_some_and(is_comment_format) { - region = Region::Paragraph; - } else if pod_command(line, b"=begin").is_some_and(is_comment_format) { - region = Region::Begin; - } else { - // `=cut` closes the block; any other line documents, or - // opens a block that never was a comment region. - return regions > 0 && line.starts_with(b"=cut"); - } - regions += 1; - } - } - } - regions > 0 -} - -/// The text after a POD command line's `=` and the whitespace -/// separating it from its argument, or `None` when `line` is not that -/// command. -fn pod_command<'s>(line: &'s [u8], command: &[u8]) -> Option<&'s [u8]> { - let rest = line.strip_prefix(command)?; - rest.first() - .is_some_and(u8::is_ascii_whitespace) - .then(|| rest.trim_ascii_start()) -} - -/// Whether a POD command's format name is `comment`. -fn is_comment_format(format: &[u8]) -> bool { - format - .strip_prefix(b"comment") - .is_some_and(|after| after.first().is_none_or(u8::is_ascii_whitespace)) -} - /// The trailing comment that heads `first`'s LDoc group, when `first` sits /// directly under a row that ends in one: LDoc lexes a line comment with /// its newline and gathers every consecutive comment token into one group @@ -1028,47 +666,6 @@ pub(crate) fn lua_group_head_above<'a, T: Checker>( .filter(|above| T::is_comment(above) && run_continues_to::(above, first, code)) } -/// An LDoc block opener, per `lang.lua`: a line comment starting with -/// three or more dashes (`start_comment_ = '^%-%-%-+'`, so a line of -/// dashes alone opens too) unless it ends in a dash run preceded by a -/// non-dash (`end_comment_ = '[^%-]%-%-+[^-]*\n$'` — `--- x ---` is an -/// end marker), or a long comment whose body opens with a dash -/// (`block_comment = '^%-%-%[=*%[%-+'`). -#[must_use] -pub(crate) fn ldoc_opener(s: &[u8]) -> bool { - if let Some(body) = lua_long_comment_body(s) { - return body.first() == Some(&b'-'); - } - s.starts_with(b"---") && !ldoc_end_marker(s) -} - -/// A Lua long comment, `--[[ … ]]` or a levelled `--[==[ … ]==]`, as -/// opposed to a line comment that merely starts `--[`. -#[must_use] -pub(crate) fn lua_long_comment(s: &[u8]) -> bool { - lua_long_comment_body(s).is_some() -} - -/// The text after a long comment's opening bracket, or `None` when `s` -/// is not a long comment. -fn lua_long_comment_body(s: &[u8]) -> Option<&[u8]> { - let rest = s.strip_prefix(b"--[")?; - let levels = rest.iter().take_while(|b| **b == b'=').count(); - rest.get(levels..)?.strip_prefix(b"[") -} - -/// `[^-]--+[^-]*$`: the line's final dash run has two or more dashes and -/// something other than a dash before it. -fn ldoc_end_marker(s: &[u8]) -> bool { - let trimmed = s.len() - s.iter().rev().take_while(|b| **b != b'-').count(); - let dashes = s[..trimmed] - .iter() - .rev() - .take_while(|b| **b == b'-') - .count(); - dashes >= 2 && trimmed > dashes -} - #[cfg(test)] mod tests { use super::*; @@ -1096,22 +693,6 @@ mod tests { assert!(binding.gathers.is_none()); } - #[test] - fn ruby_directive_opener_accepts_rdoc_and_yard_forms() { - assert!(ruby_directive_opener(b"##")); - assert!(ruby_directive_opener(b"## Section")); - assert!(ruby_directive_opener(b"# @!method foo")); - assert!(ruby_directive_opener(b"#\t@!macro bar")); - assert!(!ruby_directive_opener(b"# plain")); - assert!(!ruby_directive_opener(b"# @param x")); - assert!(!ruby_directive_opener(b"#!/usr/bin/env ruby")); - // A banner or divider: three or more `#` are neither RDoc's lone - // `##` nor YARD's `##` prefix. - assert!(!ruby_directive_opener(b"###")); - assert!(!ruby_directive_opener(b"########################")); - assert!(!ruby_directive_opener(b"##### Private helpers #####")); - } - /// The walk back from a stretch's last comment reads past the parse /// errors the scan forward from each member read past, so a member /// whose scan stopped at the next comment is decided by the stretch @@ -1145,227 +726,6 @@ mod tests { assert_eq!(credited, [1, 3]); } - #[test] - fn javadoc_block_accepts_the_block_form_only() { - assert!(javadoc_block(b"/** doc */")); - assert!(javadoc_block(b"/**\n * doc\n */")); - assert!(!javadoc_block(b"/**/"), "empty block"); - assert!(!javadoc_block(b"/*** banner */"), "three stars"); - assert!(!javadoc_block(b"/* plain */")); - assert!(!javadoc_block(b"// plain")); - assert!(!javadoc_block(b"/**"), "nothing after the marker"); - } - - #[test] - fn triple_slash_rejects_a_fourth_slash() { - assert!(triple_slash(b"/// doc")); - assert!(triple_slash(b"///"), "bare marker still opens a doc line"); - assert!(triple_slash(b"///< after member")); - assert!(!triple_slash(b"//// banner")); - assert!(!triple_slash(b"// plain")); - } - - #[test] - fn doxygen_covers_the_four_markers_and_their_member_forms() { - for doc in [ - &b"/** d */"[..], - b"/*! d */", - b"/// d", - b"//! d", - b"/**< d */", - b"/*!< d */", - b"///< d", - b"//!< d", - ] { - assert!(doxygen(doc), "{}", String::from_utf8_lossy(doc)); - } - for plain in [&b"/* d */"[..], b"// d", b"/**/", b"/*** d */", b"//// d"] { - assert!(!doxygen(plain), "{}", String::from_utf8_lossy(plain)); - } - } - - #[test] - fn java_doc_follows_javac_on_three_stars_and_jep_467() { - assert!(java_doc(b"/** d */")); - assert!(java_doc(b"/*** d */"), "javac discards leading asterisks"); - assert!(java_doc(b"/// d"), "JEP 467 Markdown doc comment"); - assert!(!java_doc(b"/**/")); - assert!(!java_doc(b"//// d")); - assert!(!java_doc(b"/* d */")); - assert!(!java_doc(b"// d")); - } - - #[test] - fn kotlin_doc_is_any_block_but_the_empty_one() { - assert!(kotlin_doc(b"/** d */")); - assert!(kotlin_doc(b"/*** d */")); - assert!(!kotlin_doc(b"/**/")); - assert!(!kotlin_doc(b"/// d"), "KDoc has no line form"); - } - - #[test] - fn phpdoc_needs_whitespace_after_the_marker() { - assert!(phpdoc(b"/** d */")); - assert!(phpdoc(b"/**\n * d\n */")); - assert!(!phpdoc(b"/**/")); - assert!(!phpdoc(b"/*** d */")); - assert!(!phpdoc(b"/**d */")); - assert!(!phpdoc(b"# d")); - assert!(!phpdoc(b"// d")); - } - - #[test] - fn csharp_doc_rejects_the_doxygen_only_spellings() { - assert!(csharp_doc(b"/// d")); - assert!(csharp_doc(b"/** d */")); - for plain in [ - &b"//! d"[..], - b"/*! d */", - b"//// d", - b"/**/", - b"/*** d */", - b"// d", - ] { - assert!(!csharp_doc(plain), "{}", String::from_utf8_lossy(plain)); - } - } - - /// Each row is what `jsdoc -X` 4.x made of the comment above a call: - /// a doclet of its own (virtual) or nothing. A name may sit on the - /// line after its tag, and a tag inline in the description is text. - #[test] - fn jsdoc_virtual_follows_jsdoc_naming_tags() { - crate::test_support::assert_classifies( - jsdoc_virtual, - &[ - b"/** @file */", - b"/**\n * @fileoverview x\n */", - b"/** @overview */", - b"/** @module */", - b"/**\n * @typedef {Object} T\n */", - b"/**\n * @typedef {Object}\n * T\n */", - b"/**\n * @function f\n */", - b"/**\n * @callback cb\n */", - b"/** @FILE */", - b"/** @FileOverview */", - b"/** @Typedef {Object} T */", - b"/** @function f **/", - ], - &[ - b"/**\n * @license MIT\n */", - b"/**\n * @typedef {Object}\n */", - b"/**\n * @function\n */", - b"/** @function **/", - // A nested `{type}` is one type: no name follows it. - b"/**\n * @typedef {{a: number}}\n */", - b"/** x @file */", - b"/** d */", - ], - ); - assert!(tsdoc_module(b"/**\n * @packageDocumentation\n */")); - assert!(tsdoc_module(b"/**\n * @module m\n */")); - assert!(!tsdoc_module(b"/**\n * @license MIT\n */")); - } - - #[test] - fn go_directive_matches_go_ast_is_directive() { - for directive in [ - &b"//go:build linux"[..], - b"//go:generate go run x", - b"//line foo.go:12", - b"//extern f", - b"//export F", - b"//nolint:errcheck", - b"//x:1", - ] { - assert!( - go_directive(directive), - "{}", - String::from_utf8_lossy(directive) - ); - } - for text in [ - &b"// go:build linux"[..], - b"//Note: not a directive", - b"//x:", - b"//exported F", - b"// d", - ] { - assert!(!go_directive(text), "{}", String::from_utf8_lossy(text)); - } - } - - /// A block is a comment only while nothing but `comment` regions fill - /// it: POD after a region's end (`=head1 NAME` past a `=for comment` - /// paragraph or an `=end comment`) is documentation pod2text renders. - #[test] - fn pod_comment_region_is_the_comment_format_only() { - for region in [ - &b"=begin comment\n\nold();\n\n=end comment\n\n=cut"[..], - b"=for comment\nold();\n\n=cut", - b"=begin comment\n=end comment\n=cut", - b"=for comment\nold();\n", - b"=begin comment\nold();\n", - b"=for comment a\n\n=for comment b\n\n=cut", - b"=begin comment\nx\n=end comment\n=for comment y\n\n=cut", - b"=for comment\r\nold();\r\n\r\n=cut\r\n", - ] { - assert!( - pod_comment_region(region), - "{}", - String::from_utf8_lossy(region) - ); - } - for doc in [ - &b"=pod\n\nText.\n\n=cut"[..], - b"=head1 NAME\n\nFoo\n\n=cut", - b"=begin html\n\n

x

\n\n=end html\n\n=cut", - b"=begin comments\n=end comments\n=cut", - b"=beginning", - b"=for comment\nInternal note.\n\n=head1 NAME\n\nFoo - bar\n\n=cut", - b"=begin comment\n\nold();\n\n=end comment\n\n=head1 X\n\nDoc.\n\n=cut", - b"=for comment\nnote\n\nPlain text paragraph.\n\n=cut", - ] { - assert!(!pod_comment_region(doc), "{}", String::from_utf8_lossy(doc)); - } - } - - /// The match is on the whole comment, because it is applied anywhere - /// in the file: a documentation row that mentions an encoding, or - /// names `frozen_string_literal` in prose, is not a magic line. - #[test] - fn ruby_magic_comment_matches_whole_magic_lines_only() { - for magic in [ - &b"#!/usr/bin/env ruby"[..], - b"# coding: utf-8", - b"# encoding: UTF-8", - b"# Encoding: binary", - b"#coding=utf-8", - b"# -*- coding: utf-8 -*-", - b"# -*- mode: ruby; encoding: utf-8 -*-", - b"# -*- frozen_string_literal: true -*-", - b"# vim: set fileencoding=utf-8 :", - b"# frozen_string_literal: true", - b"# frozen-string-literal: false", - ] { - assert!( - ruby_magic_comment(magic), - "{}", - String::from_utf8_lossy(magic) - ); - } - for doc in [ - &b"# Encoding: defaults to UTF-8."[..], - b"# Handles transcoding: see below", - b"# Pass `frozen_string_literal: true` to opt in.", - b"# Sets the encoding= attribute", - b"# d", - b"#", - ] { - assert!(!ruby_magic_comment(doc), "{}", String::from_utf8_lossy(doc)); - } - } - /// A wide parse `ERROR` holding a `package` — whose target test asks /// for a *sibling* — is judged on a derived chain that lends no /// `SiblingCache`, so the lookup seeks rather than asking a cache diff --git a/big-code-analysis-ast/src/lang_helpers/doc_text.rs b/big-code-analysis-ast/src/lang_helpers/doc_text.rs new file mode 100644 index 000000000..ce2104821 --- /dev/null +++ b/big-code-analysis-ast/src/lang_helpers/doc_text.rs @@ -0,0 +1,659 @@ +//! Doc-comment recognition by bytes, for the grammars that emit one +//! comment kind for every comment form (#1597). +//! +//! Rust (a `doc_comment` child), Groovy (`groovydoc_comment`) and +//! Perl (`pod_statement`) mark documentation structurally. Every other +//! grammar parses `/** … */`, `///` or `---` to the same `comment` node +//! as a plain comment, so the opening marker is the only evidence, and +//! each language's own tool decides which markers count: Doxygen, javac, +//! Kotlin's lexer, JSDoc, PHP's scanner, Roslyn and LDoc disagree at the +//! edges (`/***`, `////`, `/**/`), and the predicates below follow each +//! tool rather than one rule for all. The slice they inspect is the +//! comment node's own span, which never reaches `to_string_lossy` +//! (grammar-dispatch rule 10): a comment's bytes may be any encoding, +//! and a marker is ASCII. +//! +//! Deliberate deviations, all on the strict side: `////` is never +//! documentation (tree-sitter-rust and Roslyn reject it; Doxygen's and +//! javac's lexers accept it, but a line of slashes is a banner, not +//! prose); the empty `/**/` is never documentation (javac lexes it as +//! an empty doc comment, which documents nothing); and a doc marker has +//! to open the comment — a `/**` found later in a comment's text is +//! text. +//! +//! Every test here reads bytes only and never a `Node`; which of the +//! comments they accept document, and what, is the binding walk's +//! question in [`super::doc_comment`] (#1649). + +/// `/**` opened by a byte that is neither `*` nor `/`: the Javadoc / +/// JSDoc / Doxygen block form. Rejects the empty `/**/` and the banner +/// `/***`, as JSDoc ("more than 3 stars will be ignored"), Doxygen +/// (`JAVADOC_BANNER` defaults to `NO`) and Roslyn do. +#[inline] +#[must_use] +pub(crate) fn javadoc_block(s: &[u8]) -> bool { + matches!(s, [b'/', b'*', b'*', b, ..] if *b != b'*' && *b != b'/') +} + +/// A JSDoc comment as tsc reads it (`isJSDocLikeText`): `/**` followed +/// by any byte but `/`. Unlike JSDoc itself it takes the banner `/***`; +/// only the empty `/**/` is rejected. TypeScript and TSX follow tsc, since +/// tsc is what reads their doc comments (#1597). +#[inline] +#[must_use] +pub(crate) fn tsdoc_block(s: &[u8]) -> bool { + matches!(s, [b'/', b'*', b'*', b, ..] if *b != b'/') +} + +/// `///` not followed by a fourth slash: the C# / Doxygen / Rust line +/// form, including the after-member `///<`. Rejects `////`. +#[inline] +#[must_use] +fn triple_slash(s: &[u8]) -> bool { + matches!(s, [b'/', b'/', b'/', rest @ ..] if rest.first() != Some(&b'/')) +} + +/// A Doxygen comment block in a C-family language: the Javadoc block, +/// the Qt `/*!` block, and the `///` / `//!` line forms, each also in +/// its after-member `<` spelling (Doxygen, "Comment blocks for C-like +/// languages"). `////` is rejected — stricter than Doxygen's own lexer, +/// see the module doc. +#[inline] +#[must_use] +pub(crate) fn doxygen(s: &[u8]) -> bool { + javadoc_block(s) || s.starts_with(b"/*!") || triple_slash(s) || s.starts_with(b"//!") +} + +/// A Javadoc comment: any `/**` block but the empty `/**/` (a third +/// `*` is just a leading asterisk the spec discards), plus the JEP 467 +/// Markdown form `///`. Javac's lexer also takes `/**/` and `////` as +/// doc comments; both are rejected here, as the module doc says. +#[inline] +#[must_use] +pub(crate) fn java_doc(s: &[u8]) -> bool { + kotlin_doc(s) || triple_slash(s) +} + +/// A KDoc block as Kotlin's lexer tokenises it: `/**` except the empty +/// `/**/`. `/***` is KDoc; line comments never are. +#[inline] +#[must_use] +pub(crate) fn kotlin_doc(s: &[u8]) -> bool { + s.starts_with(b"/**") && s != b"/**/" +} + +/// A PHPDoc block as the Zend scanner tokenises `T_DOC_COMMENT`: `/**` +/// followed by whitespace. `/**/`, `/***` and `/**x` are plain comments. +#[inline] +#[must_use] +pub(crate) fn phpdoc(s: &[u8]) -> bool { + s.starts_with(b"/**") && s.get(3).is_some_and(u8::is_ascii_whitespace) +} + +/// A C# XML documentation comment as Roslyn lexes it: `///` (not +/// `////`) or `/**` (not `/**/` nor `/***`). `//!` and `/*!` are the +/// Doxygen spellings and are plain comments here. +#[inline] +#[must_use] +pub(crate) fn csharp_doc(s: &[u8]) -> bool { + triple_slash(s) || javadoc_block(s) +} + +/// A doc comment's line with the delimiters JSDoc's `unwrap` strips: the +/// indent, one leading `*` and the space after it. +fn jsdoc_line(line: &[u8]) -> &[u8] { + let line = line.trim_ascii_start(); + let line = line.strip_prefix(b"*").unwrap_or(line); + line.strip_prefix(b" ").unwrap_or(line) +} + +/// The lines of a `/** … */` body, delimiters and the extra stars at +/// either end stripped (`unwrap`), each with its leading ` * ` stripped +/// ([`jsdoc_line`]). `Clone`, so a tag's text — every line up to the next +/// tag, as JSDoc's `toTags` reads it — can be read in place from the line +/// that opened it. +fn jsdoc_lines(s: &[u8]) -> impl Iterator + Clone { + let body = s.strip_prefix(b"/**").unwrap_or(s); + let body = body.strip_suffix(b"*/").unwrap_or(body); + let start = body.iter().position(|&b| b != b'*').unwrap_or(body.len()); + let end = body + .iter() + .rposition(|&b| b != b'*') + .map_or(start, |at| at + 1); + body[start..end].split(|&b| b == b'\n').map(jsdoc_line) +} + +/// The `(title, text)` a line opens when it starts a tag. +fn jsdoc_tag_open(line: &[u8]) -> Option<(&[u8], &[u8])> { + let tag = line.trim_ascii_start().strip_prefix(b"@")?; + let title = tag.iter().take_while(|b| !b.is_ascii_whitespace()).count(); + (title > 0).then(|| tag.split_at(title)) +} + +/// Whether a JSDoc tag's text names a symbol: anything left once the +/// `{type}` that may lead it is set aside. +fn jsdoc_names<'a>(lines: impl IntoIterator) -> bool { + // The lines joined, read once without joining them: a `{type}` is + // skipped by brace depth, and anything non-blank after it names. + let mut bytes = lines + .into_iter() + .flat_map(|line| line.iter().copied()) + .skip_while(u8::is_ascii_whitespace) + .peekable(); + if bytes.peek() == Some(&b'{') { + let mut depth = 0_usize; + for b in bytes.by_ref() { + match b { + b'{' => depth += 1, + b'}' => depth -= 1, + _ => {} + } + if depth == 0 { + break; + } + } + } + bytes.any(|b| !b.is_ascii_whitespace()) +} + +/// A JSDoc *virtual* comment, which stands alone: it documents the symbol +/// its tags name rather than the code after it, and the code after it gets +/// no doc from it (`createSymbolDoclet` drops a comment that set a name). +/// `@file`, `@fileoverview`, `@overview` and `@module` always set one; the +/// tags that take a name set one only when given it. `@license` sets none, +/// so a licence comment documents what follows like any other (#1613). +/// Titles match in any case, as the tag dictionary's `normalize` does. +#[must_use] +pub(crate) fn jsdoc_virtual(s: &[u8]) -> bool { + const ALWAYS: [&[u8]; 4] = [b"file", b"fileoverview", b"overview", b"module"]; + const NAMING: [&[u8]; 18] = [ + b"name", + b"function", + b"func", + b"method", + b"class", + b"constructor", + b"constant", + b"const", + b"member", + b"var", + b"namespace", + b"mixin", + b"interface", + b"event", + b"external", + b"host", + b"typedef", + b"callback", + ]; + let is = |title: &[u8], names: &[&[u8]]| names.iter().any(|n| title.eq_ignore_ascii_case(n)); + let mut lines = jsdoc_lines(s); + while let Some(line) = lines.next() { + let Some((title, text)) = jsdoc_tag_open(line) else { + continue; + }; + if is(title, &ALWAYS) { + return true; + } + // The tag's text runs over every line up to the next tag. + let rest = lines.clone().take_while(|l| jsdoc_tag_open(l).is_none()); + if is(title, &NAMING) && jsdoc_names(std::iter::once(text).chain(rest)) { + return true; + } + } + false +} + +/// A TypeDoc module comment: one carrying `@module` or +/// `@packageDocumentation`, which `getCommentImpl` withholds from any +/// declaration and reads as the file's documentation (#1613). +#[must_use] +pub(crate) fn tsdoc_module(s: &[u8]) -> bool { + jsdoc_lines(s) + .filter_map(jsdoc_tag_open) + .any(|(title, _)| title == b"module" || title == b"packageDocumentation") +} + +/// A go/doc directive line, part of a lead comment group but not of +/// its text (go.dev/doc/comment, "Directives"): `//go:build`, +/// `//line `, `//extern `, `//export `, and the general +/// `//[a-z0-9]+:[a-z0-9]` form. A directive has no space after `//`. +#[must_use] +pub(crate) fn go_directive(s: &[u8]) -> bool { + let Some(rest) = s.strip_prefix(b"//") else { + return false; + }; + if rest.starts_with(b"line ") || rest.starts_with(b"extern ") || rest.starts_with(b"export ") { + return true; + } + let word = rest + .iter() + .take_while(|b| b.is_ascii_lowercase() || b.is_ascii_digit()) + .count(); + word > 0 + && rest.get(word) == Some(&b':') + && rest + .get(word.saturating_add(1)) + .is_some_and(|b| b.is_ascii_lowercase() || b.is_ascii_digit()) +} + +/// A Ruby comment that opens a documentation block whatever follows it: +/// RDoc's `##`, which introduces a metaprogrammed method's docs +/// (`RDoc::Parser::Ruby`), or a YARD `@!` directive (`# @!method`, +/// `# @!attribute`, `# @!macro`, `# @!parse`), which documents a DSL call +/// rather than a `def` (#1597). The `##` is exactly two `#`: YARD sets its +/// hash flag only for a `##` prefix and RDoc reads a lone `##` line, so a +/// `####…` banner or a `##### Helpers #####` divider opens nothing. +#[must_use] +pub(crate) fn ruby_directive_opener(s: &[u8]) -> bool { + if s.starts_with(b"##") && s.get(2) != Some(&b'#') { + return true; + } + s.strip_prefix(b"#").is_some_and(|rest| { + rest.iter() + .position(|b| *b != b' ' && *b != b'\t') + .is_some_and(|at| rest[at..].starts_with(b"@!")) + }) +} + +/// The comment lines YARD drops before documenting: a shebang, an +/// encoding line and the `frozen_string_literal` magic comment, as +/// YARD's `SHEBANG_LINE`, `ENCODING_LINE` and `FROZEN_STRING_LINE` name +/// them, case-insensitively. YARD applies them only before the first +/// token; here a line is excluded wherever it sits, so the match is on +/// the whole comment — `# coding: utf-8`, `# -*- coding: utf-8 -*-`, +/// `# vim: set fileencoding=utf-8 :`, `# frozen_string_literal: true` — +/// and a documentation row that only mentions an encoding (`# Encoding: +/// defaults to UTF-8.`) stays documentation. What this loses is a +/// comment that repeats a magic line mid-file. +#[must_use] +pub(crate) fn ruby_magic_comment(s: &[u8]) -> bool { + if s.starts_with(b"#!") { + return true; + } + s.strip_prefix(b"#") + .map(<[u8]>::trim_ascii) + .is_some_and(|body| ruby_modeline(body) || ruby_magic_setting(body)) +} + +fn contains_ci(hay: &[u8], needle: &[u8]) -> bool { + hay.windows(needle.len()) + .any(|window| window.eq_ignore_ascii_case(needle)) +} + +/// An Emacs (`-*- … -*-`) or Vim (`vim: …`) modeline that carries the +/// magic setting among others: an encoding, or for Emacs the +/// frozen-string flag. A modeline's key is never a magic one, so +/// `ruby_magic_setting` cannot also match it. +fn ruby_modeline(body: &[u8]) -> bool { + if body.len() >= 6 && body.starts_with(b"-*-") && body.ends_with(b"-*-") { + return contains_ci(body, b"coding") || contains_ci(body, b"frozen"); + } + body.get(..4) + .is_some_and(|head| head.eq_ignore_ascii_case(b"vim:")) + && contains_ci(body, b"coding") +} + +/// A comment that is one magic setting and nothing else: `key: value` +/// or `key=value`, the key a magic one and the value a single word. +fn ruby_magic_setting(body: &[u8]) -> bool { + let Some(separator) = body.iter().position(|b| matches!(b, b':' | b'=')) else { + return false; + }; + let (key, value) = body.split_at(separator); + let value = value.get(1..).unwrap_or_default().trim_ascii_start(); + [ + &b"coding"[..], + b"encoding", + b"frozen_string_literal", + b"frozen-string-literal", + ] + .iter() + .any(|known| key.trim_ascii_end().eq_ignore_ascii_case(known)) + && !value.is_empty() + && value + .iter() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) +} + +/// A POD block that is nothing but `comment` regions — `=begin comment` +/// … `=end comment` or a `=for comment` paragraph — which perlpod leaves +/// to no formatter and perlfaq7 recommends for commenting code out: POD +/// syntax, not documentation. A region covers only its own text, as +/// perlpod scopes it: a `=for comment` paragraph ends at the next blank +/// line and a `=begin comment` region at its `=end comment`, so a block +/// that goes on to document something (`=head1 NAME` …) is documentation. +#[must_use] +pub(crate) fn pod_comment_region(s: &[u8]) -> bool { + enum Region { + Outside, + Paragraph, + Begin, + } + let mut region = Region::Outside; + let mut regions = 0_usize; + for line in s.split(|&byte| byte == b'\n') { + let blank = line.trim_ascii().is_empty(); + match region { + Region::Paragraph if blank => region = Region::Outside, + Region::Begin if pod_command(line, b"=end").is_some_and(is_comment_format) => { + region = Region::Outside; + } + Region::Paragraph | Region::Begin => {} + Region::Outside if blank => {} + Region::Outside => { + if pod_command(line, b"=for").is_some_and(is_comment_format) { + region = Region::Paragraph; + } else if pod_command(line, b"=begin").is_some_and(is_comment_format) { + region = Region::Begin; + } else { + // `=cut` closes the block; any other line documents, or + // opens a block that never was a comment region. + return regions > 0 && line.starts_with(b"=cut"); + } + regions += 1; + } + } + } + regions > 0 +} + +/// The text after a POD command line's `=` and the whitespace +/// separating it from its argument, or `None` when `line` is not that +/// command. +fn pod_command<'s>(line: &'s [u8], command: &[u8]) -> Option<&'s [u8]> { + let rest = line.strip_prefix(command)?; + rest.first() + .is_some_and(u8::is_ascii_whitespace) + .then(|| rest.trim_ascii_start()) +} + +/// Whether a POD command's format name is `comment`. +fn is_comment_format(format: &[u8]) -> bool { + format + .strip_prefix(b"comment") + .is_some_and(|after| after.first().is_none_or(u8::is_ascii_whitespace)) +} + +/// An LDoc block opener, per `lang.lua`: a line comment starting with +/// three or more dashes (`start_comment_ = '^%-%-%-+'`, so a line of +/// dashes alone opens too) unless it ends in a dash run preceded by a +/// non-dash (`end_comment_ = '[^%-]%-%-+[^-]*\n$'` — `--- x ---` is an +/// end marker), or a long comment whose body opens with a dash +/// (`block_comment = '^%-%-%[=*%[%-+'`). +#[must_use] +pub(crate) fn ldoc_opener(s: &[u8]) -> bool { + if let Some(body) = lua_long_comment_body(s) { + return body.first() == Some(&b'-'); + } + s.starts_with(b"---") && !ldoc_end_marker(s) +} + +/// A Lua long comment, `--[[ … ]]` or a levelled `--[==[ … ]==]`, as +/// opposed to a line comment that merely starts `--[`. +#[must_use] +pub(crate) fn lua_long_comment(s: &[u8]) -> bool { + lua_long_comment_body(s).is_some() +} + +/// The text after a long comment's opening bracket, or `None` when `s` +/// is not a long comment. +fn lua_long_comment_body(s: &[u8]) -> Option<&[u8]> { + let rest = s.strip_prefix(b"--[")?; + let levels = rest.iter().take_while(|b| **b == b'=').count(); + rest.get(levels..)?.strip_prefix(b"[") +} + +/// `[^-]--+[^-]*$`: the line's final dash run has two or more dashes and +/// something other than a dash before it. +fn ldoc_end_marker(s: &[u8]) -> bool { + let trimmed = s.len() - s.iter().rev().take_while(|b| **b != b'-').count(); + let dashes = s[..trimmed] + .iter() + .rev() + .take_while(|b| **b == b'-') + .count(); + dashes >= 2 && trimmed > dashes +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ruby_directive_opener_accepts_rdoc_and_yard_forms() { + assert!(ruby_directive_opener(b"##")); + assert!(ruby_directive_opener(b"## Section")); + assert!(ruby_directive_opener(b"# @!method foo")); + assert!(ruby_directive_opener(b"#\t@!macro bar")); + assert!(!ruby_directive_opener(b"# plain")); + assert!(!ruby_directive_opener(b"# @param x")); + assert!(!ruby_directive_opener(b"#!/usr/bin/env ruby")); + // A banner or divider: three or more `#` are neither RDoc's lone + // `##` nor YARD's `##` prefix. + assert!(!ruby_directive_opener(b"###")); + assert!(!ruby_directive_opener(b"########################")); + assert!(!ruby_directive_opener(b"##### Private helpers #####")); + } + + #[test] + fn javadoc_block_accepts_the_block_form_only() { + assert!(javadoc_block(b"/** doc */")); + assert!(javadoc_block(b"/**\n * doc\n */")); + assert!(!javadoc_block(b"/**/"), "empty block"); + assert!(!javadoc_block(b"/*** banner */"), "three stars"); + assert!(!javadoc_block(b"/* plain */")); + assert!(!javadoc_block(b"// plain")); + assert!(!javadoc_block(b"/**"), "nothing after the marker"); + } + + #[test] + fn triple_slash_rejects_a_fourth_slash() { + assert!(triple_slash(b"/// doc")); + assert!(triple_slash(b"///"), "bare marker still opens a doc line"); + assert!(triple_slash(b"///< after member")); + assert!(!triple_slash(b"//// banner")); + assert!(!triple_slash(b"// plain")); + } + + #[test] + fn doxygen_covers_the_four_markers_and_their_member_forms() { + for doc in [ + &b"/** d */"[..], + b"/*! d */", + b"/// d", + b"//! d", + b"/**< d */", + b"/*!< d */", + b"///< d", + b"//!< d", + ] { + assert!(doxygen(doc), "{}", String::from_utf8_lossy(doc)); + } + for plain in [&b"/* d */"[..], b"// d", b"/**/", b"/*** d */", b"//// d"] { + assert!(!doxygen(plain), "{}", String::from_utf8_lossy(plain)); + } + } + + #[test] + fn java_doc_follows_javac_on_three_stars_and_jep_467() { + assert!(java_doc(b"/** d */")); + assert!(java_doc(b"/*** d */"), "javac discards leading asterisks"); + assert!(java_doc(b"/// d"), "JEP 467 Markdown doc comment"); + assert!(!java_doc(b"/**/")); + assert!(!java_doc(b"//// d")); + assert!(!java_doc(b"/* d */")); + assert!(!java_doc(b"// d")); + } + + #[test] + fn kotlin_doc_is_any_block_but_the_empty_one() { + assert!(kotlin_doc(b"/** d */")); + assert!(kotlin_doc(b"/*** d */")); + assert!(!kotlin_doc(b"/**/")); + assert!(!kotlin_doc(b"/// d"), "KDoc has no line form"); + } + + #[test] + fn phpdoc_needs_whitespace_after_the_marker() { + assert!(phpdoc(b"/** d */")); + assert!(phpdoc(b"/**\n * d\n */")); + assert!(!phpdoc(b"/**/")); + assert!(!phpdoc(b"/*** d */")); + assert!(!phpdoc(b"/**d */")); + assert!(!phpdoc(b"# d")); + assert!(!phpdoc(b"// d")); + } + + #[test] + fn csharp_doc_rejects_the_doxygen_only_spellings() { + assert!(csharp_doc(b"/// d")); + assert!(csharp_doc(b"/** d */")); + for plain in [ + &b"//! d"[..], + b"/*! d */", + b"//// d", + b"/**/", + b"/*** d */", + b"// d", + ] { + assert!(!csharp_doc(plain), "{}", String::from_utf8_lossy(plain)); + } + } + + /// Each row is what `jsdoc -X` 4.x made of the comment above a call: + /// a doclet of its own (virtual) or nothing. A name may sit on the + /// line after its tag, and a tag inline in the description is text. + #[test] + fn jsdoc_virtual_follows_jsdoc_naming_tags() { + crate::test_support::assert_classifies( + jsdoc_virtual, + &[ + b"/** @file */", + b"/**\n * @fileoverview x\n */", + b"/** @overview */", + b"/** @module */", + b"/**\n * @typedef {Object} T\n */", + b"/**\n * @typedef {Object}\n * T\n */", + b"/**\n * @function f\n */", + b"/**\n * @callback cb\n */", + b"/** @FILE */", + b"/** @FileOverview */", + b"/** @Typedef {Object} T */", + b"/** @function f **/", + ], + &[ + b"/**\n * @license MIT\n */", + b"/**\n * @typedef {Object}\n */", + b"/**\n * @function\n */", + b"/** @function **/", + // A nested `{type}` is one type: no name follows it. + b"/**\n * @typedef {{a: number}}\n */", + b"/** x @file */", + b"/** d */", + ], + ); + assert!(tsdoc_module(b"/**\n * @packageDocumentation\n */")); + assert!(tsdoc_module(b"/**\n * @module m\n */")); + assert!(!tsdoc_module(b"/**\n * @license MIT\n */")); + } + + #[test] + fn go_directive_matches_go_ast_is_directive() { + for directive in [ + &b"//go:build linux"[..], + b"//go:generate go run x", + b"//line foo.go:12", + b"//extern f", + b"//export F", + b"//nolint:errcheck", + b"//x:1", + ] { + assert!( + go_directive(directive), + "{}", + String::from_utf8_lossy(directive) + ); + } + for text in [ + &b"// go:build linux"[..], + b"//Note: not a directive", + b"//x:", + b"//exported F", + b"// d", + ] { + assert!(!go_directive(text), "{}", String::from_utf8_lossy(text)); + } + } + + /// A block is a comment only while nothing but `comment` regions fill + /// it: POD after a region's end (`=head1 NAME` past a `=for comment` + /// paragraph or an `=end comment`) is documentation pod2text renders. + #[test] + fn pod_comment_region_is_the_comment_format_only() { + for region in [ + &b"=begin comment\n\nold();\n\n=end comment\n\n=cut"[..], + b"=for comment\nold();\n\n=cut", + b"=begin comment\n=end comment\n=cut", + b"=for comment\nold();\n", + b"=begin comment\nold();\n", + b"=for comment a\n\n=for comment b\n\n=cut", + b"=begin comment\nx\n=end comment\n=for comment y\n\n=cut", + b"=for comment\r\nold();\r\n\r\n=cut\r\n", + ] { + assert!( + pod_comment_region(region), + "{}", + String::from_utf8_lossy(region) + ); + } + for doc in [ + &b"=pod\n\nText.\n\n=cut"[..], + b"=head1 NAME\n\nFoo\n\n=cut", + b"=begin html\n\n

x

\n\n=end html\n\n=cut", + b"=begin comments\n=end comments\n=cut", + b"=beginning", + b"=for comment\nInternal note.\n\n=head1 NAME\n\nFoo - bar\n\n=cut", + b"=begin comment\n\nold();\n\n=end comment\n\n=head1 X\n\nDoc.\n\n=cut", + b"=for comment\nnote\n\nPlain text paragraph.\n\n=cut", + ] { + assert!(!pod_comment_region(doc), "{}", String::from_utf8_lossy(doc)); + } + } + + /// The match is on the whole comment, because it is applied anywhere + /// in the file: a documentation row that mentions an encoding, or + /// names `frozen_string_literal` in prose, is not a magic line. + #[test] + fn ruby_magic_comment_matches_whole_magic_lines_only() { + for magic in [ + &b"#!/usr/bin/env ruby"[..], + b"# coding: utf-8", + b"# encoding: UTF-8", + b"# Encoding: binary", + b"#coding=utf-8", + b"# -*- coding: utf-8 -*-", + b"# -*- mode: ruby; encoding: utf-8 -*-", + b"# -*- frozen_string_literal: true -*-", + b"# vim: set fileencoding=utf-8 :", + b"# frozen_string_literal: true", + b"# frozen-string-literal: false", + ] { + assert!( + ruby_magic_comment(magic), + "{}", + String::from_utf8_lossy(magic) + ); + } + for doc in [ + &b"# Encoding: defaults to UTF-8."[..], + b"# Handles transcoding: see below", + b"# Pass `frozen_string_literal: true` to opt in.", + b"# Sets the encoding= attribute", + b"# d", + b"#", + ] { + assert!(!ruby_magic_comment(doc), "{}", String::from_utf8_lossy(doc)); + } + } +} diff --git a/big-code-analysis-ast/src/roles/mod.rs b/big-code-analysis-ast/src/roles/mod.rs index 936af0ac4..ba2d8e4bd 100644 --- a/big-code-analysis-ast/src/roles/mod.rs +++ b/big-code-analysis-ast/src/roles/mod.rs @@ -124,7 +124,7 @@ //! the source and legacy views; owner normalisation; and text decoding. //! Those stay the epic's mechanisms, reading the rows. The reusable //! helpers beside the rows: the marker tests in -//! `lang_helpers/doc_comment.rs`, the Doxygen entry projection and +//! `lang_helpers/doc_text.rs`, the Doxygen entry projection and //! conditional memo in `lang_helpers/doxygen.rs`, YARD's scope fold and //! call forms in `lang_helpers/ruby.rs`, and Python's run and assignment //! helpers in `lang_helpers/python.rs`. diff --git a/big-code-analysis-bench/src/shapes.rs b/big-code-analysis-bench/src/shapes.rs index 8cac9e600..32f9aa180 100644 --- a/big-code-analysis-bench/src/shapes.rs +++ b/big-code-analysis-bench/src/shapes.rs @@ -337,6 +337,50 @@ pub fn nested_fns_by_row(depth: usize) -> String { ) } +/// Bash: functions nested `depth` deep, each body opening with a +/// pipeline continued across a row that holds only `\`. +/// +/// The bare row is the `loc` question #1445 answers: it carries no node, +/// so it is found by reading the padding between a node's children. Every +/// level is a space of its own, so a scan that read each space's whole +/// byte range would read the bytes of level *k* `depth - k` times; the +/// padding of each node is read once. Affine in depth, no indentation. +#[must_use] +pub fn bash_nested_continued_fns(depth: usize) -> String { + format!( + "{}{}", + "f() {\necho a | \\\n\\\ncat\n".repeat(depth), + "}\n".repeat(depth) + ) +} + +/// Tcl: `width` comments, each continued by a row holding only `\` and +/// closed by a blank row, at the top level of one file. +/// +/// Each bare row is comment text (#1445), and telling it from a code row +/// asks whether the row before it ends inside a comment node, once per +/// bare row. `descendant_for_byte_range` answers by descending the +/// root's balanced `repeat` tree; finding the comment by walking the +/// root's children instead costs the file's width per question. +#[must_use] +pub fn tcl_wide_continued_comments(width: usize) -> String { + "# c \\\n\\\n\nputs x\n".repeat(width) +} + +/// Tcl: `width` units of a `proc` whose body never closes, each holding +/// a comment continued by a row holding only `\` and closed by a blank +/// row. +/// +/// The unclosed openers leave every unit a direct child of one flat root +/// `ERROR`, which tree-sitter never balances, so the lookup that tells a +/// bare row continuing a comment from a code row (#1445) costs that +/// root's width when it descends from the gap's owner. It descends from +/// the child before the gap instead, a constant per bare row. +#[must_use] +pub fn tcl_unclosed_continued_comments(width: usize) -> String { + "proc p {} {\n# c \\\n\\\n\n".repeat(width) +} + /// JavaScript: `function f() { function f() { … 1; … } }`. /// /// The linear control for [`nested_arrows`]: one function per nesting @@ -776,6 +820,24 @@ pub fn ruby_wide_private_defs(width: usize) -> String { source } +/// Ruby: a class body holding `depth` nested `if`s, an instance-variable +/// assignment and an `attr_reader` at every level beside the next `if`, +/// so the attribute walk reads one wrapper per level and asks the +/// `Member` row about one assignment target under each. +#[must_use] +pub fn ruby_nested_if_attrs(depth: usize) -> String { + let mut source = String::from( + "class C +", + ); + for i in 0..depth { + // `fmt::Write for String` never fails; see `wide_distinct_fns`. + let _ = writeln!(source, "if c{i}\n@a{i} = 1\nattr_reader :r{i}"); + } + source.push_str(&"end\n".repeat(depth + 1)); + source +} + /// Lua: a module block, then `width` units of a tagless `---` block /// directly under a call — no item follows it, so the first-block test /// scans back — and a documented function, whose block is the reading. @@ -2391,6 +2453,71 @@ pub const PROBES: &[Probe] = &[ row in total, and `nom/nested-fn-rows` is the same \ shape under a metric that keeps no per-row set.", }, + Probe { + name: "loc/bash-nested-continued-fn", + lang: LANG::Bash, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Loc], + reading: |m| m.loc.ploc(), + }, + render: bash_nested_continued_fns, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1445: a row holding only `\\` has no node, so `Loc` \ + finds it in the padding between a node's children. Each \ + padding byte belongs to one node, so the scan is linear \ + however deep the spaces nest. Scanning every node's \ + whole range instead, which reads level k once per \ + enclosing level, took this from 1.09 to 1.93. The \ + reading counts the bare rows too, one in five of the \ + file's code rows.", + }, + Probe { + name: "loc/tcl-wide-continued-comment", + lang: LANG::Tcl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Loc], + reading: |m| m.loc.cloc(), + }, + render: tcl_wide_continued_comments, + sizes: SLOT_SCAN_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1445: a bare `\\` row after a comment is that \ + comment's text, decided by looking up the comment node \ + under the row before it, once per bare row, from the \ + child just before the bare row's gap. Finding the \ + comment by walking the root's children instead measured \ + 2.00 against 1.01 at widths 1 000-4 000; at this probe's \ + own sizes one such walk at 16 000 exceeds the 20 s \ + budget. The reading is the comment rows, half of them \ + the bare ones.", + }, + Probe { + name: "loc/tcl-unclosed-continued-comments", + lang: LANG::Tcl, + axis: Axis::Width, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Loc], + reading: |m| m.loc.cloc(), + }, + render: tcl_unclosed_continued_comments, + sizes: SLOT_SCAN_WIDTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1445 review: an unclosed opener per unit leaves every \ + unit a direct child of one flat root `ERROR`, which \ + tree-sitter never balances, and its \ + `descendant_for_byte_range` scans each level's children \ + from the first. The comment lookup descends from the \ + child before the bare row's gap, a constant per bare \ + row. Descending from the gap's owner, the root, instead \ + measured 2.09 against 1.10. The reading is the comment \ + rows, half of them the bare ones.", + }, Probe { name: "nom/nested-fn-rows", lang: LANG::Rust, @@ -3192,6 +3319,28 @@ pub const PROBES: &[Probe] = &[ re-sought per statement. The reading is one method per \ call.", }, + Probe { + name: "npa/ruby-nested-if-attrs", + lang: LANG::Ruby, + axis: Axis::Depth, + workload: Workload::Metrics { + exclude_tests: false, + selection: &[Metric::Npa], + reading: |m| m.npa.class_na_sum(), + }, + render: ruby_nested_if_attrs, + sizes: LINEAR_DEPTHS, + max_exponent: LINEAR_BOUND, + rationale: "#1719: the Ruby attribute walk reads the statements the \ + method walk does, through every conditional at class \ + scope, and asks the `Member` row whether an assignment's \ + target is an attribute under the chain the walk holds, \ + one push and pop below the statement. Asked with \ + `Ancestors::unknown()` instead, the gate's parent lookup \ + climbs from the root and the walk measured quadratic \ + (exponent 1.98 against 1.10). The reading is two \ + attributes per level.", + }, Probe { name: "loc/lua-tagless-blocks", lang: LANG::Lua, @@ -3767,6 +3916,9 @@ mod tests { "nexits/perl-error-tail-keys", "count/perl-error-tail-calls", "dump/perl-error-tail-calls", + // The flat root `ERROR` is the shape: its unclosed openers are + // what keep tree-sitter from balancing the root's children. + "loc/tcl-unclosed-continued-comments", ]; /// Probes whose shape recovers *inside* a command rather than at @@ -4080,6 +4232,8 @@ mod tests { "npa/cpp-wide-class-members", "npm/ruby-nested-if-defs", "npm/ruby-wide-private-defs", + "npa/ruby-nested-if-attrs", + "loc/tcl-unclosed-continued-comments", "loc/cpp-nested-namespaces", "loc/ts-nested-fn", "nom/js-arrow-per-array-depth", @@ -4147,6 +4301,34 @@ mod tests { } } + /// The three #1445 bare-row probes read their bare rows: each unit's + /// reading counts the `\`-only row as well as the row it continues, + /// so a shape that lost the bare row reads `1 × size` (Tcl, comment + /// rows) or `4 × size` (Bash, code rows) instead. The timing bound + /// cannot tell — the shape would still grow — and nor can + /// `probe_workload_is_exercised` or the growth check. + #[test] + fn bare_row_probes_read_their_bare_rows() { + for (name, rows_per_unit) in [ + ("loc/bash-nested-continued-fn", 5), + ("loc/tcl-wide-continued-comment", 2), + ("loc/tcl-unclosed-continued-comments", 2), + ] { + let probe = PROBES + .iter() + .find(|p| p.name == name) + .unwrap_or_else(|| panic!("{name} is a probe")); + for size in [3, probe.sizes[0]] { + let ast = parse(probe, size); + let reading = probe + .workload + .walk(&ast, probe.workload.options()) + .expect("metrics walk"); + assert_eq!(reading, (rows_per_unit * size) as u64, "{name} at {size}"); + } + } + } + /// `cognitive/elixir-keyword-sections`: the two `do:`-bearing lists /// score one `else:` section per pair each and the third nothing, so /// the reading is exactly `2 × width` over three calls. The timing diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 120b9b7ef..2cebbc82c 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -310,6 +310,12 @@ introduced **Cognitive Complexity** in the 2017 white paper understandability*](https://www.sonarsource.com/docs/CognitiveComplexity.pdf) and the follow-up IEEE TechDebt 2018 paper [*Cognitive Complexity: An Overview and Evaluation*](https://ieeexplore.ieee.org/document/8595102/). +The white paper's link always serves the latest revision, and the +rules have been reworded since 2017: v1.5 made the multi-level +`break` / `continue` increment explicit. This chapter and the source +cite it at v1.7, 29 August 2023, where Appendix B's §B1 lists the +increments, §B2 the structures that raise the nesting level, and §B3 +the ones that pay a nesting increment. The metric was designed as a replacement for Cyclomatic Complexity in code-quality tooling. Campbell's argument is that cyclomatic @@ -422,6 +428,12 @@ since adopted it. levels up happened to be a closure. Python cannot express the shape at all: a `def` is a statement and a lambda body is a single expression. +- **PHP** scores `break N` / `continue N`, its only non-default jump + argument, at `+0`. The numeric form leaves N enclosing loops whose + nesting is already counted, so it is treated as a structured + loop-level exit; `goto label;` still adds `+1`. The paper's v1.7 §B1 + lists a jump to a NUMBER as an increment, so this is a deliberate + deviation, and an undecided one: #1765 (item 2) tracks it. - **Rust**'s `let`-`else` (`let Some(a) = y else { continue };`) postdates the specification. It is scored as the `if let` it replaces: `+1` plus the current nesting, with everything under it, @@ -1062,6 +1074,26 @@ string interpolations are all handled correctly. The per-language LLOC; this is the subtle one, because what counts as a statement is language-defined. +One kind of row has no node to identify it. In Bash, Tcl and iRules +the lexer splices a line-continuation `\` and its newline away, so a +row holding only that `\` is invisible in the tree. Such rows are found +by reading the bytes between nodes. Each one counts as PLOC, the code +it continues. The exception is a row that continues a Tcl or iRules +comment (`# note \`), which counts as CLOC. A row that continues +nothing starts a line and counts as what that line becomes: CLOC when +the row it joins is a comment, blank when that row is empty, PLOC +otherwise. The C family, Ruby and Elixir grammars splice a +continuation the same way, but their bare rows are not found yet and +still count as blank +([#1801](https://github.com/dekobon/big-code-analysis/issues/1801)). +Where CRLF line endings reach the parser unchanged, through the +library's own `analyze` or Python's `Ast.parse`, the Tcl and iRules +grammars lex the bare row as a word, so there it counts as PLOC even +after a comment +([#1802](https://github.com/dekobon/big-code-analysis/issues/1802)). +`bca`, the web server and the Python bindings' `analyze`, +`analyze_source` and `Ast.from_path` convert line endings to LF first. + Documentation rows are comment rows that are also reported on their own as DLOC (`loc.dloc`), so CLOC can be read with or without them. Each language's own documentation tool decides which comments are diff --git a/big-code-analysis-cli/src/walk.rs b/big-code-analysis-cli/src/walk.rs index f1ef33258..93736ec39 100644 --- a/big-code-analysis-cli/src/walk.rs +++ b/big-code-analysis-cli/src/walk.rs @@ -470,7 +470,7 @@ pub(crate) fn expand_seed_paths( warn("0 files matched"); } let ignored = if filters.measure_ignored { - measure_ignored_entries(&walked_dirs, &found, filters) + measure_ignored_entries(&walked_dirs, &found, filters, threads) } else { IgnoredEntries::default() }; @@ -512,56 +512,125 @@ fn admit_file_seed(seed: &Path, filters: &WalkFilters<'_>, found: &mut SeedSet) /// just listed, so the cost tracks the *kept* tree, not the no-ignore /// universe. Read errors are skipped silently: the walk already warned /// about anything it could not list, and this pass is advisory. +/// +/// The listings run on `threads` scoped workers, the walk's own budget: +/// serial, this pass was a second full enumeration of the kept tree on +/// the main thread after the parallel walk had joined, and on a +/// 65k-directory tree it cost more wall time than the walk itself +/// (#1307). Jobs are per *directory*, not per seed, because the usual +/// workload is one seed with many directories. fn measure_ignored_entries( walked_dirs: &[(PathBuf, Vec)], found: &SeedSet, filters: &WalkFilters<'_>, + threads: usize, ) -> IgnoredEntries { - let walked = WalkedSets { - dirs: walked_dirs - .iter() - .flat_map(|(_, dirs)| dirs) - .map(|dir| walk_seed::strip_cur_dir(dir)) - .collect(), - files: found - .files - .iter() - .map(|file| walk_seed::strip_cur_dir(file)) - .collect(), - explicit_canon: found - .explicit_files - .iter() - .filter_map(|path| path.canonicalize().ok()) - .collect(), - }; - let mut ignored = IgnoredEntries::default(); - for (seed, dirs) in walked_dirs { - for dir in dirs { - let Ok(entries) = std::fs::read_dir(dir) else { - continue; - }; - for entry in entries.flatten() { - classify_dropped_child(&entry, seed, filters, &walked, &mut ignored); + let walked = WalkedSets::new(walked_dirs, found); + let jobs: Vec> = walked_dirs + .iter() + .enumerate() + .flat_map(|(rank, (seed, dirs))| dirs.iter().map(move |dir| MeasureJob { rank, seed, dir })) + .collect(); + let next = AtomicUsize::new(0); + let measure = || measure_jobs(&jobs, &next, filters, &walked); + let parts = std::thread::scope(|scope| { + // The calling thread is the last worker, so one thread spawns + // nothing and `--jobs 1` stays serial. Every handle is collected + // before that share runs: joining each as it is spawned would run + // the workers one at a time. + let workers: Vec<_> = (1..threads.min(jobs.len())) + .map(|_| scope.spawn(measure)) + .collect(); + let mut parts = vec![measure()]; + // `resume_unwind` re-raises a panicking worker's own payload, + // where the scope's implicit join would raise a generic one. + parts.extend(workers.into_iter().map(|worker| { + worker + .join() + .unwrap_or_else(|payload| std::panic::resume_unwind(payload)) + })); + parts + }); + let mut files = Vec::new(); + let mut dirs = Vec::new(); + for part in parts { + files.extend(part.files); + dirs.extend(part.dirs); + } + IgnoredEntries { + files: merge_measured(files), + dirs: merge_measured(dirs), + } +} + +/// One directory [`measure_ignored_entries`] re-lists: the seed whose +/// walk entered it, and that seed's position among the directory seeds +/// — what [`merge_measured`] orders overlapping spellings by. +struct MeasureJob<'a> { + rank: usize, + seed: &'a Path, + dir: &'a Path, +} + +/// One worker's share of the measurement, each path tagged with the +/// rank of the seed it was found under. +#[derive(Default)] +struct RankedEntries { + files: Vec<(usize, PathBuf)>, + dirs: Vec<(usize, PathBuf)>, +} + +/// Claim directories from the shared job list until it is exhausted, +/// classifying each one's children. Claiming one directory at a time +/// keeps the workers balanced without knowing the directories' sizes. +fn measure_jobs( + jobs: &[MeasureJob<'_>], + next: &AtomicUsize, + filters: &WalkFilters<'_>, + walked: &WalkedSets<'_>, +) -> RankedEntries { + let mut found = RankedEntries::default(); + while let Some(job) = jobs.get(next.fetch_add(1, Ordering::Relaxed)) { + let Ok(entries) = std::fs::read_dir(job.dir) else { + continue; + }; + for entry in entries.flatten() { + match classify_dropped_child(&entry, job.seed, filters, walked) { + Some(Dropped::File(path)) => found.files.push((job.rank, path)), + Some(Dropped::Dir(path)) => found.dirs.push((job.rank, path)), + None => {} } } } - // Overlapping seeds can flag the same entry once per seed; one - // mention per path, in a stable order for the `--report-skipped` - // listing. - dedupe_measured(&mut ignored.files); - dedupe_measured(&mut ignored.dirs); - ignored + found } /// Sort and dedupe one measured list in the `./`-stripped space, so the /// two spellings overlapping seeds give one path (`-p .` yields /// `./src/f.rs` where `-p src` yields `src/f.rs`) collapse to a single -/// mention instead of being counted twice. A *stable* sort keeps the -/// surviving spelling the first-seeded one rather than whichever walker -/// thread happened to report it. -fn dedupe_measured(paths: &mut Vec) { - paths.sort_by(|a, b| walk_seed::strip_cur_dir(a).cmp(walk_seed::strip_cur_dir(b))); - paths.dedup_by(|a, b| walk_seed::strip_cur_dir(a) == walk_seed::strip_cur_dir(b)); +/// mention instead of being counted twice, in a deterministic order +/// for the `--report-skipped` listing. +/// +/// The surviving spelling is the earliest seed's. That is a tie-break +/// on the seed rank, not on input order: the input is concatenated in +/// whatever order the workers finished, so a stable sort over it would +/// let thread timing pick the spelling. The raw spelling is the final +/// key so the order is total and no tie is left to the sort. +fn merge_measured(mut ranked: Vec<(usize, PathBuf)>) -> Vec { + ranked.sort_unstable_by(|(rank_a, a), (rank_b, b)| { + walk_seed::strip_cur_dir(a) + .cmp(walk_seed::strip_cur_dir(b)) + .then(rank_a.cmp(rank_b)) + .then_with(|| a.as_os_str().cmp(b.as_os_str())) + }); + ranked.dedup_by(|(_, a), (_, b)| walk_seed::strip_cur_dir(a) == walk_seed::strip_cur_dir(b)); + ranked.into_iter().map(|(_, path)| path).collect() +} + +/// What [`classify_dropped_child`] attributes to an ignore rule. +enum Dropped { + File(PathBuf), + Dir(PathBuf), } /// What the walk produced, as membership sets: the directories it @@ -584,8 +653,39 @@ struct WalkedSets<'a> { explicit_canon: std::collections::HashSet, } -/// Classify one directory child the walk did not keep, recording it in -/// `ignored` when only an ignore rule can explain the drop. Hidden +impl<'a> WalkedSets<'a> { + fn new(walked_dirs: &'a [(PathBuf, Vec)], found: &'a SeedSet) -> Self { + Self { + dirs: walked_dirs + .iter() + .flat_map(|(_, dirs)| dirs) + .map(|dir| walk_seed::strip_cur_dir(dir)) + .collect(), + files: found + .files + .iter() + .map(|file| walk_seed::strip_cur_dir(file)) + .collect(), + explicit_canon: found + .explicit_files + .iter() + .filter_map(|path| path.canonicalize().ok()) + .collect(), + } + } + + /// Whether `path` is a file the caller named explicitly, analyzed + /// despite the ignore rule that kept it out of the walk (#726). + fn checked_explicitly(&self, path: &Path) -> bool { + !self.explicit_canon.is_empty() + && path + .canonicalize() + .is_ok_and(|canonical| self.explicit_canon.contains(&canonical)) + } +} + +/// Classify one directory child the walk did not keep, returning it +/// when only an ignore rule can explain the drop. Hidden /// entries and symlinks are what the walker skips unconditionally; /// exclude globs and unrecognized extensions are the walk's own /// filters; anything in `walked` the walk saw. What remains was pruned @@ -595,30 +695,25 @@ fn classify_dropped_child( seed: &Path, filters: &WalkFilters<'_>, walked: &WalkedSets<'_>, - ignored: &mut IgnoredEntries, -) { +) -> Option { // `DirEntry::file_type` does not follow symlinks, matching the // walk's `follow_links(false)`. - let Ok(file_type) = entry.file_type() else { - return; - }; + let file_type = entry.file_type().ok()?; // Hidden entries are dropped by `.hidden(true)` and symlinks by // `follow_links(false)` whether or not any ignore rule exists, so // neither carries an ignore signal. if walker_hides(entry) || file_type.is_symlink() { - return; + return None; } let path = entry.path(); // Both membership tests compare in the `./`-stripped space — see // [`WalkedSets`] for why the spelling cannot be trusted verbatim. if file_type.is_dir() { - if !walked.dirs.contains(walk_seed::strip_cur_dir(&path)) { - ignored.dirs.push(path); - } - return; + let pruned = !walked.dirs.contains(walk_seed::strip_cur_dir(&path)); + return pruned.then_some(Dropped::Dir(path)); } if !file_type.is_file() || walked.files.contains(walk_seed::strip_cur_dir(&path)) { - return; + return None; } // A file no parser owns would have been read and skipped, not // checked, so it is not worth reporting as a gate bypass. The @@ -626,17 +721,13 @@ fn classify_dropped_child( // handling off too, so they own their explanation; and a file the // caller named explicitly was analyzed despite the ignore rule // (#726), so accusing the run of skipping it would contradict the - // violations it just printed. - let checked_explicitly = !walked.explicit_canon.is_empty() - && path - .canonicalize() - .is_ok_and(|canonical| walked.explicit_canon.contains(&canonical)); - if !checked_explicitly - && filters.analyzable(&path) + // violations it just printed. That last test canonicalizes, a + // syscall chain per file, so it runs after the two lookups that turn + // most dropped files away. + let reportable = filters.analyzable(&path) && filters.passes(&path, &walk_seed::match_path_for(seed, &path)) - { - ignored.files.push(path); - } + && !walked.checked_explicitly(&path); + reportable.then_some(Dropped::File(path)) } /// Mirror of the walker's `.hidden(true)` filter: a dot-prefixed name diff --git a/big-code-analysis-cli/src/walk_tests.rs b/big-code-analysis-cli/src/walk_tests.rs index 0a767db4b..4b6d3a768 100644 --- a/big-code-analysis-cli/src/walk_tests.rs +++ b/big-code-analysis-cli/src/walk_tests.rs @@ -77,3 +77,141 @@ fn walk_directory_seed_returns_sorted_paths() { the sortedness assertion above would pass unsorted" ); } + +/// The ignore measurement's merge keeps the earliest seed's spelling of +/// a path both seeds reached, whatever order the parts arrive in (#1307). +/// +/// The measurement's workers hand their parts back in completion order, +/// so the input here is deliberately the later seed first — the order a +/// stable sort keyed on the path alone would turn into the output, +/// letting thread timing choose which spelling `--report-skipped` +/// prints. `p/./src` and `p/src` compare equal as paths (interior `.` +/// is not a component), so only the rank can decide between them; the +/// assertion compares raw bytes for the same reason. +#[test] +fn merge_measured_keeps_the_first_seeded_spelling() { + let later_seed = PathBuf::from("p/./src/v.rs"); + let earlier_seed = PathBuf::from("p/src/v.rs"); + assert_eq!( + later_seed, earlier_seed, + "fixture: the two spellings must be one path to Path's comparison" + ); + let merged = merge_measured(vec![ + (1, later_seed), + (1, PathBuf::from("p/./src/b.rs")), + (0, earlier_seed), + (0, PathBuf::from("./p/a.rs")), + ]); + + let spelled: Vec<&std::ffi::OsStr> = merged.iter().map(|path| path.as_os_str()).collect(); + assert_eq!( + spelled, + ["./p/a.rs", "p/./src/b.rs", "p/src/v.rs"], + "sorted in the `./`-stripped space, one mention per path, \ + and the rank-0 spelling of the shared path" + ); +} + +/// The merge sorts and dedupes in the `./`-stripped space: `./q.rs` +/// (rank 0) and `q.rs` (rank 1) are one path and sort after `p/z.rs`. +/// Keyed on the raw spelling instead, `./q.rs` sorts first and the two +/// spellings are no longer adjacent, so both survive. +#[test] +fn merge_measured_sorts_and_dedupes_in_the_stripped_space() { + let merged = merge_measured(vec![ + (1, PathBuf::from("q.rs")), + (0, PathBuf::from("p/z.rs")), + (0, PathBuf::from("./q.rs")), + ]); + let spelled: Vec<&std::ffi::OsStr> = merged.iter().map(|path| path.as_os_str()).collect(); + assert_eq!(spelled, ["p/z.rs", "./q.rs"]); +} + +/// The merge's last key, the raw spelling, leaves no tie to the sort: +/// two spellings of one path under one seed rank keep the same survivor +/// whichever the workers reported first. `p/./src/v.rs` sorts before +/// `p/src/v.rs` byte-wise (`.` < `s`), so it survives both orders; with +/// the key dropped, each order keeps whichever spelling came first. +#[test] +fn merge_measured_breaks_a_same_rank_tie_on_the_spelling() { + let dotted = PathBuf::from("p/./src/v.rs"); + let plain = PathBuf::from("p/src/v.rs"); + for input in [ + vec![(0, dotted.clone()), (0, plain.clone())], + vec![(0, plain.clone()), (0, dotted.clone())], + ] { + let merged = merge_measured(input); + let spelled: Vec<&std::ffi::OsStr> = merged.iter().map(|path| path.as_os_str()).collect(); + assert_eq!( + spelled, + ["p/./src/v.rs"], + "one survivor, the byte-wise first" + ); + } +} + +/// The parallel ignore measurement finds exactly what the serial one +/// does (#1307): every worker's share reaches the merge, including the +/// calling thread's, and no directory is skipped. +/// +/// The root and its 48 subdirectories each hold one ignored file and +/// one ignored directory beside a kept file, so every job contributes +/// exactly one of each and a lost job moves an exact count. The root +/// carries them too because it is usually the first job claimed: +/// without them, skipping job 0 went unnoticed here. A directory +/// claimed twice is not observable here: the merge dedupes the repeat, +/// so a double claim costs time, not output. +#[test] +fn parallel_ignore_measurement_matches_serial() { + const DIRS: usize = 48; + let tmp = tempfile::TempDir::new().expect("tempdir"); + let proj = tmp.path().join("proj"); + std::fs::create_dir(&proj).expect("create proj"); + std::fs::write(proj.join(".gitignore"), "*.gen.rs\nbuild/\n").expect("write .gitignore"); + let walked = + std::iter::once(proj.clone()).chain((0..DIRS).map(|d| proj.join(format!("pkg{d}")))); + for dir in walked { + std::fs::create_dir_all(dir.join("build")).expect("create dirs"); + std::fs::write(dir.join("kept.rs"), b"pub fn f() {}\n").expect("write kept"); + std::fs::write(dir.join("dropped.gen.rs"), b"pub fn f() {}\n").expect("write dropped"); + } + + let empty_include = mk_globset(Vec::new()).expect("empty globset"); + let empty_exclude = build_exclude_globset(Vec::new(), None, "--exclude-from"); + let cwd = std::env::current_dir().unwrap_or_default(); + let filters = WalkFilters { + include: &empty_include, + excludes: crate::walk_seed::AnchoredExcludes::new( + &empty_exclude, + &empty_exclude, + None, + &cwd, + None, + ), + language_forced: false, + measure_ignored: true, + }; + let mut errors = WalkErrors::default(); + let walk = walk_directory_seed(&proj, false, 4, &filters, &mut errors); + let mut found = SeedSet::default(); + for path in walk.files { + found.push_walked(path); + } + let walked_dirs = vec![(proj, walk.dirs)]; + + let serial = measure_ignored_entries(&walked_dirs, &found, &filters, 1); + let parallel = measure_ignored_entries(&walked_dirs, &found, &filters, 8); + + assert_eq!( + serial.files.len(), + DIRS + 1, + "one ignored file per walked directory" + ); + assert_eq!( + serial.dirs.len(), + DIRS + 1, + "one pruned directory per walked directory" + ); + assert_eq!(parallel.files, serial.files); + assert_eq!(parallel.dirs, serial.dirs); +} diff --git a/big-code-analysis-cli/tests/check/check_skipped_inputs.rs b/big-code-analysis-cli/tests/check/check_skipped_inputs.rs index 20f556744..228d4b372 100644 --- a/big-code-analysis-cli/tests/check/check_skipped_inputs.rs +++ b/big-code-analysis-cli/tests/check/check_skipped_inputs.rs @@ -558,3 +558,115 @@ fn excluded_and_ignored_file_is_not_counted() { .success() .stderr(predicate::str::contains("not checked").not()); } + +/// Run `bca check --report-skipped` from `cwd` over `seeds` at `jobs` +/// threads under the file's `cyclomatic=1` gate, returning stderr. The +/// seeds are relative to `cwd`, which `check()` cannot express. +fn report_skipped(cwd: &Path, seeds: &[&str], jobs: &str) -> String { + let mut args = vec!["check", "--no-config", "--threshold", "cyclomatic=1"]; + for seed in seeds { + args.extend_from_slice(&["--paths", seed]); + } + args.extend_from_slice(&["--report-skipped", "--jobs", jobs]); + let output = cli(cwd).args(&args).assert().success().get_output().clone(); + String::from_utf8(output.stderr).unwrap() +} + +/// The measurement re-lists the walked directories on the walk's whole +/// thread budget (#1307), so its output must not depend on that budget: +/// the listing is byte-identical at `--jobs 1` and `--jobs 8`, and +/// overlapping seeds keep the *first-named* seed's spelling of a path +/// both reached — `-p . -p pkg0` prints `./pkg0/…`, `-p pkg0 -p .` +/// prints `pkg0/…` — rather than whichever worker reported it first. +/// +/// The expected spellings are built with `Path::join`, as the walk builds +/// the printed paths, so they carry `\` on Windows exactly where the +/// listing does. +#[test] +fn report_skipped_is_independent_of_job_count_and_keeps_seed_order() { + // Packages under the root: with the root and the `pkg0` seed that is + // `PKGS + 2` per-directory jobs, enough to occupy all eight workers + // of the `--jobs 8` run. + const PKGS: usize = 24; + let (dir, proj) = fixture(&[("ok.rs", CLEAN), (".gitignore", "*.gen.rs\n")]); + for d in 0..PKGS { + let pkg = proj.join(format!("pkg{d}")); + fs::create_dir(&pkg).unwrap(); + fs::write(pkg.join("ok.rs"), CLEAN).unwrap(); + fs::write(pkg.join("v.gen.rs"), CLEAN).unwrap(); + } + + let dotted = format!("{}", Path::new(".").join("pkg0").join("v.gen.rs").display()); + let bare = format!(": {}", Path::new("pkg0").join("v.gen.rs").display()); + let summary = format!("bca: {PKGS} files not checked ({PKGS} ignored)"); + for (seeds, kept, dropped) in [ + (&[".", "pkg0"], &dotted, &bare), + (&["pkg0", "."], &bare, &dotted), + ] { + let serial = report_skipped(&proj, seeds, "1"); + assert!(serial.contains(&summary), "{seeds:?}: {serial}"); + assert!(serial.contains(kept), "{seeds:?} keeps {kept}: {serial}"); + assert!( + !serial.contains(dropped), + "{seeds:?} drops {dropped}: {serial}" + ); + assert_eq!( + report_skipped(&proj, seeds, "8"), + serial, + "{seeds:?}: the listing must not depend on --jobs" + ); + } + drop(dir); +} + +/// A `./`-spelled seed is measured like its bare twin, and the listing +/// keeps the spelling the walk emitted: the seed verbatim, joined to the +/// entry name (`./proj\violator.rs` on Windows, hence the `join`). +#[test] +fn dot_slash_seed_reports_the_ignored_file() { + let (dir, _proj) = fixture(&[ + ("violator.rs", &branchy("offender")), + ("ok.rs", CLEAN), + (".gitignore", "violator.rs\n"), + ]); + + let stderr = report_skipped(dir.path(), &["./proj"], "auto"); + assert!( + stderr.contains("bca: 1 file not checked (1 ignored)"), + "{stderr}" + ); + let note = format!( + "note: skipped (ignored): {}", + Path::new("./proj").join("violator.rs").display() + ); + assert!(stderr.contains(¬e), "{stderr}"); +} + +/// Ignored files whose names are not UTF-8 are each counted: the +/// measurement compares paths as `OsStr`, never through a lossy string. +/// The two names here both decode lossily to `\u{fffd}\u{fffd}.gen.rs`, +/// so a lossy sort or dedupe would merge them into one. Not on macOS, +/// whose filesystems refuse such a name with `EILSEQ`, nor Windows, +/// which has no byte-path constructor. +#[cfg(all(unix, not(any(target_os = "macos", target_os = "ios"))))] +#[test] +fn non_utf8_ignored_file_is_counted() { + use std::os::unix::ffi::OsStrExt; + + let (dir, proj) = fixture(&[("ok.rs", CLEAN), (".gitignore", "*.gen.rs\n")]); + for bytes in [&b"\xff\xfe.gen.rs"[..], &b"\xfe\xff.gen.rs"[..]] { + let name = std::ffi::OsStr::from_bytes(bytes); + // A filesystem mounted `utf8only` (ZFS) refuses the name as macOS + // does, so the fixture cannot exist there either. + if let Err(e) = fs::write(proj.join(name), CLEAN) { + eprintln!("skipping: this filesystem refuses a non-UTF-8 name: {e}"); + return; + } + } + + let stderr = report_skipped(dir.path(), &["proj"], "auto"); + assert!( + stderr.contains("bca: 2 files not checked (2 ignored)"), + "{stderr}" + ); +} diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index a54993be8..47e63d9b6 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -194,6 +194,9 @@ Read it as follows. | `loc/ruby-nested-doc-constants` | Ruby | depth | YARD's namespace scope test through nested `if`s (#1597) | linear | | `loc/lua-tagless-blocks` | Lua | width | LDoc's first-block test, scanning back to the previous block (#1597) | linear | | `loc/perl-data-pod` | Perl | width | the one-pass POD scan of the data section after `__END__` (#1608) | linear | +| `loc/bash-nested-continued-fn` | Bash | depth | the bare-`\`-row gap scan (`credit_bare_continuation_rows`), each padding byte read by one node however deep the spaces nest (#1445) | linear | +| `loc/tcl-wide-continued-comment` | Tcl | width | the comment lookup for a bare row continuing a comment, from the child before the row's gap (#1445) | linear | +| `loc/tcl-unclosed-continued-comments` | Tcl | width | the same lookup when an unclosed opener per unit leaves every unit a direct child of one flat `ERROR` (#1445) | linear | | `halstead/perl-dash-keys` | Perl | width | the dash-key workaround's `=>` lookup per key (#1545) | linear | | `abc/perl-bareword-keys` | Perl | width | the bareword-key workaround's `=>` lookup per key (#1572) | linear | | `abc/perl-nested-signed-keys` | Perl | depth | the ABC value peel's `-and` / `-not` sign test, handed the `-`'s parent rather than a chain to climb (#1603) | linear | @@ -231,6 +234,7 @@ Read it as follows. | `npa/java-deep-expression` | Java | depth | the member walks' per-body chain (`Ancestors::of_children`), built inside the body arms rather than for every node the walk visits (#1696) | linear | | `npm/ruby-nested-if-defs` | Ruby | depth | the Ruby member walk reading a `def` through nested conditionals on one moved ancestor chain (`ChildAncestors::descend` / `ascend`) rather than a chain copy per level (#1401) | linear | | `npm/ruby-wide-private-defs` | Ruby | width | the same walk reading each `private` call's argument list and re-filing the name it declares as one deferred `(covered, bit)` on the name's index entry, applied once at the tally, rather than a write per declaration — half the names distinct, half one name redefined, so both halves of the index are on the ladder (#1401) | linear | +| `npa/ruby-nested-if-attrs` | Ruby | depth | the attribute walk on the shared class-body walk, reading an `attr_*` and an `@ivar` assignment through nested conditionals and asking the `Member` row under the held chain (#1719) | linear | | `npa/cpp-wide-class-members` | C++ | width | a shape control for the C++ member walks: `cpp_declares_function` is asked of every field and static data member template by both consumers and must read only the member's own declarator chain — a walk that re-sought siblings or read a `template_declaration`'s neighbours would show here (#1299) | linear | | `nom/elixir-unquoted-def` | Elixir | depth | the `def`-under-no-`quote` question `nom/nested-quote`'s short-circuit cannot help, folded down the walker's chain once per ancestor (#1691) | linear | | `halstead/csharp-qualified-generics` | C# | depth | each generic qualifier classified without climbing the remaining qualified-name ancestry | linear | diff --git a/mise.toml b/mise.toml index f2cd333d3..8edfdc1f3 100644 --- a/mise.toml +++ b/mise.toml @@ -7,6 +7,7 @@ node = "25.9.0" # `.github/workflows/ci.yml` (the CI lint job fetches the upstream # release tarball directly rather than going through mise) and with # `RUMDL_VERSION` plus the per-arch SHA256 table in the `Dockerfile`. +# `make check-rumdl-lockstep` fails when the three disagree (#1779). rumdl = "0.3.2" # Python is required for the `big-code-analysis-py` crate # (PyO3 + maturin). Pinned to a 3.12 series — bindings declare diff --git a/src/metrics/cognitive.rs b/src/metrics/cognitive.rs index eee689359..03af0ac23 100644 --- a/src/metrics/cognitive.rs +++ b/src/metrics/cognitive.rs @@ -32,6 +32,24 @@ use crate::macros::implement_metric_trait; use crate::metrics::ConstraintReach; use crate::*; +// `§B1` / `B2` / `B3` here and in the per-language modules cite Appendix B +// ("Specification") of Campbell's *Cognitive Complexity* white paper at +// v1.7, 29 August 2023. The PDF's URL is unversioned and the rules have +// been reworded under it (v1.5 made the multi-level break/continue +// increment explicit), so check a citation against this summary of v1.7: +// B1 Increments: if, else if, else, ternary; switch; for, foreach; +// while, do while; catch; goto/break/continue to a LABEL or NUMBER; +// sequences of binary logical operators; each recursion-cycle method. +// B2 Nesting level: the B1 structures from if through catch, plus nested +// methods and lambdas. Jumps, operator sequences and recursion do not +// raise it. +// B3 Nesting increments: if, ternary, switch, for, foreach, while, +// do while, catch. Nothing else pays for its depth, so a flat +1 is +// a B1 increment that B3 omits. +// The paper's prose also ignores null-coalescing operators as shorthand, +// so scoring `??` (#230), Elvis `?:` (#239, #246) and Perl's `//` as +// boolean sequences is this crate's choice, not a B1 reading. + // No language scores the specification's recursion increment (B1, "each // method in a recursion cycle"), and this is a decision rather than a gap: // the walk is a single pass over one file's syntax tree and resolves no @@ -8053,10 +8071,12 @@ end", #[test] fn typescript_nullish_coalescing_with_if_230() { // Regression for issue #230: the example from the issue body. - // Boolean sequences pay a flat +1 (no nesting penalty) per Sonar - // B1, so the issue body's stated total of 3 was wrong — the - // correct answer is if(+1) + ?? chain (+1) = 2. Previously the - // `??` chain was not counted at all (= 1). + // Boolean sequences pay a flat +1: Sonar B1's increment, with no + // nesting increment since B3 omits them (that `??` counts at all + // is this crate's choice; see the module header). So the issue + // body's stated total of 3 was wrong — the correct answer is + // if(+1) + ?? chain (+1) = 2. Previously the `??` chain was not + // counted at all (= 1). check_metrics::( "function risky(x: string | null, fallback: string | null): string { if (x === \"y\") { // +1 @@ -8144,8 +8164,8 @@ end", #[test] fn csharp_null_coalescing_cognitive_230() { // Regression for issue #230: C# `??` must form a boolean sequence - // just like `&&` / `||`. Boolean sequences pay a flat +1 (no - // nesting penalty) per Sonar B1. + // just like `&&` / `||`. Boolean sequences pay a flat +1: Sonar + // B1's increment, with no nesting increment since B3 omits them. // if(+1) + ?? chain (+1) = 2. Previously the `??` chain // contributed nothing and the function scored 1. check_metrics::( @@ -8182,8 +8202,8 @@ end", fn php_null_coalescing_cognitive_230() { // Regression for issue #230: PHP `??` must form a boolean sequence // just like `&&` / `||`. Parallels the PHP cyclomatic - // null-coalescing handling. Boolean sequences pay a flat +1 (no - // nesting penalty) per Sonar B1. + // null-coalescing handling. Boolean sequences pay a flat +1: Sonar + // B1's increment, with no nesting increment since B3 omits them. // if(+1) + ?? chain (+1) = 2. check_metrics::( "( "def f(a, b) { if (a != null) { // +1 @@ -9151,8 +9172,9 @@ end", #[test] fn kotlin_elvis_inside_if_239() { // Regression for issue #239: Elvis chain inside an `if` body. - // Boolean sequences pay a flat +1 (no nesting penalty) per - // Sonar B1: if(+1) + ?: chain(+1) = 2. Previously the Elvis + // Boolean sequences pay a flat +1: Sonar B1's increment, with no + // nesting increment since B3 omits them. + // if(+1) + ?: chain(+1) = 2. Previously the Elvis // chain was not counted at all and the function scored 1. check_metrics::( "fun f(a: String?, b: String?): String { @@ -9885,7 +9907,9 @@ end", // PHP has no labeled break/continue; only the numeric level form // `break N;` / `continue N;`, which exits N enclosing loops already // accounted for by nesting. Per issue #435 the numeric form is a - // structured loop-level exit and adds +0. + // structured loop-level exit and adds +0. v1.7 §B1 increments a + // jump to a NUMBER, so this pins a deliberate, still undecided + // deviation, not the paper: #1765 (item 2) decides it. check_metrics::( "( "def foo\n while a\n redo\n end\n begin\n work\n rescue\n retry\n end\nend\n", "foo.rb", diff --git a/src/metrics/cognitive/php.rs b/src/metrics/cognitive/php.rs index 163b2476f..7debe72c5 100644 --- a/src/metrics/cognitive/php.rs +++ b/src/metrics/cognitive/php.rs @@ -78,7 +78,10 @@ impl Cognitive for PhpCode { // `break N;` / `continue N;`, which breaks out of N enclosing // loops. Those enclosing loops are already counted via nesting, // so the numeric form is a structured loop-level exit and adds - // +0 — only `goto` is genuinely unstructured here. + // +0 — only `goto` is genuinely unstructured here. That +0 is a + // deliberate, still undecided deviation from v1.7 §B1, which + // lists a jump to a NUMBER as an increment: #1765 (item 2) + // decides it, so do not "fix" it from the citation alone. GotoStatement => { increment_by_one(stats); } diff --git a/src/metrics/loc.rs b/src/metrics/loc.rs index 3c39805f2..93d69bd8e 100644 --- a/src/metrics/loc.rs +++ b/src/metrics/loc.rs @@ -8464,22 +8464,15 @@ EOF } /// A `\`-continuation row inside a heredoc's command prefix is code - /// (#1443, #1445). + /// (#1445). /// - /// `heredoc_redirect` spans the command-line prefix as well as the - /// literal, so crediting its whole interior over-credits a blank or - /// comment-only prefix row. #1443 narrowed the range to the literal's - /// own rows to fix that, and the narrowing was reverted: the shape it - /// fixed needs `bash -n` to reject the file, while the shape it broke - /// — this one — runs. - /// - /// A continuation row holding only `\` carries no leaf for the - /// leaf-gated catch-all to credit, so the wrapper's blanket range is - /// the only thing covering it. Narrowing took it from `ploc 5` to - /// `ploc 4`. The general gap is #1445 (`echo a | \` + `\` + - /// ` grep b` loses the row with no heredoc involved); until that - /// lands, this pins the row the blanket range saves, so a second - /// attempt at the narrowing fails here rather than shipping. + /// A row holding only `\` carries no leaf. The heredoc arm's + /// whole-interior range happens to cover this one, so the pair below + /// passes with or without `credit_bare_continuation_rows`; the + /// heredoc-free `echo a | \` fixtures are what pin that helper. This + /// test pins the row against a future narrowing of the heredoc range + /// (#1443 tried one; `bash_heredoc_body_rows_after_an_operator_are_the_literal` + /// says why it was withdrawn). #[cfg(feature = "bash")] #[test] fn bash_heredoc_continuation_row_is_code() { @@ -8509,6 +8502,439 @@ EOF ); } + /// A heredoc whose command line ends in an operator keeps every body + /// row as part of the literal. + /// + /// tree-sitter-bash parses the first body rows after a trailing `|`, + /// `&&` or `||` as the operator's right-hand command, while bash + /// starts the body on the row after the `<<` and reads the command + /// after the terminator. #1443's narrowed range, briefly restored + /// under #1445, started the credit after the last non-body child and + /// so lost one literal row per fixture: a blank body row read as + /// `blank`, a `#` body row as comment-only. Each fixture is valid + /// Bash (`bash -n` accepts it, and the `# hash line` one prints that + /// row), and the expected rows are `main`'s before #1445: every row + /// is code. The `cloc 1` of the two `#` rows is not: bash prints them + /// as data, and the comment is the upstream misparse's own `comment` + /// node (see the `HeredocRedirect` arm), pinned so a grammar fix is + /// seen rather than endorsed. + #[cfg(feature = "bash")] + #[test] + fn bash_heredoc_body_rows_after_an_operator_are_the_literal() { + let cases: [(&[u8], LocRows); 4] = [ + (b"cat < LocRows { + (loc.sloc(), loc.ploc(), loc.cloc(), loc.blank()) + } + + /// Asserts each `(source, file rows, blank before #1445)` fixture's + /// file-level `loc_rows` under `lang`, byte-for-byte: `check_metrics` + /// would normalise away the CRLF and the missing final newline that + /// two of the fixtures are about (`.claude/rules/testing.md`). + #[cfg(any( + feature = "bash", + feature = "c", + feature = "cpp", + feature = "elixir", + feature = "irules", + feature = "mozcpp", + feature = "objc", + feature = "ruby", + feature = "tcl" + ))] + #[track_caller] + fn assert_continuation_fixtures(lang: crate::LANG, fixtures: &[(&[u8], LocRows, u64)]) { + let mismatches: Vec = fixtures + .iter() + .filter_map(|&(source, expected, blank_before)| { + let loc = metrics_verbatim( + lang, + source, + MetricsOptions::default().with_only(&[crate::Metric::Loc]), + ) + .loc; + let measured = loc_rows(&loc); + (measured != expected).then(|| { + format!( + "{:?}: measured {measured:?}, expected {expected:?} \ + (blank {blank_before} before #1445)", + String::from_utf8_lossy(source) + ) + }) + }) + .collect(); + assert!( + mismatches.is_empty(), + "{lang:?}, {} of {} fixtures:\n{}", + mismatches.len(), + fixtures.len(), + mismatches.join("\n") + ); + } + + /// Asserts `(unit, nested space)` `loc_rows` for a source holding one + /// function-like space, so a bare row is shown credited to the space + /// that contains it and not only to the file (#1445): `Loc::compute` + /// writes into the current space's `Stats`, and the unit sees nested + /// rows only through `Stats::merge`. + #[cfg(any(feature = "bash", feature = "irules", feature = "tcl"))] + #[track_caller] + fn assert_nested_continuation(lang: crate::LANG, source: &[u8], expected: LocRows) { + let space = space_verbatim( + lang, + source, + MetricsOptions::default().with_only(&[crate::Metric::Loc]), + ); + let text = String::from_utf8_lossy(source); + assert_eq!(space.spaces.len(), 1, "{lang:?} {text:?}: one nested space"); + assert_eq!( + loc_rows(&space.metrics.loc), + expected, + "{lang:?} {text:?}: unit" + ); + assert_eq!( + loc_rows(&space.spaces[0].metrics.loc), + expected, + "{lang:?} {text:?}: the nested space spans every row too" + ); + } + + /// #1445: a `\`-continuation row holding only the backslash is the + /// code it continues. The lexer splices `\` + newline away, so the row + /// surfaces no node, and nothing credited it before. One that continues + /// nothing starts a logical line and is what that line becomes: code, + /// a comment, or blank. + /// + /// Every fixture is valid Bash (`bash -n`, and run) except the CRLF + /// one: bash rejects a CRLF script outright, but the grammar splices + /// `\` + CRLF the same way (`/\\\r?\n/` in its extras), and the row + /// holds text either way. + #[cfg(feature = "bash")] + #[test] + fn bash_bare_continuation_rows_are_the_line_they_join() { + assert_continuation_fixtures( + crate::LANG::Bash, + &[ + // The issue's reproduction, and the bare row indented by + // spaces and by a tab: the row is still only the `\`. + (b"echo a | \\\n\\\n grep b\n", (3, 3, 0, 0), 1), + (b"echo a | \\\n \\\n grep b\n", (3, 3, 0, 0), 1), + (b"echo a | \\\n\t\\\n grep b\n", (3, 3, 0, 0), 1), + // Before the root's first token, which is where its span + // begins, so only the root's leading padding holds it. + (b"\\\necho a\n", (2, 2, 0, 0), 1), + (b"echo a | \\\r\n\\\r\n grep b\r\n", (3, 3, 0, 0), 1), + // At EOF with no newline the grammar emits a node for the + // `\`, so this one was already code. Tcl's twin is the EOF + // row that only the root's trailing padding reaches. + (b"echo a | \\\n\\", (2, 2, 0, 0), 0), + // Between two function definitions, which the grammar + // emits as adjacent siblings with no token between: the + // row is in the root's own padding, outside both spaces. + (b"f() { :; }\n\\\ng() { :; }\n", (3, 3, 0, 0), 1), + // Controls. A blank row between continuations stays blank: + // only a row spelled `\` is credited. + (b"echo a \\\n\n\\\necho b\n", (4, 3, 0, 1), 2), + // Bash never continues a comment (`echo b` runs), so the + // bare row after one is code, not comment. + (b"# c \\\n\\\necho b\n", (3, 2, 1, 0), 1), + // A bare row that continues nothing joins the row after it, + // and bash reads that logical line as the row it joins: a + // comment (also after indentation, or after a run of bare + // rows), or an empty line, at EOF too. Each runs only its + // `echo`. + (b"\\\n# note\necho done\n", (3, 1, 2, 0), 1), + (b"\\\n # note\necho done\n", (3, 1, 2, 0), 1), + (b"\\\n\\\n# note\necho done\n", (4, 1, 3, 0), 2), + (b"\\\n\necho done\n", (3, 1, 0, 2), 2), + (b"echo a\n\\\n", (2, 1, 0, 1), 1), + // After a comment's `\`, which is comment text, the bare row + // starts a line of its own and joins the empty row. + (b"# c \\\n\\\n\necho b\n", (4, 1, 1, 2), 2), + // Two bare rows in the root's gaps three rows apart: the + // first joins a comment, the second a function, so the + // second does not inherit the first's class. + (b"\\\n# note\nf() { :; }\n\\\ng() { :; }\n", (5, 3, 2, 0), 2), + // The same joins with CRLF, which only Bash's grammar + // splices (`/\\\r?\n/`); Tcl's lexes the row as a word. + (b"\\\r\n# note\r\necho done\r\n", (3, 1, 2, 0), 1), + (b"\\\r\n\r\necho done\r\n", (3, 1, 0, 2), 2), + // Inside a `raw_string`, a leaf with no gaps, so the bare-row + // scan never sees the row: the literal's own interior + // credit makes it code. A control for that arm, not for + // the scan. + (b"echo 'a\n\\\nb'\n", (3, 3, 0, 0), 0), + ], + ); + } + + /// #1445's rows inside a function are the function's own, not only + /// the file's. + #[cfg(feature = "bash")] + #[test] + fn bash_bare_continuation_row_in_a_function_is_its_code() { + assert_nested_continuation( + crate::LANG::Bash, + b"f() {\n echo a | \\\n\\\n grep b\n}\n", + (5, 5, 0, 0), + ); + // Indented as a function body's rows usually are. + assert_nested_continuation( + crate::LANG::Bash, + b"f() {\n echo a | \\\n \\\n grep b\n}\n", + (5, 5, 0, 0), + ); + } + + /// A bare row between two function definitions is in the root's own + /// padding, so the file counts it and neither function does: each + /// space keeps its one row. + #[cfg(feature = "bash")] + #[test] + fn bash_bare_continuation_row_between_functions_is_neither_s() { + let source = b"f() { :; }\n\\\ng() { :; }\n"; + let space = space_verbatim( + crate::LANG::Bash, + source, + MetricsOptions::default().with_only(&[crate::Metric::Loc]), + ); + assert_eq!(loc_rows(&space.metrics.loc), (3, 3, 0, 0), "the file"); + assert_eq!(space.spaces.len(), 2, "one space per function"); + for function in &space.spaces { + assert_eq!( + loc_rows(&function.metrics.loc), + (1, 1, 0, 0), + "{:?} holds only its own row", + function.name + ); + } + } + + /// #1445 for Tcl, whose comments continue: `tclsh` reads `# c \` + `\` + /// as one comment, but the grammar's `comment` token (`#[^\n]*`) stops + /// at the row's end, so a bare row continuing one is comment that no + /// comment node covers. It goes to CLOC, and a run of bare rows + /// continues whatever its first one did. + /// + /// Each comment fixture ends the comment with a blank row before the + /// code that follows, because the grammar does not honour the + /// continuation past a row with text: `# c \` + `puts x` runs nothing + /// in `tclsh` and parses as a command here, a separate gap no bare-row + /// credit can fix. Every fixture was run through `tclsh`. + #[cfg(feature = "tcl")] + #[test] + fn tcl_bare_continuation_rows_are_code_or_comment() { + assert_continuation_fixtures( + crate::LANG::Tcl, + &[ + (b"puts [list a \\\n\\\n b]\n", (3, 3, 0, 0), 1), + // An indented bare row is still bare (`tclsh` prints `a b`). + (b"puts [list a \\\n \\\n b]\n", (3, 3, 0, 0), 1), + (b"\\\nputs a\n", (2, 2, 0, 0), 1), + // A bare row that continues nothing is the line it joins: a + // comment, which may itself continue, or an empty line. + (b"\\\n# note\nputs done\n", (3, 1, 2, 0), 1), + (b"\\\n\nputs done\n", (3, 1, 0, 2), 2), + (b"\\\n # c \\\n\\\n\nputs x\n", (5, 1, 3, 1), 3), + // FIXME(#1808): `tclsh` reads (3, 1, 2, 0) — the bare row + // and `# c` are one comment line — but the grammar joins + // the bare row to `puts a` before it, so `#` and `c` are + // two words of that command and the row reads as code. + (b"puts a\n\\\n# c\n", (3, 3, 0, 0), 1), + // With CRLF the grammar lexes the bare row's `\` + `\r` as + // an escaped character, a word of the surrounding command, + // so the row was already code; see the FIXME below for the + // comment position, where that word is the defect. + (b"puts [list a \\\r\n\\\r\n b]\r\n", (3, 3, 0, 0), 0), + // At EOF with no newline, which only the root's trailing + // padding reaches. `tclsh` runs it as a command named `\`. + (b"puts a\n\\", (2, 2, 0, 0), 1), + // rows: 0 comment, 1 its continuation, 2 blank, 3 code. + (b"# c \\\n\\\n\nputs x\n", (4, 1, 2, 1), 2), + (b"# c \\\n\\\n\\\n\nputs x\n", (5, 1, 3, 1), 3), + // FIXME(#1802): `tclsh` reads (4, 1, 2, 1), as for the LF + // twin above. With CRLF the grammar lexes the bare row's + // `\` + `\r` as an escaped character and emits a `command`, + // so the row is a node, not padding, and reads as code. + (b"# c \\\r\n\\\r\n\r\nputs x\r\n", (4, 2, 1, 1), 1), + // A CRLF comment row before an LF bare row: the comment + // token takes the `\r`, and the lookup steps over it to + // the `\`. Only the raw library path keeps the CR. + (b"# c \\\r\n\\\n\nputs x\n", (4, 1, 2, 1), 2), + // A comment after code on its row continues too. + (b"puts x ;# c \\\n\\\n\nputs y\n", (4, 2, 2, 1), 2), + // The comment sits in a command substitution inside a + // quoted word, which already made its rows code: the bare + // row is a comment on a code row, and row 4 stays blank. + (b"puts \"[# c \\\n\\\n\n]\"\n\nputs x\n", (6, 5, 2, 1), 1), + // Controls. An even run is escaped backslashes: the comment + // ends on its row, and the bare row continues `puts x`. + (b"# c \\\\\n\\\nputs x\n", (3, 2, 1, 0), 1), + // A braced literal credits no blank row of its own, but its + // bare row is text. + (b"set x {a\n\n\\\nb}\n", (4, 3, 0, 1), 2), + // A quoted word already credits every row it spans. + (b"puts \"a\n\\\nb\"\n", (3, 3, 0, 0), 0), + ], + ); + } + + /// #1445's rows inside a `proc` are the procedure's own: a bare code + /// row, and a bare row continuing a comment in the body. + #[cfg(feature = "tcl")] + #[test] + fn tcl_bare_continuation_rows_in_a_proc_are_its_own() { + assert_nested_continuation( + crate::LANG::Tcl, + b"proc p {} {\n puts [list a \\\n\\\n b]\n}\n", + (5, 5, 0, 0), + ); + // rows: 0 header, 1 comment, 2 its continuation, 3 blank, 4 code, + // 5 `}`. The comment sits inside the body's braced word, so the + // lookup that classifies the bare row runs below the root. + assert_nested_continuation( + crate::LANG::Tcl, + b"proc p {} {\n # c \\\n\\\n\n puts x\n}\n", + (6, 3, 2, 1), + ); + } + + /// #1445 for iRules, which forked Tcl's grammar and shares both its + /// extras and its `#[^\n]*` comment token. No iRules interpreter was + /// available to run these; the Tcl twins above were run through + /// `tclsh`. + #[cfg(feature = "irules")] + #[test] + fn irules_bare_continuation_rows_are_code_or_comment() { + assert_continuation_fixtures( + crate::LANG::Irules, + &[ + (b"set a [list a \\\n\\\n b]\n", (3, 3, 0, 0), 1), + // The line a bare row that continues nothing joins, as in Tcl. + (b"\\\n# note\nset a 1\n", (3, 1, 2, 0), 1), + (b"\\\n\nset a 1\n", (3, 1, 0, 2), 2), + // A word of the command, as in Tcl's CRLF fixture. + (b"set a [list a \\\r\n\\\r\n b]\r\n", (3, 3, 0, 0), 0), + (b"# c \\\n\\\n\nset a 1\n", (4, 1, 2, 1), 2), + // A CRLF comment row before an LF bare row, as in Tcl. + (b"# c \\\r\n\\\n\nset a 1\n", (4, 1, 2, 1), 2), + // FIXME(#1802): the same CRLF word as Tcl's twin. + (b"# c \\\r\n\\\r\n\r\nset a 1\r\n", (4, 2, 1, 1), 1), + (b"# c \\\\\n\\\nset a 1\n", (3, 2, 1, 0), 1), + ], + ); + } + + /// #1445's rows inside an event handler are the handler's own. + #[cfg(feature = "irules")] + #[test] + fn irules_bare_continuation_rows_in_a_handler_are_its_own() { + assert_nested_continuation( + crate::LANG::Irules, + b"when HTTP_REQUEST {\n log local0. [list a \\\n\\\n b]\n}\n", + (5, 5, 0, 0), + ); + assert_nested_continuation( + crate::LANG::Irules, + b"when HTTP_REQUEST {\n # c \\\n\\\n\n log local0. x\n}\n", + (6, 3, 2, 1), + ); + // A bare row that continues nothing joins the handler's comment. + assert_nested_continuation( + crate::LANG::Irules, + b"when HTTP_REQUEST {\n\\\n # note\n log local0. x\n}\n", + (5, 3, 2, 0), + ); + // The comment inside a quoted word's command substitution, as in + // Tcl's twin: the bare row is a comment on a code row, so row 5 + // stays blank. + assert_nested_continuation( + crate::LANG::Irules, + b"when HTTP_REQUEST {\n log local0. \"[# c \\\n\\\n\n]\"\n\n log local0. x\n}\n", + (8, 7, 2, 1), + ); + } + + /// FIXME(#1801): the C family, Ruby and Elixir splice `\` + newline in + /// their lexer extras as Bash does, but their `Loc` impls do not call + /// `credit_bare_continuation_rows` yet, so the bare row still reads as + /// blank. Pinned so the gap stays visible rather than endorsed: the fix + /// moves every row to `(3, 3, 0, 0)`. + /// + /// gcc, g++ and clang (`-x objective-c`) accept the C-family fixture, + /// and `ruby -c` the Ruby one, which evaluates to 3. No Elixir + /// interpreter was available; its row rests on the grammar alone. + #[cfg(all( + feature = "c", + feature = "cpp", + feature = "elixir", + feature = "mozcpp", + feature = "objc", + feature = "ruby" + ))] + #[test] + fn bare_continuation_rows_in_unwired_grammars_still_read_blank() { + let c_family: &[(&[u8], LocRows, u64)] = + &[(b"int x = 1 + \\\n\\\n 2;\n", (3, 2, 0, 1), 1)]; + for lang in [ + crate::LANG::C, + crate::LANG::Cpp, + crate::LANG::Mozcpp, + crate::LANG::Objc, + ] { + assert_continuation_fixtures(lang, c_family); + } + let expression: &[(&[u8], LocRows, u64)] = &[(b"x = 1 + \\\n\\\n 2\n", (3, 2, 0, 1), 1)]; + for lang in [crate::LANG::Ruby, crate::LANG::Elixir] { + assert_continuation_fixtures(lang, expression); + } + } + /// The orphan `heredoc_body` — the shape that makes routing the /// wrapper an *addition* rather than a replacement. /// diff --git a/src/metrics/loc/bash.rs b/src/metrics/loc/bash.rs index d37b9de07..e7a237386 100644 --- a/src/metrics/loc/bash.rs +++ b/src/metrics/loc/bash.rs @@ -24,6 +24,16 @@ impl Loc for BashCode { use Bash::*; let start = init(node, stats, is_func_space); + credit_bare_continuation_rows( + node, + code, + ancestors, + stats, + BareRowComments { + kind: Bash::Comment as u16, + continued: false, + }, + ); match node.kind_id().into() { Program => {} @@ -119,34 +129,33 @@ impl Loc for BashCode { stats.ploc.lines.insert(start); add_string_interior_ploc(node, stats, start); } - // The wrapper spans the command-line *prefix* as well as the - // literal, and the grammar lets that prefix cross rows — a - // `pipeline` is one of its children — so this range can - // over-credit. #1443 narrowed it to the literal's own rows and - // that was **reverted**; the narrowing is restored only when - // #1445 lands. The reasoning is worth keeping, because the - // narrowing looks obviously right: + // The wrapper spans the command-line prefix as well as the + // literal, and credits its whole interior. #1443 narrowed this + // to "one past the last row any non-body child occupies"; that + // narrowing is withdrawn because its premise — that a blank or + // comment-only prefix row is a bash syntax error — is false for + // a prefix ending in `|`, `|&`, `&&` or `||`. There + // tree-sitter-bash parses the *first body rows* as the + // operator's right-hand command, while bash starts the body on + // the row after the `<<` (`cat < { check_comment_ends_on_code_line(stats, start); stats.ploc.lines.insert(start); diff --git a/src/metrics/loc/irules.rs b/src/metrics/loc/irules.rs index 2351d11e4..806d6c2ba 100644 --- a/src/metrics/loc/irules.rs +++ b/src/metrics/loc/irules.rs @@ -22,6 +22,16 @@ impl Loc for IrulesCode { is_func_space: bool, ) { let start = init(node, stats, is_func_space); + credit_bare_continuation_rows( + node, + code, + ancestors, + stats, + BareRowComments { + kind: Irules::Comment as u16, + continued: true, + }, + ); match node.kind_id().into() { // Same row-terminator quirk as the Tcl impl this dialect diff --git a/src/metrics/loc/shared.rs b/src/metrics/loc/shared.rs index 304371eac..ca1b3968f 100644 --- a/src/metrics/loc/shared.rs +++ b/src/metrics/loc/shared.rs @@ -348,3 +348,360 @@ pub(crate) fn add_string_interior_ploc(node: &Node, stats: &mut Stats, start: us .lines .insert_range(start.saturating_add(1), node.end_line().saturating_sub(1)); } + +// A node's start row, the bare row and its end row: a node spanning fewer +// rows has no complete row strictly between its children to find. +const ROWS_AROUND_A_GAP_ROW: usize = 2; + +// Credits each `\`-continuation row that holds nothing but the backslash +// as the code it continues (#1445). +// +// Bash, Tcl and iRules splice `\` + newline away in the lexer — each +// grammar lists `/\\\r?\n/` among its extras — so a row spelled only `\` +// surfaces no node at all. Their `Loc` arms credit nodes, and a row no +// node covers reaches neither PLOC nor CLOC, so +// `blank = sloc - ploc - cloc` claimed it: `echo a | \` + `\` + +// ` grep b` reported `ploc 2, blank 1` for one three-row command. +// +// FIXME(#1801): C, C++, Mozcpp, Objective-C, Ruby and Elixir list the +// same extra and do not call this yet, so a bare row there still reads +// as blank — `#define` bodies included, where a bare row between the +// parameters and the body, or after a `/* … */` that ends a +// `preproc_arg`, is padding rather than macro text. +// +// A bare row that continues no row before it starts a logical line, and +// is what that line becomes once spliced: a comment where the row it joins +// opens one (`\` + `# note` is one comment line to bash and tclsh), blank +// where that row is empty or EOF (`\` + an empty row), and code otherwise. +// The joined row is the first after the run of bare rows, and since only +// padding precedes its first token, that token is the gap's next child. +// +// It reads only a node's *gaps*, the padding between its children. A +// bare row carries no token, so it lies in exactly one gap: that of the +// deepest node spanning it. That node's `stats` belongs to the innermost +// space containing the row, which is the space whose own `ploc` needs +// it, and every enclosing space receives it through `Stats::merge`. A +// per-space scan of each space's whole byte range would also be right +// and costs bytes × nesting depth, the class `loc/nested-fn-rows` exists +// to keep linear (`loc/bash-nested-continued-fn` measures it); each +// padding byte here sits in one node's gaps, so the scan is linear in the +// file. Iterating a node's children is the cost every node would pay, so +// a node too short to hold a gap row returns before it. +// +// The root alone also owns the padding before its first child and after +// its last, to EOF: its span begins at its first token, so a leading +// `\` + `echo a` has its bare row outside every node. +// +// `comments` names the grammar's comment kind and whether a `\` ending a +// comment row continues the comment. `tclsh` reads `# c \` + `\` + +// `puts x` as one comment, but the Tcl and iRules `comment` token is +// `#[^\n]*` and stops at the row's end, so a bare row continuing one is +// comment text the comment node does not cover; it is credited to CLOC. +// Bash never continues a comment — `echo a # c \` + `\` + `echo b` runs +// `echo b` — so there a bare row after a comment starts a logical line. +#[inline] +pub(crate) fn credit_bare_continuation_rows<'a>( + node: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, + stats: &mut Stats, + comments: BareRowComments, +) { + let is_root = ancestors.parent(node).is_none(); + if !is_root + && (node.child_count() == 0 + || node.end_row() < node.start_row().saturating_add(ROWS_AROUND_A_GAP_ROW)) + { + return; + } + let mut scan = GapScan { + node: node.as_tree_sitter(), + before: [None; CHILDREN_BEFORE_A_GAP], + code, + stats, + comments, + previous_bare: None, + }; + let (mut gap_start, mut gap_row) = if is_root { + (0, 0) + } else { + (node.start_byte(), node.start_row()) + }; + for child in node.children() { + scan.gap(gap_start, gap_row, Some(child.as_tree_sitter())); + gap_start = child.end_byte(); + gap_row = child.end_row(); + scan.before.rotate_right(1); + scan.before[0] = Some(child.as_tree_sitter()); + } + // A non-root node ends where its last child does; the root runs on + // to EOF, the one place a bare row can lack its newline. + if is_root { + scan.gap(gap_start, gap_row, None); + } +} + +// The grammar's comment kind, and whether a `\` ending a comment row +// carries the comment onto the next row (Tcl, iRules) or not (Bash). +#[derive(Clone, Copy)] +pub(crate) struct BareRowComments { + pub(crate) kind: u16, + pub(crate) continued: bool, +} + +// What a bare row is credited as. +#[derive(Clone, Copy)] +enum BareRow { + Code, + Comment, + Blank, +} + +// What the row read so far holds, for the bare-row test: optional +// horizontal whitespace, one `\`, and an optional `\r` before the newline. +#[derive(Clone, Copy, PartialEq, Eq)] +enum RowShape { + Blank, + Backslash, + BackslashCr, + Other, +} + +impl RowShape { + fn step(self, byte: u8) -> Self { + match (self, byte) { + (Self::Blank, b' ' | b'\t') => Self::Blank, + (Self::Blank, b'\\') => Self::Backslash, + (Self::Backslash, b'\r') => Self::BackslashCr, + _ => Self::Other, + } + } + + fn is_bare(self) -> bool { + matches!(self, Self::Backslash | Self::BackslashCr) + } +} + +// How many of a node's children before a gap `GapScan` keeps for the +// comment lookup: the one ending at the gap, the `\n` token a grammar may +// put between it and the gap, and one spare for a zero-width recovery +// token. +const CHILDREN_BEFORE_A_GAP: usize = 3; + +struct GapScan<'s, 'a> { + // The node whose gaps are scanned, and so the one holding any comment + // a bare row in them continues. + node: tree_sitter::Node<'a>, + // The node's children before the gap being scanned, nearest first. + before: [Option>; CHILDREN_BEFORE_A_GAP], + code: &'s [u8], + stats: &'s mut Stats, + comments: BareRowComments, + // The last bare row credited and what as: a run of bare rows is + // whatever its first one is. + previous_bare: Option<(usize, BareRow)>, +} + +impl<'a> GapScan<'_, 'a> { + // Scans `start` up to `next`, the child ending the gap (EOF for + // `None`); the gap's first byte sits on `row`. + fn gap(&mut self, start: usize, mut row: usize, next: Option>) { + let end = next.map_or(self.code.len(), |child| child.start_byte()); + let Some(bytes) = self.code.get(start..end) else { + return; + }; + // `None` while the row being read began inside a child, whose + // token it then carries. + let mut row_start = + (start == 0 || self.code.get(start.wrapping_sub(1)) == Some(&b'\n')).then_some(start); + let mut shape = RowShape::Blank; + for (offset, &byte) in (start..).zip(bytes) { + if byte == b'\n' { + if let Some(first) = row_start + && shape.is_bare() + { + self.credit(row, first, start, next); + } + row = row.saturating_add(1); + row_start = Some(offset.saturating_add(1)); + shape = RowShape::Blank; + } else { + shape = shape.step(byte); + } + } + if end == self.code.len() + && let Some(first) = row_start + && shape.is_bare() + { + self.credit(row, first, start, next); + } + } + + // Credits the bare `row`, which starts at byte `row_start` in the gap + // that starts at `gap_start` and ends at `next`. + fn credit( + &mut self, + row: usize, + row_start: usize, + gap_start: usize, + next: Option>, + ) { + let class = self.classify(row, row_start, gap_start, next); + match class { + // A range arm already made the row code — a Tcl quoted word + // around the command substitution holding the comment — so it + // is a comment on a code line, as `add_cloc_lines` files one; + // filing it comment-only too would subtract it from `blank` + // twice. + BareRow::Comment if self.stats.ploc.lines.contains(row) => { + add_code_comment_line(self.stats, row); + } + BareRow::Comment => add_only_comment_lines(self.stats, row, row), + // No `check_comment_ends_on_code_line`: that reclassifies a + // comment ending on this row, and nothing but the `\` is on it. + BareRow::Code => { + self.stats.ploc.lines.insert(row); + } + BareRow::Blank => {} + } + self.previous_bare = Some((row, class)); + } + + // What the bare `row` is: what the row before it continues onto it, + // or, when that row continues nothing, what the logical line it starts + // becomes. + fn classify( + &self, + row: usize, + row_start: usize, + gap_start: usize, + next: Option>, + ) -> BareRow { + if let Some((bare_row, class)) = self.previous_bare + && bare_row.saturating_add(1) == row + { + return class; + } + let Some(last) = continuing_backslash(self.code, row_start) else { + return self.spliced_into(row_start, next); + }; + let in_comment = self.holder_of(last, gap_start).is_some_and(|holder| { + holder + .descendant_for_byte_range(last, last.saturating_add(1)) + .is_some_and(|covering| covering.kind_id() == self.comments.kind) + }); + match (in_comment, self.comments.continued) { + (false, _) => BareRow::Code, + (true, true) => BareRow::Comment, + // A Bash comment's `\` is comment text, so the bare row starts + // a logical line of its own. + (true, false) => self.spliced_into(row_start, next), + } + } + + // What the logical line a run of bare rows starts at byte `row_start` + // becomes, read off the first row after the run: blank when that row + // is empty or the file ends, a comment when the gap's `next` child + // opening it is a comment, code otherwise. + fn spliced_into(&self, row_start: usize, next: Option>) -> BareRow { + let mut at = row_start; + loop { + let rest = self.code.get(at..).unwrap_or_default(); + let indent = rest + .iter() + .take_while(|&&byte| matches!(byte, b' ' | b'\t')) + .count(); + let first = at.saturating_add(indent); + match rest.get(indent..).unwrap_or_default() { + [b'\\', b'\n', ..] => at = first.saturating_add(2), + [b'\\', b'\r', b'\n', ..] => at = first.saturating_add(3), + [] | [b'\n', ..] | [b'\r', b'\n', ..] => return BareRow::Blank, + _ if next.is_some_and(|child| self.opens_comment(child, first)) => { + return BareRow::Comment; + } + _ => return BareRow::Code, + } + } + } + + // Whether `child`, or a first descendant of it, is a comment starting + // at byte `first`. Each node is asked for one gap's run at most, so + // the descent stays linear in the file. In every tree seen so far the + // comment is `child` itself; the descent runs (Tcl's `puts a` + `\` + // + `# c`, #1808, descends into a word) but has not yet found a + // comment below a node, and is kept because a grammar may wrap one. + fn opens_comment(&self, child: tree_sitter::Node<'a>, first: usize) -> bool { + let mut node = Some(child); + while let Some(current) = node + && current.start_byte() == first + { + if current.kind_id() == self.comments.kind { + return true; + } + node = current.child(0); + } + false + } + + // The node to look byte `last` up from, the `\` ending the row before + // a bare one: `None` when it is padding in the gap itself (a code + // continuation's `\`), else the child before the gap that holds it. + // + // Only the newline separates `last` from the gap's first row, so the + // child holding it is the one ending at the gap or, past a `\n` + // token, the one before. Looking up from the gap's owner instead + // costs that owner's width per bare row: tree-sitter scans each + // level's children from the first, and an unclosed opener per unit + // leaves every unit a direct child of one flat `ERROR`, which it + // never balances (`loc/tcl-unclosed-continued-comments`). The child + // must contain `last`: looked up from a child that does not, + // `descendant_for_byte_range` answers the child itself, a comment + // that ended a row earlier among them. + fn holder_of(&self, last: usize, gap_start: usize) -> Option> { + if last >= gap_start { + return None; + } + // None of the three fallbacks below is reached by any fixture or + // by the malformed inputs probed for #1445: the `\` is always + // inside the child ending at the gap or the one before. Each + // returns the safe answer rather than asserting, because a + // zero-width recovery token landing between the `\` and the gap + // would reach one, and there the answer below is the right one. + for child in self.before { + // Fewer children than slots: `last` is padding before the + // first, where no comment is. + let child = child?; + if child.start_byte() <= last { + return (last < child.end_byte()).then_some(child); + } + } + // More zero-width children than slots stand between: the owner + // still holds `last`, at the owner's width. + Some(self.node) + } +} + +// The offset of the `\` that ends the row before the one starting at byte +// `row_start`, if that `\` continues the row: a trailing `\r` is stepped +// over, and an even run is escaped backslashes rather than a continuation +// (`tclsh` runs the row after `# c \\`). +fn continuing_backslash(code: &[u8], row_start: usize) -> Option { + // `row_start - 1` is the previous row's newline. + let newline = row_start.checked_sub(1)?; + let before = code.get(..newline)?; + // Reads the row the way `RowShape` reads the bare one. It decides + // input whose comment row ends in CRLF while the bare row after it + // ends in LF: the Tcl and iRules `comment` token takes the `\r`, the + // bare row is padding, and tclsh continues the comment through it + // (`tcl_bare_continuation_rows_are_code_or_comment`). Only the raw + // library and Python `Ast.parse` paths keep a CR at all; the CLI + // normalises line endings first. An all-CRLF bare row never gets + // here: those grammars lex its `\` + `\r` as a word (#1802). + let content = before.strip_suffix(b"\r").unwrap_or(before); + let run = content + .iter() + .rev() + .take_while(|&&byte| byte == b'\\') + .count(); + (run % 2 == 1).then(|| content.len().saturating_sub(1)) +} diff --git a/src/metrics/loc/tcl.rs b/src/metrics/loc/tcl.rs index 19481133c..68c639eb6 100644 --- a/src/metrics/loc/tcl.rs +++ b/src/metrics/loc/tcl.rs @@ -22,6 +22,16 @@ impl Loc for TclCode { is_func_space: bool, ) { let start = init(node, stats, is_func_space); + credit_bare_continuation_rows( + node, + code, + ancestors, + stats, + BareRowComments { + kind: Tcl::Comment as u16, + continued: true, + }, + ); match node.kind_id().into() { // Tcl is the only grammar family here that surfaces the row diff --git a/src/metrics/npa.rs b/src/metrics/npa.rs index 66c169e6f..5f92249e2 100644 --- a/src/metrics/npa.rs +++ b/src/metrics/npa.rs @@ -384,8 +384,10 @@ macro_rules! impl_npa_java_like { }; } +mod ruby_body; mod shared; pub(crate) use python::python_unwrap_function; +pub(crate) use ruby_body::{RubyBodyConsumer, RubyBodyShape, RubyBodyState, RubyOpenCall}; pub(crate) use shared::*; // Reached by the per-language submodules through `use super::*`. @@ -763,6 +765,94 @@ mod tests { }); } + // `cpp_count_field_identifiers`'s member-pointer arm (#1718), asked + // directly. Its one caller hands it a `field_declaration`, and no + // field spelling reaches the arm under the pinned grammars — the + // parenthesised one recovers into an `ERROR` over `S::` beside a + // `pointer_declarator`, the unparenthesised one into a + // `field_identifier` `S` beside a `bitfield_clause` — so the + // templated payloads are the only real trees carrying the + // shape. Rows: a member pointer counts one; one under a global scope + // and with an array bound `N` (an `identifier`) still counts one; a + // qualified *type* holds no member pointer and adds nothing. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + fn assert_member_pointer_names_count_once(lang: LANG) { + let ast = crate::test_support::parse_named( + lang, + "foo.cpp", + "struct S {\n\ + template static int (S::*fp)(T);\n\ + template static int (::S::*fps[N])(T);\n\ + template static std::vector v;\n\ + };", + ); + let counts: Vec = ast + .root_node() + .descendants_by_kind(&["declaration"]) + .iter() + .map(cpp_count_field_identifiers) + .collect(); + assert_eq!(counts, [1, 1, 0]); + } + + #[cfg(feature = "cpp")] + #[test] + fn cpp_member_pointer_names_count_once() { + assert_member_pointer_names_count_once(LANG::Cpp); + } + + #[cfg(feature = "mozcpp")] + #[test] + fn mozcpp_member_pointer_names_count_once() { + assert_member_pointer_names_count_once(LANG::Mozcpp); + } + + // A type qualified 20,000 levels deep (`A::A::…::B v;`) is one data + // member. Every member-pointer arm asks `cpp_member_pointer` about + // each qualified name a member holds, so the scope chain is walked in + // a loop: the recursive first draft overflowed a worker's stack at + // about 11,000 levels, a depth no other metric minds (#1718 review). + // Run on a small stack so a reintroduced recursion aborts here. + // Measured with the recursion restored, in the dev profile the tests + // build: it overflows at about 2,600 levels on this 512 KiB stack + // (10,800 at the 2 MiB a spawned worker gets, 43,500 at 8 MiB), so + // `DEPTH` is ~7.7x past it. A future `DEPTH` must stay well above the + // overflow depth for the stack it sets, or a recursion passes here. + // + // expected: one static data member — na = 1, nm = 0. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + fn assert_deeply_qualified_type_is_one_attribute() { + // `A::` levels before the type's last name, far past what the + // stack below holds at one frame per level. + const DEPTH: usize = 20_000; + // A deliberately small test-thread stack. + const SMALL_STACK: usize = 512 * 1024; + let source = format!("struct X {{ static {}B v; }};\n", "A::".repeat(DEPTH)); + std::thread::Builder::new() + .stack_size(SMALL_STACK) + .spawn(move || { + check_metrics_with_npm::(&source, "foo.cpp", |metric| { + assert_eq!(metric.npa.class_na_sum(), 1); + assert_eq!(metric.npm.class_nm_sum(), 0); + }); + }) + .expect("spawn bounded-stack thread") + .join() + .expect("the member-pointer walk must not overflow a 512 KiB stack"); + } + + #[cfg(feature = "cpp")] + #[test] + fn cpp_deeply_qualified_type_is_one_attribute() { + assert_deeply_qualified_type_is_one_attribute::(); + } + + #[cfg(feature = "mozcpp")] + #[test] + fn mozcpp_deeply_qualified_type_is_one_attribute() { + assert_deeply_qualified_type_is_one_attribute::(); + } + #[cfg(feature = "java")] #[test] fn java_single_attributes() { @@ -3365,6 +3455,378 @@ mod tests { ); } + // --- Ruby attributes the body reaches through a wrapper (#1719) ------- + // + // The attribute walk reads the statements the method walk does — the + // shared body walk in `npa/ruby_body.rs` — so an `attr_*` call or an + // `@x = …` / `@@x = …` assignment counts wherever it runs at class + // scope: under a conditional, a loop, a `begin`, a `case` arm, a + // statement modifier, parentheses, an assignment's right side, or as + // a call's argument. Until #1719 only the body's direct children and + // a visibility call's direct arguments were read, and a bare marker + // was read only as a direct child, with no shadow table. These mirror + // `npm.rs`'s #1401 block. Every visibility expectation was measured + // against ruby 3.0.2 with `public_instance_methods(false)` and its + // siblings (the reader method is the attribute's visibility), the + // instance and class variables with `instance_variables` / + // `class_variables`; the oracle script and its output are recorded on + // #1719. The walk is syntactic and counts every branch, as `nom` + // does, so a branch Ruby skips still counts. + + #[cfg(feature = "ruby")] + #[test] + fn ruby_attribute_under_a_conditional_counts() { + // Both arms of an `if`, a `begin` after a `private` marker, and a + // bare `attr_reader` after it. Measured: `modern` public, `hidden` + // and `plain` private, `@wrapped` set; `legacy` sits on the branch + // Ruby did not take. Before #1719: na = 1 (`plain`), npa = 0. + // + // expected: na = 5 (modern, legacy, @wrapped, hidden, plain), + // npa = 2 (modern, legacy). + check_metrics::( + "class Cond\n if RUBY_VERSION > \"3.0\"\n attr_reader :modern\n else\n attr_reader :legacy\n end\n private\n begin\n @wrapped = 1\n attr_reader :hidden\n end\n attr_reader :plain\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 5); + assert_eq!(metric.npa.class_npa_sum(), 2); + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_attribute_under_each_statement_wrapper_counts() { + // One declaration per wrapper the walk reads through, each + // reachable by that wrapper's arm alone: `while`, `until`, `for` + // (whose body is a `do`), `unless`, the four statement modifiers + // (the `body` field), `rescue` and `ensure`, `elsif`, a `when` + // and a `case` `else`, and an `in` clause. Ruby defines the eight + // on branches it takes, every one public. Before #1719: na = 1. + // + // expected: na = 15 (twelve `attr_reader`s, three assignments), + // npa = 15. + check_metrics::( + "class W\n while false\n attr_reader :in_while\n end\n until true\n attr_reader :in_until\n end\n for i in [] do\n attr_reader :in_for\n end\n unless false\n attr_reader :in_unless\n end\n attr_reader :early unless false\n attr_reader :if_mod if true\n @late = 1 while false\n @until_mod = 1 until true\n begin\n 1\n rescue\n attr_reader :in_rescue\n ensure\n attr_reader :in_ensure\n end\n if false\n 2\n elsif true\n attr_reader :in_elsif\n end\n case 1\n when 1\n attr_reader :in_when\n else\n @in_case_else = 1\n end\n case 2\n in Integer\n attr_reader :in_clause\n end\n attr_reader :bare\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 15); + assert_eq!(metric.npa.class_npa_sum(), 15); + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_attribute_in_parentheses_a_rescue_modifier_or_an_assignment_counts() { + // A parenthesised statement list; the `rescue` modifier on both + // sides; the right side of an assignment, plain, through + // parentheses and chained (`@p = @q = 1` sets both, measured); + // and `@h = private attr_reader :h`, whose target is one + // attribute and whose right side declares a private `h`. + // Measured: `h` private, every other reader Ruby defines public. + // Before #1719: na = 3 (@p, @h, bare). + // + // expected: na = 13 (grouped, rescued, handler, @on_raise, a, @b, + // plain, @in_parens, @p, @q, @h, h, bare), npa = 12. + check_metrics::( + "class P\n (attr_reader :grouped)\n attr_reader :rescued rescue attr_reader :handler\n raise \"boom\" rescue @on_raise = 1\n (attr_reader :a; @b = 1)\n x = attr_reader :plain\n y = (@in_parens = 1)\n @p = @q = 1\n @h = private attr_reader :h\n attr_reader :bare\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 13); + assert_eq!(metric.npa.class_npa_sum(), 12); + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_attribute_in_a_call_argument_counts() { + // A call's arguments run at class scope: `memoize(attr_reader + // :a)` defines `a`, `foo(@x = 1)` sets `@x`, and two decorators + // deep is no different (measured, with `memoize` and `foo` + // defined on `Module`: `a`, `b` and `bare` public, `@x` set). An + // ordinary call governs nothing, so each takes the flag. Before + // #1719: na = 1 (`bare`). + // + // expected: na = 4, npa = 4. + check_metrics::( + "class Arg\n memoize(attr_reader :a)\n foo(@x = 1)\n memoize memoize attr_reader :b\n attr_reader :bare\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 4); + assert_eq!(metric.npa.class_npa_sum(), 4); + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_only_a_plain_ivar_assignment_is_an_attribute() { + // The scope `npa/ruby.rs` documents: a plain `@x = …` is an + // attribute, while an operator assignment (`@c ||= 1`, `@d += 1`) + // and a multiple assignment (`@a, @b = 1, 2`) are not counted, + // though Ruby sets each variable. The shared walk reads both as + // wrappers and asks the `Member` row about their target, so it + // is the row's `is_assignment_target` gate that keeps them out; + // `ruby_space_tree_rows` pins the gate, this pins npa reading it. + // + // expected: na = 1 (`@x`). + check_metrics::( + "class A\n @c ||= 1\n @d += 1\n @a, @b = 1, 2\n @x = 1\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 1); + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_attribute_in_an_ordinary_call_takes_a_private_flag() { + // The other half of `ruby_attribute_in_a_call_argument_counts`: + // an ordinary call governs nothing, so its argument takes the + // body flag even when that flag is not the default (measured, + // with `memoize` returning its argument: `a` and `b` private). + // + // expected: na = 2, npa = 0. + check_metrics::( + "class P\n private\n memoize(attr_reader :a)\n attr_reader :b\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 2); + assert_eq!(metric.npa.class_npa_sum(), 0); + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_shadowed_keyword_does_not_flip_the_attribute_flag() { + // The issue's second half: after `private = false`, the bare + // `private` statement is the local, not the marker, so `@x` and + // `y` stay public — and the method walk already said so of `m` + // (measured: `A` has `m`, `y` and `y=` public). Before #1719 the + // attribute walk had no shadow table and read the bare word as a + // direct child, so npa was 0 while npm was 1. Removing the shadow + // table fails this. + // + // expected: na = 2, npa = 2; nm = 1, npm = 1. + check_metrics_with_npm::( + "class A\n private = false\n private\n @x = 1\n attr_accessor :y\n def m; end\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 2); + assert_eq!(metric.npa.class_npa_sum(), 2); + assert_eq!(metric.npm.class_nm_sum(), 1); + assert_eq!(metric.npm.class_npm_sum(), 1); + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_shadowed_keyword_in_a_condition_is_not_a_statement() { + // The condition-is-not-a-statement control: `if private` reads + // the local, and the walk reads the `if`'s `then`, never its + // condition. Measured: `y` public (`x` sits on the branch the + // falsy local skips). What this pins is that `x` is read under + // the `if` (na = 1 without the wrapper) and that the later bare + // `private` is the local (npa = 1 without the shadow table). It + // cannot discriminate a widened `If` arm that read the condition + // as a statement: that arm would find the same shadowed local and + // change nothing, as `is_statement_of`'s comment records. + // + // expected: na = 2 (x, y), npa = 2. + check_metrics::( + "class K\n private = false\n if private\n attr_accessor :x\n end\n private\n attr_reader :y\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 2); + assert_eq!(metric.npa.class_npa_sum(), 2); + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_marker_inside_a_wrapper_sets_the_attribute_flag() { + // A marker inside a wrapper's statements runs when the wrapper + // does: a `public` under `if true` undoes the `private` before + // it, a `private` in a `begin` demotes what follows, a `public` + // in a `while` undoes that (measured: `after_branch` public, + // `after_begin` private, `after_while` public). Each marker moves + // the flag across npa's public/non-public line, so none of the + // three can be skipped unseen. Before #1719 all three read + // private under the first marker. + // + // expected: na = 3, npa = 2 (`after_branch`, `after_while`). + check_metrics::( + "class In\n private\n if true\n public\n end\n attr_reader :after_branch\n begin\n private\n end\n attr_reader :after_begin\n while true\n public\n break\n end\n attr_reader :after_while\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 3); + assert_eq!(metric.npa.class_npa_sum(), 2); + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_bare_keyword_in_an_expression_or_argument_is_the_attribute_marker() { + // Ruby's `private` is a method call wherever it is evaluated: + // `x = private` and `foo(private)` demote what follows, `y = + // public` undoes it, and a binding in an argument (`foo(private = + // 1)`) shadows the word from then on (measured: `U` has + // `after_assign` private; `E1` has `m` public; `E2` has `d` + // private). Before #1719 none of the three was read: U npa 3, + // E2 npa 2. + // + // expected: U na 3 / npa 2; E1 na 1 / npa 1; E2 na 2 / npa 1. + // Asserted per class: in the totals (na 6 / npa 4) a lost + // argument-position shadow (E1 -1) and a lost argument-position + // marker (E2 +1) cancel. + check_func_space::( + "class U\n attr_reader :before\n x = private\n attr_reader :after_assign\n y = public\n attr_reader :after_second\nend\nclass E1\n foo(private = 1)\n x = private\n attr_reader :m\nend\nclass E2\n attr_reader :a\n foo(private)\n attr_reader :d\nend\n", + "foo.rb", + |func_space| { + for (class, na, npa) in [("U", 3, 2), ("E1", 1, 1), ("E2", 2, 1)] { + let npa_stats = &child_space(&func_space, class).metrics.npa; + assert_eq!(npa_stats.class_na_sum(), na, "{class} na"); + assert_eq!(npa_stats.class_npa_sum(), npa, "{class} npa"); + } + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_destructured_for_and_rescue_bindings_shadow_the_attribute_keyword() { + // Each binding form the walk models shadows the keyword — a + // destructuring target flat, nested or splatted, a `for` variable, + // a `rescue =>` variable — so the later `x = private` is the local + // and `visible` / `v` stay public; an instance-variable target + // binds no local, so `F`'s `demoted` is private (measured: as + // listed). `F`'s `@private = 1` is itself an attribute, set + // public. Not modelled: a pattern-matching binding (`in [Integer + // => private, *]`, `1 => private`) and a regexp named capture + // create the local too, and the walk still reads a later bare + // `private` as the marker there. + // + // expected: na = 7 (visible ×4, @private, demoted, v), npa = 6. + check_metrics::( + "class D\n private, other = false, nil\n x = private\n attr_reader :visible\nend\nclass N1\n (a, private) = [1, false]\n x = private\n attr_reader :visible\nend\nclass N2\n *private = [false]\n x = private\n attr_reader :visible\nend\nclass E\n for private in [false]\n end\n y = private\n attr_reader :visible\nend\nclass F\n @private = 1\n z = private\n attr_reader :demoted\nend\nclass G\n begin\n raise \"x\"\n rescue => private\n end\n w = private\n attr_reader :v\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 7); + assert_eq!(metric.npa.class_npa_sum(), 6); + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_attribute_inside_another_space_is_not_the_class_s() { + // The boundary the walk stops at: a `def` body, a block, a lambda + // and a nested `class` / `module` body are spaces of their own. + // `Inner`'s `@v` is `Inner`'s attribute, counted once, under + // `Inner`; `M` is a namespace, which `npa` does not count. A walk + // that entered `Inner` would count `@v` twice, and one that + // entered `M` would count `q`. + // + // Ruby does set `@y` on `S` — a block's `self` is the class — but + // a block opens a space, as it does for `npm`'s `def`, and the + // walk does not read it (measured: `S.instance_variables` is + // `[:@y]`). That gap is recorded here rather than asserted. + // + // expected: na = 2 (S's `bare`, Inner's `@v`), npa = 2. + check_metrics::( + "class S\n def inner\n @x = 1\n end\n [1].each do |i|\n @y = 1\n end\n l = -> { @w = 1 }\n class Inner\n @v = 1\n end\n module M\n attr_reader :q\n end\n attr_reader :bare\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 2); + assert_eq!(metric.npa.class_npa_sum(), 2); + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_attribute_under_deeply_nested_calls_does_not_recurse() { + // The attribute walk is the method walk's explicit stack, so an + // `attr_reader` under 5,000 nested calls costs no Rust stack; run + // on a small stack so a reintroduced recursion fails here. + // + // expected: na = 2 (deep, bare), npa = 2. + let nested = format!( + "class C\n {}attr_reader :deep{}\n attr_reader :bare\nend\n", + "f(".repeat(5_000), + ")".repeat(5_000) + ); + std::thread::Builder::new() + .stack_size(512 * 1024) + .spawn(move || { + check_metrics::(&nested, "foo.rb", |metric| { + assert_eq!(metric.npa.class_na_sum(), 2); + assert_eq!(metric.npa.class_npa_sum(), 2); + }); + }) + .expect("spawn bounded-stack thread") + .join() + .expect("the attribute walk must not overflow a 512 KiB stack"); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_attr_in_a_wrapper_argument_counts() { + // A statement wrapper passed as a keyword's argument runs its + // `attr_*` at class scope, so each is an attribute: the statement + // list of `private (attr_reader :x)` (the space before the + // parenthesis makes it one) and a `begin`. + // + // Only the count is pinned, as for `def` in npm's + // `ruby_def_in_a_wrapper_argument_counts`. Ruby hands the + // wrapper's last value to `private`, so `x` and `y` are private + // (measured on ruby 3.0.2), while the walk does not resolve that + // value and both take the body-wide flag — a known gap, not a + // contract, and so not asserted. + // + // expected: na = 3 (x, y, bare). + check_metrics::( + "class A\n private (attr_reader :x)\n private(begin\n attr_reader :y\n end)\n attr_reader :bare\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 3); + }, + ); + } + + #[cfg(feature = "ruby")] + #[test] + fn ruby_attribute_and_method_walks_agree_on_visibility() { + // Grammar-dispatch rule 7's parity: one `attr_reader` beside one + // `def` in each position the walk reads — under a wrapper after a + // marker inside it, after an expression-position `public`, after + // a bare `private` that a local shadows, and after an + // argument-position `protected` — so the two walks must count + // the same and agree on which are public. The shadowed `private` + // sits where the flag is public, so reading it as the marker + // would demote `rd`/`md`. Measured: `ra`/`ma` private, `rb`/`mb` + // and `rd`/`md` public, `rc`/`mc` protected. Before #1719 the + // attribute walk read na = 2. + // + // expected: na = nm = 4, npa = npm = 2. + check_metrics_with_npm::( + "class Par\n if true\n private\n attr_reader :ra\n def ma; end\n end\n x = public\n attr_reader :rb\n def mb; end\n private = 1\n private\n attr_reader :rd\n def md; end\n foo(protected)\n begin\n attr_reader :rc\n def mc; end\n end\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 4); + assert_eq!(metric.npm.class_nm_sum(), 4); + assert_eq!(metric.npa.class_npa_sum(), 2); + assert_eq!(metric.npm.class_npm_sum(), 2); + }, + ); + } + // --------------------------------------------------------------- // Default-impl placeholder smoke tests (audited in #188). // diff --git a/src/metrics/npa/ruby.rs b/src/metrics/npa/ruby.rs index a3420ed25..1578c9fb1 100644 --- a/src/metrics/npa/ruby.rs +++ b/src/metrics/npa/ruby.rs @@ -8,6 +8,121 @@ use super::*; +// The attributes of one Ruby class or singleton-class body, counted off +// the statements the body walk `Npm` shares (`ruby_body.rs`) hands it, +// wherever that walk reads one — directly in the body, under a +// conditional, a loop, a `begin`, a `case` arm, a statement modifier or +// parentheses, on an assignment's right side, or as a call's argument: +// - a class-scope `@var = …` (`InstanceVariable`) or `@@var = …` +// (`ClassVariable`) — the `Member` row's variables where they are a +// plain assignment's sole target — is one attribute, whatever the right +// side. The other ways Ruby sets one are not counted, as before the +// shared walk: a multiple assignment (`@a, @b = 1, 2`), an operator +// assignment (`@c ||= 1`), `for @d in …` and `rescue => @e`; +// - an `attr_accessor` / `attr_reader` / `attr_writer` macro is one +// attribute per symbol it names. +// +// Visibility is the walk's: the body opens public, a bare `private` / +// `public` / `protected` sets the flag wherever Ruby evaluates it as the +// marker call — unless a local of that name shadows it — and the +// argument form (`private attr_accessor :b`) governs what its argument +// list declares and leaves the flag alone (#1255). An attribute is an +// instance-level declaration, so a class-method keyword +// (`private_class_method attr_accessor :b`) never governs one and the +// flag decides. +// +// The retroactive symbol form (`private :b` after `attr_accessor :b`) +// is deliberately not modelled. It renames the visibility of the +// generated *reader* only, leaving `b=` public, so there is no single +// answer for the attribute — unlike `Npm`, where the symbol names one +// method (see #1399). For the same reason a nested keyword chain is not +// re-filed: in `private public attr_accessor :x` (Ruby 3.1+, where +// `public` returns its argument; 3.0 raises) the attribute takes the +// innermost keyword, while `Npm` hands what `public` returns to the +// outer `private` and demotes a `def` spelled the same way. +// +// A keyword wrapping an ordinary call (`private memoize attr_reader :x`) +// is not resolved: the argument is not a declaration the walk recognises, +// so `x` takes the flag. Ruby makes it private when `memoize` returns its +// argument; `Npm` has the same gap for `private memoize def m`, so the +// two metrics still agree. So is a keyword whose argument is a statement +// wrapper (`private (attr_reader :x)` — the space makes the parentheses +// a statement list — or `private(begin; attr_reader :x; end)`): the walk +// reads the macro as one of the wrapper's statements, so it takes the +// flag, where Ruby hands the wrapper's last value to `private` and makes +// `x` private. `Npm` records the same approximation for a `def` there +// (`ruby_def_in_a_wrapper_argument_counts`). +// +// Attribute assignments to instance/class variables are visible only +// via the methods that wrap them, so the visibility flag at the point +// of declaration is what `npa` should reflect. +struct RubyAttributes<'s> { + stats: &'s mut Stats, +} + +impl RubyAttributes<'_> { + // Counts what one statement or argument declares. `keyword` is the + // visibility call `node` is an argument of, if it is one. + fn read<'a>( + &mut self, + node: &Node<'a>, + shape: RubyBodyShape, + keyword: Option, + body: &mut RubyBodyState<'a, '_>, + ) { + use crate::roles::ruby::member; + + let code = body.code(); + let flag = body.visibility(); + match shape { + RubyBodyShape::Call if ruby_is_attr_macro(node, code) => { + let public = ruby_wrapped_is_public(keyword, false, flag); + add_attributes(self.stats, ruby_attr_macro_symbol_count(node), public); + } + // An assignment is a wrapper to the walk, which reads its + // right side for the statements it may hold; the target is + // the `Member` row's question, asked under the assignment. + // Of the wrappers only an assignment has a `left`, so no other + // reaches the row. A keyword the assignment is an argument of + // (`private(@x = 1)`) is handed the value, not a name, so it + // governs nothing here and the flag decides. + RubyBodyShape::Wrapper => { + if let Some(target) = node.child_by_field_name("left") + && body.below(node, |under| member::owns(&target, code, under)) + { + add_attributes(self.stats, 1, ruby_declaration_is_public(false, flag)); + } + } + RubyBodyShape::Call | RubyBodyShape::Definition | RubyBodyShape::Leaf => {} + } + } +} + +impl<'a> RubyBodyConsumer<'a> for RubyAttributes<'_> { + type Gathered = (); + + fn statement( + &mut self, + statement: &Node<'a>, + shape: RubyBodyShape, + body: &mut RubyBodyState<'a, '_>, + ) { + self.read(statement, shape, None, body); + } + + // The argument carries the enclosing call's keyword, so `private + // attr_accessor :b` is counted once, here, as the keyword says. + fn argument( + &mut self, + argument: &Node<'a>, + shape: RubyBodyShape, + call: &mut RubyOpenCall, + body: &mut RubyBodyState<'a, '_>, + ) { + self.read(argument, shape, call.keyword, body); + } +} + impl Npa for RubyCode { fn compute<'a>( node: &Node<'a>, @@ -28,6 +143,7 @@ impl Npa for RubyCode { if !matches!(parent_kind, Class | SingletonClass) { return; } - ruby_walk_class_body(node, code, stats); + let mut attributes = RubyAttributes { stats }; + RubyBodyState::walk(code, ancestors.of_children(node), node, &mut attributes); } } diff --git a/src/metrics/npa/ruby_body.rs b/src/metrics/npa/ruby_body.rs new file mode 100644 index 000000000..b4c0b599e --- /dev/null +++ b/src/metrics/npa/ruby_body.rs @@ -0,0 +1,557 @@ +//! The walk over a Ruby class body that `Npm` and `Npa` share (#1719): +//! which statements the body holds — through the wrappers they sit in +//! and the argument lists of the calls among them — and the visibility +//! state those statements set as they run. Each consumer reads the +//! statements and arguments it is handed for its own declarations; +//! this module counts nothing. +//! +//! The protocol is three events. A *statement* is a node the body or a +//! wrapper runs at class scope; an *argument* is a named child of a +//! call's argument list, handed with the call it belongs to. Both are +//! read before the walk enters them, so a call is entered at the event +//! that reaches it, and its arguments follow as events of their own. +//! *Leaving* a call comes once its arguments are all read, which is +//! when a consumer that acts on what they named — `npm`'s re-filing of +//! `private :foo` — can act. A bare visibility marker is the walk's own +//! and reaches no consumer. +//! +//! `npm` (`npm/ruby.rs`) declares the `def`s it is handed; `npa` +//! (`npa/ruby.rs`) counts the `attr_*` calls and the instance- and +//! class-variable assignments. Because both read one walk, they agree +//! on which statements a body holds and on the visibility each runs +//! under (#1719; grammar-dispatch rule 7). +#![allow(clippy::wildcard_imports, clippy::enum_glob_use)] + +use super::*; +use crate::node::Children; + +// Whether `node` is a wrapper the walk reads through for the +// statements under it: a conditional and its arms, a `case` and +// its clauses, a loop and its `do` body, a `begin` with its +// `rescue` / `ensure` clauses, a statement modifier, a +// parenthesised statement list, and the right side of an +// assignment — `x = def b; end` and `@h = private def h; end` +// define `b` and `h` as any bare `def` would (measured), and so do +// both of `a, b = def x; end, def y; end`, whose right side is a +// `right_assignment_list` of them. A `def`'s +// or a block's children are a space of their own and are not read: +// a `def` inside `def inner` or an `items.each do … end` is that +// space's method, as `nom` counts it, not the class's. +fn reads_through(kind: u16) -> bool { + use Ruby::*; + + // role-bypass: allow(Branch, Switch, Loop, Catch, ElseIf, CaseArm, DocTarget) — the + // wrappers whose statements run at class scope, read for their + // children; not whether they branch, switch, loop, chain or arm a + // case, which is the control-flow rows' and the `CaseArm` row's + // question (#1697), nor whether an assignment takes a doc comment + matches!( + kind.into(), + If | Unless + | Elsif + | Else + | Then + | Case + | CaseMatch + | When + | InClause + | While + | Until + | For + | Do + | Begin + | Rescue + | Ensure + | IfModifier + | UnlessModifier + | WhileModifier + | UntilModifier + | RescueModifier + | RescueModifier2 + | RescueModifier3 + | ParenthesizedStatements + | Assignment + | Assignment2 + | OperatorAssignment + | OperatorAssignment2 + | RightAssignmentList + ) +} + +fn is_assignment(kind: u16) -> bool { + use Ruby::*; + + // role-bypass: allow(DocTarget) — an assignment statement for the + // class-scope ledger; not whether it takes a doc comment + matches!( + kind.into(), + Assignment | Assignment2 | OperatorAssignment | OperatorAssignment2 + ) +} + +// Whether `child` of a wrapper `parent` is one of its statements +// rather than its condition, value, pattern, header or handler. +// +// Only statement positions are read. Ruby does evaluate an `if`'s +// condition, and a bare `private` there is the marker call unless a +// local shadows it, so `if private … end` demotes what follows in Ruby +// and not here: reading conditions would mean reading every expression +// for a keyword, which this inventory of statement positions does not +// attempt. The grammar exposes each consequence as +// a dedicated node or a field, so the arms pick by role rather than +// by index (grammar-dispatch rule 3): `then` / `else` / `elsif` +// under a conditional, the clauses under a `case`, the `then` of a +// clause or a `rescue`, the `do` under a loop, the `body` field of +// a modifier, and the `right` field of an assignment, whose `left` +// is the target. The `rescue` modifier is read on both sides: its +// `body` runs, and its `handler` runs when the body raises — a +// `def` in either position is a definition at class scope, and the +// walk counts every branch. An `else`, a `then`, a `do`, an +// `ensure`, a `begin` and a parenthesised list hold statements +// directly. `rescue_modifier` is three kind ids for one rule +// (grammar-dispatch rule 1): the parser emits `RescueModifier3` in +// every position measured — statement, assignment right side, +// method body — and the other two are aliases the grammar declares +// that no spelling produced; listed defensively per rule 2, and +// pinned unreachable by `ruby_rescue_modifier_aliases_are_not_emitted`. +// +// The exclusions are chosen by role, and no fixture pins them (the +// perturbation sweep recorded on #1401): a widened arm that read a +// condition, a loop header, a `case` value or a `when` pattern as a +// statement would find, in every valid fixture, either a local that +// shadows the keyword — which the walk already ignores — or a +// bare keyword that Ruby evaluates as the marker call, which the +// widened arm would then read *correctly* where this arm does not. +// A `when` pattern and a `rescue` class list are wrapped (`pattern`, +// `exceptions`) besides. The arms stay as the statement inventory +// they are; the fixture that would discriminate them is the one +// whose answer they get wrong, and the comment above records that +// gap rather than pinning it. +fn is_statement_of(parent: &Node<'_>, parent_kind: u16, child: &Node<'_>) -> bool { + use Ruby::*; + + // role-bypass: allow(Branch, Switch, Loop, ElseIf, Catch, CaseArm) — which + // child of a wrapper is its clause and which of a clause is its + // body, for traversal; not whether the wrapper branches or the arm + // pays a decision, which is the rows' question (#1697) + child.is_named() + && match parent_kind.into() { + If | Unless | Elsif => matches!(child.kind_id().into(), Then | Else | Elsif), + Case | CaseMatch => matches!(child.kind_id().into(), When | InClause | Else), + When | InClause | Rescue => matches!(child.kind_id().into(), Then), + While | Until | For => matches!(child.kind_id().into(), Do), + IfModifier | UnlessModifier | WhileModifier | UntilModifier => parent + .child_by_field_name("body") + .is_some_and(|body| body.id() == child.id()), + RescueModifier | RescueModifier2 | RescueModifier3 => true, + _ if is_assignment(parent_kind) => parent + .child_by_field_name("right") + .is_some_and(|right| right.id() == child.id()), + _ => true, + } +} + +/// What the walk does with a node once a consumer has read it. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum RubyBodyShape { + /// A `def` — the `FuncOpener` row's callable. Its body is a space + /// of its own and is not read. + Definition, + /// A call, whose arguments are read next; a parenthesised bare + /// keyword (`private()`) has none and sets the flag instead. + Call, + /// A wrapper whose statements run at class scope, read next. + Wrapper, + /// Anything else, which the walk does not enter. + Leaf, +} + +/// A call whose arguments the walk is reading. +pub(crate) struct RubyOpenCall { + /// The visibility keyword the call spells, `None` for an ordinary + /// call. + pub(crate) keyword: Option, + /// What the consumer has gathered from the arguments read so far. + pub(crate) gathered: T, +} + +/// A reader of the statements and arguments a Ruby class body holds. +/// See the module documentation for the order of the events. +pub(crate) trait RubyBodyConsumer<'a> { + /// What the consumer gathers over one call's arguments, for + /// [`leave_call`](Self::leave_call). + type Gathered: Default; + + /// A statement of the body or of a wrapper under it, other than a + /// bare visibility marker, before the walk enters it. + fn statement( + &mut self, + statement: &Node<'a>, + shape: RubyBodyShape, + body: &mut RubyBodyState<'a, '_>, + ); + + /// One argument of `call`, other than a bare visibility marker, + /// before the walk enters it. + fn argument( + &mut self, + argument: &Node<'a>, + shape: RubyBodyShape, + call: &mut RubyOpenCall, + body: &mut RubyBodyState<'a, '_>, + ); + + /// `call`'s arguments are all read. `outer` is the call whose + /// argument it was, if it was one. A consumer that gathers nothing + /// from a call's arguments has nothing to do here. + fn leave_call( + &mut self, + _call: RubyOpenCall, + _outer: Option<&mut RubyOpenCall>, + ) { + } +} + +// What the walk is inside while it reads a node's children: the body +// or a wrapper whose statements run at class scope, or a call's +// argument list. +enum RubyFrameKind { + Statements, + Arguments(RubyOpenCall), +} + +// One node the walk is reading the children of, and those children +// still to read. Each wrapper, call and argument list is a frame on an +// explicit stack rather than a Rust frame, so a body nested or called +// arbitrarily deep costs no stack (#1401 review: 5,000 nested calls +// overflowed the recursive first draft), and each child list is read +// once through a held iterator rather than re-sought per step. +struct RubyFrame<'a, T> { + node: Node<'a>, + children: Children<'a>, + kind: RubyFrameKind, +} + +/// The state one pass over a Ruby class body carries: the body-wide +/// visibility flag, the keyword names a local has shadowed, and the +/// chain of whichever node's children are being read, moved down and +/// back up as the walk enters and leaves a frame so a row's gate reads +/// each node under its own parent (#1696) without a chain copy per +/// level. +pub(crate) struct RubyBodyState<'a, 'chain> { + code: &'a [u8], + chain: ChildAncestors<'a, 'chain>, + visibility: RubyVisibility, + // The visibility keywords a local of the same name has shadowed: + // after `private = false`, a bare `private` anywhere later in the + // body is that variable, and Ruby reads it as the marker call only + // where no such local is in scope (measured: `x = private` with no + // local demotes what follows; with the local assigned first it does + // not). A Ruby local exists from its binding to the end of its scope, + // decided at parse time, and the only scope here is the class body: + // none of the wrappers the walk reads through opens one, and the + // scopes that do (a `def`, a block, a nested class) are not read. A + // binding inside a branch therefore shadows the rest of the body + // whether or not the branch runs (measured: `if false; private = 1; + // end; private` leaves what follows public), and never unshadowing + // is exact. + shadowed: [bool; RubyVisibility::COUNT], +} + +impl<'a, 'chain> RubyBodyState<'a, 'chain> { + /// Reads every statement of `body` — a class or singleton-class + /// body, whose children `chain` describes — into `consumer`. + pub(crate) fn walk>( + code: &'a [u8], + chain: ChildAncestors<'a, 'chain>, + body: &Node<'a>, + consumer: &mut C, + ) { + let mut state = Self { + code, + chain, + // Ruby class bodies open in default-public state, whatever + // the previous body's trailing visibility was. + visibility: RubyVisibility::Public, + shadowed: [false; RubyVisibility::COUNT], + }; + state.read_body(body, consumer); + } + + /// The source the body was parsed from. + pub(crate) fn code(&self) -> &'a [u8] { + self.code + } + + /// The body-wide visibility flag where the walk stands. + pub(crate) fn visibility(&self) -> RubyVisibility { + self.visibility + } + + /// Answers `read` with the chain of `node`'s children, for a gate + /// on a child the walk does not read itself (an assignment's + /// target). `node` is the statement or argument being read. + pub(crate) fn below( + &mut self, + node: &Node<'a>, + read: impl FnOnce(Ancestors<'a, '_>) -> R, + ) -> R { + self.chain.descend(node); + let answer = read(self.chain.get()); + self.chain.ascend(); + answer + } + + // A bare `private` / `public` / `protected` sets the flag — Ruby + // evaluates it as the marker call wherever it stands — unless a + // local of that name shadows it. The walk asks this of every node + // it reads as a statement or a call argument, and of nothing else: + // a keyword in a condition, a loop header or a `case` value is + // evaluated by Ruby and not seen here (`is_statement_of` records + // the gap). Answers whether `node` was one, shadowed or not. + fn apply_marker(&mut self, node: &Node<'a>) -> bool { + let Some(marker) = ruby_visibility_marker(node, self.code) else { + return false; + }; + if !self.shadowed[marker as usize] { + self.visibility = marker; + } + true + } + + // Records every visibility keyword's name a binding target binds, + // so the bare word is a local from here on: the `left` of an + // assignment or the `pattern` of a `for`, directly (`private = 1`) + // or inside a destructuring list (`private, other = …`, `(a, + // private) = …`, `*private = …`), which the grammar nests as + // `left_assignment_list` / `destructured_left_assignment` / + // `rest_assignment` holding the identifiers. An `@private = 1` or + // `obj.private = 2` target is not an `identifier` and binds no + // local, so a later bare `private` is still the call (measured). + fn shadow_bindings_of(&mut self, target: &Node<'a>) { + use Ruby::*; + + let mut stack = vec![*target]; + while let Some(node) = stack.pop() { + if let Some(marker) = ruby_visibility_marker(&node, self.code) { + self.shadowed[marker as usize] = true; + continue; + } + if matches!( + node.kind_id().into(), + LeftAssignmentList + | DestructuredLeftAssignment + | RestAssignment + | ExceptionVariable + ) { + stack.extend(node.children().filter(Node::is_named)); + } + } + } + + // Records the locals a statement binds before its body is read: an + // assignment's `left`, a `for`'s `pattern`, a `rescue` clause's + // `=> e` variable (an `exception_variable` wrapping the target). + // All three bind at class scope; a block's parameters bind in the + // block, a space the walk does not enter. + // + // Only nodes the walk reads are asked, so a binding anywhere else + // creates the local unseen: in a condition, a `case` value, an + // operand or a literal element (`if (private = false)`, `x = + // [private = 1]`), or by a pattern match or a regexp named capture. + // A later bare keyword is then read as the marker though Ruby reads + // the local (measured on ruby 3.0.2); the same statement-position + // limit `is_statement_of` records. + fn shadow_statement_bindings(&mut self, statement: &Node<'a>, kind: u16) { + use Ruby::*; + + let field = if is_assignment(kind) { + "left" + } else if kind == For as u16 { + "pattern" + } else if kind == Rescue as u16 { + "variable" + } else { + return; + }; + if let Some(target) = statement.child_by_field_name(field) { + self.shadow_bindings_of(&target); + } + } + + // What the walk will do with `node` once it is read: a `def` — the + // `FuncOpener` row's callable — is a space of its own; a call has + // arguments to read; a wrapper has statements to read. + fn shape(&self, node: &Node<'a>, kind: u16) -> RubyBodyShape { + use crate::roles::ruby::{call, func_opener}; + + if func_opener::is_callable(node, self.code, self.chain.get()) { + return RubyBodyShape::Definition; + } + // The `Call` row is ungated, so its kind table is the whole + // answer: the visibility macros and the decorator-style calls are + // each told by name once the walk reads the call. + if call::lists_kind(kind) { + RubyBodyShape::Call + } else if reads_through(kind) { + RubyBodyShape::Wrapper + } else { + RubyBodyShape::Leaf + } + } + + // Reads one statement of the body or of a wrapper, or one argument + // of the call on top of the stack, under the chain of its siblings: + // a bare visibility marker sets the flag, anything else is handed to + // the consumer and then entered if it is a call or a wrapper. + // + // A bare `private` is the marker call wherever an expression is + // evaluated — `x = private` and `y = public` set the flag + // (measured), and so does `foo(private)` — unless a local of that + // name shadows it. The assignment, `for` or `rescue` that creates + // the local is read as a wrapper (for the `def` its right side or + // body may hold), and shadows from its target before that is read; + // a binding in an argument (`foo(private = 1)`) shadows as well. + fn read>( + &mut self, + node: &Node<'a>, + consumer: &mut C, + stack: &mut Vec>, + ) { + if self.apply_marker(node) { + return; + } + let kind = node.kind_id(); + self.shadow_statement_bindings(node, kind); + let shape = self.shape(node, kind); + match stack.last_mut() { + Some(RubyFrame { + kind: RubyFrameKind::Arguments(call), + .. + }) => consumer.argument(node, shape, call, self), + _ => consumer.statement(node, shape, self), + } + match shape { + RubyBodyShape::Call => self.open_call(node, stack), + RubyBodyShape::Wrapper => self.open_wrapper(node, stack), + RubyBodyShape::Definition | RubyBodyShape::Leaf => {} + } + } + + // Opens a call: a bare keyword call sets the flag and is done; any + // other call with an argument list is pushed, with the keyword it + // spells, so its arguments are read as the walk reaches them. + fn open_call(&mut self, call: &Node<'a>, stack: &mut Vec>) { + let keyword = match ruby_visibility_effect(call, self.code) { + Some(RubyVisibilityEffect::Flag(flag)) => { + self.visibility = flag; + return; + } + Some(RubyVisibilityEffect::Arguments(keyword)) => Some(keyword), + None => None, + }; + // The arguments sit under the call's `argument_list`: each is + // asked under its own chain, the call's plus the call and the + // list, so a gate the row grows later reads the right parent. + let Some(list) = ruby_call_argument_list(call) else { + return; + }; + self.chain.descend(call); + self.chain.descend(&list); + stack.push(RubyFrame { + node: *call, + children: list.children(), + kind: RubyFrameKind::Arguments(RubyOpenCall { + keyword, + gathered: T::default(), + }), + }); + } + + // Opens a statement wrapper: its statements are read as the walk + // reaches them, under its own chain. + fn open_wrapper(&mut self, wrapper: &Node<'a>, stack: &mut Vec>) { + self.chain.descend(wrapper); + stack.push(RubyFrame { + node: *wrapper, + children: wrapper.children(), + kind: RubyFrameKind::Statements, + }); + } + + // Reads every statement of the body, of each wrapper under it and + // of each call's argument list, in source order. The body's direct + // children are all statements; a wrapper's are the ones + // `is_statement_of` picks; an argument list's are its named + // children. + fn read_body>(&mut self, body: &Node<'a>, consumer: &mut C) { + let floor = self.chain.get().depth(); + let mut stack = vec![RubyFrame { + node: *body, + children: body.children(), + kind: RubyFrameKind::Statements, + }]; + // The top frame is taken off to read its next child and put + // back before the child is, so a reader that opens a frame of + // its own stacks it on top; a frame with no child left closes. + while let Some(mut frame) = stack.pop() { + let parent = frame.node; + let parent_kind = parent.kind_id(); + let arguments = matches!(frame.kind, RubyFrameKind::Arguments(_)); + let child = frame.children.find(|child| { + if arguments || parent.id() == body.id() { + child.is_named() + } else { + is_statement_of(&parent, parent_kind, child) + } + }); + let Some(child) = child else { + self.close_frame(frame, body, consumer, &mut stack); + continue; + }; + stack.push(frame); + self.read(&child, consumer, &mut stack); + } + // Every frame descended is ascended on its pop: a frame kind + // that forgot would leave every later gate reading the wrong + // parent, which nothing else can see. + debug_assert_eq!( + self.chain.get().depth(), + floor, + "the member walk left the chain off the body's floor" + ); + } + + // Leaves a frame whose children are all read: a call gives back the + // two chain levels it took and is handed to the consumer with the + // call it is an argument of, a wrapper gives back the level it + // took, and the body — the one frame the chain was built for — + // took none. + fn close_frame>( + &mut self, + done: RubyFrame<'a, C::Gathered>, + body: &Node<'a>, + consumer: &mut C, + stack: &mut [RubyFrame<'a, C::Gathered>], + ) { + match done.kind { + RubyFrameKind::Arguments(call) => { + self.chain.ascend(); + self.chain.ascend(); + let outer = match stack.last_mut() { + Some(RubyFrame { + kind: RubyFrameKind::Arguments(outer), + .. + }) => Some(outer), + _ => None, + }; + consumer.leave_call(call, outer); + } + RubyFrameKind::Statements => { + if done.node.id() != body.id() { + self.chain.ascend(); + } + } + } + } +} diff --git a/src/metrics/npa/shared.rs b/src/metrics/npa/shared.rs index 2a1947943..f5f128be4 100644 --- a/src/metrics/npa/shared.rs +++ b/src/metrics/npa/shared.rs @@ -201,8 +201,9 @@ pub(crate) fn ruby_method_name<'a>(method: &Node<'a>, source: &'a [u8]) -> Optio // Ruby class-body visibility state. `private` / `public` / `protected` // keywords flip this flag for every subsequent declaration in the same // body until another marker overrides them. The default at the top of -// every class body is `Public`. The discriminants index `npm`'s -// per-keyword shadow table, which is `COUNT` long. +// every class body is `Public`. The discriminants index the shared body +// walk's per-keyword shadow table (`ruby_body.rs`), which is `COUNT` +// long. #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum RubyVisibility { Public, @@ -256,9 +257,9 @@ impl RubyVisibilityCall { } } -// What a visibility-keyword `Call` in a Ruby class body does. Shared by -// `Npm` and `Npa` so the two walkers cannot drift on the same Ruby rule -// (grammar-dispatch rule 7). +// What a visibility-keyword `Call` in a Ruby class body does. Read by +// the body walk `Npm` and `Npa` share (`ruby_body.rs`), so the two +// cannot drift on the same Ruby rule (grammar-dispatch rule 7). pub(crate) enum RubyVisibilityEffect { // `private()` — the explicit-parens spelling of the bare keyword, // which sets the default for every later instance-level declaration @@ -303,21 +304,21 @@ pub(crate) fn ruby_declaration_is_public(singleton: bool, body_flag: RubyVisibil singleton || body_flag == RubyVisibility::Public } -// Whether a declaration nested in a visibility call's argument list is -// public. The keyword governs it only when it names that method family, -// so `private def self.x` leaves the singleton alone and -// `private_class_method` never reaches an instance method or an -// attribute; an ungoverned declaration falls back to its own default. -// The single source of the rule for both `Npm` and `Npa` (#1255). +// Whether a declaration is public, given the visibility call whose +// argument list it sits in, if any. The keyword governs it only when it +// names that method family, so `private def self.x` leaves the singleton +// alone and `private_class_method` never reaches an instance method or an +// attribute; an unwrapped or ungoverned declaration falls back to its own +// default. The single source of the rule for both `Npm` and `Npa` +// (#1255). pub(crate) fn ruby_wrapped_is_public( - keyword: RubyVisibilityCall, + keyword: Option, singleton: bool, body_flag: RubyVisibility, ) -> bool { - if keyword.governs(singleton) { - keyword.visibility == RubyVisibility::Public - } else { - ruby_declaration_is_public(singleton, body_flag) + match keyword { + Some(keyword) if keyword.governs(singleton) => keyword.visibility == RubyVisibility::Public, + _ => ruby_declaration_is_public(singleton, body_flag), } } @@ -361,111 +362,6 @@ pub(crate) fn add_attributes(stats: &mut Stats, count: usize, public: bool) { } } -// Attributes declared by an `attr_*` macro nested in a visibility call's -// argument list: `private attr_accessor :b` parses as a `private` call -// whose sole argument is the `attr_accessor` call, so the macro is never -// a direct child of the body (#1255). -fn ruby_wrapped_attr_count(call: &Node, source: &[u8]) -> usize { - use Ruby::*; - - // role-bypass: allow(NullSafe) — an `attr_*` macro call, which the row's safe-navigation - // owner is a kind of; the macro is told by its name - ruby_call_named_arguments(call) - .filter(|arg| { - matches!(arg.kind_id().into(), Call | Call2 | Call3 | Call4) - && ruby_is_attr_macro(arg, source) - }) - .map(|arg| ruby_attr_macro_symbol_count(&arg)) - .sum() -} - -// Walks the direct children of a Ruby class / singleton-class body -// (`BodyStatement` under `Class` / `SingletonClass`) tallying: -// - class-scope assignments to `@var` (`InstanceVariable`) and -// `@@var` (`ClassVariable`) — one attribute per assignment, regardless -// of whether the RHS is a constant or another expression. -// - `attr_accessor` / `attr_reader` / `attr_writer` macros — one -// attribute per symbol argument. -// -// Visibility flags follow Ruby's keyword-marker convention: a bare -// `private` / `public` / `protected` identifier flips the default for -// every subsequent declaration in the body. The default visibility at -// the top of every class body is `public`. The argument-form of those -// keywords (`private :foo`, `private attr_accessor :b`) does not flip -// the body-wide flag — matching Ruby's runtime behaviour — but it does -// govern the declarations it wraps, so an `attr_*` macro nested in a -// visibility call takes the keyword's visibility (#1255). -// -// The retroactive symbol form (`private :b` after `attr_accessor :b`) -// is deliberately not modelled here. It renames the visibility of the -// generated *reader* only, leaving `b=` public, so there is no single -// answer for the attribute — unlike `Npm`, where the symbol names one -// method (see #1399). -// -// Attribute assignments to instance/class variables are visible only -// via the methods that wrap them, so the visibility flag at the point -// of declaration is what `npa` should reflect. -pub(crate) fn ruby_walk_class_body(body: &Node<'_>, source: &[u8], stats: &mut Stats) { - // bca: suppress(cognitive) — grammar dispatch match, clearest whole - // One pass over the body carrying Ruby's body-wide visibility flag, with - // an arm per attribute-declaring shape (`@x = …`, `attr_*` macros). The - // flag must be readable at every arm, so the arms cannot be lifted out - // without threading it back through each helper. - use crate::roles::ruby::member; - use Ruby::*; - - let mut visibility = RubyVisibility::Public; - for child in body.children() { - if let Some(marker) = ruby_visibility_marker(&child, source) { - visibility = marker; - continue; - } - match child.kind_id().into() { - // A class-scope `@x = …` / `@@n = …`: the `Member` row's - // variable where it is an assignment target. The lhs's chain - // is the body's plus the statement, which is all the gate - // reads. - // role-bypass: allow(DocTarget) — the statement whose target the - // attribute count reads; not whether it takes a doc comment - Assignment | Assignment2 => { - let Some(lhs) = child.children().next() else { - continue; - }; - let under = [*body, child]; - if member::owns(&lhs, source, Ancestors::known(&under)) { - add_attributes(stats, 1, ruby_declaration_is_public(false, visibility)); - } - } - // role-bypass: allow(NullSafe) — the `attr_*` and visibility macro calls, which the - // row's safe-navigation owner is a kind of; each is told by its name - Call | Call2 | Call3 | Call4 if ruby_is_attr_macro(&child, source) => { - add_attributes( - stats, - ruby_attr_macro_symbol_count(&child), - ruby_declaration_is_public(false, visibility), - ); - } - // role-bypass: allow(NullSafe) — the visibility macro calls, told by name - Call | Call2 | Call3 | Call4 => match ruby_visibility_effect(&child, source) { - Some(RubyVisibilityEffect::Flag(flag)) => visibility = flag, - // `private attr_accessor :b`. An attribute is always an - // instance-level declaration, so a class-method keyword - // never governs one: the macro it wraps still declares - // its attributes, at the body-wide flag. - Some(RubyVisibilityEffect::Arguments(keyword)) => { - add_attributes( - stats, - ruby_wrapped_attr_count(&child, source), - ruby_wrapped_is_public(keyword, false, visibility), - ); - } - None => {} - }, - _ => {} - } - } -} - // Go's lexical export rule: an identifier names an exported (public) // member iff its first character is an uppercase Unicode letter // (`unicode.IsUpper`). Uses `char::is_uppercase` on the first `char` @@ -561,13 +457,55 @@ pub(crate) fn cpp_declares_function(node: &Node) -> bool { // payload. A field's own initialiser is its `default_value` // field, never an `init_declarator`, so the `field_declaration` // call site is unaffected. - "pointer_declarator" | "reference_declarator" | "declaration" | "init_declarator" => { + "pointer_declarator" | "reference_declarator" | "declaration" => { cpp_declares_function(&child) } + // A parenthesised value is a member function template's parameter + // list, read by the grammar as an expression list when every + // parameter parses as one: `template void connect(T* obj, + // void (T::*slot)());` is `init_declarator(connect, + // argument_list(…))`. A member-declarator takes only a + // brace-or-equal-initializer ([class.mem.general]), so at class + // scope, the only scope these helpers read, that value is never + // an initialiser. + "init_declarator" => { + child + .child_by_field_name("value") + .is_some_and(|value| value.kind() == "argument_list") + || cpp_declares_function(&child) + } + // A member pointer outside parentheses: `template + // static int S::*getPd();` and, under a `pointer_declarator`, + // `template int (*S::*getPp())(T);` are methods + // returning one (#1718). Any other qualified name — the `std::` + // of a field's type among them — declares no function. + "qualified_identifier" => { + cpp_member_pointer(&child).is_some_and(|pointer| cpp_declares_function(&pointer)) + } _ => false, }) } +// The `pointer_type_declarator` of a member-pointer declarator, which +// both grammars spell as a `qualified_identifier` whose `name` holds it: +// `S::*fp` is `qualified_identifier(scope: S, name: +// pointer_type_declarator(*fp))`. A nested or global scope (`N::S::*`, +// `::S::*`) nests a further `qualified_identifier` in `name`, so the walk +// follows `name` rather than reading one level. Anything else — a plain +// qualified name — has no member pointer and answers `None`. +// +// A loop, not recursion: every qualified *type* a member declaration +// holds is asked too, and `A::A::…::B` nests one `qualified_identifier` +// per `::`, so a recursive walk overflowed the stack on a deep enough +// name that every other metric reads (#1718 review). +fn cpp_member_pointer<'a>(qualified: &Node<'a>) -> Option> { + let mut name = qualified.child_by_field_name("name")?; + while name.kind() == "qualified_identifier" { + name = name.child_by_field_name("name")?; + } + (name.kind() == "pointer_type_declarator").then_some(name) +} + // Given the `parenthesized_declarator` in a `function_declarator`'s // `declarator` field, says whether the enclosing declaration is still // a function. Parentheses alone do not make it a function *pointer* — @@ -584,15 +522,33 @@ pub(crate) fn cpp_declares_function(node: &Node) -> bool { // function pointer. Neither language lets a function return a // function type, so that inner nesting is the only reading — the // same grammar fact `crate::c_declarator` relies on. +// - `template static int (S::*fp)(T);` interposes a member +// pointer, `template int (S::*get())(T);` the same around a +// further `function_declarator`: a data member and a method returning +// a member-function pointer, told apart exactly as the two above +// (#1718). Only the templated form gets here. Both grammars parse the +// non-templated `int (S::*pmf)(int);` and `static int (S::*fp)(int);` +// into an `ERROR` over `S::` beside a plain `pointer_declarator`, +// which the first arm already reads correctly. Not every +// non-templated spelling recovers that way: with two specifiers +// before the return type (`static constexpr int (S::*fp)(int) = +// nullptr;`, `static inline`, `static const`) a `function_declarator` +// nests directly in another, so the data member still reads as a +// method — a recovery of the grammar's that this predicate cannot see +// past, and older than #1718. // // Hence: every child must still read as a function. An indirection // qualifies only when `cpp_declares_function` finds a function under -// it; a bare name, an `operator_name`, or an `ms_call_modifier` -// (`int (__cdecl *f)(int)`) never disqualifies on its own. +// it; a bare name, an `operator_name`, a qualified name with no member +// pointer, or an `ms_call_modifier` (`int (__cdecl *f)(int)`) never +// disqualifies on its own. fn cpp_parenthesized_declares_function(paren: &Node) -> bool { paren.children().all(|child| match child.kind() { "pointer_declarator" | "reference_declarator" => cpp_declares_function(&child), "parenthesized_declarator" => cpp_parenthesized_declares_function(&child), + "qualified_identifier" => { + cpp_member_pointer(&child).is_none_or(|pointer| cpp_declares_function(&pointer)) + } _ => true, }) } @@ -627,6 +583,25 @@ pub(crate) fn cpp_count_field_identifiers(node: &Node) -> usize { | "reference_declarator" => { count += cpp_count_field_identifiers(&child); } + // The member-pointer data member `(S::*fp)` that + // `cpp_parenthesized_declares_function` declines (#1718). + // One member pointer declares one name, so it counts one + // without a walk: the scope `S` is never a member, and the + // name's kind does not matter (the templated parse makes it + // a `type_identifier`, which a walk for `field_identifier` + // would miss). No `field_declaration` reaches this arm under + // the pinned grammars, which never leave a member pointer as + // a direct `qualified_identifier` child of one: the + // parenthesised `int (S::*pmf)(int);` recovers into an + // `ERROR` over `S::` beside a `pointer_declarator`, and the + // unparenthesised `int S::*pd;` into a `field_identifier` + // `S`, an `ERROR` over `:` and a `bitfield_clause` — which + // counts the scope as the one attribute, right for data and + // wrong for a method (`int S::*getPd();` reads as data; older + // than #1718). It keeps the invariant above if a grammar + // stops doing so: whatever the predicate declines, this + // counts. + "qualified_identifier" => count += usize::from(cpp_member_pointer(&child).is_some()), _ => {} } } @@ -660,14 +635,9 @@ pub(crate) fn cpp_count_field_identifiers(node: &Node) -> usize { // initialiser and the parameter name of `static int (*fp)(int a)` — all // plain `identifier`s, unlike a field's `field_identifier` — from being // counted as members. Name-based like `cpp_declares_function`, for the -// same Mozcpp reason. -// -// FIXME(#1718): a pointer-to-member-function data member, -// `static int (S::*fp)(T);`, still reads as a method here and in the -// non-templated `int (S::*pmf)(int);` alike — the parenthesised -// declarator holds a `qualified_identifier` wrapping a -// `pointer_type_declarator`, an indirection -// `cpp_parenthesized_declares_function` does not recognise. +// same Mozcpp reason. A member-pointer data member template, +// `template static int (S::*fp)(T);`, is a `declaration` that +// helper declines like any other (#1718). pub(crate) fn cpp_template_declares_data_member(template: &Node) -> bool { template .children() diff --git a/src/metrics/npm.rs b/src/metrics/npm.rs index dca36e289..002375ae8 100644 --- a/src/metrics/npm.rs +++ b/src/metrics/npm.rs @@ -5183,6 +5183,210 @@ class C { ); } + // Member pointers (#1718), shared by the `cpp_*` / `mozcpp_*` pair for + // the same no-file-extension reason as the fixtures above. Valid + // C++20: `g++ -std=c++20 -fsyntax-only` accepts it as written. + // + // Both grammars spell the templated `S::*fp` as `qualified_identifier + // (scope: S, name: pointer_type_declarator)`; the predicate missed + // that indirection inside parentheses and so read every member-pointer + // data member template as a method, and outside parentheses it missed + // the function under one and read two methods as data. Before #1718 + // this class measured npa 4/4 and npm 8/6. + // + // - `fp`, `cfp`, `fps`, `gfp` and `privFp` are data in the + // parenthesised position. `gfp`'s global scope nests a second + // `qualified_identifier` in `name`, so a walk that read one level + // finds no member pointer and keeps it a method. + // - `get` and `privGet` are methods returning a member-function + // pointer: the member pointer wraps their `function_declarator`. + // A gate that declined every member pointer outright makes both + // data. + // - `getPp` (under a `pointer_declarator`) and `getPd` (unparenthesised) + // are methods reached through `cpp_declares_function`'s own + // `qualified_identifier` arm, not the parenthesised one. + // - `pd` is the data control for that arm: a member pointer with no + // function under it. + // - `pmf` and `getPmf` are the parenthesised non-templated forms, + // which both grammars recover into an `ERROR` over `S::` beside a + // plain `pointer_declarator`; they read correctly before the fix + // and pin that it left that path alone. Other non-templated + // spellings recover differently and still misread, older than the + // fix: the unparenthesised `int S::*getPd();` as data, and + // `static constexpr int (S::*fp)(int) = nullptr;` as a method — + // FIXME(#1807), pinned by `*_member_pointer_recovery_misreads`. + // - the `private:` section makes public and total differ on both + // metrics, and the two metrics differ from each other (7/6 against + // 5/4), so neither a visibility-blind arm nor a swap of the two + // counters can pass. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const MEMBER_POINTER_MEMBERS: &str = "class S {\n\ + public:\n\ + template static int (S::*fp)(T);\n\ + template static int (S::*const cfp)(T);\n\ + template static int (S::*fps[4])(T);\n\ + template static int (::S::*gfp)(T);\n\ + template static int S::*pd;\n\ + int (S::*pmf)(int);\n\ + template int (S::*get())(T);\n\ + template int (*S::*getPp())(T);\n\ + template static int S::*getPd();\n\ + int (S::*getPmf())(int);\n\ + private:\n\ + template static int (S::*privFp)(T);\n\ + template int (S::*privGet())(T);\n\ + };"; + + // Positive controls, one shape per class so neither can borrow its + // count from the other: the non-templated member-function pointer is + // one attribute, and the templated method returning one is one + // method. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const MEMBER_FUNCTION_POINTER_DATA: &str = "class S { public: int (S::*pmf)(int); };"; + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const METHOD_RETURNING_MEMBER_FUNCTION_POINTER: &str = + "class S { public: template int (S::*get())(T); };"; + + #[cfg(feature = "cpp")] + #[test] + fn cpp_member_pointer_members_split_between_npa_and_npm() { + check_metrics_with_npa::(MEMBER_POINTER_MEMBERS, "foo.cpp", |metric| { + // `fp`, `cfp`, `fps`, `gfp`, `pd`, `pmf`, and `privFp`. + assert_eq!(metric.npa.class_na_sum(), 7); + assert_eq!(metric.npa.class_npa_sum(), 6); + // `get`, `getPp`, `getPd`, `getPmf`, and `privGet`. + assert_eq!(metric.npm.class_nm_sum(), 5); + assert_eq!(metric.npm.class_npm_sum(), 4); + }); + } + + #[cfg(feature = "mozcpp")] + #[test] + fn mozcpp_member_pointer_members_split_between_npa_and_npm() { + check_metrics_with_npa::(MEMBER_POINTER_MEMBERS, "foo.cpp", |metric| { + assert_eq!(metric.npa.class_na_sum(), 7); + assert_eq!(metric.npa.class_npa_sum(), 6); + assert_eq!(metric.npm.class_nm_sum(), 5); + assert_eq!(metric.npm.class_npm_sum(), 4); + }); + } + + // Member function templates whose every parameter parses as an + // expression — a pointer, a function pointer, a member-function + // pointer — read as `init_declarator(name, argument_list(…))` in both + // grammars, which took the parameter list for a parenthesised + // initialiser that a class member cannot have. `connect` and `post` + // are methods; `x` (`= T(1)`, an expression value) and `y` (`{1}`, an + // initializer list) are the data controls, whose values are never an + // `argument_list`. Valid C++20: `g++ -std=c++20 -fsyntax-only` + // accepts it as written. Before the fix this class read npa 4 / npm 0. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const EXPRESSION_LIKE_PARAMETER_TEMPLATES: &str = "struct C {\n\ + template void connect(T* obj, void (T::*slot)());\n\ + template static void post(void (*fn)(T));\n\ + template static constexpr T x = T(1);\n\ + template static constexpr T y{1};\n\ + };"; + + #[cfg(feature = "cpp")] + #[test] + fn cpp_expression_like_parameter_templates_are_methods() { + check_metrics_with_npa::( + EXPRESSION_LIKE_PARAMETER_TEMPLATES, + "foo.cpp", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 2); + assert_eq!(metric.npm.class_nm_sum(), 2); + }, + ); + } + + #[cfg(feature = "mozcpp")] + #[test] + fn mozcpp_expression_like_parameter_templates_are_methods() { + check_metrics_with_npa::( + EXPRESSION_LIKE_PARAMETER_TEMPLATES, + "foo.cpp", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 2); + assert_eq!(metric.npm.class_nm_sum(), 2); + }, + ); + } + + #[cfg(feature = "cpp")] + #[test] + fn cpp_member_pointer_controls() { + check_metrics_with_npa::(MEMBER_FUNCTION_POINTER_DATA, "foo.cpp", |metric| { + assert_eq!(metric.npa.class_na_sum(), 1); + assert_eq!(metric.npm.class_nm_sum(), 0); + }); + check_metrics_with_npa::( + METHOD_RETURNING_MEMBER_FUNCTION_POINTER, + "foo.cpp", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 0); + assert_eq!(metric.npm.class_nm_sum(), 1); + }, + ); + } + + #[cfg(feature = "mozcpp")] + #[test] + fn mozcpp_member_pointer_controls() { + check_metrics_with_npa::(MEMBER_FUNCTION_POINTER_DATA, "foo.cpp", |metric| { + assert_eq!(metric.npa.class_na_sum(), 1); + assert_eq!(metric.npm.class_nm_sum(), 0); + }); + check_metrics_with_npa::( + METHOD_RETURNING_MEMBER_FUNCTION_POINTER, + "foo.cpp", + |metric| { + assert_eq!(metric.npa.class_na_sum(), 0); + assert_eq!(metric.npm.class_nm_sum(), 1); + }, + ); + } + + // FIXME(#1807): two valid non-templated spellings (`g++ -std=c++20 + // -fsyntax-only` accepts both) that both grammars' error recovery + // misreads. Each is alone in its class because together the class + // totals come out right (na 1, nm 1) while both members are wrong. + // The truth is the swap of each pin: `getPd` is a method (na 0, nm + // 1), `fp` a static data member (na 1, nm 0). Pinned so a grammar + // or reader fix moves these rows visibly. + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const GET_PD_METHOD: &str = "struct S {\n int S::*getPd();\n};\n"; + #[cfg(any(feature = "cpp", feature = "mozcpp"))] + const FP_DATA_MEMBER: &str = + "struct S {\n static constexpr int (S::*fp)(int) = nullptr;\n};\n"; + + #[cfg(feature = "cpp")] + #[test] + fn cpp_member_pointer_recovery_misreads() { + check_metrics_with_npa::(GET_PD_METHOD, "foo.cpp", |metric| { + assert_eq!(metric.npa.class_na_sum(), 1); + assert_eq!(metric.npm.class_nm_sum(), 0); + }); + check_metrics_with_npa::(FP_DATA_MEMBER, "foo.cpp", |metric| { + assert_eq!(metric.npa.class_na_sum(), 0); + assert_eq!(metric.npm.class_nm_sum(), 1); + }); + } + + #[cfg(feature = "mozcpp")] + #[test] + fn mozcpp_member_pointer_recovery_misreads() { + check_metrics_with_npa::(GET_PD_METHOD, "foo.cpp", |metric| { + assert_eq!(metric.npa.class_na_sum(), 1); + assert_eq!(metric.npm.class_nm_sum(), 0); + }); + check_metrics_with_npa::(FP_DATA_MEMBER, "foo.cpp", |metric| { + assert_eq!(metric.npa.class_na_sum(), 0); + assert_eq!(metric.npm.class_nm_sum(), 1); + }); + } + #[cfg(feature = "cpp")] #[test] fn cpp_non_method_template_payloads_are_not_counted() { diff --git a/src/metrics/npm/ruby.rs b/src/metrics/npm/ruby.rs index d6dd0c1e4..a55266320 100644 --- a/src/metrics/npm/ruby.rs +++ b/src/metrics/npm/ruby.rs @@ -7,14 +7,12 @@ )] use super::npa::{ - RubyVisibility, RubyVisibilityCall, RubyVisibilityEffect, ruby_call_argument_list, - ruby_declaration_is_public, ruby_method_name, ruby_symbol_name, ruby_visibility_effect, - ruby_visibility_marker, ruby_wrapped_is_public, + RubyBodyConsumer, RubyBodyShape, RubyBodyState, RubyOpenCall, RubyVisibility, + RubyVisibilityCall, ruby_method_name, ruby_symbol_name, ruby_wrapped_is_public, }; use std::collections::HashMap; use super::*; -use crate::node::Children; // The five instance methods Ruby makes private the moment they are // defined: `Class#new` calls `initialize`, and the other four are @@ -38,204 +36,24 @@ const RUBY_AUTO_PRIVATE_METHODS: [&str; 5] = [ ]; // The method names a call's arguments spell or declare, for the call -// to re-file and, where Ruby returns them, to hand up (`close_call`). +// to re-file and, where Ruby returns them, to hand up (`leave_call`). type RubyArgumentNames<'a> = Vec<&'a str>; -// What the walk is inside while it reads a node's children: the body -// or a wrapper whose statements run at class scope — a conditional, a -// loop body, a `begin`, a `case` arm, the statement under a modifier — -// or a call's argument list, where a `def` is a declaration (`private -// def x`, `memoize def x`) and a symbol names one. -enum RubyFrameKind<'a> { - Statements, - Arguments { - // The visibility keyword the call spells, `None` for an - // ordinary call. - keyword: Option, - // What the arguments read so far name, for the keyword to - // re-file once the list is read, and for the enclosing call to - // be handed if the keyword returns them. - names: RubyArgumentNames<'a>, - }, -} - -// One node the walk is reading the children of, and those children -// still to read. Each wrapper, call and argument list is a frame on an -// explicit stack rather than a Rust frame, so a body nested or called -// arbitrarily deep costs no stack (#1401 review: 5,000 nested calls -// overflowed the recursive first draft), and each child list is read -// once through a held iterator rather than re-sought per step. -struct RubyFrame<'a> { - node: Node<'a>, - children: Children<'a>, - kind: RubyFrameKind<'a>, -} - -// The statement shapes of a Ruby class body, read by the member walk -// below and nothing else of it: which wrappers hold statements that -// run at class scope, and which children of each are those statements. -// Free of the walk's state so the attribute walk can share them -// (FIXME(#1719)). - -// Whether `node` is a wrapper the walk reads through for the -// statements under it: a conditional and its arms, a `case` and -// its clauses, a loop and its `do` body, a `begin` with its -// `rescue` / `ensure` clauses, a statement modifier, a -// parenthesised statement list, and the right side of an -// assignment — `x = def b; end` and `@h = private def h; end` -// define `b` and `h` as any bare `def` would (measured), and so do -// both of `a, b = def x; end, def y; end`, whose right side is a -// `right_assignment_list` of them. A `def`'s -// or a block's children are a space of their own and are not read: -// a `def` inside `def inner` or an `items.each do … end` is that -// space's method, as `nom` counts it, not the class's. -// -// FIXME(#1719): the attribute walk, `ruby_walk_class_body` in -// `npa/shared.rs`, still reads the body's direct children only, and -// reads a bare keyword as the marker only there — without the -// expression-position reading or the shadow table the ledger keeps — -// so the two Ruby member walks can disagree on one body's visibility. -// This inventory, `is_statement_of` and `RubyMethodLedger`'s marker -// rules are what it should share. -fn reads_through(kind: u16) -> bool { - use Ruby::*; - - // role-bypass: allow(Branch, Switch, Loop, Catch, ElseIf, CaseArm, DocTarget) — the - // wrappers whose statements run at class scope, read for their - // children; not whether they branch, switch, loop, chain or arm a - // case, which is the control-flow rows' and the `CaseArm` row's - // question (#1697), nor whether an assignment takes a doc comment - matches!( - kind.into(), - If | Unless - | Elsif - | Else - | Then - | Case - | CaseMatch - | When - | InClause - | While - | Until - | For - | Do - | Begin - | Rescue - | Ensure - | IfModifier - | UnlessModifier - | WhileModifier - | UntilModifier - | RescueModifier - | RescueModifier2 - | RescueModifier3 - | ParenthesizedStatements - | Assignment - | Assignment2 - | OperatorAssignment - | OperatorAssignment2 - | RightAssignmentList - ) -} - -fn is_assignment(kind: u16) -> bool { - use Ruby::*; - - // role-bypass: allow(DocTarget) — an assignment statement for the - // class-scope ledger; not whether it takes a doc comment - matches!( - kind.into(), - Assignment | Assignment2 | OperatorAssignment | OperatorAssignment2 - ) -} - -// Whether `child` of a wrapper `parent` is one of its statements -// rather than its condition, value, pattern, header or handler. -// -// Only statement positions are read. Ruby does evaluate an `if`'s -// condition, and a bare `private` there is the marker call unless a -// local shadows it, so `if private … end` demotes what follows in Ruby -// and not here: reading conditions would mean reading every expression -// for a keyword, which this inventory of statement positions does not -// attempt. The grammar exposes each consequence as -// a dedicated node or a field, so the arms pick by role rather than -// by index (grammar-dispatch rule 3): `then` / `else` / `elsif` -// under a conditional, the clauses under a `case`, the `then` of a -// clause or a `rescue`, the `do` under a loop, the `body` field of -// a modifier, and the `right` field of an assignment, whose `left` -// is the target. The `rescue` modifier is read on both sides: its -// `body` runs, and its `handler` runs when the body raises — a -// `def` in either position is a definition at class scope, and the -// walk counts every branch. An `else`, a `then`, a `do`, an -// `ensure`, a `begin` and a parenthesised list hold statements -// directly. `rescue_modifier` is three kind ids for one rule -// (grammar-dispatch rule 1): the parser emits `RescueModifier3` in -// every position measured — statement, assignment right side, -// method body — and the other two are aliases the grammar declares -// that no spelling produced; listed defensively per rule 2, and -// pinned unreachable by `ruby_rescue_modifier_aliases_are_not_emitted`. -// -// The exclusions are chosen by role, and no fixture pins them (the -// perturbation sweep recorded on #1401): a widened arm that read a -// condition, a loop header, a `case` value or a `when` pattern as a -// statement would find, in every valid fixture, either a local that -// shadows the keyword — which the ledger already ignores — or a -// bare keyword that Ruby evaluates as the marker call, which the -// widened arm would then read *correctly* where this arm does not. -// A `when` pattern and a `rescue` class list are wrapped (`pattern`, -// `exceptions`) besides. The arms stay as the statement inventory -// they are; the fixture that would discriminate them is the one -// whose answer they get wrong, and `is_statement_of`'s comment -// above records that gap rather than pinning it. -fn is_statement_of(parent: &Node<'_>, parent_kind: u16, child: &Node<'_>) -> bool { - use Ruby::*; - - // role-bypass: allow(Branch, Switch, Loop, ElseIf, Catch, CaseArm) — which - // child of a wrapper is its clause and which of a clause is its - // body, for traversal; not whether the wrapper branches or the arm - // pays a decision, which is the rows' question (#1697) - child.is_named() - && match parent_kind.into() { - If | Unless | Elsif => matches!(child.kind_id().into(), Then | Else | Elsif), - Case | CaseMatch => matches!(child.kind_id().into(), When | InClause | Else), - When | InClause | Rescue => matches!(child.kind_id().into(), Then), - While | Until | For => matches!(child.kind_id().into(), Do), - IfModifier | UnlessModifier | WhileModifier | UntilModifier => parent - .child_by_field_name("body") - .is_some_and(|body| body.id() == child.id()), - RescueModifier | RescueModifier2 | RescueModifier3 => true, - _ if is_assignment(parent_kind) => parent - .child_by_field_name("right") - .is_some_and(|right| right.id() == child.id()), - _ => true, - } -} - // The methods a Ruby class body has declared so far and what each is -// worth: the running body-wide visibility flag, the keyword names a -// local has shadowed, and the declarations with their public bit. -// Visibility cannot be settled arm-by-arm — `private :foo` demotes a -// method declared *earlier* in the same body — so the tally is taken -// once the whole body has been read (#1255). +// worth: the declarations with their public bit, and the re-filing the +// visibility keywords applied to them. The statements, the wrappers +// they sit in and the visibility flag are the shared body walk's +// (`npa/ruby_body.rs`); this is what `npm` reads off them. Visibility +// cannot be settled arm-by-arm — `private :foo` demotes a method +// declared *earlier* in the same body — so the tally is taken once the +// whole body has been read (#1255). struct RubyMethodLedger<'a> { code: &'a [u8], // `class << x` rather than `class X`. Ruby's automatic-private rule // does not reach a singleton class, so the ledger needs to know // which of the two it is keeping. in_singleton_class: bool, - visibility: RubyVisibility, - // The visibility keywords a local of the same name has shadowed: - // after `private = false`, a bare `private` anywhere later in the - // body is that variable, and Ruby reads it as the marker call only - // where no such local is in scope (measured: `x = private` with no - // local demotes what follows; with the local assigned first it does - // not). A Ruby local lives to the end of the body it was assigned - // in, which here is the class body or a wrapper under it — the - // walk does not unshadow at a wrapper's end, which over-shadows - // only the body after an assignment made *inside* a branch. - shadowed: [bool; RubyVisibility::COUNT], - // Every declaration so far, in order: the position of the body's - // visibility *epoch* when it was declared and its own public bit. + // Every declaration so far, in order: its own public bit. public: Vec, // Every named declaration, by name and family (instance or // singleton): the declarations' indices into `public`, and the bit @@ -268,10 +86,6 @@ impl<'a> RubyMethodLedger<'a> { Self { code, in_singleton_class, - // Ruby class bodies open in default-public state, whatever - // the previous body's trailing visibility was. - visibility: RubyVisibility::Public, - shadowed: [false; RubyVisibility::COUNT], public: Vec::new(), by_name: HashMap::new(), } @@ -288,75 +102,6 @@ impl<'a> RubyMethodLedger<'a> { && name.is_some_and(|n| RUBY_AUTO_PRIVATE_METHODS.contains(&n)) } - // A bare `private` / `public` / `protected` sets the flag — Ruby - // evaluates it as the marker call wherever it stands — unless a - // local of that name shadows it. The walk asks this of every node - // it reads as a statement or a call argument, and of nothing else: - // a keyword in a condition, a loop header or a `case` value is - // evaluated by Ruby and not seen here (`is_statement_of` records - // the gap). Answers whether `node` was one, shadowed or not. - fn apply_marker(&mut self, node: &Node<'a>) -> bool { - let Some(marker) = ruby_visibility_marker(node, self.code) else { - return false; - }; - if !self.shadowed[marker as usize] { - self.visibility = marker; - } - true - } - - // Records every visibility keyword's name a binding target binds, - // so the bare word is a local from here on: the `left` of an - // assignment or the `pattern` of a `for`, directly (`private = 1`) - // or inside a destructuring list (`private, other = …`, `(a, - // private) = …`, `*private = …`), which the grammar nests as - // `left_assignment_list` / `destructured_left_assignment` / - // `rest_assignment` holding the identifiers. An `@private = 1` or - // `obj.private = 2` target is not an `identifier` and binds no - // local, so a later bare `private` is still the call (measured). - fn shadow_bindings_of(&mut self, target: &Node<'a>) { - use Ruby::*; - - let mut stack = vec![*target]; - while let Some(node) = stack.pop() { - if let Some(marker) = ruby_visibility_marker(&node, self.code) { - self.shadowed[marker as usize] = true; - continue; - } - if matches!( - node.kind_id().into(), - LeftAssignmentList - | DestructuredLeftAssignment - | RestAssignment - | ExceptionVariable - ) { - stack.extend(node.children().filter(Node::is_named)); - } - } - } - - // Records the locals a statement binds before its body is read: an - // assignment's `left`, a `for`'s `pattern`, a `rescue` clause's - // `=> e` variable (an `exception_variable` wrapping the target). - // All three bind at class scope; a block's parameters bind in the - // block, a space the walk does not enter. - fn shadow_statement_bindings(&mut self, statement: &Node<'a>, kind: u16) { - use Ruby::*; - - let field = if is_assignment(kind) { - "left" - } else if kind == For as u16 { - "pattern" - } else if kind == Rescue as u16 { - "variable" - } else { - return; - }; - if let Some(target) = statement.child_by_field_name(field) { - self.shadow_bindings_of(&target); - } - } - // Records a `method` / `singleton_method` node, taking its // visibility from `keyword` when a visibility call wraps it and from // the body-wide flag otherwise. Answers the declared name, for the @@ -364,15 +109,21 @@ impl<'a> RubyMethodLedger<'a> { fn declare( &mut self, method: &Node<'a>, - singleton: bool, keyword: Option, + body_flag: RubyVisibility, ) -> Option<&'a str> { + use Ruby::*; + + // A `def` is the `FuncOpener` row's callable, which the walk + // has already asked; the row's two callables differ in the + // bucket (`def self.` is the singleton), which is this + // consumer's reading. + // role-bypass: allow(FuncOpener) — the singleton bucket + let singleton = method.kind_id() == SingletonMethod as u16; let name = ruby_method_name(method, self.code); let named_by_keyword = keyword.is_some_and(|kw| kw.governs(singleton)); - let public = keyword.map_or_else( - || ruby_declaration_is_public(singleton, self.visibility), - |kw| ruby_wrapped_is_public(kw, singleton, self.visibility), - ) && (named_by_keyword || !self.is_auto_private(name, singleton)); + let public = ruby_wrapped_is_public(keyword, singleton, body_flag) + && (named_by_keyword || !self.is_auto_private(name, singleton)); if let Some(name) = name { self.by_name .entry((name, singleton)) @@ -409,56 +160,67 @@ impl<'a> RubyMethodLedger<'a> { } } -// One pass over a Ruby class body: the ledger it fills, and the chain -// of whichever node's children are being read, moved down and back up -// as the walk enters and leaves a frame so a row's gate reads each node -// under its own parent (#1696) without a chain copy per level. -struct RubyClassBody<'a, 'chain> { - ledger: RubyMethodLedger<'a>, - chain: ChildAncestors<'a, 'chain>, -} +impl<'a> RubyBodyConsumer<'a> for RubyMethodLedger<'a> { + type Gathered = RubyArgumentNames<'a>; -impl<'a, 'chain> RubyClassBody<'a, 'chain> { - fn new(code: &'a [u8], in_singleton_class: bool, chain: ChildAncestors<'a, 'chain>) -> Self { - Self { - ledger: RubyMethodLedger::new(code, in_singleton_class), - chain, + // A `def` the body or a wrapper runs declares a method under the + // body-wide flag. + fn statement( + &mut self, + statement: &Node<'a>, + shape: RubyBodyShape, + body: &mut RubyBodyState<'a, '_>, + ) { + if shape == RubyBodyShape::Definition { + self.declare(statement, None, body.visibility()); } } - // Opens a call: a bare keyword call sets the flag and is done; any - // other call with an argument list is pushed, with the keyword it - // spells, so its arguments are read as the walk reaches them. - fn open_call(&mut self, call: &Node<'a>, stack: &mut Vec>) { - let keyword = match ruby_visibility_effect(call, self.ledger.code) { - Some(RubyVisibilityEffect::Flag(flag)) => { - self.ledger.visibility = flag; - return; + // One argument of a call: a `def` (`private def x; end`, `memoize + // def x; end`) declares under the call's keyword and names itself + // to the call, and a symbol names a method. A nested call names + // what it returns when it is left; a wrapper passed as an argument + // — `private((def x; end))`, `memoize(begin … end)`, `memoize(x = + // def y; end)` — holds statements that run at class scope exactly + // as a bare one does, so the `def`s inside are declared. What the + // wrapper *evaluates to* — the value of its last statement, which + // Ruby hands to the call (`private((def a; end; def b; end))` + // demotes `b` alone, measured) — is not resolved: the `def`s take + // the body-wide flag and the enclosing keyword re-files nothing + // through them. Counting them at all is the correction the #1401 + // review asked for; their visibility under a wrapping keyword is the + // approximation recorded in `ruby_def_in_a_wrapper_argument_counts`. + fn argument( + &mut self, + argument: &Node<'a>, + shape: RubyBodyShape, + call: &mut RubyOpenCall, + body: &mut RubyBodyState<'a, '_>, + ) { + use Ruby::*; + + let names = &mut call.gathered; + match shape { + RubyBodyShape::Definition => { + names.extend(self.declare(argument, call.keyword, body.visibility())); } - Some(RubyVisibilityEffect::Arguments(keyword)) => Some(keyword), - None => None, - }; - // The arguments sit under the call's `argument_list`: each is - // asked under its own chain, the call's plus the call and the - // list, so a gate the row grows later reads the right parent. - let Some(list) = ruby_call_argument_list(call) else { - return; - }; - self.chain.descend(call); - self.chain.descend(&list); - stack.push(RubyFrame { - node: *call, - children: list.children(), - kind: RubyFrameKind::Arguments { - keyword, - names: RubyArgumentNames::new(), + RubyBodyShape::Call | RubyBodyShape::Wrapper => {} + RubyBodyShape::Leaf => match argument.kind_id().into() { + // role-bypass: allow(StringLit) — a `%i[]` array names several + // methods; the row owns it as a literal, a different question + SymbolArray => names.extend( + argument + .children() + .filter_map(|sym| ruby_symbol_name(&sym, self.code)), + ), + _ => names.extend(ruby_symbol_name(argument, self.code)), }, - }); + } } - // Closes a call whose arguments have all been read: the keyword, if + // Leaves a call whose arguments have all been read: the keyword, if // any, re-files what they named, and what the call returns is handed - // to the enclosing call's frame when that is what the parent is. + // to the enclosing call when that is what the call is an argument of. // // A visibility keyword governs a nested `def` only when it names that // method family: `private def self.x` demotes the *instance* method @@ -481,216 +243,23 @@ impl<'a, 'chain> RubyClassBody<'a, 'chain> { // (batch perf audit: 16,000 × 16,000 took 20 s). The semantics // require each governing level to act, real code nests two deep, // and no probe reads the shape; recorded, not restructured. - fn close_call( + fn leave_call( &mut self, - keyword: Option, - names: RubyArgumentNames<'a>, - stack: &mut [RubyFrame<'a>], + call: RubyOpenCall, + outer: Option<&mut RubyOpenCall>, ) { - self.chain.ascend(); - self.chain.ascend(); - let Some(keyword) = keyword else { + let Some(keyword) = call.keyword else { return; }; let public = keyword.visibility == RubyVisibility::Public; - for name in &names { - self.ledger.refile(name, keyword.targets_singleton, public); + for name in &call.gathered { + self.refile(name, keyword.targets_singleton, public); } if keyword.targets_singleton { return; } - if let Some(RubyFrame { - kind: RubyFrameKind::Arguments { names: outer, .. }, - .. - }) = stack.last_mut() - { - outer.extend(names); - } - } - - // Reads one argument of the call on top of the stack: a `def` - // declares and names itself to the call, a nested call is opened, - // and a symbol names a method. - fn read_argument(&mut self, arg: &Node<'a>, stack: &mut Vec>) { - use crate::roles::ruby::func_opener; - use Ruby::*; - - let code = self.ledger.code; - let Some(RubyFrame { - kind: RubyFrameKind::Arguments { keyword, .. }, - .. - }) = stack.last() - else { - return; - }; - let keyword = *keyword; - // An argument is evaluated as any expression is: a bare keyword - // there is the marker call (`foo(private)` demotes what follows, - // measured), a binding there (`foo(private = 1)`) shadows it from - // then on. Read before anything else, as a statement's are. - if self.ledger.apply_marker(arg) { - return; - } - let kind = arg.kind_id(); - self.ledger.shadow_statement_bindings(arg, kind); - // A `def` passed to a call (`private def x; end`, `memoize def - // x; end`) is the `FuncOpener` row's callable, as a bare one is. - // Which of its two callables decides the singleton bucket is - // this consumer's reading. A nested call is opened as a frame of - // its own; the names it returns reach this one when it closes. - let named: Vec<&'a str> = if func_opener::is_callable(arg, code, self.chain.get()) { - // role-bypass: allow(FuncOpener) — the singleton bucket - self.ledger - .declare(arg, kind == SingletonMethod as u16, keyword) - .into_iter() - .collect() - } else { - match kind.into() { - // role-bypass: allow(NullSafe) — a call nested in an argument - // list, which the row's safe-navigation owner is a kind of; - // read as the body's own calls are - Call | Call2 | Call3 | Call4 => return self.open_call(arg, stack), - // role-bypass: allow(StringLit) — a `%i[]` array names several - // methods; the row owns it as a literal, a different question - SymbolArray => arg - .children() - .filter_map(|sym| ruby_symbol_name(&sym, code)) - .collect(), - // A wrapper passed as an argument — `private((def x; end))`, - // `memoize(begin … end)`, `memoize(x = def y; end)` — holds - // statements that run at class scope exactly as a bare one - // does, so the `def`s inside are declared. What the wrapper - // *evaluates to* — the value of its last statement, which - // Ruby hands to the call (`private((def a; end; def b; - // end))` demotes `b` alone, measured) — is not resolved: the - // `def`s take the body-wide flag and the enclosing keyword - // re-files nothing through them. Counting them at all is the - // correction the #1401 review asked for; their visibility - // under a wrapping keyword is the approximation recorded in - // `ruby_def_in_a_wrapper_argument_counts`. - _ if reads_through(kind) => return self.open_wrapper(arg, stack), - _ => ruby_symbol_name(arg, code).into_iter().collect(), - } - }; - if let Some(RubyFrame { - kind: RubyFrameKind::Arguments { names, .. }, - .. - }) = stack.last_mut() - { - names.extend(named); - } - } - - // Reads one statement of the body or of a wrapper, under the chain - // of its siblings: a bare visibility marker sets the flag, a `def` - // declares, a call is opened for its arguments, and a wrapper is - // pushed so its statements are read in turn. - fn read_statement(&mut self, child: &Node<'a>, stack: &mut Vec>) { - use crate::roles::ruby::func_opener; - use Ruby::*; - - // A bare `private` is the marker call wherever an expression is - // evaluated — `x = private` and `y = public` set the flag - // (measured) — unless a local of that name shadows it. The - // assignment or `for` that creates the local is read as a - // wrapper (for the `def` its right side or body may hold), and - // shadows from its target before that is read. - if self.ledger.apply_marker(child) { - return; - } - let kind = child.kind_id(); - self.ledger.shadow_statement_bindings(child, kind); - // A `def` — the `FuncOpener` row's callable — declares a method; - // the row's two callables differ in the bucket (`def self.` is - // the singleton), which is this consumer's reading. - if func_opener::is_callable(child, self.ledger.code, self.chain.get()) { - // role-bypass: allow(FuncOpener) — the singleton bucket - self.ledger - .declare(child, kind == SingletonMethod as u16, None); - return; - } - match kind.into() { - // role-bypass: allow(NullSafe) — the visibility macro calls and - // the decorator-style ones, which the row's safe-navigation - // owner is a kind of; each is told by its name - Call | Call2 | Call3 | Call4 => self.open_call(child, stack), - _ if reads_through(kind) => self.open_wrapper(child, stack), - _ => {} - } - } - - // Opens a statement wrapper: its statements are read as the walk - // reaches them, under its own chain. - fn open_wrapper(&mut self, wrapper: &Node<'a>, stack: &mut Vec>) { - self.chain.descend(wrapper); - stack.push(RubyFrame { - node: *wrapper, - children: wrapper.children(), - kind: RubyFrameKind::Statements, - }); - } - - // Reads every statement of the body, of each wrapper under it and - // of each call's argument list, in source order. The body's direct - // children are all statements; a wrapper's are the ones - // `is_statement_of` picks; an argument list's are its named - // children. - fn walk(&mut self, body: &Node<'a>) { - let floor = self.chain.get().depth(); - let mut stack = vec![RubyFrame { - node: *body, - children: body.children(), - kind: RubyFrameKind::Statements, - }]; - // The top frame is taken off to read its next child and put - // back before the child is, so a reader that opens a frame of - // its own stacks it on top; a frame with no child left closes. - while let Some(mut frame) = stack.pop() { - let parent = frame.node; - let parent_kind = parent.kind_id(); - let arguments = matches!(frame.kind, RubyFrameKind::Arguments { .. }); - let child = frame.children.find(|child| { - if arguments || parent.id() == body.id() { - child.is_named() - } else { - is_statement_of(&parent, parent_kind, child) - } - }); - let Some(child) = child else { - self.close_frame(frame, body, &mut stack); - continue; - }; - stack.push(frame); - if arguments { - self.read_argument(&child, &mut stack); - } else { - self.read_statement(&child, &mut stack); - } - } - // Every frame descended is ascended on its pop: a frame kind - // that forgot would leave every later gate reading the wrong - // parent, which nothing else can see. - debug_assert_eq!( - self.chain.get().depth(), - floor, - "the member walk left the chain off the body's floor" - ); - } - - // Leaves a frame whose children are all read: a call closes with - // its keyword's re-filing, a wrapper gives back the chain level it - // took, and the body — the one frame the chain was built for — - // took none. - fn close_frame(&mut self, done: RubyFrame<'a>, body: &Node<'a>, stack: &mut [RubyFrame<'a>]) { - match done.kind { - RubyFrameKind::Arguments { keyword, names } => { - self.close_call(keyword, names, stack); - } - RubyFrameKind::Statements => { - if done.node.id() != body.id() { - self.chain.ascend(); - } - } + if let Some(outer) = outer { + outer.gathered.extend(call.gathered); } } } @@ -739,14 +308,10 @@ impl Npm for RubyCode { } // role-bypass: allow(Container) — the singleton-class bucket - let mut body = RubyClassBody::new( - code, - matches!(parent_kind, SingletonClass), - ancestors.of_children(node), - ); - body.walk(node); + let mut ledger = RubyMethodLedger::new(code, matches!(parent_kind, SingletonClass)); + RubyBodyState::walk(code, ancestors.of_children(node), node, &mut ledger); - let (count, public) = body.ledger.tally(); + let (count, public) = ledger.tally(); stats.class_nm += count; stats.class_npm += public; } diff --git a/utils/check-rumdl-lockstep-test.py b/utils/check-rumdl-lockstep-test.py new file mode 100644 index 000000000..928371cd8 --- /dev/null +++ b/utils/check-rumdl-lockstep-test.py @@ -0,0 +1,519 @@ +#!/usr/bin/env python3 +"""Tests for check-rumdl-lockstep.py. + +Three kinds of test, matching the check-ruff-lockstep-test.py pattern: + +* Unit tests against synthetic file contents, weighted toward the shapes + a line-oriented read can get wrong — a sibling tool's identically + shaped pin, a ``${RUMDL_VERSION}`` expansion that is not a key, the + arm64 hash sitting beside the x86_64 one — and toward the malformed + inputs on which a version gate must fail loudly rather than pass. +* ``main()`` tests over a synthetic repository root, covering each drift + branch and the file it names. +* A smoke test running the real gate against the real repository, + asserting a clean tree reports OK. + +Run with: + python3 -m unittest -q utils/check-rumdl-lockstep-test.py +""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import pathlib +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +UTILS_DIR = pathlib.Path(__file__).resolve().parent +REPO_ROOT = UTILS_DIR.parent +SCRIPT_SRC = UTILS_DIR / "check-rumdl-lockstep.py" + + +def _load_module(): # type: ignore[no-untyped-def] + spec = importlib.util.spec_from_file_location("check_rumdl_lockstep", SCRIPT_SRC) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +GATE = _load_module() + +X86_SHA = "a" * 64 +ARM_SHA = "b" * 64 +# A sibling tool's hash, deliberately equal in shape: the gate must not +# reach it however the files are ordered. +OTHER_SHA = "c" * 64 + +MISE_SAMPLE = """\ +[tools] +node = "25.9.0" +# Keep in lockstep with ci.yml and the Dockerfile. +rumdl = "0.3.2" +python = "3.12" +"pipx:ruff" = "latest" +""" + +# The rumdl step sits between two siblings with identically shaped env +# maps, and the comment above it expands `${RUMDL_VERSION}` the way the +# real one does and spells the key itself after a `#`. +CI_SAMPLE = f"""\ +jobs: + lint: + steps: + - name: Install actionlint + env: + ACTIONLINT_VERSION: "1.7.12" + ACTIONLINT_SHA256: "{OTHER_SHA}" + run: echo + # RUMDL_VERSION: bump with mise.toml and the Dockerfile. + # Regenerate with: + # curl -fsSL https://example.invalid/v${{RUMDL_VERSION}}/rumdl.tar.gz.sha256 + - name: Install rumdl + env: + RUMDL_VERSION: "0.3.2" + RUMDL_SHA256: "{X86_SHA}" + run: | + url="https://example.invalid/v${{RUMDL_VERSION}}/rumdl-v${{RUMDL_VERSION}}-x86_64-unknown-linux-gnu.tar.gz" + echo "${{RUMDL_SHA256}} /tmp/rumdl.tgz" | sha256sum -c - + - name: Install checkmake + env: + CHECKMAKE_VERSION: "0.2.2" + CHECKMAKE_SHA256: "{OTHER_SHA}" + run: echo +""" + +# Two heredoc RUN blocks of the same `case` shape, rumdl second, so the +# block selection — not position — decides which hash is read. +DOCKER_SAMPLE = f"""\ +FROM debian:trixie +# Toolchain / release pins. +# RUMDL/…/ACTIONLINT -> .github/workflows/ci.yml lint job +ENV RUMDL_VERSION=0.3.2 +ENV ACTIONLINT_VERSION=1.7.12 + +RUN <<'EOF' +set -eux +case $(dpkg --print-architecture) in + amd64) ACTIONLINT_ARCH="amd64"; SHA256="{OTHER_SHA}" ;; + arm64) ACTIONLINT_ARCH="arm64"; SHA256="{OTHER_SHA}" ;; +esac +wget -O /tmp/actionlint.tgz "https://example.invalid/v${{ACTIONLINT_VERSION}}/actionlint.tar.gz" +EOF + +# rumdl (Markdown linter/formatter) +RUN <<'EOF' +set -eux +case $(dpkg --print-architecture) in + amd64) TRIPLE="x86_64-unknown-linux-gnu"; SHA256="{X86_SHA}" ;; + arm64) TRIPLE="aarch64-unknown-linux-gnu"; SHA256="{ARM_SHA}" ;; + *) echo "Unsupported architecture" && exit 1 ;; +esac +wget -O /tmp/rumdl.tgz "https://example.invalid/v${{RUMDL_VERSION}}/rumdl-v${{RUMDL_VERSION}}-${{TRIPLE}}.tar.gz" +echo "${{SHA256}} /tmp/rumdl.tgz" | sha256sum --check +EOF +""" + + +class MiseVersionTest(unittest.TestCase): + def test_the_tools_entry_is_returned(self) -> None: + self.assertEqual(GATE.read_mise_version(MISE_SAMPLE), "0.3.2") + + def test_a_missing_entry_is_a_hard_error(self) -> None: + text = MISE_SAMPLE.replace('rumdl = "0.3.2"\n', "") + with self.assertRaises(SystemExit) as caught: + GATE.read_mise_version(text) + self.assertIn("under [tools]", str(caught.exception)) + + def test_a_non_string_entry_is_a_hard_error(self) -> None: + # mise accepts a table form too; this gate reads only the string + # form, and must say so rather than compare a dict to a version. + text = MISE_SAMPLE.replace('rumdl = "0.3.2"', 'rumdl = { version = "0.3.2" }') + with self.assertRaises(SystemExit): + GATE.read_mise_version(text) + + def test_an_entry_outside_tools_is_not_read(self) -> None: + text = '[env]\nrumdl = "0.3.2"\n' + with self.assertRaises(SystemExit): + GATE.read_mise_version(text) + + def test_invalid_toml_is_a_hard_error(self) -> None: + with self.assertRaises(SystemExit) as caught: + GATE.read_mise_version("[tools\n") + self.assertIn("not valid TOML", str(caught.exception)) + + +class CiPinTest(unittest.TestCase): + def test_the_rumdl_step_env_is_returned(self) -> None: + # The sample has `${RUMDL_VERSION}` in a comment and in `run:`; + # only the `env:` keys count, so exactly one of each is found. + self.assertEqual(GATE.read_ci_pin(CI_SAMPLE), ("0.3.2", X86_SHA)) + + def test_an_unquoted_or_commented_value_is_read(self) -> None: + text = CI_SAMPLE.replace( + 'RUMDL_VERSION: "0.3.2"', "RUMDL_VERSION: 0.3.2 # pin" + ) + self.assertEqual(GATE.read_ci_pin(text)[0], "0.3.2") + + def test_crlf_line_endings_are_handled(self) -> None: + self.assertEqual( + GATE.read_ci_pin(CI_SAMPLE.replace("\n", "\r\n")), ("0.3.2", X86_SHA) + ) + + def test_a_missing_version_key_is_a_hard_error(self) -> None: + text = CI_SAMPLE.replace(' RUMDL_VERSION: "0.3.2"\n', "") + with self.assertRaises(SystemExit) as caught: + GATE.read_ci_pin(text) + self.assertIn("RUMDL_VERSION", str(caught.exception)) + self.assertIn("found 0", str(caught.exception)) + + def test_a_duplicated_step_is_a_hard_error(self) -> None: + # Ambiguous rather than wrong: picking either would let a stale + # duplicate decide what the gate compares. + text = CI_SAMPLE + ' - env:\n RUMDL_SHA256: "' + X86_SHA + '"\n' + with self.assertRaises(SystemExit) as caught: + GATE.read_ci_pin(text) + self.assertIn("RUMDL_SHA256", str(caught.exception)) + self.assertIn("found 2", str(caught.exception)) + + def test_a_truncated_hash_is_a_hard_error(self) -> None: + text = CI_SAMPLE.replace(X86_SHA, X86_SHA[:40]) + with self.assertRaises(SystemExit) as caught: + GATE.read_ci_pin(text) + self.assertIn("not a 64-digit", str(caught.exception)) + + +class DockerVersionTest(unittest.TestCase): + def test_the_env_line_is_returned(self) -> None: + self.assertEqual(GATE.read_docker_version(DOCKER_SAMPLE), "0.3.2") + + def test_the_legacy_space_form_is_read(self) -> None: + text = DOCKER_SAMPLE.replace( + "ENV RUMDL_VERSION=0.3.2", "ENV RUMDL_VERSION 0.3.2" + ) + self.assertEqual(GATE.read_docker_version(text), "0.3.2") + + def test_a_quoted_value_is_unquoted(self) -> None: + text = DOCKER_SAMPLE.replace("RUMDL_VERSION=0.3.2", 'RUMDL_VERSION="0.3.2"') + self.assertEqual(GATE.read_docker_version(text), "0.3.2") + + def test_a_missing_env_line_is_a_hard_error(self) -> None: + text = DOCKER_SAMPLE.replace("ENV RUMDL_VERSION=0.3.2\n", "") + with self.assertRaises(SystemExit) as caught: + GATE.read_docker_version(text) + self.assertIn("found 0", str(caught.exception)) + + def test_two_env_lines_are_a_hard_error(self) -> None: + text = DOCKER_SAMPLE + "ENV RUMDL_VERSION=0.3.1\n" + with self.assertRaises(SystemExit) as caught: + GATE.read_docker_version(text) + self.assertIn("found 2", str(caught.exception)) + + +class DockerShaTest(unittest.TestCase): + def test_the_x86_64_arm_of_the_rumdl_block_is_returned(self) -> None: + # The first `amd64)` arm in the file is actionlint's, and the + # rumdl block's other arm is arm64's: a position-based read would + # return one of them instead. + self.assertEqual(GATE.read_docker_x86_64_sha(DOCKER_SAMPLE), X86_SHA) + + def test_reordered_arms_still_resolve_to_the_triple(self) -> None: + amd = f' amd64) TRIPLE="x86_64-unknown-linux-gnu"; SHA256="{X86_SHA}" ;;\n' + arm = f' arm64) TRIPLE="aarch64-unknown-linux-gnu"; SHA256="{ARM_SHA}" ;;\n' + text = DOCKER_SAMPLE.replace(amd + arm, arm + amd) + self.assertNotEqual(text, DOCKER_SAMPLE) + self.assertEqual(GATE.read_docker_x86_64_sha(text), X86_SHA) + + def test_an_unquoted_delimiter_is_recognised(self) -> None: + text = DOCKER_SAMPLE.replace("RUN <<'EOF'", "RUN < None: + text = DOCKER_SAMPLE.replace( + "RUN <<'EOF'", "RUN --mount=type=cache,target=/tmp/c <<'EOF'" + ) + self.assertEqual(GATE.read_docker_x86_64_sha(text), X86_SHA) + + def test_no_rumdl_block_is_a_hard_error(self) -> None: + text = DOCKER_SAMPLE.replace("${RUMDL_VERSION}", "0.3.2") + with self.assertRaises(SystemExit) as caught: + GATE.read_docker_x86_64_sha(text) + self.assertIn("found 0", str(caught.exception)) + + def test_a_block_without_an_x86_64_arm_is_a_hard_error(self) -> None: + text = DOCKER_SAMPLE.replace('TRIPLE="x86_64-unknown-linux-gnu"', 'TRIPLE="x"') + with self.assertRaises(SystemExit) as caught: + GATE.read_docker_x86_64_sha(text) + # The arm's own error, not the block-count one, which also ends + # in "found 0" and is what a fixture that lost the block raises. + self.assertIn("SHA256=", str(caught.exception)) + self.assertIn(GATE.X86_64_TRIPLE, str(caught.exception)) + self.assertIn("found 0", str(caught.exception)) + + def test_an_unterminated_heredoc_yields_no_block(self) -> None: + # The closing `EOF` is the only thing that delimits a block; a + # body that never closes must not be read as one. + text = DOCKER_SAMPLE.rsplit("EOF", 1)[0] + with self.assertRaises(SystemExit) as caught: + GATE.read_docker_x86_64_sha(text) + self.assertIn("found 0", str(caught.exception)) + + def test_a_malformed_hash_is_a_hard_error(self) -> None: + text = DOCKER_SAMPLE.replace(X86_SHA, X86_SHA.upper()) + with self.assertRaises(SystemExit) as caught: + GATE.read_docker_x86_64_sha(text) + self.assertIn("not a 64-digit", str(caught.exception)) + + +class DriftedTest(unittest.TestCase): + A, B, C = (pathlib.Path(name) for name in ("a", "b", "c")) + + def test_agreement_names_nothing(self) -> None: + self.assertEqual(GATE.drifted({self.A: "1", self.B: "1", self.C: "1"}), []) + + def test_the_odd_one_out_is_named_alone(self) -> None: + # Each position in turn, so the answer cannot come from order. + for odd in (self.A, self.B, self.C): + versions = {path: "1" for path in (self.A, self.B, self.C)} + versions[odd] = "2" + with self.subTest(odd=odd): + self.assertEqual(GATE.drifted(versions), [odd]) + + def test_three_different_values_name_every_file(self) -> None: + self.assertEqual( + GATE.drifted({self.A: "1", self.B: "2", self.C: "3"}), + [self.A, self.B, self.C], + ) + + +class RepoCase(unittest.TestCase): + """A synthetic repository root for `main()` to read.""" + + def _repo( + self, + mise: str = MISE_SAMPLE, + ci: str = CI_SAMPLE, + docker: str = DOCKER_SAMPLE, + baseline: str | None = None, + ) -> None: + root = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory())) + (root / GATE.CI_WORKFLOW).parent.mkdir(parents=True) + (root / GATE.MISE_TOML).write_text(mise, encoding="utf-8") + (root / GATE.CI_WORKFLOW).write_text(ci, encoding="utf-8") + (root / GATE.DOCKERFILE).write_text(docker, encoding="utf-8") + if baseline is None: + baseline = GATE.format_baseline("0.3.2", X86_SHA) + (root / GATE.BASELINE).write_text(baseline, encoding="utf-8") + self.enterContext(mock.patch.object(GATE, "REPO_ROOT", root)) + + def _run(self, *argv: str) -> tuple[int, str, str]: + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = GATE.main(list(argv)) + return code, out.getvalue(), err.getvalue() + + +class MainTest(RepoCase): + """`main()` over a synthetic repository root.""" + + def _drifted_line(self, err: str) -> str: + lines = [line for line in err.splitlines() if "drifted:" in line] + self.assertEqual(len(lines), 1, err) + return lines[0] + + def test_an_aligned_tree_passes(self) -> None: + self._repo() + code, out, _ = self._run() + self.assertEqual(code, 0) + self.assertIn("rumdl lockstep OK", out) + # The version, not just the word OK: a pass that names the wrong + # version is the drift this gate exists to report. + self.assertIn("rumdl 0.3.2", out) + + def test_a_forgotten_dockerfile_is_named(self) -> None: + # #1653's near miss: mise.toml and ci.yml bumped, Dockerfile not. + self._repo( + mise=MISE_SAMPLE.replace("0.3.2", "0.3.3"), + ci=CI_SAMPLE.replace('"0.3.2"', '"0.3.3"'), + ) + code, _, err = self._run() + self.assertEqual(code, 1) + line = self._drifted_line(err) + self.assertIn("Dockerfile", line) + self.assertNotIn("mise.toml", line) + self.assertNotIn("ci.yml", line) + + def test_each_drifted_file_is_named_alone(self) -> None: + cases = { + "mise.toml": {"mise": MISE_SAMPLE.replace("0.3.2", "0.3.1")}, + "ci.yml": {"ci": CI_SAMPLE.replace('"0.3.2"', '"0.3.1"')}, + "Dockerfile": {"docker": DOCKER_SAMPLE.replace("=0.3.2", "=0.3.1")}, + } + for name, files in cases.items(): + with self.subTest(drifted=name): + self._repo(**files) + code, _, err = self._run() + self.assertEqual(code, 1) + line = self._drifted_line(err) + for other in cases: + if other == name: + self.assertIn(other, line) + else: + self.assertNotIn(other, line) + # The whole table is printed, not only the verdict. + self.assertIn("rumdl 0.3.1", err) + self.assertIn("rumdl 0.3.2", err) + + def test_a_hash_pasted_into_one_file_only_fails(self) -> None: + # Same version everywhere, but the Dockerfile kept the old hash. + self._repo(docker=DOCKER_SAMPLE.replace(X86_SHA, "d" * 64)) + code, _, err = self._run() + self.assertEqual(code, 1) + self.assertNotIn("drifted:", err) + # The digest comparison's own sentence; a bare "SHA256" is also in + # the footer every failure prints. + self.assertIn(f"differs from {GATE.CI_WORKFLOW} {GATE.SHA256_VAR}", err) + self.assertIn("d" * 64, err) + self.assertIn(X86_SHA, err) + + def test_the_arm64_hash_is_not_compared(self) -> None: + # No offline counterpart exists; the image build verifies it. + self._repo(docker=DOCKER_SAMPLE.replace(ARM_SHA, "e" * 64)) + code, _, _ = self._run() + self.assertEqual(code, 0) + + def test_every_problem_is_reported_in_one_run(self) -> None: + self._repo( + mise=MISE_SAMPLE.replace("0.3.2", "0.3.1"), + ci=CI_SAMPLE.replace(X86_SHA, "d" * 64), + ) + code, _, err = self._run() + self.assertEqual(code, 1) + self.assertIn("mise.toml", self._drifted_line(err)) + self.assertIn("d" * 64, err) + + def test_an_absent_input_file_is_a_hard_error(self) -> None: + self._repo() + (GATE.REPO_ROOT / GATE.DOCKERFILE).unlink() + with self.assertRaises(SystemExit) as caught: + self._run() + self.assertIn("cannot read Dockerfile", str(caught.exception)) + + +class BaselineTest(RepoCase): + """The recorded version/hash pair (#1779 review): a bump must move both.""" + + NEW_SHA = "d" * 64 + + def _bumped(self, *, rehash: bool) -> None: + # All three versions moved to 0.3.3; the x86_64 hash in ci.yml and + # the Dockerfile moved with them only when `rehash`. + ci, docker = CI_SAMPLE.replace("0.3.2", "0.3.3"), DOCKER_SAMPLE.replace("0.3.2", "0.3.3") + if rehash: + ci, docker = ci.replace(X86_SHA, self.NEW_SHA), docker.replace(X86_SHA, self.NEW_SHA) + self._repo(mise=MISE_SAMPLE.replace("0.3.2", "0.3.3"), ci=ci, docker=docker) + + def _recorded(self) -> tuple[str, str]: + return GATE.read_baseline((GATE.REPO_ROOT / GATE.BASELINE).read_text(encoding="utf-8")) + + def test_a_version_bump_with_stale_hashes_fails_by_name(self) -> None: + self._bumped(rehash=False) + code, out, err = self._run() + self.assertEqual(code, 1) + self.assertNotIn("lockstep OK", out) + self.assertIn( + "version bumped to 0.3.3 but the x86_64-unknown-linux-gnu hash is still 0.3.2's", + err, + ) + self.assertIn("paste its sha256 into", err) + + def test_an_honest_bump_fails_until_recorded(self) -> None: + # Both moved, but the record is still 0.3.2's: left that way, the + # next bump would be checked against the wrong release's hash. + self._bumped(rehash=True) + code, out, err = self._run() + self.assertEqual(code, 1) + self.assertNotIn("lockstep OK", out) + self.assertIn("still records 0.3.2", err) + self.assertIn("--update", err) + self.assertEqual(self._recorded(), ("0.3.2", X86_SHA), "a plain run writes nothing") + + def test_update_records_an_honest_bump(self) -> None: + self._bumped(rehash=True) + code, out, err = self._run("--update") + self.assertEqual(code, 0, err) + self.assertIn("recorded rumdl 0.3.3", out) + self.assertEqual(self._recorded(), ("0.3.3", self.NEW_SHA)) + # The next run checks against the new pair and has nothing to say. + code, out, err = self._run() + self.assertEqual((code, err), (0, "")) + + def test_update_refuses_a_stale_bump(self) -> None: + self._bumped(rehash=False) + code, _, err = self._run("--update") + self.assertEqual(code, 1) + self.assertIn("hash is still 0.3.2's", err) + self.assertEqual(self._recorded(), ("0.3.2", X86_SHA), "a stale pair is never recorded") + + def test_update_with_nothing_moved_is_refused(self) -> None: + self._repo() + code, _, err = self._run("--update") + self.assertEqual(code, 1) + self.assertIn("already records rumdl 0.3.2", err) + + def test_a_new_hash_without_a_bump_fails(self) -> None: + self._repo( + ci=CI_SAMPLE.replace(X86_SHA, self.NEW_SHA), + docker=DOCKER_SAMPLE.replace(X86_SHA, self.NEW_SHA), + ) + code, _, err = self._run() + self.assertEqual(code, 1) + self.assertIn("hash changed but rumdl is still 0.3.2", err) + + def test_a_malformed_baseline_is_a_hard_error(self) -> None: + self._repo(baseline="# header only\n") + with self.assertRaises(SystemExit) as caught: + self._run() + self.assertIn("found 0", str(caught.exception)) + self.assertIn(str(GATE.BASELINE), str(caught.exception)) + + def test_the_committed_baseline_is_what_update_writes(self) -> None: + text = (REPO_ROOT / GATE.BASELINE).read_text(encoding="utf-8") + self.assertEqual(text, GATE.format_baseline(*GATE.read_baseline(text))) + + +class RealRepositoryTest(unittest.TestCase): + def test_clean_tree_passes(self) -> None: + result = subprocess.run( + [sys.executable, str(SCRIPT_SRC)], + capture_output=True, + check=False, + text=True, + cwd=REPO_ROOT, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("rumdl lockstep OK", result.stdout) + self.assertIn("3 sites checked", result.stdout) + + def test_it_runs_from_an_unrelated_cwd(self) -> None: + # `REPO_ROOT` is derived from the script's own location, so the + # gate must not depend on being invoked from the repository root + # (AGENTS.md, "Project layout"). + with tempfile.TemporaryDirectory() as elsewhere: + result = subprocess.run( + [sys.executable, str(SCRIPT_SRC)], + capture_output=True, + check=False, + text=True, + cwd=elsewhere, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/check-rumdl-lockstep.py b/utils/check-rumdl-lockstep.py new file mode 100644 index 000000000..75ce235ba --- /dev/null +++ b/utils/check-rumdl-lockstep.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +"""check-rumdl-lockstep + +One rumdl release lints this repository's Markdown, and it is pinned in +three files that only comments kept in step: + +* ``mise.toml`` — ``[tools] rumdl = "X.Y.Z"``, what ``mise install`` + gives a contributor; +* ``.github/workflows/ci.yml`` — the ``lint`` job's ``RUMDL_VERSION`` / + ``RUMDL_SHA256`` step env, what CI downloads and verifies; +* ``Dockerfile`` — ``ENV RUMDL_VERSION`` plus the per-arch SHA256 + ``case`` in the rumdl ``RUN`` block, what the dev container installs. + +#1653 shows the cost. Its work list named ``mise.toml`` and ``ci.yml`` +and left out the ``Dockerfile``, which the bump caught only by grepping. +Had it been missed, the container would lint on a different rumdl than +CI, and the only symptom would be "clean in the container, red in CI" +on the first document that tripped a rule the two versions disagree on. + +Two invariants are checked. + +**The three versions are equal.** There is no natural anchor — no file +is derived from another — so a disagreement names the file that differs +from the other two, or all three when no two agree. Every value is +printed either way, so the reader sees the whole table rather than a +verdict about which side is "right". + +**The Dockerfile's x86_64 SHA256 equals ``ci.yml``'s ``RUMDL_SHA256``.** +Both verify ``rumdl-v-x86_64-unknown-linux-gnu.tar.gz``, so +they must match byte for byte. This catches a version bump that pasted +only one hash, offline. The arm64 hash has no CI counterpart to compare +against; the image build's ``sha256sum --check`` already covers it. + +**A version bump moves the hash too.** The two checks above pass a bump +that changed all three versions and left both hashes alone, because the +stale hash still agrees with itself. ``.rumdl-lockstep-baseline.txt`` +records the version and x86_64 SHA256 the gate last accepted, and the +gate fails by name when the version has moved but the hash has not, or +the hash has moved but the version has not (a release's tarball does not +change). A bump that moves both fails too, until ``--update`` records +the new pair in the same commit: a record left at the old release would +check the *next* bump against the wrong hash, and a passing run's note +is read by no one. ``--update`` refuses a pair that moved only one of +the two, and refuses when nothing moved. + +**Why rumdl alone, not every pin the Dockerfile header mirrors from +``ci.yml``.** rumdl is the only tool declared in all three files. taplo, +shfmt and shellcheck reach CI through ``taiki-e/install-action``, which +carries no SHA256 to compare. checkmake's arm64 container path installs +a different fork at a different version from CI's, so "the container +matches CI" is false for it by design. actionlint does have the same +``ci.yml`` / ``Dockerfile`` version-and-hash pair; it is two files, not +three, and is left for a follow-up rather than generalising this gate +on spec. + +``mise.toml`` is parsed with ``tomllib``. ``ci.yml`` and the +``Dockerfile`` are parsed by line, because PyYAML is not in the +standard library and nothing parses a Dockerfile: this gate runs under a +contributor's bare ``python3`` and as a ``language: system`` pre-commit +hook. Each read is anchored to a shape that occurs exactly once and +hard-errors on anything else — no match, two matches, a malformed hash — +because a version gate that silently passes on input it does not +recognise is the bug being fixed, not a lenient version of the fix. + +Exit status: 0 when the pins agree (or ``--update`` recorded a new pair), +1 on drift, on a refused ``--update`` or on an input the gate cannot +read, 2 when ``tomllib`` is unavailable. + +See `#1779`, and ``check-ruff-lockstep.py`` (#1230) for the sibling +gate this one is modelled on. +""" + +from __future__ import annotations + +import argparse +import collections +import pathlib +import re +import sys + +# tomllib landed in 3.11. On older Python, fall back to the external +# `tomli` package (same API), matching check-ruff-lockstep.py. +try: + import tomllib +except ImportError: + try: + import tomli as tomllib # type: ignore[import-not-found,no-redef] + except ImportError: + sys.stderr.write( + "error: check-rumdl-lockstep.py requires Python 3.11+\n" + " (tomllib lives in the standard library starting at 3.11).\n" + " On older Python, install `tomli` and retry:\n" + " pip install tomli\n" + ) + sys.exit(2) + +# `parents[1]`, not `parent`: this gate lives in `utils/` but every path +# it reads is anchored at the repository root. +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] + +MISE_TOML = pathlib.Path("mise.toml") +CI_WORKFLOW = pathlib.Path(".github/workflows/ci.yml") +DOCKERFILE = pathlib.Path("Dockerfile") +BASELINE = pathlib.Path(".rumdl-lockstep-baseline.txt") + +BASELINE_HEADER = """\ +# rumdl lockstep baseline: the rumdl release and its x86_64 tarball +# SHA256 that check-rumdl-lockstep.py last accepted (#1779). +# Maintained by `python3 utils/check-rumdl-lockstep.py --update`; see the +# script's docstring. +# +# The gate fails when the version pinned in mise.toml, ci.yml and the +# Dockerfile has moved but the hash in ci.yml and the Dockerfile is still +# the one recorded here: the bump kept the old release's hash. A bump +# that pastes the new release's hash into both files fails too, until +# `--update` records the new pair; commit this file with that bump. +# +# One record: ` `. +""" + +RUMDL = "rumdl" +VERSION_VAR = "RUMDL_VERSION" +SHA256_VAR = "RUMDL_SHA256" + +# The release artifact both CI and the container's amd64 arm download. +# Keying the Dockerfile arm on the triple rather than on `amd64)` is +# what makes the comparison mean "the same tarball": an arm spelled +# `x86_64)` or reordered still resolves to the hash of this file. +X86_64_TRIPLE = "x86_64-unknown-linux-gnu" + +# A step-level `env:` key in the workflow. Anchored to leading +# whitespace and a trailing `:` so the `${RUMDL_VERSION}` expansions in +# the `run:` script and the comment above the step cannot match. +_CI_KEY_RE = { + name: re.compile(rf"^\s*{name}:\s*(\S.*)$") for name in (VERSION_VAR, SHA256_VAR) +} +# `ENV RUMDL_VERSION=0.3.2`, or the legacy `ENV RUMDL_VERSION 0.3.2`. +_DOCKER_ENV_RE = re.compile(rf"^ENV\s+{VERSION_VAR}(?:=|\s+)(\S+)\s*$") +# `RUN <<'EOF'` / `RUN < str: + """Reduce a YAML scalar to its value: no comment, no quotes. + + A version or a hex digest never contains whitespace or ``#``, so the + first whitespace-delimited token before any comment is the value. + """ + words = raw.split("#", 1)[0].split() + return words[0].strip("\"'") if words else "" + + +def _sha256(value: str, where: str) -> str: + """Return ``value`` if it is a lowercase hex SHA256, else hard-error. + + Equality alone would bless two identically truncated pastes, which + ``sha256sum --check`` would then reject at install time instead. + """ + if not _SHA256_HEX_RE.match(value): + raise SystemExit( + f"error: {where} is not a 64-digit lowercase hex SHA256: {value!r}" + ) + return value + + +def read_mise_version(text: str) -> str: + """Return ``[tools] rumdl`` from ``mise.toml``.""" + try: + data = tomllib.loads(text) + except tomllib.TOMLDecodeError as exc: + raise SystemExit(f"error: {MISE_TOML} is not valid TOML: {exc}") from exc + tools = data.get("tools") + version = tools.get(RUMDL) if isinstance(tools, dict) else None + if not isinstance(version, str) or not version: + raise SystemExit( + f'error: expected `{RUMDL} = "X.Y.Z"` under [tools] in {MISE_TOML}.' + ) + return version + + +def _one_ci_value(text: str, name: str) -> str: + values = [ + _scalar(match.group(1)) + for line in text.splitlines() + if (match := _CI_KEY_RE[name].match(line)) + ] + if len(values) != 1: + raise SystemExit( + f"error: expected exactly one `{name}:` key in {CI_WORKFLOW},\n" + f" found {len(values)}." + ) + return values[0] + + +def read_ci_pin(text: str) -> tuple[str, str]: + """Return ``(RUMDL_VERSION, RUMDL_SHA256)`` from the CI workflow. + + Exactly one of each is required. The lint job's sibling install + steps carry their own ``*_VERSION`` / ``*_SHA256`` env maps, and a + duplicated rumdl step would leave it ambiguous which one CI runs. + """ + version = _one_ci_value(text, VERSION_VAR) + digest = _one_ci_value(text, SHA256_VAR) + return version, _sha256(digest, f"{CI_WORKFLOW} {SHA256_VAR}") + + +def read_docker_version(text: str) -> str: + """Return the Dockerfile's ``ENV RUMDL_VERSION``.""" + values = [ + match.group(1).strip("\"'") + for line in text.splitlines() + if (match := _DOCKER_ENV_RE.match(line)) + ] + if len(values) != 1: + raise SystemExit( + f"error: expected exactly one `ENV {VERSION_VAR}=` line in " + f"{DOCKERFILE},\n found {len(values)}." + ) + return values[0] + + +def _heredoc_blocks(text: str) -> list[list[str]]: + """Split a Dockerfile into the bodies of its heredoc ``RUN`` blocks.""" + blocks: list[list[str]] = [] + body: list[str] | None = None + delimiter = "" + for line in text.splitlines(): + if body is None: + match = _DOCKER_HEREDOC_RE.match(line) + if match: + body, delimiter = [], match.group(2) + elif line.strip() == delimiter: + blocks.append(body) + body = None + else: + body.append(line) + return blocks + + +def read_docker_x86_64_sha(text: str) -> str: + """Return the SHA256 the Dockerfile's rumdl block pins for x86_64. + + The block is the heredoc ``RUN`` whose download URL interpolates + ``RUMDL_VERSION``; within it, the arm is the one naming the x86_64 + triple. Anything other than exactly one of each is a hard error. + """ + rumdl_blocks = [ + block + for block in _heredoc_blocks(text) + if any(_DOCKER_VERSION_REF_RE.search(line) for line in block) + ] + if len(rumdl_blocks) != 1: + raise SystemExit( + f"error: expected exactly one heredoc RUN block in {DOCKERFILE}\n" + f" whose download URL names ${{{VERSION_VAR}}}, " + f"found {len(rumdl_blocks)}." + ) + digests = [ + match.group(2) + for line in rumdl_blocks[0] + if X86_64_TRIPLE in line and (match := _DOCKER_SHA_RE.search(line)) + ] + if len(digests) != 1: + raise SystemExit( + f"error: expected exactly one `SHA256=` on a line naming " + f"{X86_64_TRIPLE}\n in {DOCKERFILE}'s rumdl block, " + f"found {len(digests)}." + ) + return _sha256(digests[0], f"{DOCKERFILE} {X86_64_TRIPLE} SHA256") + + +def drifted(versions: dict[pathlib.Path, str]) -> list[pathlib.Path]: + """Return the files whose version disagrees with the rest. + + That is the files outside the majority, or every file when no value + is shared by more than one — with three sites the odd one out is + named by itself, which is what #1653's forgotten Dockerfile needed. + """ + (common, count), *_ = collections.Counter(versions.values()).most_common(1) + if count == len(versions): + return [] + return [ + path for path, version in versions.items() if count == 1 or version != common + ] + + +def read_repo_file(relative: pathlib.Path) -> str: + """Read a repository file, hard-erroring when it is absent.""" + path = REPO_ROOT / relative + try: + return path.read_text(encoding="utf-8") + except OSError as exc: + raise SystemExit(f"error: cannot read {relative}: {exc}") from exc + + +def read_baseline(text: str) -> tuple[str, str]: + """Return the ``(version, sha256)`` record of the baseline file.""" + records = [ + line.split() + for line in text.splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + if len(records) != 1 or len(records[0]) != 2: + raise SystemExit( + f"error: expected exactly one ` ` record in " + f"{BASELINE}, found {len(records)}." + ) + version, digest = records[0] + return version, _sha256(digest, f"{BASELINE} record") + + +def format_baseline(version: str, digest: str) -> str: + """The baseline file recording ``version`` and ``digest``.""" + return f"{BASELINE_HEADER}{version} {digest}\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Check that mise.toml, ci.yml and the Dockerfile pin one rumdl release." + ) + parser.add_argument( + "--update", + action="store_true", + help=f"record the live version and x86_64 hash in {BASELINE} after a bump that moved both", + ) + args = parser.parse_args(argv) + docker_text = read_repo_file(DOCKERFILE) + ci_version, ci_sha = read_ci_pin(read_repo_file(CI_WORKFLOW)) + docker_sha = read_docker_x86_64_sha(docker_text) + versions = { + MISE_TOML: read_mise_version(read_repo_file(MISE_TOML)), + CI_WORKFLOW: ci_version, + DOCKERFILE: read_docker_version(docker_text), + } + + problems: list[str] = [] + + odd = drifted(versions) + if odd: + table = "\n".join(f" {path}: {RUMDL} {v}" for path, v in versions.items()) + problems.append( + f" {RUMDL} versions disagree; drifted: " + + ", ".join(str(path) for path in odd) + + f"\n{table}" + ) + + if docker_sha != ci_sha: + problems.append( + f" {DOCKERFILE}: {X86_64_TRIPLE} SHA256 {docker_sha}\n" + f" differs from {CI_WORKFLOW} {SHA256_VAR} {ci_sha}.\n" + f" Both verify the same tarball; one hash was not updated." + ) + + if problems: + print( + "error: rumdl pin declarations disagree:\n" + + "\n".join(problems) + + f"\n\nBump {MISE_TOML}, the {CI_WORKFLOW} lint job's " + f"{VERSION_VAR} / {SHA256_VAR},\nand the {DOCKERFILE}'s " + f"{VERSION_VAR} plus per-arch SHA256 table together,\n" + "in the same commit. See #1779.", + file=sys.stderr, + ) + return 1 + + base_version, base_sha = read_baseline(read_repo_file(BASELINE)) + bumped = ci_version != base_version + rehashed = ci_sha != base_sha + if bumped and not rehashed: + print( + f"error: {RUMDL} version bumped to {ci_version} but the " + f"{X86_64_TRIPLE} hash is still {base_version}'s;\n" + f" download the new release and paste its sha256 into " + f"{CI_WORKFLOW} and the {DOCKERFILE}.\n" + f" ({BASELINE} records {RUMDL} {base_version} SHA256 " + f"{base_sha[:12]}...; see #1779.)", + file=sys.stderr, + ) + return 1 + if rehashed and not bumped: + print( + f"error: the {X86_64_TRIPLE} hash changed but {RUMDL} is still " + f"{ci_version};\n" + f" a release's tarball does not change, so either the version or " + f"the hash in {CI_WORKFLOW} and the {DOCKERFILE} is wrong.\n" + f" ({BASELINE} records SHA256 {base_sha[:12]}... for " + f"{base_version}; see #1779.)", + file=sys.stderr, + ) + return 1 + + if args.update: + if not bumped: + print( + f"error: {BASELINE} already records {RUMDL} {ci_version}; " + "nothing to update.", + file=sys.stderr, + ) + return 1 + (REPO_ROOT / BASELINE).write_text(format_baseline(ci_version, ci_sha), encoding="utf-8") + print( + f"rumdl lockstep: recorded {RUMDL} {ci_version}, {X86_64_TRIPLE} " + f"SHA256 {ci_sha[:12]}... in {BASELINE}." + ) + return 0 + + if bumped: + print( + f"error: {RUMDL} bumped to {ci_version} (SHA256 {ci_sha[:12]}...) " + f"but {BASELINE} still records {base_version};\n" + " run `python3 utils/check-rumdl-lockstep.py --update` and commit " + "the result with this bump. See #1779.", + file=sys.stderr, + ) + return 1 + + print( + f"rumdl lockstep OK ({RUMDL} {ci_version}, {X86_64_TRIPLE} " + f"SHA256 {ci_sha[:12]}..., 3 sites checked)." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/utils/perturb-test.py b/utils/perturb-test.py index 55824a0e1..8172320cf 100644 --- a/utils/perturb-test.py +++ b/utils/perturb-test.py @@ -270,6 +270,169 @@ def test_record_defaults_into_the_backup_directory_not_the_repo(self) -> None: self.assertFalse((self.repo.root / "perturb-results.json").exists(), "no litter in the repo") self._drop(backup) + def _cargo_stub(self) -> pathlib.Path: + """An executable named `cargo`, outside the repository, that + records it ran and reports one failing test; the argv check keys + on the program's basename, so only this shape reaches it.""" + bin_dir = pathlib.Path(self._tmp.name) / "bin" + bin_dir.mkdir() + stub = bin_dir / "cargo" + ran = pathlib.Path(self._tmp.name) / "cargo-ran" + stub.write_text( + f"#!{sys.executable}\nimport pathlib, sys\npathlib.Path({str(ran)!r}).write_text(' '.join(sys.argv[1:]))\n" + f"sys.stdout.write('test a::b ... FAILED\\n{RED}')\nsys.exit(101)\n" + ) + stub.chmod(0o755) + return stub + + def test_refuses_a_multi_target_argv_before_any_git_read_or_write(self) -> None: + stub = self._cargo_stub() + # Dirty too: the argv refusal must come first, so the target + # check's message is absent and no `git` ran to produce it. + dirty = ORIGINAL + b"// uncommitted\n" + self.repo.target.write_bytes(dirty) + manifest = {**self.repo.manifest("", 0, [_case()]), "cargo": [str(stub), "test", "-p", "a", "-p", "b", "--lib"]} + results, rc, _, err = self._sweep(manifest) + self.assertEqual((results, rc), ([], 2)) + self.assertIn("without --no-fail-fast: 2 package selectors", err) + self.assertNotIn("not tracked and clean", err) + self.assertEqual(self.repo.target.read_bytes(), dirty) + self.assertEqual(list(self.backup.iterdir()), [], "nothing was written") + self.assertFalse((pathlib.Path(self._tmp.name) / "cargo-ran").exists(), "cargo never ran") + + def test_an_accepted_cargo_argv_runs(self) -> None: + # The positive control for the refusal above: the same stub with + # the flag, and a single-target argv without it, reach the run. + stub = self._cargo_stub() + ran = pathlib.Path(self._tmp.name) / "cargo-ran" + for tail in (["-p", "a", "-p", "b", "--lib", "--no-fail-fast"], ["-p", "a", "--lib"]): + ran.unlink(missing_ok=True) + manifest = {**self.repo.manifest("", 0, [_case(expect=["a::b"])]), "cargo": [str(stub), "test", *tail]} + results, rc, _, err = self._sweep(manifest) + self.assertEqual((rc, results[0]["verdict"]), (0, "caught"), err) + self.assertEqual(ran.read_text(), " ".join(["test", *tail])) + self.assertEqual(self.repo.target.read_bytes(), ORIGINAL) + + def test_argv_rule_refuses_every_multi_target_shape(self) -> None: + refused = { + "two packages": ["-p", "a", "-p", "b", "--lib"], + "no target selector": ["-p", "a"], + "no package selector": ["--lib"], + "nothing selected": [], + "two targets": ["-p", "a", "--lib", "--test", "t"], + # Each beside an otherwise single-target argv, so the flag + # itself is the only reason it is refused. Spelled out rather + # than read from `perturb.MULTI_TARGET_FLAGS`: a table built + # from the set under test loses a case with every member the + # set loses. + **{ + flag: ["-p", "a", "--lib", flag] + for flag in ("--workspace", "--all", "--bins", "--tests", "--examples", "--benches", "--all-targets") + }, + "--package= spelling": ["--package=a", "--package=b", "--doc"], + "-pNAME spelling": ["-pa", "-pb", "--doc"], + # cargo reads `-qpa` as `-q -p a`, so these are two packages. + "a -p inside a cluster": ["-qpa", "-p", "b", "--lib"], + "two clusters": ["-qpa", "-vpb", "--lib"], + # Each triple is a test executable of its own. + "two triples": ["-p", "a", "--lib", "--target", "x86_64-unknown-linux-gnu", "--target", "i686-unknown-linux-gnu"], + "--target= spelling": ["-p", "a", "--lib", "--target=x86_64-unknown-linux-gnu", "--target=i686-unknown-linux-gnu"], + } + for label, tail in refused.items(): + self.assertIsNotNone(perturb.cargo_argv_problem(["cargo", "test", *tail]), label) + # The flag anywhere before `--` accepts each one. + for at in (0, len(tail)): + flagged = [*tail[:at], "--no-fail-fast", *tail[at:]] + self.assertIsNone(perturb.cargo_argv_problem(["cargo", "test", *flagged]), (label, at)) + + def test_argv_rule_accepts_one_package_with_one_target(self) -> None: + for tail in ( + ["-p", "a", "--lib"], + ["-p", "a", "--doc"], + ["--package", "a", "--test", "t"], + ["--package=a", "--test=t"], + ["-pa", "--bin", "b"], + ["-p=a", "--example", "e"], + ["-p", "a", "--bench", "b"], + # Feature, profile and filter arguments do not add targets. + ["-p", "a", "--lib", "--all-features", "-F", "x,y", "--release", "some_filter"], + # Short-option clusters, read as cargo reads them. + ["-qp", "a", "--lib"], + ["-qpa", "--lib"], + ["-vqp=a", "--lib"], + ["-p", "a", "--lib", "-j8", "-Fx"], + # A value holding a `p` is a value, not a second `-p`: the + # letters after `-F`, `-j` or `-m` are its argument. + ["-p", "a", "--lib", "-Fpx"], + ["-p", "a", "--lib", "-jp"], + ["-p", "a", "--lib", "-mp"], + # One triple is one build. + ["-p", "a", "--lib", "--target", "x86_64-unknown-linux-gnu"], + ): + self.assertIsNone(perturb.cargo_argv_problem(["cargo", "test", *tail]), tail) + + def test_argv_rule_ignores_everything_after_a_bare_double_dash(self) -> None: + # Those go to the test binary: neither a selector nor the flag. + self.assertIsNone(perturb.cargo_argv_problem(["cargo", "test", "-p", "a", "--lib", "--", "--test", "x", "--workspace"])) + self.assertIn( + "2 package selectors", perturb.cargo_argv_problem(["cargo", "test", "-p", "a", "-p", "b", "--lib", "--", "--no-fail-fast"]) or "" + ) + + def test_argv_rule_refuses_a_glob_selector(self) -> None: + # Cargo expands these, so one selector can still run many targets. + for tail in ( + ["-p", "bca-*", "--lib"], + ["-p", "a?", "--lib"], + ["-p", "a", "--test", "*"], + ["-p", "a", "--test=t[12]"], + ["-p", "a", "--bin", "b*"], + ): + problem = perturb.cargo_argv_problem(["cargo", "test", *tail]) + self.assertIn("is a glob", problem or "", tail) + self.assertIsNone(perturb.cargo_argv_problem(["cargo", "test", *tail, "--no-fail-fast"]), tail) + + def test_argv_rule_recognises_every_spelling_of_cargo_test(self) -> None: + multi = ["-p", "a", "-p", "b", "--lib"] + for head in ( + ["cargo", "test"], + ["cargo.exe", "test"], + ["/home/u/.cargo/bin/cargo", "test"], + ["cargo", "+nightly", "test"], + ["cargo", "t"], + ["cargo", "--locked", "test"], + ["cargo", "--color", "never", "-Z", "unstable-options", "test"], + ["cargo", "+nightly", "-q", "--config=build.jobs=2", "test"], + # A global option's value as a separate token, which the + # skip must consume with it. + ["cargo", "--config", "build.jobs=2", "test"], + ["cargo", "-C", "sub", "test"], + # A wrapper in front of cargo is seen through, including one + # whose own argument also ends in `cargo`. + ["timeout", "1800", "cargo", "test"], + ["env", "RUST_BACKTRACE=1", "cargo", "test"], + ["nice", "cargo", "test"], + ["rustup", "run", "nightly", "cargo", "test"], + ["env", "CARGO=/usr/bin/cargo", "cargo", "test"], + ): + self.assertIsNotNone(perturb.cargo_argv_problem([*head, *multi]), head) + + def test_argv_rule_leaves_other_programs_and_subcommands_alone(self) -> None: + # The self-tests' own stub argv, and cargo subcommands that are + # not `cargo test`, carry no target rule. + self.assertIsNone(perturb.cargo_argv_problem(self.repo.manifest(RED, 101, [_case()])["cargo"])) + for argv in ( + [], + ["cargo"], + ["cargo", "build", "-p", "a", "-p", "b"], + ["cargo", "nextest", "run", "--workspace"], + ["cargo-test", "-p", "a", "-p", "b"], + # A shell string is not parsed, and a wrapped cargo that does + # not run `test` is not `cargo test`. + ["sh", "-c", "cargo test --workspace"], + ["timeout", "60", "cargo", "build", "-p", "a", "-p", "b"], + ): + self.assertIsNone(perturb.cargo_argv_problem(argv), argv) + def test_help_and_a_missing_manifest_are_usage_not_a_traceback(self) -> None: for argv in (["--help"], []): err, out = io.StringIO(), io.StringIO() diff --git a/utils/perturb.py b/utils/perturb.py index 43c5db4d4..ca0049cb0 100755 --- a/utils/perturb.py +++ b/utils/perturb.py @@ -26,14 +26,26 @@ The failure parser reads `cargo test` output only: the `test ... FAILED` lines and the `test result:` summary of each target. A `cargo nextest` argv prints neither, so every red run under it is -`HARNESS-ERROR` — loud, but not a result. A `cargo` argv naming more -than one target must carry `--no-fail-fast`, or the targets after the -first failing one never run and their guards read as dead -(`.claude/rules/testing.md`, "So is the invocation"). +`HARNESS-ERROR` — loud, but not a result. -The refusals and classifications are the acceptance criteria of #1744, -each one a failure the rules file has already paid for: +The refusals and classifications are the acceptance criteria of #1744 +and #1787, each one a failure the rules file has already paid for: +- A `cargo test` argv that can run more than one test target must carry + `--no-fail-fast`, or the targets after the first failing one never + run and their guards read as dead (`.claude/rules/testing.md`, "So is + the invocation"). Anything but exactly one `-p` with exactly one of + `--lib`, `--doc`, `--test`, `--bin`, `--example` or `--bench` is + refused without it, before anything is read or run; so is a glob + (`*`, `?`, `[`) in either value, since cargo expands it, and a second + `--target`, since each triple builds test executables of its own. A + `-p` inside a short-option cluster (`-qpa`) counts like any other. The + flag is never added on the manifest's behalf: the manifest says what + it runs. A wrapper in front of cargo (`timeout 1800 cargo test`, `env`, + `nice`, `rustup run nightly cargo test`) is seen through; an argv with + no `cargo` program in it (the self-test's stub, a `sh -c` string) or + whose subcommand is not `test` (a cargo `[alias]` included) is not + checked. - Every mutation target must be tracked and clean in git: `git ls-files --error-unmatch` refuses an untracked, ignored or nonexistent path, and `git status --porcelain` a modified one, both before the first @@ -80,6 +92,115 @@ ) TEST_RESULT_RE = re.compile(r"^test result: \w+\. (\d+) passed; (\d+) failed;", re.M) +CARGO_PROGRAMS = frozenset({"cargo", "cargo.exe"}) +# `t` is cargo's built-in alias for `test`. +TEST_SUBCOMMANDS = frozenset({"test", "t"}) +# Options cargo accepts before the subcommand whose value is the next +# token; skipping them is what finds `test` in `cargo --color never test`. +CARGO_GLOBAL_VALUE_OPTS = frozenset({"--color", "--config", "-C", "-Z"}) +PACKAGE_OPTS = frozenset({"-p", "--package"}) +SINGLE_TARGET_FLAGS = frozenset({"--lib", "--doc"}) +SINGLE_TARGET_OPTS = frozenset({"--test", "--bin", "--example", "--bench"}) +MULTI_TARGET_FLAGS = frozenset({"--workspace", "--all", "--bins", "--tests", "--examples", "--benches", "--all-targets"}) +# `cargo test`'s short options that take a value (`-C` and `-Z` are +# cargo's global ones). In a cluster such as `-qpa`, clap reads every +# letter as a flag up to the first of these, whose value is the rest of +# the token, or the next token when nothing is left. +SHORT_VALUE_OPTS = frozenset("pFjmCZ") +GLOB_CHARS = frozenset("*?[") + + +def split_short_cluster(tok: str) -> list[str]: + """A short-option token as clap reads it: `-qpa` is `-q -p a`, `-vv` + is `-v -v`, and `-p=a` is `-p a`.""" + split: list[str] = [] + for at, letter in enumerate(tok[1:], start=1): + split.append(f"-{letter}") + if letter in SHORT_VALUE_OPTS: + value = tok[at + 1 :].removeprefix("=") + if value: + split.append(value) + break + return split + + +def cargo_test_args(cargo: list[str]) -> list[str] | None: + """The `cargo test` options of `cargo`, before any bare `--` and with + `--opt=value` and short-option clusters split into separate tokens; + None when the argv runs no `cargo test`. + + Every token named `cargo` is tried in turn and the first that starts a + `cargo test` invocation is read, which sees through a wrapper program + in front of it. Trying each matters: an assignment such as + `env CARGO=/usr/bin/cargo cargo test` also names `cargo`.""" + for start, program in enumerate(cargo): + if pathlib.PurePath(program).name not in CARGO_PROGRAMS: + continue + rest = cargo[start + 1 :] + if rest and rest[0].startswith("+"): + rest = rest[1:] + while rest and rest[0].startswith("-"): + rest = rest[2:] if rest[0] in CARGO_GLOBAL_VALUE_OPTS else rest[1:] + if rest and rest[0] in TEST_SUBCOMMANDS: + return split_test_options(rest[1:]) + return None + + +def split_test_options(tokens: list[str]) -> list[str]: + """`tokens` up to a bare `--`, with `--opt=value` and short-option + clusters split the way cargo reads them.""" + args: list[str] = [] + for tok in tokens: + if tok == "--": + break + if tok.startswith("--"): + args.extend(tok.split("=", 1)) + elif tok.startswith("-") and len(tok) > 2: + args.extend(split_short_cluster(tok)) + else: + args.append(tok) + return args + + +def cargo_argv_problem(cargo: list[str]) -> str | None: + """Why `cargo` can run more than one test target with no + `--no-fail-fast`, or None when it cannot (or is not `cargo test`). + + One target is exactly one package selector with exactly one of + `--lib`, `--doc`, `--test N`, `--bin N`, `--example N`, `--bench N`, + neither value a glob, built for at most one `--target` triple. + Feature and profile options do not change the target count, so they + are not read; nor is a `build.target` list in cargo's config.""" + args = cargo_test_args(cargo) + if args is None or "--no-fail-fast" in args: + return None + reasons: list[str] = [] + packages = targets = triples = 0 + it = iter(args) + for tok in it: + if tok in MULTI_TARGET_FLAGS: + reasons.append(f"`{tok}` selects several targets") + elif tok in SINGLE_TARGET_FLAGS: + targets += 1 + elif tok == "--target": + next(it, "") + triples += 1 + elif tok in PACKAGE_OPTS or tok in SINGLE_TARGET_OPTS: + value = next(it, "") + if GLOB_CHARS.intersection(value): + reasons.append(f"`{tok} {value}` is a glob, which cargo expands to every match") + if tok in PACKAGE_OPTS: + packages += 1 + else: + targets += 1 + if packages != 1: + reasons.append(f"{packages} package selectors, need exactly one `-p`") + if targets != 1: + reasons.append(f"{targets} target selectors, need exactly one of --lib/--doc/--test/--bin/--example/--bench") + if triples > 1: + reasons.append(f"{triples} `--target` triples, each building its own test executables") + return "; ".join(reasons) or None + def target_problems(cwd: pathlib.Path, files: list[str]) -> list[str]: """Why the targets cannot be mutated: one line per untracked, @@ -162,6 +283,15 @@ def sweep(manifest: dict, backup_dir: pathlib.Path) -> tuple[list[dict], int]: any case is not a clean `caught`, or a harness problem occurred).""" cwd = pathlib.Path(manifest["cwd"]).resolve() cargo = list(manifest["cargo"]) + argv_problem = cargo_argv_problem(cargo) + if argv_problem is not None: + print(f"refusing: the cargo argv can run several test targets without --no-fail-fast: {argv_problem}", file=sys.stderr) + print( + "a failing target stops the run and every later target's guards read as dead; " + "add --no-fail-fast to the manifest's argv — perturb.py never adds it on your behalf", + file=sys.stderr, + ) + return [], 2 cases = manifest["cases"] files = sorted({case["file"] for case in cases}) problems = target_problems(cwd, files)