mirror of
https://github.com/newliver666/apk-reverse.git
synced 2026-09-28 21:56:59 +02:00
inliver233
dd8057cfcb
CI went red on `1dc1d9b` in both jobs while the identical command passed locally, and the reason is
worth keeping: the checks depended on state that only exists on a developer's machine.
check_commands.py -- the gate's verdict depended on a gitignored directory
The previous commit's fix recognised a workbench script by matching its name against a walk of
`tools/`. That tree is gitignored, so a clean checkout has none of it: `dcc.py` matched locally and
skipped, and on CI nothing matched, it fell through to `missing-script`, and the gate reported
drift for three quoted lines. A check whose verdict changes with a directory CI does not have is
not a check. Two changes remove the dependency:
* `qualified_basenames()` collects every `.py` the *document itself* writes with a path, so a
bare name the document has already located elsewhere is a skip, computed from the text.
* `./dcc.py` is a declared upstream entry point, checked before the workbench-name test.
* a bare name after a `cd` that resolves nowhere is `after-cd-unresolved` -- unverifiable, which
is not the same claim as broken, and is now reported as such instead of as drift.
Verified by running the whole CI sequence against a tree with `tools/` hidden: five gates exit 0,
the suite passes. That experiment is the one that should have been run before the first push.
check_budget.py -- the measurement was installation-dependent
The gate measures the always-loaded body in tokens, preferring `tiktoken` and falling back to
`characters / N`. `N` had been guessed at 3.6 rather than calibrated, so a machine with tiktoken
saw ~11.1k tokens and passed while a machine without it saw ~14.2k and warned -- and CI's 3.11 job
is the one without it. `N` is now the measured 4.285 characters per token (47,629 chars -> 11,114
tokens on SKILL.md), both paths agree within one token, and `tests/unit/test_budget_measurement.py`
forces both paths and asserts they do not straddle the budget.
CI now installs `tiktoken` so the comparison exercises the real tokenizer instead of skipping.
Also: two `.py` files are formatted for consistency, and the reconstruction record documents the
CI-only failure, since "it passed on my machine" is the lesson rather than an aside.
Gates: check_repo 0 problem, check_refs 0 dangling / 0 warning, check_routing coherent,
check_budget 0 failure / 0 warning (both measurement paths), check_commands 154 checked / 0 drift,
pytest 617 passed on 3.14 and 609 on 3.11.
91 lines
3.8 KiB
Python
91 lines
3.8 KiB
Python
"""The budget gate must reach the same verdict whether or not `tiktoken` is installed.
|
|
|
|
This is a small test for a defect that cost two red CI runs. `check_budget.py` measures the
|
|
always-loaded body in tokens, preferring `tiktoken` and falling back to `characters / N` when it is
|
|
absent. The first version's `N` was guessed (3.6) instead of calibrated, so a machine with tiktoken
|
|
saw ~10,900 tokens and passed while a machine without it saw ~13,200 and warned -- and this
|
|
repository's gate runner (CI's 3.11 job, and any contributor's box) is the one without it.
|
|
|
|
A gate whose answer depends on which packages the runner happens to have installed is not a gate, so
|
|
both paths are forced here and compared.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from conftest import REPO_ROOT
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
@pytest.fixture
|
|
def budget():
|
|
sys.path.insert(0, str(REPO_ROOT))
|
|
return importlib.import_module("check_budget")
|
|
|
|
|
|
def _has_tiktoken():
|
|
"""Whether a real tokenizer is available, reported rather than assumed.
|
|
|
|
The comparison needs *two* implementations; where there is only one, the honest outcome is a
|
|
skip with the install command, not a failure and not a silent pass. CI installs it, so the
|
|
property is verified where it matters; a contributor without it has both gates agreeing anyway,
|
|
because the fallback is calibrated to the same constant.
|
|
"""
|
|
try:
|
|
import tiktoken # noqa: F401
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
def test_the_fallback_estimate_agrees_with_the_real_tokenizer(budget):
|
|
"""Both measurement paths must land on the same side of the soft budget."""
|
|
if not _has_tiktoken():
|
|
pytest.skip("tiktoken is not installed here, so there is no measurement to compare against; "
|
|
"install it with `python -m pip install tiktoken`")
|
|
text = (REPO_ROOT / "skills" / "apk-reverse" / "SKILL.md").read_text(encoding="utf-8")
|
|
|
|
real, how_real = budget.token_estimate(text)
|
|
assert "tiktoken" in how_real, how_real
|
|
|
|
# Force the fallback path by handing the module a tokenizer that cannot be imported.
|
|
saved = sys.modules.get("tiktoken")
|
|
sys.modules["tiktoken"] = None # `import tiktoken` raises here
|
|
try:
|
|
fallback, how_fallback = budget.token_estimate(text)
|
|
finally:
|
|
if saved is not None:
|
|
sys.modules["tiktoken"] = saved
|
|
else:
|
|
sys.modules.pop("tiktoken", None)
|
|
|
|
assert "estimate" in how_fallback, how_fallback
|
|
drift = abs(real - fallback) / max(1, real)
|
|
assert drift < 0.05, (
|
|
"the fallback is %.1f%% away from tiktoken (%d vs %d): recalibrate TOKENS_PER_CHAR "
|
|
"against a fresh measurement rather than a guess" % (100 * drift, fallback, real))
|
|
assert (real <= budget.TOKEN_SOFT) == (fallback <= budget.TOKEN_SOFT), (
|
|
"the two paths straddle the soft budget (%d real / %d fallback vs %d): the gate would "
|
|
"answer differently on CI than on a developer machine"
|
|
% (real, fallback, budget.TOKEN_SOFT))
|
|
|
|
|
|
def test_the_fallback_constant_is_the_calibrated_one(budget):
|
|
"""Pin the constant to a measurement: `chars / tokens` on SKILL.md, recomputed here."""
|
|
text = (REPO_ROOT / "skills" / "apk-reverse" / "SKILL.md").read_text(encoding="utf-8")
|
|
real, how = budget.token_estimate(text)
|
|
if "tiktoken" not in how:
|
|
pytest.skip("no tiktoken available, so there is no measurement to calibrate against")
|
|
expected = len(text) / real
|
|
assert abs(budget.TOKENS_PER_CHAR - expected) < 0.25, (
|
|
"TOKENS_PER_CHAR is %.3f but this SKILL.md measures %.3f characters per token -- the "
|
|
"document changed, so recalibrate" % (budget.TOKENS_PER_CHAR, expected))
|