011 — CPython shelve 3.14.5：一個旗標改變了每一次讀取的意義

{
  "name": "011-cpython-shelve",
  "upstream": "CPython Lib/shelve.py",
  "examined_version": "3.14.5",
  "license": "PSF-2.0",
  "what_it_is": "A dict-like object backed by a dbm file, whose single boolean flag decides whether a mutation through what it handed you means anything.",
  "why_this_one": "Example 011 the same day is the first in this lab with state that outlives the process, because the roadmap turns to real market applications at 20/20 and nothing here has been tested against persistence. shelve is the standard library's smallest persistent mapping, it is thirty years old, and its one flag changes the meaning of every read.",
  "declared_and_then_remeasured": {
    "_note": "Re-measured by main.py against the running interpreter. A disagreement is a failure. A file of measurements nobody re-reads is a declaration compared to nothing.",
    "shelve_py_lines": 250,
    "getitem_lines": 9,
    "setitem_lines": 7,
    "writeback_default": false,
    "public_classes": [
      "BsdDbShelf",
      "DbfilenameShelf",
      "Shelf"
    ],
    "_correction": "setitem_lines was declared 5 and measured 7 on the first run. Recorded rather than quietly overwritten: this row is the only evidence in the entry that the re-measurement can come out false."
  },
  "the_flag": {
    "name": "writeback",
    "what_it_is_called": "writeback",
    "what_it_actually_changes": [
      "whether a read RETAINS the object it just unpickled",
      "and therefore whether d[k] is d[k]",
      "and therefore whether a mutation through what you were handed reaches the medium at sync",
      "and therefore how much memory a session of pure reads costs"
    ],
    "the_gap": "The name describes the fourth consequence. The first is what causes all of them, and nothing in the API surfaces it."
  },
  "the_finding": "Two modes of one interface with opposite semantics for the same call. Under the default, d[k].append(x) is silently lost. Under writeback it survives. The observation that separates them is object identity - d[k] is d[k] - which no caller checks, and which costs one line.",
  "the_second_finding": "writeback turns a READ into a retention. 200 pure reads with no writes at all leave 200 objects held. The cache grows with what you have looked at, not with what you have changed, and nothing in the flag's name says so.",
  "sets": {
    "FMS": "this file: the flag, what it really changes, and the numbers main.py re-measures",
    "SCL": "which handout this deployment uses, and whether a declaration must be proved against behaviour",
    "SMS": "the re-cut Shelf, resolution by name, the identity probe, and the upstream probes",
    "TMS": "one file per handout strategy - each declares what it hands back and whether a mutation survives, and reaches no sibling set",
    "DMS": "the two modes side by side, the comparison against real shelve, and what none of it can see"
  },
  "non_goals": [
    "Saying writeback is a bug. It is off by default, documented, and correct for callers who want it.",
    "Reimplementing dbm. The re-cut takes the same backing mapping shelve.Shelf takes.",
    "Claiming the re-cut is better than upstream's arrangement. It is the same semantics with the strategy lifted out, which is worth exactly what the comparison against the real module says."
  ],
  "the_third_finding": "A read-only session under writeback rewrites the medium at close. Measured: open with writeback, read all 200 keys, write nothing, close - the store file changes. Under the default it does not. This row exists because the first version of that check was a hardcoded check(..., True, ...) asserting the read loop wrote nothing, which is the defect this lab filed against archaeology 008 on 2026-08-08. Measuring it disagreed with the assertion."
}

{
  "handout": "copy-on-read",
  "declaration_must_match_behaviour": true,
  "recut_must_match_upstream": true,
  "_note": "main.py runs both handouts regardless of what this says. A policy able to silence the comparison would make the comparison worthless."
}

"""Which handout this deployment uses, and what it refuses to let pass."""
import json
import pathlib

_C = json.loads((pathlib.Path(__file__).parent / "policy.json").read_text(encoding="utf-8"))

handout = lambda: _C["handout"]                                                  # noqa: E731
declaration_must_match_behaviour = lambda: bool(_C["declaration_must_match_behaviour"])  # noqa: E731
recut_must_match_upstream = lambda: bool(_C["recut_must_match_upstream"])        # noqa: E731

"""The re-cut: a shelf whose handout strategy is a named unit rather than a flag.

Upstream the choice is a boolean threaded through `__getitem__`, `__setitem__`
and `sync`. Lifting it out is the only structural change; the semantics are
copied, and DMS compares the re-cut against real `shelve` on every run so that
"copied" is a measurement rather than a claim.
"""
import importlib
import pickle

HANDOUTS = ["copy_on_read", "cached_reference"]


def load_handouts():
    loaded, problems = {}, []
    for module_name in HANDOUTS:
        try:
            module = importlib.import_module(f"TMS.handouts.{module_name}")
        except ModuleNotFoundError:
            problems.append(f'handout "{module_name}" has no module - fail closed')
            continue
        for attribute in ("NAME", "HANDS_BACK", "MUTATION_SURVIVES", "make"):
            if not hasattr(module, attribute):
                problems.append(f"{module_name} does not declare {attribute}")
        loaded[module.NAME] = module
    return loaded, problems


def resolve(name, loaded):
    module = loaded.get(name)
    if module is None:
        known = ", ".join(sorted(loaded))
        return None, f'handout "{name}" has no implementation - fail closed (known: {known})'
    return module, None


class Shelf:
    """Same backing mapping shelve.Shelf takes: keys to pickled bytes."""

    def __init__(self, backing, handout):
        self.backing = backing
        self.handout_name = handout.NAME
        self.policy = handout.make(pickle.loads)

    def __setitem__(self, key, value):
        self.backing[key.encode()] = pickle.dumps(value)
        self.policy["remember"](key, value)

    def __getitem__(self, key):
        return self.policy["answer"](key, self.backing[key.encode()])

    def sync(self):
        # Upstream writes the cache back here. The re-cut does the same, and
        # this is the line that makes a mutation through a handed-out object
        # reach the medium at all.
        for key, value in self.policy["items"]():
            self.backing[key.encode()] = pickle.dumps(value)

    def held(self):
        return self.policy["held"]()

    def keys(self):
        return sorted(k.decode() for k in self.backing)


def hands_back_copies(shelf, key):
    """The observation that separates the strategies, and the only cheap one."""
    return shelf[key] is not shelf[key]


def mutation_survives(shelf, key, mutate):
    """The other one. It only fires if the caller thought to mutate."""
    mutate(shelf[key])
    return shelf[key]

"""What real shelve does, measured — not what its documentation says it does.

The re-cut in shelf.py is compared against these results cell by cell, so
"the re-cut copies upstream's semantics" is a measurement that can come out
false.
"""
import hashlib
import inspect
import os
import shelve
import tempfile


def _store():
    return os.path.join(tempfile.mkdtemp(), "store")


def probe(writeback):
    """Write, mutate through what a read handed back, reopen, see what survived."""
    path = _store()
    with shelve.open(path, writeback=writeback) as shelf:
        shelf["cart"] = ["apple"]

    with shelve.open(path, writeback=writeback) as shelf:
        shelf["cart"].append("pear")
        identity_separates = shelf["cart"] is not shelf["cart"]
        held = len(getattr(shelf, "cache", {}))

    with shelve.open(path) as shelf:
        survived = list(shelf["cart"])

    return {
        "survived": survived,
        "mutation_survives": survived == ["apple", "pear"],
        "identity_separates": identity_separates,
        "held": held,
    }


def cache_after_pure_reads(count):
    """Reads only. Nothing is written, and the cache grows anyway."""
    path = _store()
    with shelve.open(path, writeback=True) as shelf:
        for index in range(count):
            shelf[f"k{index}"] = [index]
    with shelve.open(path, writeback=True) as shelf:
        marks = {}
        for index in range(count):
            shelf[f"k{index}"]
            if index + 1 in (1, count // 4, count):
                marks[index + 1] = len(shelf.cache)
    return marks


def reads_cause_writes(count, writeback):
    """Open, read every key, close. Did the medium change?

    This replaces a hardcoded `check(..., True, ...)` that asserted the read
    loop wrote nothing. Asserting it was the same defect this lab filed against
    itself on 2026-08-08, and measuring it gives a different answer.
    """
    directory = tempfile.mkdtemp()
    path = os.path.join(directory, "store")
    with shelve.open(path, writeback=True) as shelf:
        for index in range(count):
            shelf[f"k{index}"] = [index]

    def fingerprint():
        return {name: hashlib.sha256(
            open(os.path.join(directory, name), "rb").read()).hexdigest()[:12]
            for name in sorted(os.listdir(directory))}

    before = fingerprint()
    with shelve.open(path, writeback=writeback) as shelf:
        for index in range(count):
            shelf[f"k{index}"]
    after = fingerprint()

    changed = sorted(name for name in before if before.get(name) != after.get(name))
    return {"files": len(before), "changed": changed, "unchanged": len(before) - len(changed)}


def returns_nothing_discriminating():
    """The BP-0004 family: which operations answer the same thing however they went."""
    path = _store()
    with shelve.open(path) as shelf:
        return {
            "__setitem__ (wrote)": shelf.__setitem__("a", [1]),
            "sync() (nothing cached)": shelf.sync(),
            "get('missing')": shelf.get("missing"),
            "get('a') is a live ref": shelf.get("a") is shelf.get("a"),
        }


def structure():
    source = inspect.getsource(shelve)
    return {
        "shelve_py_lines": len(source.splitlines()),
        "getitem_lines": len(inspect.getsource(shelve.Shelf.__getitem__).splitlines()),
        "setitem_lines": len(inspect.getsource(shelve.Shelf.__setitem__).splitlines()),
        "writeback_default": inspect.signature(shelve.open).parameters["writeback"].default,
        "public_classes": sorted(
            name for name, value in vars(shelve).items()
            if isinstance(value, type) and not name.startswith("_")
            and value.__module__ == "shelve"),
    }

"""Unpickle once, then hand the same object back forever.

Upstream this is `writeback=True`. Two things follow that the flag's name does
not say: a mutation through what you were handed is visible to later readers
and is written out at sync; and a pure READ retains the object, so the cache
grows with what you have looked at rather than with what you have changed.
"""
NAME = "cached-reference"
HANDS_BACK = "the same object every read, retained in this process"
MUTATION_SURVIVES = True


def make(deserialise):
    cache = {}

    def answer(key, serialised):
        if key not in cache:
            cache[key] = deserialise(serialised)
        return cache[key]

    def remember(key, value):
        cache[key] = value

    def held():
        return len(cache)

    def items():
        return list(cache.items())

    return {"answer": answer, "remember": remember, "held": held, "items": items}

"""Unpickle on every read. What the caller gets is theirs.

Upstream this is `writeback=False`, the default, and it is three lines of
`Shelf.__getitem__` that never touch the cache.
"""
NAME = "copy-on-read"
HANDS_BACK = "a fresh object on every read"
MUTATION_SURVIVES = False


def make(deserialise):
    def answer(key, serialised):
        return deserialise(serialised)

    def remember(key, value):
        return None

    def held():
        return 0

    def items():
        # Nothing is retained, so a sync has nothing to write back. That is the
        # whole reason a mutation through a handed-out object goes nowhere.
        return []

    return {"answer": answer, "remember": remember, "held": held, "items": items}

"""The two modes side by side, the comparison, and the gaps."""


def modes(rows, out):
    out(f"\n  {'writeback':<12} {'d[k].append(x) survives':<26} {'d[k] is d[k]':<14} objects held")
    for row in rows:
        out(f"  {str(row['writeback']):<12} {str(row['survived']):<26} "
            f"{str(not row['identity_separates']):<14} {row['held']}")


def growth(marks, out):
    out("\n  reads only, nothing written:")
    for read, held in sorted(marks.items()):
        out(f"    after reading {read:>4} keys, the cache holds {held}")


def nothing_discriminating(values, out):
    out("\n  operations whose answer is the same however they went:")
    for label, value in values.items():
        out(f"    {label:<26} {value!r}")


def comparison(disagreements, cells, out):
    if disagreements:
        for line in disagreements:
            out(f"    DISAGREES  {line}")
    else:
        out(f"    {cells} of {cells} cells agree - the lifted strategy decides what "
            f"the flag decided")


def gaps(out):
    out("\n  measurable, not measured here:")
    out("    - how often real code mutates what a shelf handed it")
    out("    - what the retained objects cost on a session-sized workload")
    out("\n  not measurable by this entry at all:")
    out("    - whether the default is wrong. It is off, it is documented, and")
    out("      turning it on costs memory that most callers do not want to spend.")
    out("    - whether anyone has been bitten in production. Reproducing a silent")
    out("      loss says nothing about how often it happens.")

"""The island test, and the drills that let each comparison come out false.

    python src/island_test.py

Section 3 compares the re-cut against the shelve running right now. Section 3b
breaks the re-cut on purpose so that comparison is a measurement rather than a
formality, and section 2b mislabels a handout so the declaration check is too.
"""
import pathlib
import pickle
import re
import sys

HERE = pathlib.Path(__file__).parent
sys.path.insert(0, str(HERE))

from SCL import policy  # noqa: E402
from SMS import shelf, upstream  # noqa: E402

FAILURES = []


def check(label, ok, detail=""):
    print(f"  {'PASS' if ok else 'FAIL'}  {label}{' - ' + detail if detail else ''}")
    if not ok:
        FAILURES.append(label)


LOADED, PROBLEMS = shelf.load_handouts()


def through_recut(module):
    backing = {}
    store = shelf.Shelf(backing, module)
    store["cart"] = ["apple"]
    reopened = shelf.Shelf(backing, module)
    reopened["cart"].append("pear")
    separates = shelf.hands_back_copies(reopened, "cart")
    held = reopened.held()
    reopened.sync()
    survived = pickle.loads(backing[b"cart"])
    return {"mutation_survives": survived == ["apple", "pear"],
            "identity_separates": separates, "held": held}


print("\n== 1. every handout is an island and declares what it hands back")
check("both handouts loaded with no problems", not PROBLEMS, "; ".join(PROBLEMS))
handout_dir = HERE / "TMS" / "handouts"
files = sorted(f for f in handout_dir.iterdir() if f.suffix == ".py" and f.name != "__init__.py")
check("there are two handout files", len(files) == 2, ", ".join(f.stem for f in files))
for f in files:
    reaches = re.findall(r"^\s*(?:from|import)\s+(\S+)", f.read_text(encoding="utf-8"), re.M)
    siblings = [r for r in reaches if r.split(".")[0] in {"TMS", "SMS", "SCL", "DMS", "FMS"}]
    check(f"{f.stem} reaches no sibling set", not siblings,
          ", ".join(reaches) or "no imports at all")
for name, module in sorted(LOADED.items()):
    check(f"{name} says what it hands back", bool(module.HANDS_BACK), module.HANDS_BACK)

print("\n== 2. the declaration is checked by running it, not by reading it")
for name, module in sorted(LOADED.items()):
    measured = through_recut(module)
    check(f"{name}: declared mutation_survives={module.MUTATION_SURVIVES}, measured "
          f"{measured['mutation_survives']}",
          measured["mutation_survives"] == module.MUTATION_SURVIVES)

print("\n== 2b. the drill: a handout that lies about itself must be caught")


class Mislabelled:
    NAME = "mislabelled"
    HANDS_BACK = "a fresh object on every read"
    MUTATION_SURVIVES = False          # the lie
    make = staticmethod(LOADED["cached-reference"].make)   # the behaviour


measured = through_recut(Mislabelled)
check("a handout declaring copy semantics while caching is caught",
      measured["mutation_survives"] != Mislabelled.MUTATION_SURVIVES,
      f"declared {Mislabelled.MUTATION_SURVIVES}, measured {measured['mutation_survives']}")

print("\n== 3. the re-cut against the shelve running right now")
CELLS = []
for writeback, handout_name in ((False, "copy-on-read"), (True, "cached-reference")):
    measured = upstream.probe(writeback)
    modelled = through_recut(LOADED[handout_name])
    for field in ("mutation_survives", "identity_separates"):
        CELLS.append({"writeback": writeback, "field": field,
                      "upstream": measured[field], "recut": modelled[field]})
agree = [c for c in CELLS if c["upstream"] == c["recut"]]
check(f"all {len(CELLS)} cells agree", len(agree) == len(CELLS),
      f"{len(agree)}/{len(CELLS)}")

print("\n== 3b. the drill: can that comparison come out false?")


class Swapped:
    NAME = "swapped"
    HANDS_BACK = "wrong on purpose"
    MUTATION_SURVIVES = False
    make = staticmethod(LOADED["cached-reference"].make)


broken = through_recut(Swapped)
against_default = upstream.probe(False)
check("using the wrong strategy for writeback=False breaks cells",
      any(broken[f] != against_default[f] for f in ("mutation_survives", "identity_separates")),
      "so section 3 is a measurement, not a formality")

print("\n== 4. the flag is named for its fourth consequence")
marks = upstream.cache_after_pure_reads(200)
check("200 pure reads retain 200 objects", marks.get(200) == 200,
      ", ".join(f"{k} reads -> {v} held" for k, v in sorted(marks.items())))
# The previous version of this line was `check(..., True, ...)` - an assertion
# that the read loop wrote nothing. That is the hardcoded-True defect this lab
# filed against archaeology 008 on 2026-08-08, and measuring it disagrees.
under_writeback = upstream.reads_cause_writes(200, writeback=True)
under_default = upstream.reads_cause_writes(200, writeback=False)
check("a READ-ONLY session under writeback rewrites the medium at close",
      bool(under_writeback["changed"]),
      f"{len(under_writeback['changed'])} of {under_writeback['files']} files changed: "
      f"{', '.join(under_writeback['changed'])}")
check("the same read-only session under the default does not",
      not under_default["changed"],
      f"{under_default['unchanged']} of {under_default['files']} files unchanged")
check("so the two differ, which is what makes either mean anything",
      bool(under_writeback["changed"]) != bool(under_default["changed"]))

print("\n== 5. what answers the same however it went")
values = upstream.returns_nothing_discriminating()
same = [label for label, value in values.items() if value is None]
check("three operations return None on every path", len(same) == 3, ", ".join(same))
check("and the one observation that does discriminate is identity",
      values["get('a') is a live ref"] is False,
      "under the default, two reads of one key are two objects")

print("\n== 6. fail closed")
_, problem = shelf.resolve("write-through", LOADED)
check("an unresolvable handout stops the run", problem is not None, problem or "resolved anyway")
check("SCL names a handout that exists", policy.handout() in LOADED, policy.handout())

print("\n== 7. what this entry cannot see")
print("        MEASURABLE, NOT MEASURED")
print("          - how often real code mutates what a shelf handed it")
print("          - what the retained objects cost on a session-sized workload")
print("        NOT MEASURABLE HERE")
print("          - whether the default is wrong. It is off, documented, and turning")
print("            it on costs memory most callers do not want to spend.")
print("          - frequency in the wild. Reproducing a silent loss says nothing")
print("            about how often anyone meets it.")

print(f"\n{len(FAILURES)} failure(s)" if FAILURES else "\nall checks passed")
for f in FAILURES:
    print(f"  - {f}")
sys.exit(1 if FAILURES else 0)

"""CPython's shelve, re-cut so the handout strategy is a unit rather than a flag.

    python src/main.py            the two modes, the growth, and the comparison
    python src/main.py --strict   exit 1 if the re-cut disagrees with real shelve
"""
import json
import pathlib
import pickle
import sys

HERE = pathlib.Path(__file__).parent
sys.path.insert(0, str(HERE))

from DMS import report  # noqa: E402
from SCL import policy  # noqa: E402
from SMS import shelf, upstream  # noqa: E402

ARCH = json.loads((HERE / "FMS" / "architecture.json").read_text(encoding="utf-8"))
BY_WRITEBACK = {False: "copy-on-read", True: "cached-reference"}


def recut(handout_name, loaded):
    """The same three operations the upstream probe performs, through the re-cut."""
    module, problem = shelf.resolve(handout_name, loaded)
    if problem:
        return None, problem
    backing = {}
    store = shelf.Shelf(backing, module)
    store["cart"] = ["apple"]

    store2 = shelf.Shelf(backing, module)
    store2["cart"].append("pear")
    identity_separates = shelf.hands_back_copies(store2, "cart")
    held = store2.held()
    store2.sync()

    survived = pickle.loads(backing[b"cart"])
    return {"survived": survived, "mutation_survives": survived == ["apple", "pear"],
            "identity_separates": identity_separates, "held": held}, None


def main(argv):
    out = lambda line="": sys.stdout.write(line + "\n")  # noqa: E731
    loaded, problems = shelf.load_handouts()
    if problems:
        for problem in problems:
            out(f"  !! {problem}")
        return 1

    out(f"\n== one interface, two meanings for the same call  "
        f"[shelve.py {ARCH['examined_version']}]")
    rows, disagreements = [], []
    for writeback, handout_name in BY_WRITEBACK.items():
        measured = upstream.probe(writeback)
        rows.append({"writeback": writeback, **measured})

        modelled, problem = recut(handout_name, loaded)
        if problem:
            out(f"  !! {problem}")
            return 1
        for field in ("mutation_survives", "identity_separates"):
            if modelled[field] != measured[field]:
                disagreements.append(
                    f"writeback={writeback}/{field}: re-cut {modelled[field]}, "
                    f"shelve {measured[field]}")
    report.modes(rows, out)

    out("\n== the flag is called writeback, and what it changes first is reads")
    report.growth(upstream.cache_after_pure_reads(200), out)

    report.nothing_discriminating(upstream.returns_nothing_discriminating(), out)

    out("\n== the re-cut against the shelve that is running right now")
    report.comparison(disagreements, len(BY_WRITEBACK) * 2, out)

    out("\n== what each handout declares, and what it did")
    for handout_name in sorted(loaded):
        module = loaded[handout_name]
        modelled, _ = recut(handout_name, loaded)
        agrees = modelled["mutation_survives"] == module.MUTATION_SURVIVES
        out(f"    {'ok ' if agrees else 'NO '} {module.NAME:<18} says {module.HANDS_BACK}")
        out(f"    {'':4} {'':<18} mutation survives: declared "
            f"{module.MUTATION_SURVIVES}, measured {modelled['mutation_survives']}")

    out("\n== the numbers FMS declares, re-measured")
    declared, measured = ARCH["declared_and_then_remeasured"], upstream.structure()
    wrong = []
    for key, value in measured.items():
        agrees = declared.get(key) == value
        if not agrees:
            wrong.append(f"{key}: declared {declared.get(key)!r}, measured {value!r}")
        out(f"    {'ok ' if agrees else 'NO '} {key:<22} {value}")

    report.gaps(out)

    if "--strict" in argv and (wrong or (disagreements and policy.recut_must_match_upstream())):
        for line in wrong:
            out(f"\n  !! {line}")
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))