018 — Who a declaration serves is decided by whoever reads it

{
  "name": "018-who-the-declaration-serves",
  "what_it_is": "The same three sources under two defensible deployment policies, where one unit declares itself incomplete and the example measures what that declaration cost it.",
  "the_structural_decision": "The direction of a declaration is not a property of the declaration. It is a property of the declaration together with the policy that consumes it — so it cannot be assumed, and it can be measured. The measurement is a counterfactual: run the pipeline again with that unit's declaration suppressed, and compare what the unit contributes.",
  "why_this_one": "改良點 15 takes a self-penalising declaration on trust. I wrote on the board the same day that self-penalising is a judgement about incentives rather than a property of the code, and that I could not construct the counter-case convincingly. This is the counter-case, and it lands on the sensible policy rather than a contrived one.",
  "status": "candidate",

  "the_two_policies": {
    "refuse-declared": "a source that declared itself incomplete is dropped from the export — defensible for a compliance export",
    "retry-declared": "a source that declared itself incomplete is re-run with a larger budget — defensible, and common, because the declaration is exactly the signal that spending more will yield more"
  },
  "neither_is_a_strawman": "Both are policies a careful person would write. The finding is not that one of them is wrong.",

  "sources": {
    "full-page":   {"holds": 3, "declares": "never — it always returns everything it has"},
    "honest-page": {"holds": 6, "declares": "more-after-cursor when the budget cuts it short"},
    "silent-page": {"holds": 6, "declares": "nothing, ever", "note": "the control — it holds exactly what honest-page holds and hands over exactly as much per unit of budget, so any difference between the two is attributable to the declaration and to nothing else"}
  },

  "the_measurement": {
    "definition": "incentive(unit) = records the unit contributes WITH its declaration - records it contributes with the declaration SUPPRESSED",
    "under_refuse_declared": {"honest-page": -3, "silent-page": 0},
    "under_retry_declared":  {"honest-page": 3,  "silent-page": 0},
    "note": "The numbers are emitted by the run, not typed here — the island test asserts these against what the pipeline computes, so a change in the code turns this file red."
  },

  "the_finding": "Under retry-declared the honest unit contributes 6 and the silent unit, holding identical data, contributes 3. Declaring made the unit that declared better off, and the unit that stayed quiet worse off. The trust criterion of 改良點 15 does not hold under that policy, and nothing about the policy is unreasonable.",

  "the_repair": "SCL states the assumption (`assumes_declarations_are`), the build measures it with the counterfactual, and a contradiction is fatal. The judgement is not removed — it is written down where something can disagree with it.",

  "sets": {
    "FMS": "this file: the two policies, what each source holds and declares, the definition of the measurement, and the units map",
    "SCL": "which policy this deployment applies, what it assumes about declarations, and what a contradiction means here",
    "SMS": "running the sources, applying the named policy, and computing the counterfactual",
    "TMS": "one file per source and one per policy — each declares itself and reaches no sibling",
    "DMS": "the per-unit contribution, and the incentive shown as the pair it was computed from"
  },

  "units": {
    "TMS/sources": ["full_page.mjs", "honest_page.mjs", "silent_page.mjs"],
    "TMS/policies": ["refuse_declared.mjs", "retry_declared.mjs"]
  },

  "non_goals": [
    "Deciding which policy is right. Both are defensible and SCL picks one; the example takes no position.",
    "Claiming the counterfactual measures the incentive in general. It measures it for that unit, under that policy, on that run's data. A unit whose declaration cannot be suppressed cannot be measured this way at all.",
    "Detecting a unit that declares strategically. Nothing here reads intent; it reads outcome, and a unit that benefits by accident is indistinguishable from one that planned to."
  ]
}

{
  "deployment": "catalogue-sync",
  "policy": "retry-declared",
  "assumes_declarations_are": "self-penalising",
  "why_this_policy": "A declaration of incompleteness is exactly the signal that spending more on this source will yield more, so this deployment spends it. Nothing about that is unreasonable, and that is the point: the finding lands on the sensible configuration and not on a contrived one.",
  "why_the_assumption_is_written_down": "改良點 15 takes a self-penalising declaration on trust. That is a claim about incentives, and a claim about incentives can be false. Writing it into SCL is what lets the build measure it instead of inheriting it.",
  "on_contradiction": "fatal"
}

import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";

const here = path.dirname(fileURLToPath(import.meta.url));
export const POLICY = JSON.parse(fs.readFileSync(path.join(here, "policy.json"), "utf8"));

export const policyName = () => POLICY.policy;
export const assumption = () => POLICY.assumes_declarations_are;
export const isFatal = () => POLICY.on_contradiction === "fatal";
export const describe = () =>
  `${POLICY.deployment}: apply ${POLICY.policy}, assuming declarations are ${POLICY.assumes_declarations_are}`;

// The assumption, made checkable. A self-penalising declaration must not leave
// the unit that made it better off than suppressing it would have.
export function contradicts(measured) {
  if (POLICY.assumes_declarations_are !== "self-penalising") return null;
  return measured.delta > 0
    ? `declaring left ${measured.source} better off by ${measured.delta} record(s) `
      + `(${measured.suppressed} suppressed -> ${measured.declared} declared)`
    : null;
}

// Run the sources, apply the policy SCL names, and measure the incentive.
//
// The measurement is the point. Example 017 asserted that a unit declaring
// itself incomplete is penalising itself, and called that a reason to take the
// declaration on trust. That was a judgement about incentives, not a property
// of anything in the code — so this module computes it instead.
//
//   incentive(unit) = what the unit contributes WITH its declaration
//                   - what it contributes with the declaration SUPPRESSED
//
// Negative means declaring cost the unit. Positive means declaring paid.
import * as fullPage from "../TMS/sources/full_page.mjs";
import * as honestPage from "../TMS/sources/honest_page.mjs";
import * as silentPage from "../TMS/sources/silent_page.mjs";
import * as refuseDeclared from "../TMS/policies/refuse_declared.mjs";
import * as retryDeclared from "../TMS/policies/retry_declared.mjs";

const SOURCE_MODULES = [fullPage, honestPage, silentPage];
const POLICY_MODULES = [refuseDeclared, retryDeclared];

export function load(extraSources = [], extraPolicies = []) {
  const sources = {};
  const policies = {};
  const problems = [];

  for (const module of [...SOURCE_MODULES, ...extraSources]) {
    for (const attribute of ["NAME", "CAN_FAIL_WITH", "HELD", "collect"]) {
      if (module[attribute] === undefined) problems.push(`a source does not declare ${attribute}`);
    }
    if (!module.CAN_FAIL_WITH?.length) {
      problems.push(`${module.NAME}: CAN_FAIL_WITH is empty`);
    }
    for (const vouch of ["COMPLETE", "IS_COMPLETE", "RETURNS_EVERYTHING"]) {
      if (module[vouch] === true) {
        problems.push(`${module.NAME}: declares ${vouch} - a unit may declare itself incomplete `
          + `and may not declare itself complete (改良點 15)`);
      }
    }
    sources[module.NAME] = module;
  }

  for (const module of [...POLICY_MODULES, ...extraPolicies]) {
    if (!module.WHAT_IT_DOES_WITH_A_DECLARATION) {
      problems.push(`${module.POLICY}: does not say what it does with a declaration`);
    }
    policies[module.POLICY] = module;
  }
  return { sources, policies, problems };
}

export function runOne(module, budget = 1, { suppressDeclaration = false } = {}) {
  const result = module.collect({ budget });
  return {
    source: module.NAME,
    held: module.HELD,
    records: result.records,
    incomplete_because: suppressDeclaration ? null : result.incomplete_because,
    suppressed: suppressDeclaration && result.incomplete_because !== null,
  };
}

export function runAll(sources, { suppress = [] } = {}) {
  return Object.keys(sources).sort()
    .map((name) => runOne(sources[name], 1, { suppressDeclaration: suppress.includes(name) }));
}

export function through(policyName, sources, options = {}) {
  const policy = policyName;
  const runs = runAll(sources, options);
  const rerun = (name, budget) => runOne(sources[name], budget);
  const applied = policy.apply(runs, rerun);
  return {
    rows: applied,
    total: applied.reduce((n, row) => n + row.kept.length, 0),
  };
}

export function contribution(result, name) {
  return result.rows.find((row) => row.source === name)?.kept.length ?? 0;
}

// The counterfactual. Same policy, same data, one declaration removed.
export function incentive(policy, sources, name) {
  const declared = contribution(through(policy, sources), name);
  const suppressed = contribution(through(policy, sources, { suppress: [name] }), name);
  return { declared, suppressed, delta: declared - suppressed };
}

// A source that declared itself incomplete is dropped from the export.
//
// Defensible: a compliance export may be smaller than expected but may not be
// quietly partial, so anything known-partial is excluded and named.
export const POLICY = "refuse-declared";
export const WHAT_IT_DOES_WITH_A_DECLARATION = "drops the source";

export function apply(runs) {
  return runs.map((run) => (run.incomplete_because
    ? { ...run, kept: [], note: `dropped: ${run.incomplete_because}` }
    : { ...run, kept: run.records, note: null }));
}

// A source that declared itself incomplete is run again with a larger budget.
//
// Defensible, and common: the declaration is exactly the signal that spending
// more on this source will yield more. Nothing here is a strawman.
export const POLICY = "retry-declared";
export const WHAT_IT_DOES_WITH_A_DECLARATION = "re-runs the source with more budget";
export const RETRY_BUDGET = 2;

export function apply(runs, rerun) {
  return runs.map((run) => {
    if (!run.incomplete_because) return { ...run, kept: run.records, note: null };
    const second = rerun(run.source, RETRY_BUDGET);
    return { ...run, kept: second.records, note: `re-run at budget ${RETRY_BUDGET}` };
  });
}

// Returns everything behind it, in one call, and declares nothing.
export const NAME = "full-page";
export const CAN_FAIL_WITH = ["unreadable-page"];
export const HELD = 3;

export function collect({ budget = 1 } = {}) {
  const take = Math.min(HELD, budget * 3);
  return {
    records: Array.from({ length: take }, (_, n) => ({ from: NAME, id: `f-${n + 1}` })),
    incomplete_because: take < HELD ? "budget-exhausted" : null,
  };
}

// Holds six records, hands over three per unit of budget, and SAYS SO.
//
// This is the unit example 017 called self-penalising. Whether that is true is
// what this example measures rather than assumes.
export const NAME = "honest-page";
export const CAN_FAIL_WITH = ["unreadable-page", "cursor-expired"];
export const HELD = 6;

export function collect({ budget = 1 } = {}) {
  const take = Math.min(HELD, budget * 3);
  return {
    records: Array.from({ length: take }, (_, n) => ({ from: NAME, id: `h-${n + 1}` })),
    incomplete_because: take < HELD ? "more-after-cursor" : null,
  };
}

// The control, and the comparison the whole example turns on.
//
// It holds exactly what honest-page holds and hands over exactly as much per
// unit of budget. The ONLY difference between the two units is that this one
// never declares. So any difference in what they end up contributing is
// attributable to the declaration and to nothing else.
export const NAME = "silent-page";
export const CAN_FAIL_WITH = ["unreadable-page", "cursor-expired"];
export const HELD = 6;

export function collect({ budget = 1 } = {}) {
  const take = Math.min(HELD, budget * 3);
  return {
    records: Array.from({ length: take }, (_, n) => ({ from: NAME, id: `s-${n + 1}` })),
    incomplete_because: null,
  };
}

// What a person is shown.
//
// An incentive is never printed as a single number. It is printed as the pair
// it was computed from, because "+3" without "3 -> 6" is a claim nobody can
// re-derive.
const pad = (v, w) => String(v).padEnd(w);

export function rows(result) {
  const lines = ["  source        held   kept   declared                note"];
  for (const row of result.rows) {
    lines.push(`  ${pad(row.source, 13)} ${pad(row.held, 6)} ${pad(row.kept.length, 6)} `
      + `${pad(row.incomplete_because ?? "-", 23)} ${row.note ?? ""}`.trimEnd());
  }
  lines.push(`  total kept: ${result.total}`);
  return lines.join("\n");
}

export function incentives(measured) {
  const lines = [
    "  what declaring cost or paid, measured by suppressing it and re-running:",
    "    unit          suppressed   declared   delta   reading",
  ];
  for (const m of measured) {
    const reading = m.delta < 0 ? "declaring COST it"
      : m.delta > 0 ? "declaring PAID"
        : "declared nothing - the control";
    lines.push(`    ${pad(m.source, 13)} ${pad(m.suppressed, 12)} ${pad(m.declared, 10)} `
      + `${pad(m.delta > 0 ? `+${m.delta}` : m.delta, 7)} ${reading}`);
  }
  return lines.join("\n");
}

// The island test.
//
//   node src/island_test.mjs
//
// Section 2 is the control that makes everything after it mean something.
// Section 4 is the finding. Section 6 proves the contradiction check reads SCL
// rather than a constant.
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";

import * as pipeline from "./SMS/pipeline.mjs";
import * as policy from "./SCL/policy.mjs";
import * as honestPage from "./TMS/sources/honest_page.mjs";
import * as silentPage from "./TMS/sources/silent_page.mjs";
import * as refuseDeclared from "./TMS/policies/refuse_declared.mjs";
import * as retryDeclared from "./TMS/policies/retry_declared.mjs";

const here = path.dirname(fileURLToPath(import.meta.url));
const CONTRACT = JSON.parse(fs.readFileSync(path.join(here, "FMS", "contract.json"), "utf8"));
const failures = [];
let ran = 0;
const check = (label, ok, detail = "") => {
  ran += 1;
  process.stdout.write(`  ${ok ? "PASS" : "FAIL"}  ${label}${detail ? ` - ${detail}` : ""}\n`);
  if (!ok) failures.push(label);
};
const say = (line = "") => process.stdout.write(`${line}\n`);

const { sources, policies, problems } = pipeline.load();
const incentiveUnder = (module, name) => pipeline.incentive(module, sources, name);

say("\n== 1. every unit is an island, and FMS matches the tree");
check("loading raised no problems", problems.length === 0, problems.join("; "));
for (const [unit, declared] of Object.entries(CONTRACT.units)) {
  const dir = path.join(here, ...unit.split("/"));
  const onDisk = fs.readdirSync(dir).filter((n) => n.endsWith(".mjs")).sort();
  check(`${unit}: FMS declares what is on disk`,
    JSON.stringify(onDisk) === JSON.stringify([...declared].sort()),
    `disk ${onDisk.join(", ")} | FMS ${[...declared].sort().join(", ")}`);
  for (const file of onDisk) {
    const body = fs.readFileSync(path.join(dir, file), "utf8");
    const siblings = onDisk.filter((n) => n !== file).map((n) => n.replace(/\.mjs$/, ""));
    check(`${unit}/${file} reaches no sibling`,
      !siblings.some((s) => new RegExp(`from\\s+["'][^"']*${s}`).test(body)));
    check(`${unit}/${file} reaches no other set`,
      !/from\s+["'][^"']*\/(SMS|DMS|SCL|FMS)\//.test(body));
  }
}

say("\n== 2. the control - two units that differ only in the declaration");
check("honest-page and silent-page hold the same number of records",
  honestPage.HELD === silentPage.HELD, `${honestPage.HELD} and ${silentPage.HELD}`);
for (const budget of [1, 2, 3]) {
  check(`at budget ${budget} they hand over the same number`,
    honestPage.collect({ budget }).records.length === silentPage.collect({ budget }).records.length,
    `${honestPage.collect({ budget }).records.length}`);
}
check("and at budget 1 exactly one of them declares",
  Boolean(honestPage.collect({ budget: 1 }).incomplete_because)
  !== Boolean(silentPage.collect({ budget: 1 }).incomplete_because));
check("so any difference between them is attributable to the declaration alone",
  honestPage.HELD === silentPage.HELD
  && honestPage.collect({ budget: 1 }).records.length === silentPage.collect({ budget: 1 }).records.length);

say("\n== 3. under refuse-declared, declaring costs the unit");
const refusedHonest = incentiveUnder(refuseDeclared, "honest-page");
const refusedSilent = incentiveUnder(refuseDeclared, "silent-page");
check("honest-page contributes nothing once it declares", refusedHonest.declared === 0);
check("and would have contributed 3 with the declaration suppressed", refusedHonest.suppressed === 3);
check("so the incentive is negative", refusedHonest.delta < 0, `${refusedHonest.delta}`);
check("the control moves by nothing", refusedSilent.delta === 0,
  `${refusedSilent.suppressed} -> ${refusedSilent.declared}`);
check("which is what says the machinery is not just moving numbers",
  refusedSilent.delta === 0 && refusedHonest.delta !== 0);

say("\n== 4. under retry-declared, the same declaration pays");
const retriedHonest = incentiveUnder(retryDeclared, "honest-page");
const retriedSilent = incentiveUnder(retryDeclared, "silent-page");
check("honest-page contributes 6 once it declares", retriedHonest.declared === 6);
check("and would have contributed 3 with the declaration suppressed", retriedHonest.suppressed === 3);
check("so the incentive is POSITIVE", retriedHonest.delta > 0, `+${retriedHonest.delta}`);
check("the control still moves by nothing", retriedSilent.delta === 0);
const applied = pipeline.through(retryDeclared, sources);
check("and the honest unit ends up AHEAD of the silent one holding identical data",
  pipeline.contribution(applied, "honest-page") > pipeline.contribution(applied, "silent-page"),
  `${pipeline.contribution(applied, "honest-page")} vs ${pipeline.contribution(applied, "silent-page")}`);
check("the same declaration points opposite ways under the two policies",
  Math.sign(refusedHonest.delta) === -Math.sign(retriedHonest.delta));
check("and neither policy withheld what it does with a declaration",
  Boolean(refuseDeclared.WHAT_IT_DOES_WITH_A_DECLARATION)
  && Boolean(retryDeclared.WHAT_IT_DOES_WITH_A_DECLARATION));

say("\n== 5. FMS's numbers are the run's numbers");
const fmsRefuse = CONTRACT.the_measurement.under_refuse_declared;
const fmsRetry = CONTRACT.the_measurement.under_retry_declared;
check("FMS's refuse-declared figures match the pipeline",
  fmsRefuse["honest-page"] === refusedHonest.delta && fmsRefuse["silent-page"] === refusedSilent.delta,
  `${JSON.stringify(fmsRefuse)} vs ${refusedHonest.delta}/${refusedSilent.delta}`);
check("FMS's retry-declared figures match the pipeline",
  fmsRetry["honest-page"] === retriedHonest.delta && fmsRetry["silent-page"] === retriedSilent.delta,
  `${JSON.stringify(fmsRetry)} vs ${retriedHonest.delta}/${retriedSilent.delta}`);

say("\n== 6. SCL states the assumption, and the check reads SCL");
const inForce = policies[policy.policyName()];
const measuredInForce = { source: "honest-page", ...incentiveUnder(inForce, "honest-page") };
check("the deployment in force is retry-declared", policy.policyName() === "retry-declared");
check("SCL writes down what it assumes", policy.assumption() === "self-penalising");
check("and the measurement contradicts it", policy.contradicts(measuredInForce) !== null,
  policy.contradicts(measuredInForce) ?? "");
check("a negative incentive does NOT contradict it",
  policy.contradicts({ source: "x", declared: 0, suppressed: 3, delta: -3 }) === null);
check("and neither does zero",
  policy.contradicts({ source: "x", declared: 3, suppressed: 3, delta: 0 }) === null);
check("the deployment calls a contradiction fatal", policy.isFatal());

say("\n== 7. the guards from 013 and 015 still hold");
const vouching = {
  NAME: "drill-vouch", CAN_FAIL_WITH: ["x"], HELD: 1, COMPLETE: true,
  collect: () => ({ records: [], incomplete_because: null }),
};
check("DRILL: a source declaring COMPLETE is still refused",
  pipeline.load([vouching]).problems.some((p) => /declares COMPLETE/.test(p)));
const mute = {
  NAME: "drill-mute", CAN_FAIL_WITH: [], HELD: 1,
  collect: () => ({ records: [], incomplete_because: null }),
};
check("DRILL: an empty CAN_FAIL_WITH is still refused",
  pipeline.load([mute]).problems.some((p) => /CAN_FAIL_WITH is empty/.test(p)));
const secretive = {
  POLICY: "drill-secretive",
  apply: (runs) => runs.map((r) => ({ ...r, kept: r.records, note: null })),
};
check("DRILL: a policy that will not say what it does with a declaration is refused",
  pipeline.load([], [secretive]).problems.some((p) => /does not say what it does/.test(p)));

say("\n== 8. what the measurement cannot do, asserted so it stays measured");
const neverDeclares = {
  NAME: "drill-never", CAN_FAIL_WITH: ["x"], HELD: 2,
  collect: () => ({ records: [{ from: "drill-never", id: "n-1" }], incomplete_because: null }),
};
const withExtra = pipeline.load([neverDeclares]);
const flat = pipeline.incentive(retryDeclared, withExtra.sources, "drill-never");
check("a unit that never declares measures as zero", flat.delta === 0,
  `${flat.suppressed} -> ${flat.declared}`);
check("and so does the control, so a zero reading does not say WHICH it is",
  flat.delta === refusedSilent.delta,
  "the limit named in FMS non_goals - if this goes red, the limit changed and the text must too");
check("nothing here reads intent - the reading is outcome only",
  !/\bintent\b|\bdeliberate\b|\bstrategic\b/
    .test(fs.readFileSync(path.join(here, "SMS", "pipeline.mjs"), "utf8")));
check("and the report never prints a delta without the pair it came from",
  /suppressed/.test(fs.readFileSync(path.join(here, "DMS", "report.mjs"), "utf8"))
  && /declared/.test(fs.readFileSync(path.join(here, "DMS", "report.mjs"), "utf8")));

say("");
if (failures.length > 0) {
  say(`  ${failures.length} FAILED: ${failures.join(" | ")}`);
  process.exitCode = 1;
} else {
  say(`  ${ran} checks passed - ${Object.keys(sources).length} sources, ${Object.keys(policies).length} policies`);
}

// Three sources, two defensible policies, and one declaration whose direction
// depends entirely on which policy is in force.
//
//   node src/main.mjs             the run under the policy SCL names
//   node src/main.mjs --compare   both policies, side by side
//   node src/main.mjs --strict    exit 1 when the measurement contradicts SCL's assumption
import * as report from "./DMS/report.mjs";
import * as policy from "./SCL/policy.mjs";
import * as pipeline from "./SMS/pipeline.mjs";

const say = (line = "") => process.stdout.write(`${line}\n`);

function measure(policyModule, sources) {
  return Object.keys(sources).sort()
    .map((name) => ({ source: name, ...pipeline.incentive(policyModule, sources, name) }));
}

function main(argv) {
  const { sources, policies, problems } = pipeline.load();
  if (problems.length > 0) {
    for (const problem of problems) say(`  REFUSED: ${problem}`);
    return 1;
  }

  if (argv.includes("--compare")) {
    say("\n  the same three sources, the same data, two defensible policies:\n");
    for (const name of Object.keys(policies).sort()) {
      const module = policies[name];
      say(`  ${name} - ${module.WHAT_IT_DOES_WITH_A_DECLARATION}`);
      say(report.rows(pipeline.through(module, sources)));
      say("");
      say(report.incentives(measure(module, sources)));
      say("");
    }
    say("  Same declaration. Opposite direction. Neither policy is unreasonable.");
    return 0;
  }

  const inForce = policies[policy.policyName()];
  const result = pipeline.through(inForce, sources);
  const measured = measure(inForce, sources);

  say(`\n  ${policy.describe()}\n`);
  say(report.rows(result));
  say("");
  say(report.incentives(measured));

  const contradictions = measured.map((m) => policy.contradicts(m)).filter(Boolean);
  if (contradictions.length > 0) {
    say("");
    for (const line of contradictions) say(`  CONTRADICTS SCL: ${line}`);
    say(`  SCL assumes declarations are ${policy.assumption()}; under this policy they are not.`);
  }

  if (argv.includes("--strict") && contradictions.length > 0 && policy.isFatal()) {
    say(`\n  --strict: the measurement contradicts a written assumption and that is fatal here`);
    return 1;
  }
  return 0;
}

process.exitCode = main(process.argv.slice(2));