Theme A: Uncertainty on a Rate

# learning/code/ex-01-two-runs-two-numbers/two_runs.py
"""Worked Example 1: Two Runs, Two Numbers."""  # => co-03: this file's own restated purpose, doubling as its module __doc__
 
from __future__ import annotations  # => hygiene: postpones annotation evaluation, interpreter-version-agnostic
 
import random  # => co-03: stands in for drawing two independent samples of the same unchanged system
 
TRUE_PASS_RATE = 0.85  # => co-02: the system's real, unobservable pass rate -- fixed and UNCHANGED across both runs
SAMPLE_SIZE = 40  # => co-03: how many cases each run draws -- a typical small eval-set size
 
 
def run_eval(true_rate: float, n: int, *, seed: int) -> float:  # => co-02: simulates one eval run against the SAME system
    """Draw n Bernoulli(true_rate) trials and return the observed pass rate."""  # => co-02: documents run_eval's contract -- no runtime output, just sets its __doc__
    rng = random.Random(seed)  # => co-03: a fresh generator per run -- seed stands in for "which cases happened to be sampled"
    passes = sum(1 for _ in range(n) if rng.random() < true_rate)  # => co-02: one Bernoulli draw per case, summed
    return passes / n  # => co-02: the OBSERVED pass rate -- a sample proportion, not the true rate itself
 
 
if __name__ == "__main__":  # => co-03: entry point -- runs only when this file executes directly, not on import
    rate_run_1 = run_eval(TRUE_PASS_RATE, SAMPLE_SIZE, seed=1)  # => co-03: "run 1" -- same system, same n, different sample
    rate_run_2 = run_eval(TRUE_PASS_RATE, SAMPLE_SIZE, seed=2)  # => co-03: "run 2" -- same system, same n, DIFFERENT sample
    print(f"True (unobservable) pass rate: {TRUE_PASS_RATE:.2%}")  # => co-02: the ground truth this demo controls
    print(f"Run 1 observed pass rate: {rate_run_1:.2%}")  # => co-03: run 1's number
    print(f"Run 2 observed pass rate: {rate_run_2:.2%}")  # => co-03: run 2's DIFFERENT number
    difference = abs(rate_run_1 - rate_run_2)  # => co-03: the spread between two runs of an UNCHANGED system
    print(f"Difference between two runs of the SAME unchanged system: {difference:.2%}")  # => co-03
    assert rate_run_1 != rate_run_2, "two independent samples of the same system must differ for this demo to make its point"  # => co-03
    print("MATCH: the system never changed, yet the two numbers differ -- that spread is sampling error, not a bug")  # => co-03
    # => co-02,co-03: a pass rate is a sample proportion of an unknown true rate -- re-sampling moves the number even when nothing else does

True (unobservable) pass rate: 85.00%
Run 1 observed pass rate: 92.50%
Run 2 observed pass rate: 80.00%
Difference between two runs of the SAME unchanged system: 12.50%
MATCH: the system never changed, yet the two numbers differ -- that spread is sampling error, not a bug

# learning/code/ex-02-pass-rate-is-an-estimate/pass_rate_estimate.py
"""Worked Example 2: Pass Rate Is an Estimate."""  # => co-02: this file's own restated purpose, doubling as its module __doc__
 
from __future__ import annotations  # => hygiene: postpones annotation evaluation, interpreter-version-agnostic
 
import random  # => co-02: draws many independent samples to reveal the estimator's own behavior
import statistics  # => co-02: averages many observed pass rates to approximate the true rate
 
TRUE_PASS_RATE = 0.73  # => co-02: the system's real, unobservable pass rate -- known here only because this is a simulation
SAMPLE_SIZE = 50  # => co-02: cases per single eval run
NUM_REPEATS = 4000  # => co-02: how many independent runs to average over -- large enough for the law of large numbers to bite
 
 
def observed_pass_rate(true_rate: float, n: int, *, seed: int) -> float:  # => co-02: ONE run -- what a team actually sees
    """Draw n Bernoulli(true_rate) trials and return the sample proportion that passed."""  # => co-02: documents observed_pass_rate's contract -- no runtime output, just sets its __doc__
    rng = random.Random(seed)  # => co-02: a fresh generator per run
    passes = sum(1 for _ in range(n) if rng.random() < true_rate)  # => co-02: count of Bernoulli successes in this run
    return passes / n  # => co-02: the sample proportion -- a single run's ESTIMATE of the true rate, not the true rate itself
 
 
if __name__ == "__main__":  # => co-02: entry point -- runs only when this file executes directly, not on import
    one_run = observed_pass_rate(TRUE_PASS_RATE, SAMPLE_SIZE, seed=0)  # => co-02: what a single team actually observes, once
    print(f"True (unobservable) pass rate: {TRUE_PASS_RATE:.4f}")  # => co-02: the ground truth this simulation controls
    print(f"One run's observed pass rate: {one_run:.4f}")  # => co-02: a single sample proportion -- one team's whole evidence
    many_runs = [observed_pass_rate(TRUE_PASS_RATE, SAMPLE_SIZE, seed=i) for i in range(NUM_REPEATS)]  # => co-02: thousands of INDEPENDENT teams running the same eval
    average_of_runs = statistics.mean(many_runs)  # => co-02: the estimator's own long-run behavior -- what "unbiased" means
    print(f"Average pass rate across {NUM_REPEATS} independent runs: {average_of_runs:.4f}")  # => co-02: converges toward the true rate
    gap_to_truth = abs(average_of_runs - TRUE_PASS_RATE)  # => co-02: how close the LONG-RUN average lands to the true rate
    print(f"Gap between the long-run average and the true rate: {gap_to_truth:.4f}")  # => co-02: should be small -- the estimator is unbiased
    assert gap_to_truth < 0.01, "the average of many independent pass-rate estimates must land close to the true rate"  # => co-02
    print("MATCH: no single run equals the true rate, but the estimator is unbiased across many runs")  # => co-02
    # => co-02: a pass rate IS a sample proportion of an unknown true rate -- treating one run's number as the truth itself is the mistake

True (unobservable) pass rate: 0.7300
One run's observed pass rate: 0.6400
Average pass rate across 4000 independent runs: 0.7297
Gap between the long-run average and the true rate: 0.0003
MATCH: no single run equals the true rate, but the estimator is unbiased across many runs

# learning/code/ex-03-simulate-sampling-error/sampling_error.py
"""Worked Example 3: Simulate Sampling Error."""  # => co-03: this file's own restated purpose, doubling as its module __doc__
 
from __future__ import annotations  # => hygiene: postpones annotation evaluation, interpreter-version-agnostic
 
import random  # => co-03: draws repeated samples at each sample size
import statistics  # => co-03: measures the spread (population standard deviation) of the resulting pass rates
 
TRUE_PASS_RATE = 0.85  # => co-03: the system's real, unobservable pass rate -- fixed across every sample size tested
SAMPLE_SIZES = (10, 40, 160, 640)  # => co-03: quadrupling n each step -- sampling error should shrink roughly like 1/sqrt(n)
REPEATS_PER_SIZE = 200  # => co-03: independent draws per sample size, enough to estimate the spread itself precisely
 
 
def observed_pass_rate(true_rate: float, n: int, *, seed: int) -> float:  # => co-03: ONE simulated eval run at size n
    """Draw n Bernoulli(true_rate) trials and return the observed pass rate."""  # => co-03: documents observed_pass_rate's contract -- no runtime output, just sets its __doc__
    rng = random.Random(seed)  # => co-03: a fresh generator per (size, repeat) draw
    passes = sum(1 for _ in range(n) if rng.random() < true_rate)  # => co-03: count of Bernoulli successes
    return passes / n  # => co-03: this run's observed pass rate
 
 
if __name__ == "__main__":  # => co-03: entry point -- runs only when this file executes directly, not on import
    spreads: dict[int, float] = {}  # => co-03: sample size -> spread (pstdev) of observed pass rates at that size
    for n in SAMPLE_SIZES:  # => co-03: one spread measurement per candidate sample size
        rates = [observed_pass_rate(TRUE_PASS_RATE, n, seed=1000 * n + i) for i in range(REPEATS_PER_SIZE)]  # => co-03: many independent runs at this n
        spread = statistics.pstdev(rates)  # => co-03: how much the observed rate bounces around, at this n
        spreads[n] = spread  # => co-03: record it for the shrink-rate check below
        bar = "#" * round(spread * 400)  # => co-03: a plain-text bar -- longer bar means more sampling error at this n
        print(f"n={n:>4} | spread={spread:.4f} | {bar}")  # => co-03: prints size, spread, and its ASCII-bar visualization
    assert spreads[SAMPLE_SIZES[0]] > spreads[SAMPLE_SIZES[-1]], "spread must shrink as n grows"  # => co-03: the qualitative claim
    ratio_10_to_640 = spreads[10] / spreads[640]  # => co-03: sqrt(640/10) = 8 is the theoretical shrink factor for a 64x larger n
    print(f"Spread ratio (n=10 / n=640): {ratio_10_to_640:.2f} (theoretical 1/sqrt(n) factor: {(640 / 10) ** 0.5:.2f})")  # => co-03
    assert 6.0 < ratio_10_to_640 < 10.0, "the shrink ratio must be close to the theoretical sqrt(n) factor"  # => co-03: sanity-checks the simulation itself
    print("MATCH: sampling error shrinks roughly like 1/sqrt(n) -- quantifiable, not mysterious")  # => co-03
    # => co-03: this is WHY a confidence interval (ex-04 onward) has a width that depends on n, not just on the observed rate

n=  10 | spread=0.1140 | ##############################################
n=  40 | spread=0.0604 | ########################
n= 160 | spread=0.0317 | #############
n= 640 | spread=0.0131 | #####
Spread ratio (n=10 / n=640): 8.73 (theoretical 1/sqrt(n) factor: 8.00)
MATCH: sampling error shrinks roughly like 1/sqrt(n) -- quantifiable, not mysterious

# learning/code/ex-04-interval-from-definition/interval_from_definition.py
"""Worked Example 4: Interval From Definition."""  # => co-04: this file's own restated purpose, doubling as its module __doc__
 
from __future__ import annotations  # => hygiene: postpones annotation evaluation, interpreter-version-agnostic
 
import math  # => co-04: sqrt for the standard error, no third-party dependency needed for the from-definition half
 
from statsmodels.stats.proportion import proportion_confint  # => co-04: the pinned library's own binomial-interval function
 
PASSES = 34  # => co-04: cases the system passed
TOTAL = 40  # => co-04: cases attempted -- a typical small eval-set size
Z_95 = 1.959963984540054  # => co-04: the two-sided 95% critical value of the standard normal distribution -- a fixed constant
 
 
def normal_interval_from_definition(passes: int, total: int, *, z: float) -> tuple[float, float]:  # => co-04: the textbook formula, in code
    """Return the normal-approximation confidence interval p_hat +/- z * standard_error."""  # => co-04: documents normal_interval_from_definition's contract -- no runtime output, just sets its __doc__
    p_hat = passes / total  # => co-04: the point estimate -- the observed pass rate itself
    standard_error = math.sqrt(p_hat * (1 - p_hat) / total)  # => co-04: how much p_hat is expected to vary run to run, from ex-03's own spread
    margin = z * standard_error  # => co-04: the interval's half-width
    return p_hat - margin, p_hat + margin  # => co-04: returns this computed value to the caller
 
 
if __name__ == "__main__":  # => co-04: entry point -- runs only when this file executes directly, not on import
    p_hat = PASSES / TOTAL  # => co-04: the point estimate this interval is centered on
    print(f"Point estimate: {PASSES}/{TOTAL} = {p_hat:.4f}")  # => co-04: the bare number ex-01's overview warns is not yet enough
    lo_def, hi_def = normal_interval_from_definition(PASSES, TOTAL, z=Z_95)  # => co-04: computed from the textbook formula directly
    print(f"From definition: [{lo_def:.4f}, {hi_def:.4f}]")  # => co-04: the interval this file derives itself
    lo_lib, hi_lib = proportion_confint(PASSES, TOTAL, method="normal")  # => co-04: the SAME computation, called from the pinned library
    print(f"From statsmodels (method='normal'): [{lo_lib:.4f}, {hi_lib:.4f}]")  # => co-04: the library's answer to the identical question
    assert math.isclose(lo_def, lo_lib, abs_tol=1e-9), "the from-definition lower bound must match the library's"  # => co-04
    assert math.isclose(hi_def, hi_lib, abs_tol=1e-9), "the from-definition upper bound must match the library's"  # => co-04
    print("MATCH: the hand-derived interval and the library's interval agree to within floating-point precision")  # => co-04
    # => co-04: this interval says the true pass rate is plausibly anywhere from ~70% to ~96% -- NOT that it is exactly 85%

Point estimate: 34/40 = 0.8500
From definition: [0.7393, 0.9607]
From statsmodels (method='normal'): [0.7393, 0.9607]
MATCH: the hand-derived interval and the library's interval agree to within floating-point precision

# learning/code/ex-05-normal-approximation-breaks/normal_approximation_breaks.py
"""Worked Example 5: The Normal Approximation Breaks."""  # => co-05: this file's own restated purpose, doubling as its module __doc__
 
from __future__ import annotations  # => hygiene: postpones annotation evaluation, interpreter-version-agnostic
 
import math  # => co-05: sqrt for the from-definition standard error
 
from statsmodels.stats.proportion import proportion_confint  # => co-05: the pinned library's own binomial-interval function
 
PASSES = 38  # => co-05: near-ceiling performance -- exactly the regime the normal approximation struggles with
TOTAL = 40  # => co-05: still a small n -- eval datasets are almost always this size or smaller
Z_95 = 1.959963984540054  # => co-05: the two-sided 95% critical value of the standard normal distribution
 
 
def normal_interval_unclipped(passes: int, total: int, *, z: float) -> tuple[float, float]:  # => co-05: the RAW textbook formula, no safety clip
    """Return p_hat +/- z * standard_error, exactly as the formula gives it -- may exceed [0, 1]."""  # => co-05: documents normal_interval_unclipped's contract -- no runtime output, just sets its __doc__
    p_hat = passes / total  # => co-05: the point estimate -- 0.95 here, right at the edge of the possible range
    standard_error = math.sqrt(p_hat * (1 - p_hat) / total)  # => co-05: shrinks toward zero as p_hat approaches 0 or 1 -- part of the problem
    margin = z * standard_error  # => co-05: the interval's half-width
    return p_hat - margin, p_hat + margin  # => co-05: UNCLIPPED -- may land outside the valid [0, 1] probability range
 
 
if __name__ == "__main__":  # => co-05: entry point -- runs only when this file executes directly, not on import
    p_hat = PASSES / TOTAL  # => co-05: 0.95 -- near the boundary where the normal approximation is known to misbehave
    print(f"Point estimate: {PASSES}/{TOTAL} = {p_hat:.4f}")  # => co-05: a rate near the ceiling, on a small n
    lo_def, hi_def = normal_interval_unclipped(PASSES, TOTAL, z=Z_95)  # => co-05: the raw, unclipped formula result
    print(f"From definition (unclipped): [{lo_def:.4f}, {hi_def:.4f}]")  # => co-05: prints the interval as the formula actually computes it
    assert hi_def > 1.0, "the unclipped normal-approximation upper bound must extend past the valid [0, 1] range"  # => co-05: the breakage this example demonstrates
    print(f"Upper bound exceeds 1.0 by {hi_def - 1.0:.4f} -- a probability greater than 100% is nonsense")  # => co-05: names the nonsensical result explicitly
 
    lo_lib, hi_lib = proportion_confint(PASSES, TOTAL, method="normal")  # => co-05: the SAME computation, called from the pinned library
    print(f"From statsmodels (method='normal'): [{lo_lib:.4f}, {hi_lib:.4f}]")  # => co-05: the library's own answer
    assert hi_lib == 1.0, "statsmodels silently clips the normal-approximation upper bound to 1.0"  # => co-05: the library hides the symptom rather than fixing it
    print("MATCH: the raw formula breaks past 1.0; the library's convenience clip HIDES that breakage rather than fixing it")  # => co-05
    # => co-05: clipping to [0, 1] is not a fix -- it silently discards the information that the method itself misbehaved here; ex-06 is the actual fix

Point estimate: 38/40 = 0.9500
From definition (unclipped): [0.8825, 1.0175]
Upper bound exceeds 1.0 by 0.0175 -- a probability greater than 100% is nonsense
From statsmodels (method='normal'): [0.8825, 1.0000]
MATCH: the raw formula breaks past 1.0; the library's convenience clip HIDES that breakage rather than fixing it

# learning/code/ex-06-wilson-and-clopper-pearson/wilson_and_clopper_pearson.py
"""Worked Example 6: Wilson and Clopper-Pearson."""  # => co-05: this file's own restated purpose, doubling as its module __doc__
 
from __future__ import annotations  # => hygiene: postpones annotation evaluation, interpreter-version-agnostic
 
from statsmodels.stats.proportion import proportion_confint  # => co-05: the pinned library's own binomial-interval function, three methods compared
 
PASSES = 38  # => co-05: the SAME near-ceiling data ex-05 showed breaking the normal approximation
TOTAL = 40  # => co-05: the SAME small n
 
 
if __name__ == "__main__":  # => co-05: entry point -- runs only when this file executes directly, not on import
    p_hat = PASSES / TOTAL  # => co-05: the point estimate this interval is centered on
    print(f"Point estimate: {PASSES}/{TOTAL} = {p_hat:.4f}")  # => co-05: the same 0.95 that broke the normal approximation
 
    lo_normal, hi_normal = proportion_confint(PASSES, TOTAL, method="normal")  # => co-05: the BROKEN method, for direct comparison
    print(f"normal (broken, clipped): [{lo_normal:.4f}, {hi_normal:.4f}]")  # => co-05: repeats ex-05's clipped result for contrast
 
    lo_wilson, hi_wilson = proportion_confint(PASSES, TOTAL, method="wilson")  # => co-05: the Wilson score interval -- the standard small-n corrective
    print(f"wilson: [{lo_wilson:.4f}, {hi_wilson:.4f}]")  # => co-05: the Wilson interval's bounds
    assert 0.0 <= lo_wilson and hi_wilson <= 1.0, "the Wilson interval must stay within the valid [0, 1] range"  # => co-05: the property ex-05's normal interval lacked
 
    lo_beta, hi_beta = proportion_confint(PASSES, TOTAL, method="beta")  # => co-05: the Clopper-Pearson (exact, beta-distribution-based) interval
    print(f"clopper-pearson (beta): [{lo_beta:.4f}, {hi_beta:.4f}]")  # => co-05: the Clopper-Pearson interval's bounds
    assert 0.0 <= lo_beta and hi_beta <= 1.0, "the Clopper-Pearson interval must stay within the valid [0, 1] range"  # => co-05: same property, different method
 
    assert hi_wilson < 1.0, "Wilson must give a genuinely sub-1.0 upper bound, not a clip"  # => co-05: distinguishes a real fix from a cosmetic clip
    assert hi_beta < 1.0, "Clopper-Pearson must give a genuinely sub-1.0 upper bound, not a clip"  # => co-05: same distinction
    print("MATCH: both Wilson and Clopper-Pearson stay in [0, 1] by CONSTRUCTION, not by clipping a broken result")  # => co-05
    # => co-05: Clopper-Pearson is provably conservative (coverage >= 95%); Wilson is closer to exactly 95% on average -- both are correct where 'normal' is not

Point estimate: 38/40 = 0.9500
normal (broken, clipped): [0.8825, 1.0000]
wilson: [0.8350, 0.9862]
clopper-pearson (beta): [0.8308, 0.9939]
MATCH: both Wilson and Clopper-Pearson stay in [0, 1] by CONSTRUCTION, not by clipping a broken result

# learning/code/ex-07-interval-width-vs-n/interval_width_vs_n.py
"""Worked Example 7: Interval Width vs. n."""  # => co-04: this file's own restated purpose, doubling as its module __doc__
 
from __future__ import annotations  # => hygiene: postpones annotation evaluation, interpreter-version-agnostic
 
from statsmodels.stats.proportion import proportion_confint  # => co-04: the pinned library's own binomial-interval function
 
PASS_RATE = 0.85  # => co-06: held fixed across every n below -- only the sample size changes
SAMPLE_SIZES = (10, 20, 40, 80, 160, 320, 640)  # => co-06: doubling each step, spanning a small eval set to a large one
 
 
if __name__ == "__main__":  # => co-06: entry point -- runs only when this file executes directly, not on import
    widths: list[float] = []  # => co-04: one Wilson-interval width per sample size, for the diminishing-return check below
    for n in SAMPLE_SIZES:  # => co-06: one interval computation per candidate sample size
        passes = round(PASS_RATE * n)  # => co-06: keep the observed rate fixed near 0.85 at every n
        lo, hi = proportion_confint(passes, n, method="wilson")  # => co-04: the well-behaved small-n interval from ex-06
        width = hi - lo  # => co-04: the interval's full width -- what a reader actually cares about
        widths.append(width)  # => co-04: recorded for the shrink-rate check below
        bar = "#" * round(width * 200)  # => co-04: a plain-text bar -- longer bar means a wider, less informative interval
        print(f"n={n:>4} | width={width:.4f} | {bar}")  # => co-04: prints size, width, and its ASCII-bar visualization
    assert widths[0] > widths[-1], "interval width must shrink as n grows"  # => co-04: the qualitative claim
    halving_index = next(i for i in range(1, len(widths)) if widths[i] < widths[0] / 2)  # => co-06: where width first drops below half its n=10 value
    print(f"Width first drops below half its n=10 value at n={SAMPLE_SIZES[halving_index]}")  # => co-06: names the diminishing-return point
    last_gain = widths[-2] - widths[-1]  # => co-06: the width saved by the LAST doubling (320 -> 640)
    first_gain = widths[0] - widths[1]  # => co-06: the width saved by the FIRST doubling (10 -> 20)
    print(f"Width saved by 10->20: {first_gain:.4f} | width saved by 320->640: {last_gain:.4f}")  # => co-06: shows diminishing returns
    assert first_gain > last_gain, "the first doubling must buy more width reduction than the last doubling"  # => co-06: the diminishing-return claim itself
    print("MATCH: each doubling of n buys a shrinking amount of extra precision -- diminishing, not free")  # => co-06
    # => co-04,co-06: this shape is WHY ex-08 asks 'how many cases for THIS effect' instead of 'as many cases as possible'

n=  10 | width=0.4532 | ###########################################################################################
n=  20 | width=0.3081 | ##############################################################
n=  40 | width=0.2201 | ############################################
n=  80 | width=0.1562 | ###############################
n= 160 | width=0.1106 | ######################
n= 320 | width=0.0782 | ################
n= 640 | width=0.0553 | ###########
Width first drops below half its n=10 value at n=40
Width saved by 10->20: 0.1451 | width saved by 320->640: 0.0229
MATCH: each doubling of n buys a shrinking amount of extra precision -- diminishing, not free

# learning/code/ex-08-how-many-cases-for-this-effect/how_many_cases.py
"""Worked Example 8: How Many Cases Do I Need For This Effect."""  # => co-06: this file's own restated purpose, doubling as its module __doc__
 
from __future__ import annotations  # => hygiene: postpones annotation evaluation, interpreter-version-agnostic
 
import math  # => co-06: ceil -- a sample size must be a whole number of cases
 
from statsmodels.stats.proportion import proportion_confint, samplesize_confint_proportion  # => co-06: the pinned library's sample-size solver
 
ANTICIPATED_RATE = 0.85  # => co-06: the pass rate the team expects, from a pilot run or a prior eval
TARGET_HALF_WIDTH = 0.05  # => co-06: the precision demanded -- "I want the interval within +/-5 points"
 
 
if __name__ == "__main__":  # => co-06: entry point -- runs only when this file executes directly, not on import
    raw_n = samplesize_confint_proportion(proportion=ANTICIPATED_RATE, half_length=TARGET_HALF_WIDTH, alpha=0.05, method="normal")  # => co-06: solves the CI-width formula for n
    print(f"Solved n (continuous): {raw_n:.2f}")  # => co-06: the exact real-valued solution to the width equation
    required_n = math.ceil(raw_n)  # => co-06: round UP -- a fractional case cannot be collected, and rounding down would miss the target
    print(f"Required n (rounded up to a whole case count): {required_n}")  # => co-06: the number a team actually plans to collect
    assert required_n == 196, "the required n for this anticipated rate and target half-width must be 196"  # => co-06: pins the exact planning number
 
    passes_at_required_n = round(ANTICIPATED_RATE * required_n)  # => co-06: what collecting exactly required_n cases would look like
    lo, hi = proportion_confint(passes_at_required_n, required_n, method="normal")  # => co-06: the SAME normal method this n was solved for
    achieved_half_width = (hi - lo) / 2  # => co-06: verify the PLANNED n actually delivers the target precision
    print(f"Achieved half-width at n={required_n}: {achieved_half_width:.4f} (target was {TARGET_HALF_WIDTH})")  # => co-06
    assert achieved_half_width <= TARGET_HALF_WIDTH + 1e-3, "collecting the solved n must actually achieve the target half-width"  # => co-06: closes the loop
 
    smaller_n = required_n - 100  # => co-06: a plausible temptation -- "let's just collect fewer cases"
    lo_small, hi_small = proportion_confint(round(ANTICIPATED_RATE * smaller_n), smaller_n, method="normal")  # => co-06: what a shortcut n actually buys
    print(f"With only n={smaller_n}, half-width would be: {(hi_small - lo_small) / 2:.4f} -- misses the {TARGET_HALF_WIDTH} target")  # => co-06
    assert (hi_small - lo_small) / 2 > TARGET_HALF_WIDTH, "an under-sized n must miss the precision target"  # => co-06: names the shortcut's real cost
    print("MATCH: the required n is a design question answered BEFORE collecting, not a guess made after")  # => co-06
    # => co-06: this is the number a sampling plan (ex-20) states and justifies, in writing, before a single case is drawn

Solved n (continuous): 195.91
Required n (rounded up to a whole case count): 196
Achieved half-width at n=196: 0.0497 (target was 0.05)
With only n=96, half-width would be: 0.0706 -- misses the 0.05 target
MATCH: the required n is a design question answered BEFORE collecting, not a guess made after

# learning/code/ex-09-forty-cases-cannot-see-it/forty_cases_cannot_see_it.py
"""Worked Example 9: Forty Cases Cannot See It."""  # => co-19: this file's own restated purpose, doubling as its module __doc__
 
from __future__ import annotations  # => hygiene: postpones annotation evaluation, interpreter-version-agnostic
 
import random  # => co-06: draws each simulated system's cases
 
from statsmodels.stats.proportion import proportions_ztest  # => co-19: the pinned library's own two-proportion significance test
 
TRUE_RATE_BASELINE = 0.78  # => co-19: system A's real, unobservable pass rate
TRUE_RATE_CANDIDATE = 0.90  # => co-19: system B's real, unobservable pass rate -- a genuine 12-point improvement
ILLUSTRATION_SEED = 9  # => co-06: a specific, fixed seed pair used for the single-run illustration below
 
 
def draw_passes(true_rate: float, n: int, *, seed: int) -> int:  # => co-06: one simulated eval run's pass COUNT
    """Draw n Bernoulli(true_rate) trials and return the number that passed."""  # => co-06: documents draw_passes's contract -- no runtime output, just sets its __doc__
    rng = random.Random(seed)  # => co-06: a fresh generator per (system, n, seed) draw
    return sum(1 for _ in range(n) if rng.random() < true_rate)  # => co-06: count of Bernoulli successes
 
 
def run_significance_test(n: int, *, seed: int) -> float:  # => co-19: one full A/B test at sample size n -- returns its p-value
    """Draw n cases for each system and return the two-proportion z-test p-value."""  # => co-19: documents run_significance_test's contract -- no runtime output, just sets its __doc__
    passes_a = draw_passes(TRUE_RATE_BASELINE, n, seed=seed * 2)  # => co-06: system A's simulated pass count at this n
    passes_b = draw_passes(TRUE_RATE_CANDIDATE, n, seed=seed * 2 + 1)  # => co-06: system B's simulated pass count at this n
    _, p_value = proportions_ztest([passes_a, passes_b], [n, n])  # => co-19: the real, genuine 12-point effect, tested at this n
    return p_value  # => co-19: returns this computed value to the caller
 
 
if __name__ == "__main__":  # => co-19: entry point -- runs only when this file executes directly, not on import
    p_at_40 = run_significance_test(40, seed=ILLUSTRATION_SEED)  # => co-19: ONE specific illustrative run at the small size
    p_at_400 = run_significance_test(400, seed=ILLUSTRATION_SEED)  # => co-19: the SAME true effect, the SAME seed family, ten times the cases
    print(f"n=40  -> p-value {p_at_40:.4f} ({'significant' if p_at_40 < 0.05 else 'NOT significant'})")  # => co-19: prints the small-n verdict
    print(f"n=400 -> p-value {p_at_400:.4f} ({'significant' if p_at_400 < 0.05 else 'NOT significant'})")  # => co-19: prints the large-n verdict
    assert p_at_40 >= 0.05, "the illustrative n=40 run must fail to detect the real effect"  # => co-19: the invisible-at-n=40 claim
    assert p_at_400 < 0.05, "the illustrative n=400 run must detect the same real effect"  # => co-19: the visible-at-n=400 claim
 
    trials = 500  # => co-19: repeats for the Monte Carlo detection-rate check below -- one illustrative run could be a fluke either way
    detections_40 = sum(1 for t in range(1000, 1000 + trials) if run_significance_test(40, seed=t) < 0.05)  # => co-19: how OFTEN n=40 catches this real effect
    detections_400 = sum(1 for t in range(1000, 1000 + trials) if run_significance_test(400, seed=t) < 0.05)  # => co-19: how often n=400 catches it
    print(f"Detection rate across {trials} independent trials: n=40 -> {detections_40 / trials:.1%} | n=400 -> {detections_400 / trials:.1%}")  # => co-19
    assert detections_40 / trials < 0.5, "n=40 must detect this real effect LESS than half the time across repeated trials"  # => co-19
    assert detections_400 / trials > 0.95, "n=400 must detect the same real effect in almost every trial"  # => co-19
    print("MATCH: the SAME real 12-point effect is invisible most of the time at n=40, and reliable at n=400")  # => co-19
    # => co-06,co-19: a non-significant result at n=40 is not 'no effect' -- it is 'not enough cases to see this effect', which ex-40 revisits

n=40  -> p-value 0.1045 (NOT significant)
n=400 -> p-value 0.0000 (significant)
Detection rate across 500 independent trials: n=40 -> 30.6% | n=400 -> 100.0%
MATCH: the SAME real 12-point effect is invisible most of the time at n=40, and reliable at n=400

# learning/code/ex-10-overlapping-intervals-are-not-a-test/overlapping_intervals.py
"""Worked Example 10: Overlapping Intervals Are Not a Test."""  # => co-18: this file's own restated purpose, doubling as its module __doc__
 
from __future__ import annotations  # => hygiene: postpones annotation evaluation, interpreter-version-agnostic
 
from statsmodels.stats.contingency_tables import mcnemar  # => co-18: the pinned library's own PAIRED significance test
from statsmodels.stats.proportion import proportion_confint  # => co-04: the pinned library's own binomial-interval function
 
N = 50  # => co-04: fifty paired cases -- baseline and candidate run on the IDENTICAL cases
BOTH_PASS = 29  # => co-18: cases where baseline AND candidate both passed
ONLY_BASELINE_PASSED = 3  # => co-18: cases where baseline passed but candidate REGRESSED
ONLY_CANDIDATE_PASSED = 13  # => co-18: cases where baseline failed but candidate FIXED it
BOTH_FAIL = 5  # => co-18: cases where both still fail -- BOTH_PASS + ONLY_BASELINE_PASSED + ONLY_CANDIDATE_PASSED + BOTH_FAIL must equal N
 
 
if __name__ == "__main__":  # => co-18: entry point -- runs only when this file executes directly, not on import
    assert BOTH_PASS + ONLY_BASELINE_PASSED + ONLY_CANDIDATE_PASSED + BOTH_FAIL == N, "the four paired-outcome counts must sum to N"  # => co-04: sanity check on the fixture
    baseline_passes = BOTH_PASS + ONLY_BASELINE_PASSED  # => co-04: baseline's OWN total pass count, ignoring pairing
    candidate_passes = BOTH_PASS + ONLY_CANDIDATE_PASSED  # => co-04: candidate's OWN total pass count, ignoring pairing
    print(f"Baseline: {baseline_passes}/{N} = {baseline_passes / N:.2%} | Candidate: {candidate_passes}/{N} = {candidate_passes / N:.2%}")  # => co-04
 
    lo_b, hi_b = proportion_confint(baseline_passes, N, method="wilson")  # => co-04: baseline's interval, treating the two runs as INDEPENDENT samples
    lo_c, hi_c = proportion_confint(candidate_passes, N, method="wilson")  # => co-04: candidate's interval, same (wrong) independence assumption
    print(f"Baseline Wilson CI:  [{lo_b:.4f}, {hi_b:.4f}]")  # => co-04: prints baseline's unpaired-style interval
    print(f"Candidate Wilson CI: [{lo_c:.4f}, {hi_c:.4f}]")  # => co-04: prints candidate's unpaired-style interval
    intervals_overlap = lo_c <= hi_b  # => co-04: the naive eyeball check a team might reach for
    print(f"Intervals overlap: {intervals_overlap}")  # => co-04: True here -- looks like "maybe no real difference"
    assert intervals_overlap, "the two independent-style intervals must overlap for this demo to make its point"  # => co-04: the setup this example needs
 
    table = [[BOTH_PASS, ONLY_BASELINE_PASSED], [ONLY_CANDIDATE_PASSED, BOTH_FAIL]]  # => co-18: the 2x2 PAIRED contingency table McNemar actually needs
    result = mcnemar(table, exact=True)  # => co-18: uses ONLY the discordant pairs -- where baseline and candidate actually disagreed
    print(f"Paired McNemar test: statistic={result.statistic}, p-value={result.pvalue:.4f}")  # => co-18: the test that actually respects the pairing
    assert result.pvalue < 0.05, "the paired McNemar test must find a significant difference despite the overlapping unpaired intervals"  # => co-18
    print("MATCH: overlapping unpaired intervals said 'maybe no difference' -- the paired test says 'yes, significant difference'")  # => co-18
    # => co-04,co-18: eyeballing two intervals for overlap ignores that the SAME cases were measured twice; McNemar uses that pairing directly, and is far more sensitive

Baseline: 32/50 = 64.00% | Candidate: 42/50 = 84.00%
Baseline Wilson CI:  [0.5014, 0.7586]
Candidate Wilson CI: [0.7149, 0.9166]
Intervals overlap: True
Paired McNemar test: statistic=3.0, p-value=0.0213
MATCH: overlapping unpaired intervals said 'maybe no difference' -- the paired test says 'yes, significant difference'

# learning/code/ex-11-interval-figure/interval_figure.py
"""Worked Example 11: A Pass-Rate Figure With Intervals."""  # => co-01: this file's own restated purpose, doubling as its module __doc__
 
from __future__ import annotations  # => hygiene: postpones annotation evaluation, interpreter-version-agnostic
 
from statsmodels.stats.proportion import proportion_confint  # => co-04: the pinned library's own binomial-interval function
 
VARIANTS = {  # => co-24: three prompt variants, each with its own pass count and its own n -- eval sets are rarely all the same size
    "prompt-v1 (baseline)": (30, 40),  # => co-24: (passes, n) -- a genuinely small pilot eval set
    "prompt-v2": (36, 40),  # => co-24: same n as the baseline -- directly comparable
    "prompt-v3": (58, 70),  # => co-24: a DIFFERENT n -- collected later, on a larger dataset
}  # => co-24: closes VARIANTS
 
 
def render_row(name: str, passes: int, n: int) -> str:  # => co-01: renders ONE labeled row of the figure -- estimate, bar, interval, n, method
    """Return one printable row: the point estimate, an ASCII bar, and the labeled Wilson interval."""  # => co-01: documents render_row's contract -- no runtime output, just sets its __doc__
    p_hat = passes / n  # => co-04: this variant's own point estimate
    lo, hi = proportion_confint(passes, n, method="wilson")  # => co-05: Wilson -- the small-n-appropriate method every row uses, uniformly
    bar = "#" * round(p_hat * 40)  # => co-01: a plain-text bar proportional to the point estimate
    return f"{name:<22} | {bar:<40} | {p_hat:.2%}  CI=[{lo:.2%}, {hi:.2%}]  n={n}  method=wilson"  # => co-24: EVERY field the reporting discipline demands, in one row
 
 
if __name__ == "__main__":  # => co-01: entry point -- runs only when this file executes directly, not on import
    print("Pass rate by prompt variant (95% Wilson interval, labeled with n and method):")  # => co-24: states the method up front, once, for the whole figure
    rows = [render_row(name, passes, n) for name, (passes, n) in VARIANTS.items()]  # => co-01: one rendered row per variant
    for row in rows:  # => co-01: prints the whole figure, one labeled row per variant
        print(row)  # => co-01: the figure itself -- this IS the artifact a reader would see
    for row in rows:  # => co-24: every row must carry n and method, never just a bar and a percentage
        assert "n=" in row and "method=wilson" in row, "every row must state its own n and its own method"  # => co-24
    print("MATCH: every bar is labeled with its own point estimate, interval, n, and method -- no row is a bare number")  # => co-01
    # => co-01,co-24: a figure of bars with no interval invites reading noise as a real ranking; this figure structurally cannot omit the interval

Pass rate by prompt variant (95% Wilson interval, labeled with n and method):
prompt-v1 (baseline)   | ##############################           | 75.00%  CI=[59.81%, 85.81%]  n=40  method=wilson
prompt-v2              | ####################################     | 90.00%  CI=[76.95%, 96.04%]  n=40  method=wilson
prompt-v3              | #################################        | 82.86%  CI=[72.38%, 89.91%]  n=70  method=wilson
MATCH: every bar is labeled with its own point estimate, interval, n, and method -- no row is a bare number

# learning/code/ex-12-report-estimate-interval-n-method/reporting_helper.py
"""Worked Example 12: A Reporting Helper -- Estimate, Interval, n, Method."""  # => co-24: this file's own restated purpose, doubling as its module __doc__
 
from __future__ import annotations  # => hygiene: postpones annotation evaluation, interpreter-version-agnostic
 
from dataclasses import dataclass  # => co-24: a typed record -- every field below is REQUIRED, none optional
 
from statsmodels.stats.proportion import proportion_confint  # => co-04: the pinned library's own binomial-interval function
 
 
@dataclass(frozen=True)  # => co-24: frozen -- a reported figure is a fact about a completed run, not something later code can quietly mutate
class ReportedRate:  # => co-24: the reportable unit this whole course insists on -- never just a percentage
    estimate: float  # => co-01: the point estimate itself
    ci_low: float  # => co-04: the interval's lower bound
    ci_high: float  # => co-04: the interval's upper bound
    n: int  # => co-24: the sample size this estimate rests on
    method: str  # => co-24: the exact method used to compute the interval -- "wilson", "beta", never left implicit
 
    def __str__(self) -> str:  # => co-24: how this record prints -- every field visible, every time
        return f"{self.estimate:.2%} CI=[{self.ci_low:.2%}, {self.ci_high:.2%}] n={self.n} method={self.method}"  # => co-24: a bare number CANNOT escape this format
 
 
def report_rate(passes: int, n: int, *, method: str = "wilson") -> ReportedRate:  # => co-24: the ONE function every pass-rate report in this course routes through
    """Compute a pass rate and package it with its interval, n, and method -- never returns a bare float."""  # => co-24: documents report_rate's contract -- no runtime output, just sets its __doc__
    estimate = passes / n  # => co-01: the point estimate
    ci_low, ci_high = proportion_confint(passes, n, method=method)  # => co-04: the interval, computed with the STATED method
    return ReportedRate(estimate=estimate, ci_low=ci_low, ci_high=ci_high, n=n, method=method)  # => co-24: returns this computed value to the caller -- ALWAYS all four fields
 
 
if __name__ == "__main__":  # => co-24: entry point -- runs only when this file executes directly, not on import
    report = report_rate(34, 40)  # => co-24: a normal-sized eval run
    print(f"Report: {report}")  # => co-24: prints every required field, via ReportedRate.__str__
    assert isinstance(report, ReportedRate), "report_rate must always return a ReportedRate, never a bare float"  # => co-24: the type-level guarantee
 
    tiny_report = report_rate(9, 10)  # => co-05: a near-ceiling, very small n -- exactly where Wilson matters most
    print(f"Tiny-n report: {tiny_report}")  # => co-24: the same four fields, at a much smaller n
    assert tiny_report.n == 10 and tiny_report.method == "wilson", "n and method must be exactly what was passed in, never inferred silently"  # => co-24
    assert 0.0 <= tiny_report.ci_low <= tiny_report.estimate <= tiny_report.ci_high <= 1.0, "the interval must bracket the estimate and stay in [0, 1]"  # => co-04
    print("MATCH: both reports carry estimate, interval, n, and method -- neither can be quoted as a bare percentage")  # => co-24
    # => co-01,co-04,co-24: this is the function every LATER worked example in this course calls to print a pass rate -- never print(f'{rate:.0%}') alone again

Report: 85.00% CI=[70.93%, 92.94%] n=40 method=wilson
Tiny-n report: 90.00% CI=[59.58%, 98.21%] n=10 method=wilson
MATCH: both reports carry estimate, interval, n, and method -- neither can be quoted as a bare percentage