solution-erp/scripts/applied-eval-nokey.ps1

# applied-eval-nokey.ps1 - Harness-16 UPDATE: Applied% WITHOUT an API key + adversarial gate - S102 (2026-07-06)
#
# ADAPT of AI_INFRA broadcast H16-update (owner-directive, 2026-07-04/05). SISTER-side KHUNG,
# tailored to SOLUTION_ERP (self=se). Function-floor kept; form tailored. NO copy-paste.
#
# ============================================================================
# WHAT THIS IS (and, loudly, what it is NOT)
# ============================================================================
#   Two SEPARATE things live in this file, and they must never be confused:
#
#   [A] SCRIPTABLE / DETERMINISTIC ($0)  <-- the REAL teeth of this script, runs here, now.
#       An ADVERSARIAL GATE + a MERGE + a 3-case SELFTEST. Given a set of judge
#       verdicts (each {id, score, is_control, control_expected_low}), the gate
#       decides BLOCK / PASS, and the merge folds surviving real questions into a
#       single Applied% number. Zero model calls. Pure arithmetic + set logic.
#
#   [B] RUNTIME / 2-AGENT-PASS  <-- DOCUMENTED ONLY here; runs later, via a workflow,
#       when the sample-questions corpus is mature. NOT executed by this script.
#       This block exists so the mechanism is written down and auditable, not so a
#       number is produced today.
#
#   The honesty ceiling (printed by -Honest and at the end of -Selftest):
#     "We have an adversarial GATE (MANUAL: you supply the verdicts + the control
#      list) + a 3-case selftest. We do NOT claim any live number has passed this
#      gate. Applied% numbers are DEFERRED until the sample-questions mature AND an
#      independent, different-session judge scores them. This file is a MECHANISM,
#      NOT a number."
#
# ============================================================================
# [B] THE NO-KEY APPLIED% FLOW (documented; RUNTIME, not run here)
# ============================================================================
#   The point: measure "does the hot-feed actually get APPLIED" without any paid
#   API / external key -- using two in-harness sub-agents instead of an oracle key.
#
#   STEP 1 - PROBE with the ANSWER BLANKED (the "candidate" sub-agent)
#     * Take a sample question that has a known answer-anchor (the gotcha #, the
#       mark-id, the file:line, the root-cause phrase). BLANK IT OUT: strip the
#       answer-anchor from the probe so the candidate cannot read the answer back.
#     * Spawn a sub-agent IN THE RIGHT PERSONA for that question (the role whose
#       diary is supposed to carry that memory), carrying that role's diary as it
#       normally would. The candidate is told NOTHING about being graded and sees
#       NO answer key -- it just does the task and emits its working (a "diary" of
#       what it recalled / applied for that role).
#
#   STEP 2 - JUDGE in a CLEAN, SEPARATE CONTEXT (the "referee" sub-agent)
#     * A DIFFERENT sub-agent, in a FRESH context window, DIFFERENT session/turn
#       from the candidate (no shared scratch, no candidate's chain-of-thought),
#       is GIVEN the ground-truth answer-anchor and scores the candidate 0..1 on
#       whether the required memory was actually recalled/applied.
#     * Control/sentinel probes are mixed in (see below): questions the referee
#       SHOULD score low (answer deliberately absent from any diary). They are the
#       adversarial tripwire for a too-lenient judge.
#
#   STEP 3 - MERGE LOCALLY ($0) -> Applied%
#     * The verdicts come back here. The GATE (block A, deterministic) runs FIRST:
#       drop the whole batch if any control scored high, or if any control was
#       silently skipped. Only then MERGE: strip controls, average the real
#       scores, Applied% = round(mean * 100). Unscored (null) real questions are
#       NOT invented -- excluded + counted (N excluded).
#
# ============================================================================
# 4 CAVEATS (MANDATORY -- this measurement is weak in exactly these 4 ways)
# ============================================================================
#   (a) POINT-IN-TIME PROXY, not real behavior. A judged score at one instant is a
#       proxy for "was it applied", NOT observed downstream behavior over a real
#       task. High Applied% != the memory changed what the system actually did.
#
#   (b) SHARED MODEL-FAMILY NOT NEUTRALIZED. Candidate and referee share the same
#       base model + the same corpus/RAG. Blanking the ANSWER-ANCHOR does NOT blank
#       the WHOLE CORPUS -- the referee (and even the candidate) may reconstruct the
#       answer from common training / shared store rather than from the diary under
#       test. The ONLY model-independent signal is RECURRENCE-FREQUENCY anchoring
#       (does the SAME bug/mark stop recurring over sessions) -- that lives in the
#       OLD measurable-now block (recurrence tokens / AS-class registry), NOT here.
#       This no-key score is the WEAKER, model-entangled cousin of that.
#
#   (c) BLINDING IS AN INSTRUCTION, NOT AN ENFORCEMENT. "Candidate doesn't see the
#       answer / doesn't know it's graded" and "referee is a clean separate context"
#       are DIRECTIVES to the spawn, not a sandbox that makes leakage impossible. A
#       shared file, a chatty prompt, or a resumed session can defeat it. Treat the
#       separation as best-effort, and prefer the control tripwire over trusting it.
#
#   (d) NO AGE DIMENSION YET, SMALL SAMPLE. This does not measure retention-over-age
#       (does an OLD memory still get applied), and the sample-question set is small.
#       So even a clean pass is a thin, young signal -- a FLAG, never a grade.
#
# ============================================================================
# NON-NEGOTIABLES (Harness-11 / Harness-16 house rules)
# ============================================================================
#   (1) NO-API  : deterministic arithmetic + set logic ONLY. NEVER calls a model.
#                 (The 2-agent pass in block [B] is RUNTIME via a workflow, not here.)
#   (2) READ-ONLY on the budget : this script NEVER reads-to-tune nor writes the
#       memory-budget.json caps. Budget numbers are the OWNER's authority (anh /
#       chu-du-an); scripts only ever measure-and-report. This file does not even
#       need the budget -- it touches NO owner-set number.
#   (3) PS 5.1, ASCII-only script body (gotcha #30). Any glyph via [char]0xXXXX
#       code-point; any target file read with -Encoding UTF8 explicitly.
#   (4) Exit 0 in normal + -Honest runs (measure-and-report, NOT a build gate).
#       -Selftest is the ONE exception: it exits 0 only if all 3 cases behave as
#       specified, non-zero otherwise, so CI/a human can trust the gate has teeth.
#
# Usage:
#   powershell.exe -ExecutionPolicy Bypass -File scripts\applied-eval-nokey.ps1 -Selftest
#   powershell.exe -ExecutionPolicy Bypass -File scripts\applied-eval-nokey.ps1 -Honest
#   powershell.exe -ExecutionPolicy Bypass -File scripts\applied-eval-nokey.ps1            # prints honesty rung + usage

param(
    [switch]$Selftest = $false,
    [switch]$Honest   = $false
)

$ErrorActionPreference = 'Stop'
$CHECK = [char]0x2705   # green check, kept as code-point (ASCII body, gotcha #30)
$CROSS = [char]0x274C   # red cross,  kept as code-point

# ---------------------------------------------------------------------------
# Constant: a control (sentinel) probe should score BELOW this to be "correctly
# rejected". At-or-above = the referee wrongly credited a control = too lenient.
# ---------------------------------------------------------------------------
Set-Variable -Name CONTROL_HIGH_THRESHOLD -Value 0.5 -Option Constant

# ===========================================================================
# [A] ADVERSARIAL GATE  (deterministic -- the real, tested teeth)
# ===========================================================================
# INPUT  : array of verdict objects (PSCustomObject or hashtable), each with
#            id                    : string, probe id
#            score                 : double 0..1, OR $null = judge did not score it
#            is_control            : bool,   true = sentinel/control probe
#            control_expected_low  : bool,   (advisory) a control is expected low
# OUTPUT : PSCustomObject { Decision = 'BLOCK'|'PASS'; Reasons = string[];
#                           Controls = int; ControlsHighOrMissing = int }
#
# LOGIC (fail-closed -- any doubt about the referee's rigor => BLOCK the batch):
#   * If ANY control has a null/missing score  -> BLOCK
#     (a control that was silently dropped means the judge quietly skipped the
#      tripwire; we do NOT let a batch through with an un-checked control.)
#   * If ANY control scored >= CONTROL_HIGH_THRESHOLD -> BLOCK
#     (the judge credited a probe whose answer is deliberately absent = too
#      lenient => every real score from this same judge is suspect => drop all.)
#   * Else (every control present AND every control < threshold) -> PASS.
# ---------------------------------------------------------------------------
function Get-ScoreOrNull($v) {
    # Read .score whether $v is a hashtable or a PSCustomObject; return $null if
    # absent or explicitly null. Do NOT coerce a missing key into 0 (that would
    # hide a skipped control).
    if ($null -eq $v) { return $null }
    if ($v -is [System.Collections.IDictionary]) {
        if (-not $v.Contains('score')) { return $null }
        $s = $v['score']
    } else {
        $prop = $v.PSObject.Properties['score']
        if ($null -eq $prop) { return $null }
        $s = $prop.Value
    }
    if ($null -eq $s) { return $null }
    return [double]$s
}

function Get-BoolField($v, [string]$name) {
    if ($null -eq $v) { return $false }
    if ($v -is [System.Collections.IDictionary]) {
        if (-not $v.Contains($name)) { return $false }
        return [bool]$v[$name]
    }
    $prop = $v.PSObject.Properties[$name]
    if ($null -eq $prop) { return $false }
    return [bool]$prop.Value
}

function Invoke-AdversarialGate {
    param([object[]]$Verdicts)

    $reasons               = New-Object System.Collections.Generic.List[string]
    $controls              = 0
    $controlsHighOrMissing = 0

    foreach ($v in $Verdicts) {
        $isControl = Get-BoolField $v 'is_control'
        if (-not $isControl) { continue }
        $controls++
        $id    = if (Get-BoolField $v 'id') { $v.id } else { $v.id }   # id passthrough (may be $null)
        $score = Get-ScoreOrNull $v

        if ($null -eq $score) {
            $controlsHighOrMissing++
            $reasons.Add("control '$($v.id)' has NO score (silently skipped) -> BLOCK")
            continue
        }
        if ($score -ge $CONTROL_HIGH_THRESHOLD) {
            $controlsHighOrMissing++
            $reasons.Add("control '$($v.id)' scored $score >= $CONTROL_HIGH_THRESHOLD (judge too lenient) -> BLOCK")
        }
    }

    if ($controls -eq 0) {
        # No tripwire at all: cannot certify the judge's rigor. Fail-closed.
        $reasons.Add("no control probes present -> cannot certify judge rigor -> BLOCK")
        return [pscustomobject]@{
            Decision = 'BLOCK'; Reasons = $reasons.ToArray()
            Controls = 0; ControlsHighOrMissing = 0
        }
    }

    $decision = if ($controlsHighOrMissing -gt 0) { 'BLOCK' } else { 'PASS' }
    return [pscustomobject]@{
        Decision = $decision; Reasons = $reasons.ToArray()
        Controls = $controls; ControlsHighOrMissing = $controlsHighOrMissing
    }
}

# ===========================================================================
# MERGE  (deterministic -- only reached AFTER the gate returns PASS)
# ===========================================================================
# Strip controls, average the REAL scores, Applied% = round(mean * 100).
# A real question the judge did not score (null) is NOT invented -> excluded +
# counted in ExcludedNull. If, after excluding controls + nulls, no real scored
# question remains, AppliedPct = $null (report "no scored real questions"),
# NEVER a fabricated 0.
# ---------------------------------------------------------------------------
function Merge-AppliedScore {
    param([object[]]$Verdicts)

    $realScores   = New-Object System.Collections.Generic.List[double]
    $excludedNull = 0
    $controlCount = 0

    foreach ($v in $Verdicts) {
        if (Get-BoolField $v 'is_control') { $controlCount++; continue }
        $score = Get-ScoreOrNull $v
        if ($null -eq $score) { $excludedNull++; continue }   # do NOT fabricate a number
        $realScores.Add($score)
    }

    if ($realScores.Count -eq 0) {
        return [pscustomobject]@{
            AppliedPct = $null; RealN = 0; ExcludedNull = $excludedNull
            ControlN = $controlCount; Mean = $null
        }
    }
    $mean = ($realScores | Measure-Object -Average).Average
    return [pscustomobject]@{
        AppliedPct   = [int][Math]::Round($mean * 100)
        RealN        = $realScores.Count
        ExcludedNull = $excludedNull
        ControlN     = $controlCount
        Mean         = [Math]::Round($mean, 4)
    }
}

# ===========================================================================
# Honesty rung (printed on plain run + at end of -Selftest)
# ===========================================================================
function Write-HonestyRung {
    Write-Host ""
    Write-Host "--- HONESTY RUNG (applied-eval-nokey) ---"
    Write-Host "We HAVE : an adversarial GATE (MANUAL -- you supply the verdicts + the control list)"
    Write-Host "          + a 3-case selftest proving the gate BLOCKs a lenient/dropped control."
    Write-Host "We do NOT claim any live number has passed this gate."
    Write-Host "Applied% numbers are DEFERRED until (a) the sample-questions mature AND"
    Write-Host "          (b) an independent, DIFFERENT-SESSION judge scores them."
    Write-Host "This file is a MECHANISM, NOT a number."
    Write-Host ""
    Write-Host "4 CAVEATS baked in: (a) point-in-time proxy != real behavior; (b) shared model-family"
    Write-Host "          not neutralized (blank-ANSWER != blank-CORPUS; only recurrence-freq anchoring"
    Write-Host "          is model-independent, and that lives in the OLD measurable-now block);"
    Write-Host "          (c) blinding is an instruction, not an enforcement; (d) no age dimension yet,"
    Write-Host "          small sample -- a FLAG, never a grade."
}

# ===========================================================================
# SELFTEST  (3 cases, internal fake data -- the ONE path that may exit non-zero)
# ===========================================================================
function Invoke-Selftest {
    # $script:-scoped so the nested Assert-Case can flip it and the flip survives
    # back here (a plain local would NOT propagate from the nested function --
    # caught by fault-injection: a sabotaged gate printed CASE FAIL yet the summary
    # still said ALL PASS + exit 0 when this was function-local).
    $script:allPass = $true

    function Assert-Case {
        param([string]$Name, [bool]$Condition, [string]$Detail)
        if ($Condition) {
            Write-Host ("{0} CASE {1}: PASS  ({2})" -f $CHECK, $Name, $Detail)
        } else {
            Write-Host ("{0} CASE {1}: FAIL  ({2})" -f $CROSS, $Name, $Detail)
            $script:allPass = $false
        }
    }

    Write-Host "=== applied-eval-nokey SELFTEST (3 cases) ==="

    # -- Case (a): a control scored HIGH (0.9) -> MUST BLOCK -------------------
    $caseA = @(
        [pscustomobject]@{ id='q1';   score=0.8; is_control=$false; control_expected_low=$false }
        [pscustomobject]@{ id='q2';   score=0.7; is_control=$false; control_expected_low=$false }
        [pscustomobject]@{ id='ctlA'; score=0.9; is_control=$true;  control_expected_low=$true  }
    )
    $rA = Invoke-AdversarialGate -Verdicts $caseA
    Assert-Case -Name 'a-control-HIGH' -Condition ($rA.Decision -eq 'BLOCK') `
        -Detail ("control 0.9 -> Decision=$($rA.Decision) (expect BLOCK); reason=" + ($rA.Reasons -join '; '))

    # -- Case (b): all controls LOW (0.1) -> MUST PASS + produce Applied% ------
    $caseB = @(
        [pscustomobject]@{ id='q1';   score=0.8; is_control=$false; control_expected_low=$false }
        [pscustomobject]@{ id='q2';   score=0.6; is_control=$false; control_expected_low=$false }
        [pscustomobject]@{ id='ctlB'; score=0.1; is_control=$true;  control_expected_low=$true  }
    )
    $rB = Invoke-AdversarialGate -Verdicts $caseB
    $mB = $null
    if ($rB.Decision -eq 'PASS') { $mB = Merge-AppliedScore -Verdicts $caseB }
    # real scores 0.8,0.6 -> mean 0.7 -> Applied% = 70; control excluded.
    $bOk = ($rB.Decision -eq 'PASS') -and ($null -ne $mB) -and ($mB.AppliedPct -eq 70) -and ($mB.ControlN -eq 1)
    $bDetail = if ($null -ne $mB) {
        "Decision=$($rB.Decision); Applied%=$($mB.AppliedPct) (expect 70); RealN=$($mB.RealN); ControlN=$($mB.ControlN)"
    } else {
        "Decision=$($rB.Decision) (expect PASS) -- merge not reached"
    }
    Assert-Case -Name 'b-control-LOW' -Condition $bOk -Detail $bDetail

    # -- Case (c): a control was FORGOTTEN (null score) -> MUST BLOCK ----------
    $caseC = @(
        [pscustomobject]@{ id='q1';   score=0.9;   is_control=$false; control_expected_low=$false }
        [pscustomobject]@{ id='q2';   score=0.8;   is_control=$false; control_expected_low=$false }
        [pscustomobject]@{ id='ctlC'; score=$null; is_control=$true;  control_expected_low=$true  }
    )
    $rC = Invoke-AdversarialGate -Verdicts $caseC
    Assert-Case -Name 'c-control-FORGOTTEN' -Condition ($rC.Decision -eq 'BLOCK') `
        -Detail ("control score=null -> Decision=$($rC.Decision) (expect BLOCK); reason=" + ($rC.Reasons -join '; '))

    Write-Host ""
    if ($script:allPass) {
        Write-Host ("{0} SELFTEST: ALL 3 CASES PASS" -f $CHECK)
    } else {
        Write-Host ("{0} SELFTEST: FAILURE -- at least one case did not behave as specified" -f $CROSS)
    }

    Write-HonestyRung
    return $script:allPass
}

# ===========================================================================
# Entry point
# ===========================================================================
if ($Selftest) {
    $ok = Invoke-Selftest
    if ($ok) { exit 0 } else { exit 1 }
}

if ($Honest) {
    Write-HonestyRung
    exit 0
}

# Plain run: no live data path exists yet (numbers DEFERRED). Print the rung + usage.
Write-Host "applied-eval-nokey.ps1 -- MECHANISM for a no-API Applied% measurement (adversarial gate + merge)."
Write-Host "No live numbers are produced: the 2-agent probe/judge pass is RUNTIME (via a workflow) and"
Write-Host "runs only when the sample-questions corpus matures. This script contributes the DETERMINISTIC"
Write-Host "gate + merge + selftest that the runtime verdicts flow through."
Write-HonestyRung
Write-Host ""
Write-Host "Run the selftest:  powershell.exe -ExecutionPolicy Bypass -File scripts\applied-eval-nokey.ps1 -Selftest"
exit 0