295c70c904
All checks were successful
Deploy SOLUTION_ERP / build-deploy (push) Successful in 5m30s
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
306 lines
17 KiB
PowerShell
306 lines
17 KiB
PowerShell
# mfe-eval.ps1 - Harness-16 Memory-Fidelity-EVAL (MFE) - S93 (2026-06-29)
|
|
#
|
|
# WHAT THIS IS (and is NOT):
|
|
# MFE = COVERAGE / RETENTION eval: when hot-memory (always-loaded Tier-1) is
|
|
# provisioned, how much of the REQUIRED governance set actually FITS, and (per
|
|
# role) how much is still CARRIED in the persistent diary. It answers a question
|
|
# the %-print (size-by-length) cannot: provisioned != remembered != applied.
|
|
# This is DISTINCT from H6.7 "memoryDelta-routing-fidelity" (the right delta
|
|
# landing in the right agent-memory). DO NOT conflate the two senses.
|
|
#
|
|
# NON-NEGOTIABLES (Harness-11 / Harness-16):
|
|
# (1) NO-API : Select-String + byte/file parse ONLY. NEVER calls a model.
|
|
# (2) READ-ONLY on token_governor : MFE consumes the budget caps (single-source),
|
|
# NEVER writes/auto-tunes them (owner-authority floor, budget.json role_boundary_note).
|
|
# (3) PS 5.1, ASCII-only script body (gotcha #30). Target files read -Encoding UTF8.
|
|
# (4) Exit 0 always : a measure-and-report tool, NOT a build gate.
|
|
#
|
|
# DETERMINISTIC ($0) block (Branches: lead Coverage + age-band + Goodhart; sub per-role coverage).
|
|
# Branch-A judge (recall/apply) = SCAFFOLD only (-Judge) : numbers MEANINGLESS until
|
|
# sample-questions mature AND an independent cross-session judge runs (see honest_caveats).
|
|
#
|
|
# Usage:
|
|
# powershell.exe -ExecutionPolicy Bypass -File scripts\mfe-eval.ps1 # all tiers
|
|
# powershell.exe -ExecutionPolicy Bypass -File scripts\mfe-eval.ps1 -Tier lead
|
|
# powershell.exe -ExecutionPolicy Bypass -File scripts\mfe-eval.ps1 -Tier sub
|
|
# powershell.exe -ExecutionPolicy Bypass -File scripts\mfe-eval.ps1 -Judge # show judge scaffold (no scoring)
|
|
|
|
param(
|
|
[string]$RepoRoot = "$PSScriptRoot\..",
|
|
[ValidateSet('lead','sub','all')] [string]$Tier = 'all',
|
|
[switch]$Judge = $false,
|
|
[datetime]$Today = (Get-Date)
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
$CHECK = [char]0x2705 # green check glyph, kept as code-point (ASCII body, gotcha #30)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# resolve paths + load single-source config
|
|
# ---------------------------------------------------------------------------
|
|
$RepoRoot = (Resolve-Path $RepoRoot).Path
|
|
$budgetPath = Join-Path $RepoRoot '.claude\agent-memory\memory-budget.json'
|
|
if (-not (Test-Path $budgetPath)) { Write-Host "memory-budget.json not found: $budgetPath"; exit 0 }
|
|
$budget = Get-Content $budgetPath -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
$cfg = $budget.mfe
|
|
if ($null -eq $cfg) { Write-Host "memory-budget.json missing 'mfe' config block (Harness-16)"; exit 0 }
|
|
|
|
$leadCap = [int]$budget.token_governor.tier1_hotfeed_tokens.lead_tokens # LIVE-read, never hardcode
|
|
$memCap = [int]$budget.token_governor.tier1_hotfeed_tokens.memory_sub_tokens
|
|
$marksPath = Join-Path $RepoRoot '.claude\governance\ACTIVE-MARKS.md'
|
|
$ledgerPath = Join-Path $RepoRoot 'docs\governance\error-ledger.md'
|
|
$gotchaPath = Join-Path $RepoRoot 'docs\gotchas.md'
|
|
$statePath = Join-Path $RepoRoot $cfg.state_file
|
|
|
|
function Read-Utf8([string]$p) { if (Test-Path $p) { return Get-Content $p -Encoding UTF8 } else { return @() } }
|
|
function Bytes([string]$s) { return [Text.Encoding]::UTF8.GetByteCount($s) }
|
|
function Cells([string]$line) {
|
|
# split a markdown table row into inner cells (drop leading/trailing empties)
|
|
$parts = $line.Trim().Trim('|') -split '\|'
|
|
return ($parts | ForEach-Object { $_.Trim() })
|
|
}
|
|
function Remove-Diacritics([string]$t) {
|
|
$n = $t.Normalize([Text.NormalizationForm]::FormD)
|
|
$sb = New-Object Text.StringBuilder
|
|
foreach ($c in $n.ToCharArray()) {
|
|
if ([Globalization.CharUnicodeInfo]::GetUnicodeCategory($c) -ne [Globalization.UnicodeCategory]::NonSpacingMark) { [void]$sb.Append($c) }
|
|
}
|
|
return $sb.ToString().ToLowerInvariant()
|
|
}
|
|
function Content-Words([string]$s, [string[]]$stop) {
|
|
$folded = Remove-Diacritics $s
|
|
$words = $folded -split '[^a-z0-9]+' | Where-Object { $_ -and $_.Length -ge 2 -and ($stop -notcontains $_) }
|
|
return ($words | Select-Object -Unique)
|
|
}
|
|
|
|
Write-Host ""
|
|
Write-Host "=== MFE (Memory-Fidelity-EVAL) - Harness-16 - $($Today.ToString('yyyy-MM-dd')) ==="
|
|
Write-Host " (coverage/retention eval; NOT memoryDelta-routing-fidelity H6.7)"
|
|
Write-Host " lead hot-feed cap (live) = $leadCap tok | mem-sub cap = $memCap tok"
|
|
Write-Host ""
|
|
|
|
# ===========================================================================
|
|
# LEAD TIER : must-remember denominator -> Coverage(fit) + age-band + Goodhart
|
|
# ===========================================================================
|
|
function Measure-Lead {
|
|
# --- Source A: Active marks (status Active-High / Active) ---
|
|
$marks = @(); $markBytes = 0; $markDates = @()
|
|
$inSec = $false
|
|
foreach ($ln in (Read-Utf8 $marksPath)) {
|
|
if ($ln -match '^##\s') { $inSec = ($ln -match 'ACTIVE') ; continue } # ACTIVE-HIGH + ACTIVE; skips MEDIUM/SUPERSEDED
|
|
if ($inSec -and $ln -match '^\|\s*`(RC-[a-z0-9-]+)`') {
|
|
$rid = $matches[1]
|
|
$c = Cells $ln
|
|
$status = $c[$c.Count-1]
|
|
if ($status -match 'Active-High' -or $status -match 'Active\b') {
|
|
$marks += $rid; $markBytes += (Bytes $ln)
|
|
if ($rid -match '-(\d{2})-(\d{2})-(\d{4})-\d{2}-\d{2}-\d{2}$') {
|
|
$markDates += [datetime]::new([int]$matches[3],[int]$matches[2],[int]$matches[1])
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# --- Source B: error-ledger Active-Guards + AS-table + RCA strikes ---
|
|
$guards = @(); $guardBytes = 0; $guardCounters = @{}
|
|
$asIds = @(); $asGotchaRefs = New-Object System.Collections.Generic.HashSet[string]
|
|
$asBytes = 0; $strikesTotal = 0; $rcaCount = 0
|
|
$section = ''
|
|
foreach ($ln in (Read-Utf8 $ledgerPath)) {
|
|
if ($ln -match '^##\s') {
|
|
if ($ln -match 'Active-Guards') { $section = 'guards' }
|
|
elseif ($ln -match 'L\.a') { $section = 'as' }
|
|
else { $section = '' }
|
|
continue
|
|
}
|
|
if ($ln -match '^### (E-\d+)') { $rcaCount++ ; continue }
|
|
if ($section -eq 'guards' -and $ln -match '^\|' -and $ln -match [regex]::Escape($CHECK)) {
|
|
$c = Cells $ln
|
|
if ($c.Count -ge 5 -and $c[0] -notmatch '^-+$' -and $c[0] -ne 'Guard') {
|
|
$net = $c[$c.Count-1]
|
|
if ($net -notmatch '^\s*-' -and $net -notmatch 'retire') {
|
|
$g = $c[0]; $guards += $g; $guardBytes += (Bytes $ln); $guardCounters[$g] = $c[1]
|
|
if ($c[3] -match '(\d+)') { $strikesTotal += [int]$matches[1] } # Strikes col
|
|
}
|
|
}
|
|
}
|
|
if ($section -eq 'as' -and $ln -match '^\|\s*(AS-\d+)\b') {
|
|
$asIds += $matches[1]; $asBytes += (Bytes $ln)
|
|
foreach ($m in [regex]::Matches($ln, '#(\d+)')) { [void]$asGotchaRefs.Add($m.Groups[1].Value) }
|
|
}
|
|
}
|
|
|
|
# --- Source C: recurring gotchas ---
|
|
$reTokens = @($cfg.recurrence_tokens)
|
|
$allGotchas = 0; $recurGotchaNums = @(); $recurBytes = 0
|
|
$lines = Read-Utf8 $gotchaPath
|
|
for ($i = 0; $i -lt $lines.Count; $i++) {
|
|
if ($lines[$i] -match '^### (\d+)\.') {
|
|
$allGotchas++
|
|
$num = $matches[1]
|
|
$blk = $lines[$i]
|
|
for ($j = $i+1; $j -lt $lines.Count -and $lines[$j] -notmatch '^### '; $j++) { $blk += "`n" + $lines[$j] }
|
|
$folded = Remove-Diacritics $blk
|
|
$isRecur = $false
|
|
foreach ($t in $reTokens) { if ($folded.Contains((Remove-Diacritics $t))) { $isRecur = $true; break } }
|
|
if ($isRecur) { $recurGotchaNums += $num; $recurBytes += (Bytes ($lines[$i])) }
|
|
}
|
|
}
|
|
|
|
# --- DEDUP (merge_guard_to_as): collapse gotchas/guards that an AS-row references ---
|
|
$merge = [bool]$cfg.merge_guard_to_as
|
|
$netRecur = @($recurGotchaNums | Where-Object { -not $asGotchaRefs.Contains($_) })
|
|
$netGuards = @()
|
|
foreach ($g in $guards) {
|
|
$ctr = [string]$guardCounters[$g]
|
|
$absorbed = $false
|
|
if ($merge) {
|
|
if ($ctr -match 'AS-\d+') { $absorbed = $true }
|
|
else { foreach ($m in [regex]::Matches($ctr, '#(\d+)')) { if ($asGotchaRefs.Contains($m.Groups[1].Value)) { $absorbed = $true; break } } }
|
|
}
|
|
if (-not $absorbed) { $netGuards += $g }
|
|
}
|
|
|
|
$denomCount = $marks.Count + $asIds.Count + $netGuards.Count + $netRecur.Count
|
|
$denomBytes = $markBytes + $asBytes + $guardBytes + $recurBytes
|
|
$tokLow = [math]::Round($denomBytes / 4.0)
|
|
$tokHigh = [math]::Round($denomBytes / 3.0)
|
|
$pctHigh = if ($leadCap -gt 0) { [math]::Round(100.0 * $tokHigh / $leadCap, 2) } else { 0 }
|
|
$fits = ($tokHigh -le $leadCap)
|
|
|
|
Write-Host "[LEAD] must-remember denominator (deduped, merge_guard_to_as=$merge):"
|
|
Write-Host " marks(Active+High)=$($marks.Count) AS-classes=$($asIds.Count) net-extra-guards=$($netGuards.Count) net-extra-recurring-gotchas=$($netRecur.Count)"
|
|
Write-Host " => DENOMINATOR = $denomCount items (raw before dedup: marks $($marks.Count) + AS $($asIds.Count) + guards $($guards.Count) + recurring $($recurGotchaNums.Count))"
|
|
Write-Host ""
|
|
Write-Host "[LEAD] Coverage-FIT (does the set FIT the hot-feed cap?):"
|
|
Write-Host " span bytes=$denomBytes => ~$tokLow - $tokHigh tok (RANGE, see caveat) / cap $leadCap tok = $pctHigh% worst-case"
|
|
if ($fits) { Write-Host " FIT = PASS (worst-case fits; headroom huge - must-remember set is tiny vs cap)" }
|
|
else { Write-Host " FIT = OVER (worst-case exceeds cap) -> decision: INCREASE budget (lack-of-SPACE case)" }
|
|
Write-Host " CAVEAT: char/4 is NOT real tokens; VN-diacritic ~3.0-3.5 byte/tok => byte/4 = upper-bound headroom."
|
|
Write-Host " Reported as a RANGE [bytes/4 .. bytes/3.0]; FIT uses bytes/3.0 (worst). Real count inside the band."
|
|
Write-Host ""
|
|
|
|
# --- age-band : FLAG old-but-still-required, NEVER cut (mark RC-...10-29-11) ---
|
|
$oldN = 0
|
|
foreach ($d in $markDates) { if (($Today - $d).TotalDays -gt 30) { $oldN++ } }
|
|
Write-Host "[LEAD] age-band (FLAG only - age=false-proxy, mark RC-...10-29-11):"
|
|
Write-Host " marks with parseable date=$($markDates.Count); >30d old but still Active=$oldN -> KEPT (status-driven, age-blind)"
|
|
Write-Host " (guards/gotchas carry session-refs not dates -> age = status-driven only; drop on status-change, never by age)"
|
|
Write-Host ""
|
|
|
|
# --- Goodhart anchor : print recurring-error reality next to coverage ---
|
|
$lastStrikes = -1
|
|
if (Test-Path $statePath) { try { $lastStrikes = [int]((Get-Content $statePath -Raw -Encoding UTF8 | ConvertFrom-Json).strikes_total) } catch { } }
|
|
Write-Host "[LEAD] Goodhart anchor (NO self-grading - real recurring-error signal):"
|
|
Write-Host " error-ledger: strikes_total=$strikesTotal RCA_entries=$rcaCount AS-classes=$($asIds.Count)"
|
|
if ($lastStrikes -ge 0 -and $strikesTotal -gt $lastStrikes -and $fits) {
|
|
Write-Host " GOODHART-WARN: strikes rose ($lastStrikes -> $strikesTotal) while set still FITS -> coverage may LIE (set 'fits' but errors recur = not actually applied)"
|
|
} else {
|
|
Write-Host " (rule: if strikes rise across runs while coverage stays high -> the score lies. last_run_strikes=$lastStrikes)"
|
|
}
|
|
# persist strikes for cross-run compare (MFE state only; does NOT touch token_governor)
|
|
# Write LF + UTF8-no-BOM explicitly (erratum EOL 2026-07-16 lo #6 point-of-generation floor, S129):
|
|
# PS5.1 Set-Content -Encoding UTF8 wrote BOM+CRLF -> worktree CRLF churn AND a 3-byte BOM in the
|
|
# committed blob. Reader at line 193 pins -Encoding UTF8 in the SAME commit (no BOM-sniff dependency).
|
|
try {
|
|
$st = @{ strikes_total = $strikesTotal; rca_entries = $rcaCount; at = $Today.ToString('yyyy-MM-dd') }
|
|
$json = (($st | ConvertTo-Json) -replace "`r`n", "`n") + "`n"
|
|
[System.IO.File]::WriteAllText($statePath, $json, (New-Object System.Text.UTF8Encoding($false)))
|
|
} catch { }
|
|
Write-Host ""
|
|
}
|
|
|
|
# ===========================================================================
|
|
# SUB TIER : per-role coverage (denominator from role .md, numerator from diary)
|
|
# ===========================================================================
|
|
function Measure-Sub {
|
|
$stop = @($cfg.sub_denominator.stop_list)
|
|
$anchors = @($cfg.sub_denominator.denom_header_anchors)
|
|
$minWords = [int]$cfg.sub_denominator.content_word_min
|
|
$roleFiles = Get-ChildItem (Join-Path $RepoRoot '.claude\agents\*.md') -ErrorAction SilentlyContinue | Where-Object { $_.Name -ne 'README.md' }
|
|
|
|
Write-Host "[SUB] per-role memory coverage (denominator = role-file anti-patterns/baseline; numerator = carried in diary):"
|
|
Write-Host " match = exact-id OR >=$minWords distinct content-words after stop-list (leading-verb-only = false-max, defeated)"
|
|
Write-Host ""
|
|
foreach ($rf in $roleFiles) {
|
|
$role = $rf.BaseName
|
|
$rlines = Read-Utf8 $rf.FullName
|
|
# denominator: numbered '^N.' lines UNDER a denom-anchored '##' header, + a NEVER:/scope line
|
|
$denom = @(); $inDenom = $false
|
|
foreach ($ln in $rlines) {
|
|
if ($ln -match '^##\s') {
|
|
$inDenom = $false
|
|
foreach ($a in $anchors) { if ($ln -match [regex]::Escape($a)) { $inDenom = $true; break } }
|
|
continue
|
|
}
|
|
if ($inDenom -and $ln -match '^\s*\d+\.\s+(.+)') { $denom += $matches[1] }
|
|
if ($ln -match 'NEVER' -and $ln -match ':') { $denom += ($ln -replace '.*NEVER\s*:?\s*','') }
|
|
}
|
|
$denom = @($denom | Where-Object { $_ -and $_.Trim().Length -gt 3 })
|
|
# diary (THE agent diary - NOT project/user MEMORY.md)
|
|
$diaryPath = Join-Path $RepoRoot ".claude\agent-memory\$role\MEMORY.md"
|
|
if (-not (Test-Path $diaryPath)) {
|
|
Write-Host (" {0,-22} N/A (no persistent diary -> ephemeral/unspawned)" -f $role)
|
|
continue
|
|
}
|
|
$diaryRaw = (Read-Utf8 $diaryPath) -join "`n"
|
|
$diaryFold = Remove-Diacritics $diaryRaw
|
|
$hit = 0
|
|
foreach ($item in $denom) {
|
|
$words = Content-Words $item $stop
|
|
$matched = $false
|
|
if ($words.Count -ge $minWords) {
|
|
$found = 0
|
|
foreach ($w in $words) { if ($diaryFold -match ('(?<![a-z0-9])' + [regex]::Escape($w) + '(?![a-z0-9])')) { $found++ } }
|
|
if ($found -ge $minWords) { $matched = $true }
|
|
} elseif ($words.Count -ge 1) {
|
|
# all-verb / thin item: require the single content word present
|
|
if ($diaryFold -match ('(?<![a-z0-9])' + [regex]::Escape($words[0]) + '(?![a-z0-9])')) { $matched = $true }
|
|
}
|
|
if ($matched) { $hit++ }
|
|
}
|
|
$den = $denom.Count
|
|
if ($den -eq 0) {
|
|
# 0 denominator = NOT-EXTRACTABLE (role-file uses prose, no numbered/NEVER block) -> N/A, NOT '0% retention'
|
|
Write-Host (" {0,-22} N/A (no numbered anti-pattern / NEVER block in role-file to extract)" -f $role)
|
|
continue
|
|
}
|
|
$pct = [math]::Round(100.0 * $hit / $den)
|
|
$flag = if ($pct -lt 60) { " <- LOW (real retention signal, not a target-miss)" } else { "" }
|
|
Write-Host (" {0,-22} {1,3}/{2,-3} = {3,3}% (MEASURED, not target-then-force){4}" -f $role, $hit, $den, $pct, $flag)
|
|
}
|
|
Write-Host ""
|
|
Write-Host "[SUB-WORKFLOW] N/A by design: fan-out workflow agents (hmw.js sub-<role>-<i>) are EPHEMERAL"
|
|
Write-Host " (no persistent diary; their only memory channel is the lead-injected memory-pack -> source-vs-cap is meaningless)."
|
|
Write-Host ""
|
|
}
|
|
|
|
# ===========================================================================
|
|
# JUDGE scaffold (Branch A) : -Judge only. SCAFFOLD - numbers MEANINGLESS.
|
|
# ===========================================================================
|
|
function Show-JudgeScaffold {
|
|
$sqPath = Join-Path $RepoRoot $cfg.sample_questions
|
|
Write-Host "[JUDGE-SCAFFOLD] Branch-A recall/apply (OPT-IN, cadence-gated, NO-API at scaffold level):"
|
|
if (Test-Path $sqPath) {
|
|
$sq = Get-Content $sqPath -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
Write-Host " sample-questions loaded: $($sq.questions.Count) (seeded $($sq.seeded_date), anchored by stable-id)"
|
|
Write-Host " scorer = EMPTY (no answers recorded). recalled/applied = (none) ; meaningful = FALSE"
|
|
} else {
|
|
Write-Host " sample-questions file NOT found: $sqPath"
|
|
}
|
|
Write-Host " HONEST: a same-session self-grade is MEANINGLESS (the agent just read the answers)."
|
|
Write-Host " Real numbers need (a) sample-questions matured in age AND (b) an independent"
|
|
Write-Host " cross-session / different-model judge. Until then this is plumbing-smoke only."
|
|
Write-Host ""
|
|
}
|
|
|
|
# ===========================================================================
|
|
# dispatch
|
|
# ===========================================================================
|
|
if ($Tier -eq 'lead' -or $Tier -eq 'all') { Measure-Lead }
|
|
if ($Tier -eq 'sub' -or $Tier -eq 'all') { Measure-Sub }
|
|
if ($Judge) { Show-JudgeScaffold }
|
|
|
|
Write-Host "=== MFE done (deterministic block; READ-ONLY on budget; exit 0) ==="
|
|
exit 0
|