{
  "schemaVersion": "flare-public-evaluation-scorecard/v1",
  "publishedAt": "2026-10-28",
  "classification": "Internal diagnostic results; not independent validation",
  "methodologyUrl": "https://tryflare.ai/evaluations",
  "baselineCommit": "b3ab13c9a6ac72729f476c692b47dd016ef530b7",
  "fullRunCommit": "99770601deca618387ee5b109d6d1805c97d8cd0",
  "repairCommit": "2099e4a9f7a01f2cb413700544f10b044f2fcf35",
  "deploymentCommit": "aa8531f59f70967234e5025e237ecc259fd0b084",
  "modelConfiguration": {
    "producer": "claude-sonnet-4-6",
    "judge": "claude-opus-4-6",
    "sameModelFamily": true
  },
  "scale": {
    "minimum": 1,
    "maximum": 5,
    "interpretation": "Case-specific rubric grades, not accuracy percentages or compromise probabilities"
  },
  "matchedComparison": [
    {
      "theme": "Detection",
      "before": 4.933,
      "after": 4.982,
      "validPairs": 224
    },
    {
      "theme": "Factuality",
      "before": 4.983,
      "after": 5,
      "validPairs": 238
    },
    {
      "theme": "Risk ranking",
      "before": 4.929,
      "after": 5,
      "validPairs": 14
    },
    {
      "theme": "Explanation",
      "before": 4.513,
      "after": 4.866,
      "validPairs": 238
    }
  ],
  "expandedMatrix": [
    {
      "theme": "Detection",
      "mean": 4.99,
      "validCases": 390
    },
    {
      "theme": "Factuality",
      "mean": 5,
      "validCases": 524
    },
    {
      "theme": "Risk ranking",
      "mean": 4.973,
      "validCases": 37
    },
    {
      "theme": "Explanation",
      "mean": 4.903,
      "validCases": 524
    }
  ],
  "fullRunValidity": {
    "currentValidPackets": 524,
    "currentTotalPackets": 525
  },
  "frozenDetectionLabels": {
    "beforePassed": 218,
    "afterPassed": 218,
    "total": 224,
    "note": "One regression in the full run was subsequently repaired; the original aggregate is retained."
  },
  "rankingOrderProbe": {
    "pairs": 5,
    "orderAndEvidenceConsistent": 5,
    "numericalScoresIdentical": 1
  },
  "targetedRepairOverlay": [
    {
      "case": "AWS role-tag change with explicit ABAC policy",
      "detection": {
        "before": 1,
        "after": 5
      },
      "explanation": {
        "before": 1,
        "after": 5
      }
    },
    {
      "case": "Workspace two customers",
      "explanation": {
        "before": 3,
        "after": 5
      }
    },
    {
      "case": "Workspace missing customer identity",
      "explanation": {
        "before": 3,
        "after": 5
      }
    },
    {
      "case": "Workspace prior-answer correction",
      "explanation": {
        "before": 2,
        "after": 5
      }
    },
    {
      "case": "Workspace independent profile bursts",
      "priorJudgment": "excluded",
      "answerReusedUnchanged": true,
      "validGrades": {
        "detection": 5,
        "factuality": 5,
        "ranking": 4,
        "explanation": 4
      }
    }
  ],
  "limitations": [
    "References were authored by the implementing agent; scenarios include reused and correlated variants.",
    "Producer and judge are from the same model family; independent evaluator qualification is incomplete.",
    "Impossible-counter control received factuality 2/5 rather than the predeclared 1/5 target; exact severity calibration remains failed.",
    "Full evaluation exercised deployed source through an engine harness with live inference, not a complete hosted-UI replay.",
    "The targeted repair overlay does not constitute a fresh full-suite aggregate for deployment aa8531f5.",
    "No real-world precision/recall, matched competitor ranking, matched latency benchmark or customer outcome claim is established.",
    "Artifact hashes identify retained internal evidence; this public summary does not make the full private corpus publicly reproducible."
  ],
  "provenance": [
    {
      "artifact": "deployed-comprehensive-evals-20261027-v2/REPORT.md",
      "description": "Full engine-harness evaluation",
      "sha256": "60c9e3ff7e3a541e7e33022e47ef90574de379e04d691b400ee64eb2ec3cfcc2"
    },
    {
      "artifact": "abac-workspace-fix-20261027-v1/REPORT.md",
      "description": "Targeted repair overlay",
      "sha256": "87cb9038ff188ce4c50b5335a3f40ac83bfc8a6b85125331150d95b322a7bbd1"
    },
    {
      "artifact": "abac-workspace-deploy-20261028-v1/REPORT.md",
      "description": "Saved deployment verification",
      "sha256": "4cd0da47e6a482159839da0e465ad97bb4d1213e722fe188c926a3a83cc34354"
    }
  ]
}