{
  "schemaVersion": "1.1",
  "title": "use.fo Auto Research Local STT Selection Methodology Manifest",
  "publishedAt": "2026-07-24",
  "lastVerifiedAt": "2026-07-24",
  "publisher": "use.fo product team",
  "disclosure": {
    "scope": "First-party methodology, local-model selection, and evidence-line manifest",
    "doesNotClaim": [
      "Universal STT accuracy",
      "Local-versus-cloud percentage accuracy",
      "Latency on every device",
      "Independent certification"
    ],
    "protectedMaterial": "Raw transcripts, reference answers, per-input direct and STT-derived answers, judge decisions, and scoring configuration remain internal so candidates can continue to be tested against held-back examples."
  },
  "evidenceTracks": [
    {
      "id": "answer-parity",
      "purpose": "Measure whether the speech-to-text step materially changes a downstream answer for the same intended input.",
      "referenceHardware": {
        "device": "MacBook Pro",
        "processor": "Apple M2 Pro",
        "memory": "64 GB"
      },
      "workflow": [
        "Generate an answer from the original input using the configured frontier answer model.",
        "Generate a second answer after speech-to-text.",
        "Use ChatGPT 5.5 in the local research environment to judge the difference under the Auto Research rubric.",
        "Produce a modification suggestion when the rubric threshold is exceeded."
      ],
      "answerModelExamples": ["Gemini 3.1 Pro", "ChatGPT 5.5"],
      "publicAggregateScore": null,
      "dataAvailability": "Internal only"
    },
    {
      "id": "local-model-selection",
      "purpose": "Choose a practical local Whisper route by checking transcript usefulness, held-back examples, completion time, and intended hardware together.",
      "referenceHardware": {
        "device": "MacBook Pro",
        "processor": "Apple M2 Pro",
        "memory": "64 GB"
      },
      "decisionRules": [
        "Use visible examples to understand a candidate, then confirm it against hidden references before promotion.",
        "Keep, review, or discard is decided from the complete visible-and-hidden evidence, not from a GEPA reflection.",
        "Treat GEPA as a tool for proposing the next narrow experiment only.",
        "Consider transcript fidelity, names and numbers, mixed-language terms, completion time, and hardware fit together."
      ],
      "selectedCandidate": {
        "model": "mlx-community/whisper-large-v3-turbo-asr-fp16",
        "runtime": "mlx-audio-direct / MLX Metal",
        "configurationDirection": "deterministic decoding; forced language; narrow en-US/de/zh-Hant prompt allowlist; 120-second chunk for audio at least 180 seconds",
        "result": {
          "visibleScore": 88.23,
          "hiddenScore": 88.77,
          "combinedScore": 88.5,
          "averageSttLatencyMs": 1870.97
        },
        "decision": "keep"
      },
      "priorDecisionRecords": [
        {
          "candidate": "MLX Audio Whisper Large V3 Turbo ASR FP16 R10 baseline",
          "combinedScore": 87.98,
          "averageSttLatencyMs": 1861.26,
          "decision": "keep as regression baseline"
        },
        {
          "candidate": "MLX Whisper Large V3 Turbo Q4 warmed process",
          "hiddenScore": 55.92,
          "averageSttLatencyMs": 5138.71,
          "decision": "discard",
          "comparability": "Earlier measurement phase; retained as a decision record, not a direct leaderboard comparison."
        },
        {
          "candidate": "whisper.cpp ggml-base",
          "hiddenScore": 48.3,
          "averageSttLatencyMs": 26076,
          "decision": "discard",
          "comparability": "Earlier measurement phase; retained as a decision record, not a direct leaderboard comparison."
        }
      ],
      "dataAvailability": "Aggregate result and method published; protected per-input corpus remains internal"
    },
    {
      "id": "stt-evidence-chat-rewriter",
      "purpose": "Evaluate transcript rewriting from metadata-rich STT source artifacts with hidden-first decisions.",
      "sourceArtifact": {
        "provider": "Groq",
        "model": "whisper-large-v3-turbo",
        "responseFormat": "verbose_json",
        "timestampGranularities": ["segment"],
        "rawArtifactCount": 80,
        "manifestGeneratedAt": "2026-05-25T14:02:28.173Z"
      },
      "fixedEvaluationLine": {
        "caseIds": ["14", "20", "25", "29", "33", "40", "48", "49", "61", "80"],
        "splits": ["visible development", "hidden evaluation"],
        "decisionDiscipline": "hidden-first keep / review / discard"
      },
      "outcomeScope": "Transcript rewrite quality; this is not the same metric as the answer-parity track.",
      "dataAvailability": "Internal only"
    }
  ]
}