{
  "recorded_dates": [
    "2026-10-13",
    "2026-10-14"
  ],
  "local_model": "Mistral Small 3.2 · 24B · Q4_K_M",
  "hardware": "Apple M5 · 32 Go",
  "source_files": {
    "agent": {
      "path": "eval_results/technical/mistral-alone-20261014/local-v1/results.json",
      "sha256": "ef6acce69ea96af37b6af90ab68bab033053739d5b737dd514b491506ec5b1f1"
    },
    "agent_review": {
      "path": "eval_results/technical/mistral-alone-20261014/review-v1.json",
      "sha256": "7f606fac192d59aa914d9c5d5d5c8166169664671df7f31692e77ffa049f5480"
    },
    "agent_summary": {
      "path": "eval_results/technical/mistral-alone-20261014/summary.json",
      "sha256": "34354406197046b8dd57b2679adf8fb2150626b7f8fe186851a30767a2491613"
    },
    "other_routes": {
      "path": "eval_results/technical/mistral-alone-20261014/remote-v1/results.json",
      "sha256": "39770199cbc9c19cbee9fb176615481ee3d4c34d8411dcbeec05416dcce39004"
    },
    "current_report": {
      "path": "docs/mistral-jev-et-embeddings-20261014.md",
      "sha256": "ce887110409d22baf7cc667855142ada364528c1862ec5ada5a354019b5eb52d"
    },
    "embeddings": {
      "path": "eval_results/technical/embedding-model-comparison-20261014/summary.json",
      "sha256": "026422d7b253626887c5918853529ff21c4a2ff6d262ecb95b0a424cbda23056"
    },
    "writing": {
      "path": "eval_results/technical/rag-mistral-20261014/final-replay.json",
      "sha256": "4635ff33dfd81ab763a9c7e9b7bc7bd59a21d170b21545b0246f3cc91f6b2966"
    },
    "retrieval": {
      "path": "docs/technical-retrieval-benchmark-20261013.md",
      "sha256": "b40befc4ea716db3f849fd6130e00cc07500e3882389b95277a9cc3de2cca217"
    }
  },
  "tool_agent": {
    "context": 16384,
    "deadline_seconds": 30,
    "timeout_cases": 6,
    "attempted_cases": 8,
    "fully_correct_cases": 0,
    "review_status": "Reviewed by an assistant against source-linked references",
    "cases": [
      {
        "id": "Q01",
        "status": "completed",
        "answer": "La recherche est en cours.",
        "timing": {
          "full_answer_seconds": 29.75158075010404,
          "model_seconds": 29.749873957014643,
          "tool_seconds": 0.0,
          "ttft_seconds": null,
          "first_useful_result_seconds": null
        },
        "review": {
          "candidate": "ollama:mistral-small3.2:latest",
          "case_id": "Q01",
          "turn": 1,
          "category": "incorrect",
          "reason": "Final submission says the search is in progress, without a count or a tool call."
        }
      },
      {
        "id": "Q10",
        "status": "timeout",
        "answer": "",
        "timing": {
          "full_answer_seconds": 30.01392529194709,
          "model_seconds": 29.47761804203037,
          "tool_seconds": 0.5333042920101434,
          "ttft_seconds": null,
          "first_useful_result_seconds": null
        },
        "review": {
          "candidate": "ollama:mistral-small3.2:latest",
          "case_id": "Q10",
          "turn": 1,
          "category": "timeout",
          "reason": "No completed final answer within the identical 30-second turn budget."
        }
      },
      {
        "id": "Q17",
        "status": "timeout",
        "answer": "",
        "timing": {
          "full_answer_seconds": 30.00488295906689,
          "model_seconds": 29.438587916083634,
          "tool_seconds": 0.5650034589925781,
          "ttft_seconds": null,
          "first_useful_result_seconds": null
        },
        "review": {
          "candidate": "ollama:mistral-small3.2:latest",
          "case_id": "Q17",
          "turn": 1,
          "category": "timeout",
          "reason": "No completed final answer within the identical 30-second turn budget."
        }
      },
      {
        "id": "Q24",
        "status": "timeout",
        "answer": "",
        "timing": {
          "full_answer_seconds": 30.003124332986772,
          "model_seconds": 29.361425998969935,
          "tool_seconds": 0.6411494999192655,
          "ttft_seconds": null,
          "first_useful_result_seconds": null
        },
        "review": {
          "candidate": "ollama:mistral-small3.2:latest",
          "case_id": "Q24",
          "turn": 1,
          "category": "timeout",
          "reason": "No completed final answer within the identical 30-second turn budget."
        }
      },
      {
        "id": "Q27",
        "status": "timeout",
        "answer": "",
        "timing": {
          "full_answer_seconds": 30.004848124925047,
          "model_seconds": 29.89961583307013,
          "tool_seconds": 0.1046066660201177,
          "ttft_seconds": null,
          "first_useful_result_seconds": null
        },
        "review": {
          "candidate": "ollama:mistral-small3.2:latest",
          "case_id": "Q27",
          "turn": 1,
          "category": "timeout",
          "reason": "No completed final answer within the identical 30-second turn budget."
        }
      },
      {
        "id": "Q31",
        "status": "timeout",
        "answer": "",
        "timing": {
          "full_answer_seconds": 30.003362208022736,
          "model_seconds": 29.425490458030254,
          "tool_seconds": 0.5773779170121998,
          "ttft_seconds": null,
          "first_useful_result_seconds": null
        },
        "review": {
          "candidate": "ollama:mistral-small3.2:latest",
          "case_id": "Q31",
          "turn": 1,
          "category": "timeout",
          "reason": "No completed final answer within the identical 30-second turn budget."
        }
      },
      {
        "id": "Q34",
        "status": "completed",
        "answer": "La Honda CB 500 F édition ZXQ-FICTIVE-999 de 2020 est inconnue.",
        "timing": {
          "full_answer_seconds": 28.421562292031012,
          "model_seconds": 28.374383791000582,
          "tool_seconds": 0.046635584090836346,
          "ttft_seconds": null,
          "first_useful_result_seconds": null
        },
        "review": {
          "candidate": "ollama:mistral-small3.2:latest",
          "case_id": "Q34",
          "turn": 1,
          "category": "partial",
          "reason": "Correctly reports nonexistent edition without substitution; does not explicitly answer the equipment unknown."
        }
      },
      {
        "id": "Q37",
        "status": "timeout",
        "answer": "",
        "timing": {
          "full_answer_seconds": 30.0047179160174,
          "model_seconds": 29.954851208021864,
          "tool_seconds": 0.04932091699447483,
          "ttft_seconds": null,
          "first_useful_result_seconds": null
        },
        "review": {
          "candidate": "ollama:mistral-small3.2:latest",
          "case_id": "Q37",
          "turn": 1,
          "category": "timeout",
          "reason": "No completed final answer within the identical 30-second turn budget."
        }
      }
    ]
  },
  "other_routes": {
    "jev": {
      "workflow": "Jev decisions + existing SQL router and renderer; no Mistral synthesis",
      "fully_correct_cases": 4,
      "cases": 8,
      "turns": 9,
      "correct_turns": 5,
      "median_seconds_per_turn": 0.2740397499874234,
      "reviews": [
        {
          "candidate": "existing-jev-workflow",
          "case_id": "Q01",
          "turn": 1,
          "category": "correct",
          "reason": "Correct source count 8."
        },
        {
          "candidate": "existing-jev-workflow",
          "case_id": "Q10",
          "turn": 1,
          "category": "correct",
          "reason": "Correct 689 cc and source."
        },
        {
          "candidate": "existing-jev-workflow",
          "case_id": "Q17",
          "turn": 1,
          "category": "correct",
          "reason": "785/805 mm and correct 20 mm difference."
        },
        {
          "candidate": "existing-jev-workflow",
          "case_id": "Q24",
          "turn": 1,
          "category": "incorrect",
          "reason": "Claims absent catalog fiches although both requested identities exist."
        },
        {
          "candidate": "existing-jev-workflow",
          "case_id": "Q27",
          "turn": 1,
          "category": "incorrect",
          "reason": "Asks for the year instead of finding earliest documented DCT mention."
        },
        {
          "candidate": "existing-jev-workflow",
          "case_id": "Q31",
          "turn": 1,
          "category": "correct",
          "reason": "Both documented Z650 2020 variants returned."
        },
        {
          "candidate": "existing-jev-workflow",
          "case_id": "Q34",
          "turn": 1,
          "category": "incorrect",
          "reason": "Silently uses standard CB500F source instead of rejecting fictional edition."
        },
        {
          "candidate": "existing-jev-workflow",
          "case_id": "Q37",
          "turn": 1,
          "category": "correct",
          "reason": "Correct list of eight."
        },
        {
          "candidate": "existing-jev-workflow",
          "case_id": "Q37",
          "turn": 2,
          "category": "incorrect",
          "reason": "Follow-up loses Yamaha/2020/ABS/shifter scope and returns 8697 rather than 3."
        }
      ]
    },
    "mistral_small4_remote": {
      "fully_correct_answers": 1,
      "provider_failure_cases": 7,
      "correct_answer_seconds": 3.900473291054368,
      "reviews": [
        {
          "candidate": "openrouter:mistralai/mistral-small-2603",
          "case_id": "Q01",
          "turn": 1,
          "category": "provider_failure",
          "reason": "HTTP 429; no final answer. This does not measure intrinsic reasoning quality."
        },
        {
          "candidate": "openrouter:mistralai/mistral-small-2603",
          "case_id": "Q10",
          "turn": 1,
          "category": "correct",
          "reason": "689 cm3 with valid MT-07 2020 source evidence."
        },
        {
          "candidate": "openrouter:mistralai/mistral-small-2603",
          "case_id": "Q17",
          "turn": 1,
          "category": "provider_failure",
          "reason": "HTTP 429; no final answer. This does not measure intrinsic reasoning quality."
        },
        {
          "candidate": "openrouter:mistralai/mistral-small-2603",
          "case_id": "Q24",
          "turn": 1,
          "category": "provider_failure",
          "reason": "HTTP 429; no final answer. This does not measure intrinsic reasoning quality."
        },
        {
          "candidate": "openrouter:mistralai/mistral-small-2603",
          "case_id": "Q27",
          "turn": 1,
          "category": "provider_failure",
          "reason": "HTTP 429; no final answer. This does not measure intrinsic reasoning quality."
        },
        {
          "candidate": "openrouter:mistralai/mistral-small-2603",
          "case_id": "Q31",
          "turn": 1,
          "category": "provider_failure",
          "reason": "HTTP 429; no final answer. This does not measure intrinsic reasoning quality."
        },
        {
          "candidate": "openrouter:mistralai/mistral-small-2603",
          "case_id": "Q34",
          "turn": 1,
          "category": "provider_failure",
          "reason": "HTTP 429; no final answer. This does not measure intrinsic reasoning quality."
        },
        {
          "candidate": "openrouter:mistralai/mistral-small-2603",
          "case_id": "Q37",
          "turn": 1,
          "category": "provider_failure",
          "reason": "HTTP 429; no final answer. This does not measure intrinsic reasoning quality."
        }
      ],
      "additional_outcomes": [
        {
          "case": "Q01",
          "category": "provider_failure",
          "error": "openrouter HTTP 429"
        },
        {
          "case": "Q17",
          "category": "provider_failure",
          "error": "openrouter HTTP 429"
        },
        {
          "case": "Q24",
          "category": "provider_failure",
          "error": "openrouter HTTP 429"
        },
        {
          "case": "Q27",
          "category": "provider_failure",
          "error": "openrouter HTTP 429"
        }
      ]
    },
    "limits": "Known development cases; single run per case; no general accuracy claim. Two common-tool agents versus existing Jev decisions + SQL renderer; workflow differences are explicit"
  },
  "rag_writing": {
    "context": 8192,
    "generation_cases": 12,
    "median_seconds": 25.796105603978503,
    "maximum_seconds": 68.42685662501026,
    "above_30_seconds": 4,
    "cases": [
      {
        "id": "new-01",
        "question": "Pour la BMW G 310 R 2023, explique ce que décrit la fiche sur le moteur, son refroidissement et les mentions A2.",
        "seconds": 68.42685662501026
      },
      {
        "id": "new-02",
        "question": "BMW G 310 GS 2023 : résume la hauteur de selle et le réservoir, puis dis si tu retrouves une mention A2 dans cette fiche précise.",
        "seconds": 21.76426866708789
      },
      {
        "id": "new-03",
        "question": "Sur la Suzuki GSX-8S 800 2024 pour A2, le shifter est-il annoncé de série et la fiche précise-t-elle s'il fonctionne dans les deux sens ?",
        "seconds": 30.769576707971282
      },
      {
        "id": "new-04",
        "question": "Kawasaki Ninja 500 2025 : raconte-moi simplement ce que la fiche indique sur le couple, la capacité du réservoir et les permis, en contexte A2.",
        "seconds": 27.53900683298707
      },
      {
        "id": "new-05",
        "question": "Pour la Honda CB 750 Hornet 2025 en contexte A2, distingue ce qui est documenté sur le shifter et sur des poignées chauffantes montées de série.",
        "seconds": 22.64310891693458
      },
      {
        "id": "new-06",
        "question": "La fiche Ducati Monster 2024 donne-t-elle un certificat qui prouve le bridage A2 de mon exemplaire ? Résume ses mentions de permis sans certifier ma moto.",
        "seconds": 17.927345041069202
      },
      {
        "id": "new-07",
        "question": "Compare BMW G 310 R 2024 et Kawasaki Ninja 500 2025 pour A2 : explique leurs hauteurs de selle et leurs réservoirs avec une source pour chacune.",
        "seconds": 43.96069708396681
      },
      {
        "id": "new-08",
        "question": "Décris la transmission et le refroidissement de la Suzuki GSX-8S 800 2024 à partir de sa fiche, sans en déduire des sensations de conduite.",
        "seconds": 21.869566082954407
      },
      {
        "id": "new-09",
        "question": "Explique ce que décrivent le freinage et les suspensions de la Kawasaki Ninja 500 2025 ; si la fiche ne mesure pas le confort, indique ce manque.",
        "seconds": 38.648023750050925
      },
      {
        "id": "browser-01",
        "question": "Honda CL 500 2024 : résume le refroidissement et la hauteur de selle, puis précise si la fiche contient une mention A2.",
        "seconds": 24.053204374969937
      },
      {
        "id": "browser-02",
        "question": "Sur la BMW F 900 R 2024 pour A2, explique ce que dit la fiche sur la boîte et le shifter, en distinguant série et option. Est-ce un certificat de bridage pour ma moto ?",
        "seconds": 28.685028124949895
      },
      {
        "id": "browser-03",
        "question": "Kawasaki Z 500 2024 : donne-moi une réponse rédigée sur la puissance et les permis indiqués dans la fiche, avec le réservoir, pour mon dossier A2.",
        "seconds": 23.370342792011797
      }
    ],
    "limits": "Replay after corrections; first generation uses a new process; not an independent accuracy evaluation."
  },
  "retrieval": {
    "date": "2026-10-13",
    "scope": "Exact identities supplied by annotations",
    "cases_with_evidence": 44,
    "methods": [
      {
        "method": "Mots-clés · BM25",
        "covered": 19,
        "total": 44,
        "median_ms": 0.4
      },
      {
        "method": "BGE-M3",
        "covered": 40,
        "total": 44,
        "median_ms": 16.7
      },
      {
        "method": "Hybride",
        "covered": 34,
        "total": 44,
        "median_ms": 16.8
      },
      {
        "method": "Hybride + reranking",
        "covered": 41,
        "total": 44,
        "median_ms": 130.2
      }
    ]
  },
  "embedding_comparison": {
    "date": "2026-10-14",
    "passages": 4096,
    "questions_with_evidence": 44,
    "methods": [
      {
        "model": "BGE-M3",
        "first_hit_subset": 27,
        "first_hit_supplied": 40,
        "all_evidence_at10_supplied": 44,
        "median_subset_ms": 15.833771030884236,
        "dense_results": {
          "supplied": {
            "scope": "supplied",
            "method": "dense",
            "positive_questions": 44,
            "hit_at1": 0.9090909090909091,
            "recall_at5": 0.9412878787878788,
            "recall_at10": 0.9962121212121212,
            "mrr_at10": 0.9450757575757575,
            "all_groups_at10": 1.0,
            "median_seconds": 0.015666167018935084
          },
          "candidate_subset": {
            "scope": "candidate_subset",
            "method": "dense",
            "positive_questions": 44,
            "hit_at1": 0.6136363636363636,
            "recall_at5": 0.7064393939393939,
            "recall_at10": 0.8371212121212122,
            "mrr_at10": 0.7303030303030303,
            "all_groups_at10": 0.6818181818181818,
            "median_seconds": 0.015833771030884236
          }
        }
      },
      {
        "model": "Qwen3-Embedding 0.6B",
        "first_hit_subset": 34,
        "first_hit_supplied": 39,
        "all_evidence_at10_supplied": 43,
        "median_subset_ms": 35.82556248875335,
        "dense_results": {
          "supplied": {
            "scope": "supplied",
            "method": "dense",
            "positive_questions": 44,
            "hit_at1": 0.8863636363636364,
            "recall_at5": 0.918560606060606,
            "recall_at10": 0.9867424242424242,
            "mrr_at10": 0.9261363636363636,
            "all_groups_at10": 0.9772727272727273,
            "median_seconds": 0.03285512502770871
          },
          "candidate_subset": {
            "scope": "candidate_subset",
            "method": "dense",
            "positive_questions": 44,
            "hit_at1": 0.7727272727272727,
            "recall_at5": 0.7007575757575757,
            "recall_at10": 0.8522727272727273,
            "mrr_at10": 0.8524891774891774,
            "all_groups_at10": 0.7272727272727273,
            "median_seconds": 0.03582556248875335
          }
        }
      },
      {
        "model": "E5-large-instruct",
        "first_hit_subset": 22,
        "first_hit_supplied": 43,
        "all_evidence_at10_supplied": 43,
        "median_subset_ms": 24.310312466695905,
        "dense_results": {
          "supplied": {
            "scope": "supplied",
            "method": "dense",
            "positive_questions": 44,
            "hit_at1": 0.9772727272727273,
            "recall_at5": 0.9375,
            "recall_at10": 0.9943181818181818,
            "mrr_at10": 0.9886363636363636,
            "all_groups_at10": 0.9772727272727273,
            "median_seconds": 0.021298937033861876
          },
          "candidate_subset": {
            "scope": "candidate_subset",
            "method": "dense",
            "positive_questions": 44,
            "hit_at1": 0.5,
            "recall_at5": 0.6666666666666666,
            "recall_at10": 0.8068181818181818,
            "mrr_at10": 0.6399891774891775,
            "all_groups_at10": 0.6363636363636364,
            "median_seconds": 0.024310312466695905
          }
        }
      }
    ],
    "limits": "Dense retrieval only; known questions and a selected subset, not the complete 105687-passage corpus. Supplied identities are an oracle; no identity-resolution claim."
  },
  "comparability": "Different experiments and routes. Do not infer general accuracy or a speedup ratio across them."
}