{
  "schema": 1,
  "suite": "v2",
  "date": "2026-10-15",
  "fixtures": {
    "kind": "synthetic",
    "count": 30,
    "url": "https://corpus.betteroffice.dev/eval/formats/v2/",
    "metadata": "https://corpus.betteroffice.dev/eval/formats/v2/metadata.json",
    "description": [
      "The v2 generator creates synthetic files with fixed timestamps. DOCX fixtures start as hand-written OOXML, are opened and saved by BetterOffice, and carry paragraph ids. The manifest pins each published file by size and SHA-256. Downloads and local fixtures are verified against those pins.",
      "Regenerated bytes can change when engine output changes. A changed fixture set needs a new version prefix and updated pins. Fixed timestamps do not guarantee identical bytes across engine versions."
    ]
  },
  "harness": "codex exec",
  "limits": [
    "The v2 results cover small synthetic fixtures, one run per task and one model. They measure the agent and tools together. They do not establish performance on real-world documents, other models or repeated runs. The checkers validate defined edits rather than visual rendering quality.",
    "Tool counts differ by arm. Timings cannot support a speed comparison across arms. Wilson intervals do not account for task selection or predict results on other document sets."
  ],
  "reproduce": [
    "for p in fonts docx xlsx pptx agents; do bun run --filter \"@betteroffice/$p\" build; done\neval/formats/run.sh \"node packages/agents/dist/cli.js\" /tmp/format-eval gpt-6-luna",
    "eval/formats/run.sh --arm generic --python /path/to/venv/bin/python /tmp/format-eval-generic gpt-6-luna",
    "python3 eval/formats/check.py xlsx-04-style-header /tmp/format-eval/xlsx-04-style-header/output.xlsx\npython3 eval/formats/validate-checkers.py\npython3 eval/formats/test-runner.py\nbun eval/formats/generate-fixtures.ts /tmp/format-fixtures"
  ],
  "arms": [
    {
      "id": "generic",
      "label": "Generic file tools",
      "summary": "Shell and Python with python-docx, openpyxl and python-pptx, without network access.",
      "details": [
        "The agent uses shell commands and Python with python-docx, openpyxl and python-pptx in a workspace-write sandbox, with no network or MCP servers. Logged commands are audited for access to the repository or task metadata. Reads are not isolated by the filesystem sandbox."
      ],
      "tool_calls": "Shell commands and file edits; one command can run a whole script.",
      "comparison_label": "generic file tools"
    },
    {
      "id": "betteroffice",
      "label": "BetterOffice MCP",
      "summary": "Only the BetterOffice MCP server for document access.",
      "details": [
        "The agent runs in a read-only sandbox. The required BetterOffice MCP server is rooted at the task directory and is the only writer. Other document access tools are disabled. Code mode can call tools but has no file or network access. Server startup and required tools are checked before tasks run."
      ],
      "tool_calls": "BetterOffice MCP calls.",
      "comparison_label": "the BetterOffice MCP server"
    }
  ],
  "tasks": [
    {
      "id": "docx-01-fix-heading",
      "format": "docx",
      "title": "Fix a misspelled heading",
      "prompt": "In input.docx, the heading Sumary is misspelled. Change it to Summary. Edit the document directly, without tracked changes. Save the result as output.docx, leaving input.docx unchanged and preserving everything else.",
      "fixture": "docx-01-fix-heading.docx",
      "bytes": 4651,
      "sha256": "6c812dc00b94a1c33aa83910d0e1bf12e0ea8936b1a176ec617a51fcd71024e9"
    },
    {
      "id": "docx-02-replace-term",
      "format": "docx",
      "title": "Rename a project everywhere",
      "prompt": "In input.docx, replace every occurrence of the project name Atlas with Beacon, in the title, the body text, the list and the table. The match is case-sensitive, so leave the lowercase word atlas unchanged. Edit the document directly, without tracked changes. Save the result as output.docx, leaving input.docx unchanged and preserving everything else.",
      "fixture": "docx-02-replace-term.docx",
      "bytes": 4684,
      "sha256": "a9a126e5d154bd1b228df0b2d876c78b3a17dd9324867dc94b4e47cd3687dd1e"
    },
    {
      "id": "docx-03-correct-cell",
      "format": "docx",
      "title": "Correct a table value",
      "prompt": "In input.docx, correct the Budget table: the Actual amount for Supplies is 130, not 100, so the Actual total is 660, not 630. Edit the document directly, without tracked changes. Save the result as output.docx, leaving input.docx unchanged and preserving everything else.",
      "fixture": "docx-03-correct-cell.docx",
      "bytes": 4652,
      "sha256": "68c84d53acedffa2a2db3d28224ff54a2f9445cfa3c08a76b79f2f119c97ecb3"
    },
    {
      "id": "docx-04-add-row",
      "format": "docx",
      "title": "Add a table row above the total",
      "prompt": "In input.docx, add a row to the Budget table directly above the Total row: Item Training, Owner Avery, Planned 80, Actual 75. Format it like the other expense rows, not like the bold Total row. Then update the Total row to Planned 700 and Actual 705. Edit the document directly, without tracked changes. Save the result as output.docx, leaving input.docx unchanged and preserving everything else.",
      "fixture": "docx-04-add-row.docx",
      "bytes": 4652,
      "sha256": "68c84d53acedffa2a2db3d28224ff54a2f9445cfa3c08a76b79f2f119c97ecb3"
    },
    {
      "id": "docx-05-delete-row",
      "format": "docx",
      "title": "Delete a table row",
      "prompt": "In input.docx, the Supplies line was cancelled. Delete its row from the Budget table and update the Total row to Planned 500 and Actual 530. Edit the document directly, without tracked changes. Save the result as output.docx, leaving input.docx unchanged and preserving everything else.",
      "fixture": "docx-05-delete-row.docx",
      "bytes": 4652,
      "sha256": "68c84d53acedffa2a2db3d28224ff54a2f9445cfa3c08a76b79f2f119c97ecb3"
    },
    {
      "id": "docx-06-heading-style",
      "format": "docx",
      "title": "Promote a paragraph to a heading",
      "prompt": "In input.docx, make the paragraph Risks and mitigations a Heading 2, a subsection heading under Next steps, using the document's Heading 2 style. Keep its text. Edit the document directly, without tracked changes. Save the result as output.docx, leaving input.docx unchanged and preserving everything else.",
      "fixture": "docx-06-heading-style.docx",
      "bytes": 4726,
      "sha256": "b293864ebb97cf0ba02effc092cf4a45864d661ea3bf8e508a1ed167797edda7"
    },
    {
      "id": "docx-07-bold-phrase",
      "format": "docx",
      "title": "Bold a phrase",
      "prompt": "In input.docx, in the paragraph that begins The pilot started, make only the phrase ten participants bold. Edit the document directly, without tracked changes. Save the result as output.docx, leaving input.docx unchanged and preserving everything else.",
      "fixture": "docx-07-bold-phrase.docx",
      "bytes": 4652,
      "sha256": "68c84d53acedffa2a2db3d28224ff54a2f9445cfa3c08a76b79f2f119c97ecb3"
    },
    {
      "id": "docx-08-insert-paragraph",
      "format": "docx",
      "title": "Insert a paragraph below a heading",
      "prompt": "In input.docx, insert a new paragraph directly below the heading Next steps with the text The steering group meets on 20 October. Use the normal body text style. Edit the document directly, without tracked changes. Save the result as output.docx, leaving input.docx unchanged and preserving everything else.",
      "fixture": "docx-08-insert-paragraph.docx",
      "bytes": 4652,
      "sha256": "68c84d53acedffa2a2db3d28224ff54a2f9445cfa3c08a76b79f2f119c97ecb3"
    },
    {
      "id": "docx-09-delete-note",
      "format": "docx",
      "title": "Delete a draft note",
      "prompt": "In input.docx, delete the paragraph that starts with Draft note. Edit the document directly, without tracked changes. Save the result as output.docx, leaving input.docx unchanged and preserving everything else.",
      "fixture": "docx-09-delete-note.docx",
      "bytes": 4710,
      "sha256": "ad66af5f6b56c6730070b3647d2a0b37f55980f43e1da86a5adf8ccf5e82a76f"
    },
    {
      "id": "docx-10-finalize-report",
      "format": "docx",
      "title": "Finalize a report",
      "prompt": "In input.docx, finalize the report: replace the placeholder [DATE] with 14 October 2026, delete the line that only says DRAFT, and make the first line, Pilot status report, a Heading 1. Edit the document directly, without tracked changes. Save the result as output.docx, leaving input.docx unchanged and preserving everything else.",
      "fixture": "docx-10-finalize-report.docx",
      "bytes": 4670,
      "sha256": "e0456aef604fea54dab0b5bcef7c67e0ec129304d4d37e3b4b6efc6785c09387"
    },
    {
      "id": "pptx-01-change-title",
      "format": "pptx",
      "title": "Change a slide title",
      "prompt": "In input.pptx, change the title on slide 1 from Quarterly update to Quarterly results. Save the result as output.pptx, leaving input.pptx unchanged and preserving everything else.",
      "fixture": "pptx-01-change-title.pptx",
      "bytes": 8728,
      "sha256": "5eed523d96b75aff660002a0a4ec46150b36a1f7ac80d1c36e44b8e607ec372e"
    },
    {
      "id": "pptx-02-update-revenue",
      "format": "pptx",
      "title": "Update a reported figure",
      "prompt": "In input.pptx, on slide 2 change Revenue: $1.2 million to Revenue: $1.5 million. Leave the margin and retention figures intact. Save the result as output.pptx, leaving input.pptx unchanged and preserving everything else.",
      "fixture": "pptx-02-update-revenue.pptx",
      "bytes": 8739,
      "sha256": "c7ac976d205fea42a39927de648eceb3907843fccfab779e4aaa849ec247d999"
    },
    {
      "id": "pptx-03-replace-bullets",
      "format": "pptx",
      "title": "Replace a bullet list",
      "prompt": "In input.pptx, replace the three bullets on slide 2, Next steps, with these three bullets in order: Finish the pilot; Collect feedback; Launch in October. Keep the existing bullet formatting. Save the result as output.pptx, leaving input.pptx unchanged and preserving everything else.",
      "fixture": "pptx-03-replace-bullets.pptx",
      "bytes": 8739,
      "sha256": "7e04f86313b452c12145790521497790df98ff18ac62b406b1754acc80593482"
    },
    {
      "id": "pptx-04-replace-term",
      "format": "pptx",
      "title": "Rename a project in slide text",
      "prompt": "In input.pptx, replace Atlas with Beacon everywhere in the visible slide titles and bullets. Leave speaker notes as they are. Save the result as output.pptx, leaving input.pptx unchanged and preserving everything else.",
      "fixture": "pptx-04-replace-term.pptx",
      "bytes": 8752,
      "sha256": "d91755dc608dee0b5ce4afffdd79381bcd5dca6613d00f824caa95c6775cdc26"
    },
    {
      "id": "pptx-05-finalize-titles",
      "format": "pptx",
      "title": "Finalize two slide titles",
      "prompt": "In input.pptx, change slide 2 title Draft results to Results and slide 3 title Draft next steps to Next steps. Leave mentions of those phrases in bullets and speaker notes unchanged, and keep both slides in their current positions. Save the result as output.pptx, leaving input.pptx unchanged and preserving everything else.",
      "fixture": "pptx-05-finalize-titles.pptx",
      "bytes": 8754,
      "sha256": "0603a7585e1767dd6f3e61eb738c3829fed63bfb223af6006e7bb07a8308959d"
    },
    {
      "id": "pptx-06-delete-appendix",
      "format": "pptx",
      "title": "Remove an appendix slide",
      "prompt": "In input.pptx, delete slide 3, titled Appendix. Keep the other slides in their current order. Save the result as output.pptx, leaving input.pptx unchanged and preserving everything else.",
      "fixture": "pptx-06-delete-appendix.pptx",
      "bytes": 10687,
      "sha256": "0908ae052c951bd591562b240a812add351e0eafe08c8fdcf09ea2798ee11d1b"
    },
    {
      "id": "pptx-07-reorder-slides",
      "format": "pptx",
      "title": "Reorder a presentation",
      "prompt": "In input.pptx, arrange the slides in this order by title: Overview, Agenda, Results, Next steps. Keep each slide's content and speaker notes attached to that slide. Save the result as output.pptx, leaving input.pptx unchanged and preserving everything else.",
      "fixture": "pptx-07-reorder-slides.pptx",
      "bytes": 10681,
      "sha256": "45956dea07f2aef6e8ef5411f4669ccf6877019d363bfcf210c77ce21ff4adeb"
    },
    {
      "id": "pptx-08-speaker-notes",
      "format": "pptx",
      "title": "Write speaker notes",
      "prompt": "In input.pptx, replace the speaker notes on slide 2, Results, with exactly two paragraphs: Pause for questions after the chart. and Ask which metric needs a follow-up. Put each sentence in its own paragraph. Save the result as output.pptx, leaving input.pptx unchanged and preserving everything else.",
      "fixture": "pptx-08-speaker-notes.pptx",
      "bytes": 8716,
      "sha256": "2528fdfb1ee7f72e5aed5f0441b3df99fcfa5a996c1225809f07c2b274fe9fee"
    },
    {
      "id": "pptx-09-position-textbox",
      "format": "pptx",
      "title": "Add a positioned text box",
      "prompt": "In input.pptx, add a text box on slide 1 containing Internal draft. Place its top-left corner 0.5 inches from the left edge and 5 inches from the top edge, with width 3 inches and height 0.5 inches. Use 18-point text in #666666. Save the result as output.pptx, leaving input.pptx unchanged and preserving everything else.",
      "fixture": "pptx-09-position-textbox.pptx",
      "bytes": 8719,
      "sha256": "a877e4275184d507f05cb3087c05aa38bb15ebeb04b93749e22de7bdf9a39bf2"
    },
    {
      "id": "pptx-10-insert-slide",
      "format": "pptx",
      "title": "Insert a slide with bullets and notes",
      "prompt": "In input.pptx, insert a new slide immediately after slide 2, Agenda. Give it the title Pilot plan and three separate bullets in this order: Start with ten users; Collect feedback daily; Report results in two weeks. Add exactly this speaker note to the new slide: Keep the pilot to two weeks. Save the result as output.pptx, leaving input.pptx unchanged and preserving everything else.",
      "fixture": "pptx-10-insert-slide.pptx",
      "bytes": 10682,
      "sha256": "51e7cde1b8c55a63a3745c80326a9832306bd68d2483b29174ff6b5cf0fc8f9d"
    },
    {
      "id": "xlsx-01-correct-value",
      "format": "xlsx",
      "title": "Correct a budget amount",
      "prompt": "In input.xlsx, change the planned Supplies amount in Budget!B3 to 180. Keep the totals as formulas and update their calculated results. Save the result as output.xlsx, leaving input.xlsx unchanged and preserving everything else.",
      "fixture": "xlsx-01-correct-value.xlsx",
      "bytes": 2195,
      "sha256": "00d9aafc2a10e7c02553342d71952be66a6e65626d7a34b902d2c7b47cf39b3b"
    },
    {
      "id": "xlsx-02-fix-total",
      "format": "xlsx",
      "title": "Repair a total formula",
      "prompt": "In input.xlsx, the Actual total in Budget!C5 leaves out Travel. Replace it with =SUM(C2:C4) and update its calculated result. Save the result as output.xlsx, leaving input.xlsx unchanged and preserving everything else.",
      "fixture": "xlsx-02-fix-total.xlsx",
      "bytes": 2196,
      "sha256": "99c7d9fefd3e60d0f6338cb00f9db4e6fe9ac19d4c8790cf1effecdda2b27a46"
    },
    {
      "id": "xlsx-03-replace-product",
      "format": "xlsx",
      "title": "Rename a product across sheets",
      "prompt": "In input.xlsx, replace every occurrence of the case-sensitive product name Trail Mix with Forest Mix on both Catalog and Orders. Keep the rest of each cell intact. Save the result as output.xlsx, leaving input.xlsx unchanged and preserving everything else.",
      "fixture": "xlsx-03-replace-product.xlsx",
      "bytes": 2535,
      "sha256": "f95454bc34bc618e590cd34b25a6178f02d91c6dd21cf92aab2f9403acbfd554"
    },
    {
      "id": "xlsx-04-style-header",
      "format": "xlsx",
      "title": "Format a header",
      "prompt": "In input.xlsx, make Sales!A1:C1 bold and give those three header cells a solid light blue fill, color #D9EAF7. Save the result as output.xlsx, leaving input.xlsx unchanged and preserving everything else.",
      "fixture": "xlsx-04-style-header.xlsx",
      "bytes": 2145,
      "sha256": "5f0714d386cff1fa13071bf6450db38e367f2ca10c63c80df4fef0039d6fc762"
    },
    {
      "id": "xlsx-05-sort-inventory",
      "format": "xlsx",
      "title": "Sort an inventory table",
      "prompt": "In input.xlsx, sort Inventory!A2:C5 by Stock in column C, smallest to largest. Move each complete row together and leave the header in row 1. Save the result as output.xlsx, leaving input.xlsx unchanged and preserving everything else.",
      "fixture": "xlsx-05-sort-inventory.xlsx",
      "bytes": 2116,
      "sha256": "0923ec7e724f2b794a7e0e3d72956a0baea34cd3469a20345a4e9d9f3c0432fd"
    },
    {
      "id": "xlsx-06-add-totals",
      "format": "xlsx",
      "title": "Add a currency total row",
      "prompt": "In input.xlsx, add a total row on Sales at row 5: A5 should say Total, B5 should contain =SUM(B2:B4), and C5 should contain =SUM(C2:C4). Format B5:C5 as dollars with two decimal places and a thousands separator, using $#,##0.00, and save calculated results. Save the result as output.xlsx, leaving input.xlsx unchanged and preserving everything else.",
      "fixture": "xlsx-06-add-totals.xlsx",
      "bytes": 2145,
      "sha256": "5f0714d386cff1fa13071bf6450db38e367f2ca10c63c80df4fef0039d6fc762"
    },
    {
      "id": "xlsx-07-growth-column",
      "format": "xlsx",
      "title": "Insert a growth column",
      "prompt": "In input.xlsx, insert a column immediately after Revenue column C, moving the existing Status column to E. Name D1 Growth %. In D2:D4 use =(C2-B2)/B2, =(C3-B3)/B3, and =(C4-B4)/B4 respectively. Format D2:D4 as percentages with one decimal place, using 0.0%, and save calculated results. Save the result as output.xlsx, leaving input.xlsx unchanged and preserving everything else.",
      "fixture": "xlsx-07-growth-column.xlsx",
      "bytes": 2175,
      "sha256": "8a5a914abb3a1fb8bd054e267145ed5d3038bd678c05c36aeab2752823f43360"
    },
    {
      "id": "xlsx-08-organize-sheets",
      "format": "xlsx",
      "title": "Rename and reorder sheets",
      "prompt": "In input.xlsx, rename Forecast to Outlook. Arrange the tabs in this order: Actuals, Outlook, Assumptions. Keep the formulas on Assumptions working by changing its B2 reference to ='Outlook'!B2; keep all other formulas and values intact. Save the result as output.xlsx, leaving input.xlsx unchanged and preserving everything else.",
      "fixture": "xlsx-08-organize-sheets.xlsx",
      "bytes": 2853,
      "sha256": "b0cdefaed16ca5f9845bfffd5732a8bfcebbab560242f663c3f80e5fbab896c1"
    },
    {
      "id": "xlsx-09-budget-summary",
      "format": "xlsx",
      "title": "Build a linked budget summary",
      "prompt": "In input.xlsx, add an Expenses total row at row 5 with A5 Total, B5 =SUM(B2:B4), and C5 =SUM(C2:C4). On Summary, set B2 to ='Expenses'!B5, B3 to ='Expenses'!C5, and B4 to =B3-B2. Format Expenses!B5:C5 and Summary!B2:B4 as dollars using $#,##0.00 and save calculated results. Save the result as output.xlsx, leaving input.xlsx unchanged and preserving everything else.",
      "fixture": "xlsx-09-budget-summary.xlsx",
      "bytes": 2576,
      "sha256": "0ae172cfa1b2b8a6aceacf660b0446f75548de191fd1762aa730532b9c80141e"
    },
    {
      "id": "xlsx-10-quarter-report",
      "format": "xlsx",
      "title": "Prepare a quarterly report",
      "prompt": "In input.xlsx on Quarterly, insert Growth % in a new column D, moving Owner to E. Sort the three data rows by Q2 descending, keeping each owner with the product. Set D2:D4 to =(C2-B2)/B2, =(C3-B3)/B3, and =(C4-B4)/B4. Add row 5 with A5 Total, B5 =SUM(B2:B4), C5 =SUM(C2:C4), and D5 =(C5-B5)/B5. Use 0.0% in D2:D5 and $#,##0.00 in B5:C5. Make A1:E1 bold with solid #D9EAF7 fill. Save calculated results. Save the result as output.xlsx, leaving input.xlsx unchanged and preserving everything else.",
      "fixture": "xlsx-10-quarter-report.xlsx",
      "bytes": 2178,
      "sha256": "e682a4b942f7908234f3152281d65ee35ed348ebdcf6d4e027dfe1b377b8413e"
    }
  ],
  "runs": [
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "docx-01-fix-heading",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 2,
      "seconds": 9.345
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "docx-02-replace-term",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 3,
      "seconds": 10.104
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "docx-03-correct-cell",
      "run": 1,
      "date": "2026-10-15",
      "pass": false,
      "check": "document.body[11].rows[3].cells[4].blocks[1].attributes: fields differ",
      "reason": "python-docx cell.text replaced the edited cell paragraphs and dropped their paragraph ids, right alignment, run size and total bold formatting.",
      "cause": "library",
      "tool_calls": 2,
      "seconds": 9.01
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "docx-04-add-row",
      "run": 1,
      "date": "2026-10-15",
      "pass": false,
      "check": "document.body[11].rows[5].cells[1].blocks[1].attributes: paraId reuses an input paragraph id",
      "reason": "The model copied a template row with its paragraph ids intact, duplicating them, and added an empty run to the Total row.",
      "cause": "model",
      "tool_calls": 2,
      "seconds": 14.981
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "docx-05-delete-row",
      "run": 1,
      "date": "2026-10-15",
      "pass": false,
      "check": "document.body[11].rows[4].cells[3].blocks[1].attributes: fields differ",
      "reason": "python-docx cell.text replaced the total cell paragraphs and dropped their paragraph ids, right alignment, run size and bold formatting.",
      "cause": "library",
      "tool_calls": 2,
      "seconds": 9.102
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "docx-06-heading-style",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 2,
      "seconds": 7.706
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "docx-07-bold-phrase",
      "run": 1,
      "date": "2026-10-15",
      "pass": false,
      "check": "document.body[4].content[3].content: count differs (1 expected, 2 found)",
      "reason": "The model created a suffix run starting with a space without xml:space=\"preserve\", so that space was no longer preserved.",
      "cause": "model",
      "tool_calls": 2,
      "seconds": 16.807
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "docx-08-insert-paragraph",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 2,
      "seconds": 10.757
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "docx-09-delete-note",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 2,
      "seconds": 8.007
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "docx-10-finalize-report",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 2,
      "seconds": 7.051
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "pptx-01-change-title",
      "run": 1,
      "date": "2026-10-13",
      "pass": false,
      "check": "slide 1.shape 1.paragraphs[1].props: count differs (4 expected, invalid found)",
      "reason": "The model assigned shape.text to rebuild the title paragraph, dropping its paragraph and run formatting.",
      "cause": "model",
      "tool_calls": 2,
      "seconds": 9.964
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "pptx-02-update-revenue",
      "run": 1,
      "date": "2026-10-13",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 2,
      "seconds": 24.051
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "pptx-03-replace-bullets",
      "run": 1,
      "date": "2026-10-13",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 2,
      "seconds": 18.976
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "pptx-04-replace-term",
      "run": 1,
      "date": "2026-10-13",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 4,
      "seconds": 21.971
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "pptx-05-finalize-titles",
      "run": 1,
      "date": "2026-10-13",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 2,
      "seconds": 29.401
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "pptx-06-delete-appendix",
      "run": 1,
      "date": "2026-10-13",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 2,
      "seconds": 12.319
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "pptx-07-reorder-slides",
      "run": 1,
      "date": "2026-10-13",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 3,
      "seconds": 21.298
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "pptx-08-speaker-notes",
      "run": 1,
      "date": "2026-10-13",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 2,
      "seconds": 27.241
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "pptx-09-position-textbox",
      "run": 1,
      "date": "2026-10-13",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 2,
      "seconds": 11.132
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "pptx-10-insert-slide",
      "run": 1,
      "date": "2026-10-13",
      "pass": false,
      "check": "layouts and masters: fields differ",
      "reason": "python-pptx created an extra notes master and theme when the model added speaker notes to the new slide.",
      "cause": "library",
      "tool_calls": 8,
      "seconds": 52.57
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "xlsx-01-correct-value",
      "run": 1,
      "date": "2026-10-13",
      "pass": false,
      "check": "document.sheets[1].metadata: count differs (2 expected, 4 found)",
      "reason": "openpyxl saved extra theme and document property parts, page margins and a calcId, changing unrelated workbook state.",
      "cause": "library",
      "tool_calls": 3,
      "seconds": 33.275
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "xlsx-02-fix-total",
      "run": 1,
      "date": "2026-10-13",
      "pass": false,
      "check": "Budget!B5: empty cached value",
      "reason": "openpyxl cleared the existing Budget!B5 formula cache on save while the model restored only the requested C5 result.",
      "cause": "library",
      "tool_calls": 5,
      "seconds": 43.237
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "xlsx-03-replace-product",
      "run": 1,
      "date": "2026-10-13",
      "pass": false,
      "check": "document.sheets[1].metadata: count differs (2 expected, 4 found)",
      "reason": "openpyxl saved extra theme and document property parts, page margins and a calcId while replacing product text.",
      "cause": "library",
      "tool_calls": 2,
      "seconds": 10.588
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "xlsx-04-style-header",
      "run": 1,
      "date": "2026-10-13",
      "pass": false,
      "check": "document.sheets[1].metadata: count differs (2 expected, 4 found)",
      "reason": "openpyxl saved extra theme and document property parts, page margins and a calcId while styling the header.",
      "cause": "library",
      "tool_calls": 1,
      "seconds": 11.179
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "xlsx-05-sort-inventory",
      "run": 1,
      "date": "2026-10-13",
      "pass": false,
      "check": "document.sheets[1].metadata: count differs (2 expected, 4 found)",
      "reason": "openpyxl saved extra theme and document property parts, page margins and a calcId while sorting the inventory.",
      "cause": "library",
      "tool_calls": 2,
      "seconds": 8.836
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "xlsx-06-add-totals",
      "run": 1,
      "date": "2026-10-13",
      "pass": false,
      "check": "Sales!B5: empty cached value",
      "reason": "The model wrote total formulas but left their calculated results empty, relying on a later recalculation instead of saving the requested values.",
      "cause": "model",
      "tool_calls": 3,
      "seconds": 24.875
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "xlsx-07-growth-column",
      "run": 1,
      "date": "2026-10-13",
      "pass": false,
      "check": "document.sheets[1].metadata: count differs (2 expected, 4 found)",
      "reason": "openpyxl saved extra theme and document property parts, page margins and a calcId while inserting the growth column.",
      "cause": "library",
      "tool_calls": 5,
      "seconds": 29.456
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "xlsx-08-organize-sheets",
      "run": 1,
      "date": "2026-10-13",
      "pass": false,
      "check": "Assumptions!B2: empty cached value",
      "reason": "openpyxl cleared the Assumptions!B2 formula cache and renumbered sheet ids on save while the model renamed and reordered sheets.",
      "cause": "library",
      "tool_calls": 2,
      "seconds": 20.476
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "xlsx-09-budget-summary",
      "run": 1,
      "date": "2026-10-13",
      "pass": false,
      "check": "document.sheets[1].metadata: count differs (2 expected, 4 found)",
      "reason": "openpyxl saved extra theme and document property parts, page margins and a calcId while building the budget summary.",
      "cause": "library",
      "tool_calls": 4,
      "seconds": 42.963
    },
    {
      "model": "gpt-6-luna",
      "arm": "generic",
      "task": "xlsx-10-quarter-report",
      "run": 1,
      "date": "2026-10-13",
      "pass": false,
      "check": "document.sheets[1].cells.A1.style.base.fill: expected None, found 'FFD9EAF7'",
      "reason": "The model changed a shared default fill color through a shallow copy, changing the base fill outside the requested header.",
      "cause": "model",
      "tool_calls": 5,
      "seconds": 53.009
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "docx-01-fix-heading",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 6,
      "seconds": 21.496
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "docx-02-replace-term",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 7,
      "seconds": 16.89
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "docx-03-correct-cell",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 12,
      "seconds": 26.461
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "docx-04-add-row",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 10,
      "seconds": 22.077
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "docx-05-delete-row",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 12,
      "seconds": 31.31
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "docx-06-heading-style",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 5,
      "seconds": 12.608
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "docx-07-bold-phrase",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 5,
      "seconds": 11.022
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "docx-08-insert-paragraph",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 5,
      "seconds": 11.334
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "docx-09-delete-note",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 5,
      "seconds": 9.358
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "docx-10-finalize-report",
      "run": 1,
      "date": "2026-10-15",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 11,
      "seconds": 26.385
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "pptx-01-change-title",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 5,
      "seconds": 24.633
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "pptx-02-update-revenue",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 7,
      "seconds": 22.954
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "pptx-03-replace-bullets",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 7,
      "seconds": 25.755
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "pptx-04-replace-term",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 11,
      "seconds": 28.911
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "pptx-05-finalize-titles",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 7,
      "seconds": 19.406
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "pptx-06-delete-appendix",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 5,
      "seconds": 18.417
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "pptx-07-reorder-slides",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 8,
      "seconds": 33.421
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "pptx-08-speaker-notes",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 6,
      "seconds": 19.461
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "pptx-09-position-textbox",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 5,
      "seconds": 26.722
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "pptx-10-insert-slide",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 9,
      "seconds": 35.729
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "xlsx-01-correct-value",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 7,
      "seconds": 20.83
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "xlsx-02-fix-total",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 6,
      "seconds": 19.21
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "xlsx-03-replace-product",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 6,
      "seconds": 26.795
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "xlsx-04-style-header",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 6,
      "seconds": 20.676
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "xlsx-05-sort-inventory",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 6,
      "seconds": 19.896
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "xlsx-06-add-totals",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 7,
      "seconds": 24.398
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "xlsx-07-growth-column",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 8,
      "seconds": 26.306
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "xlsx-08-organize-sheets",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 9,
      "seconds": 20.422
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "xlsx-09-budget-summary",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 8,
      "seconds": 22.484
    },
    {
      "model": "gpt-6-luna",
      "arm": "betteroffice",
      "task": "xlsx-10-quarter-report",
      "run": 1,
      "date": "2026-10-14",
      "pass": true,
      "check": "exact requested content, formulas, formatting, order, notes, and preserved content verified",
      "reason": null,
      "cause": null,
      "tool_calls": 10,
      "seconds": 30.374
    }
  ],
  "checker_validation": {
    "accepted": 65,
    "untouched_rejected": 30,
    "corrupted_rejected": 157
  }
}