在这个教程中，我们执行 OmniParse Bench, Datalab 打开的基准测试文档解析器，该解析器的输出分级为16,288个小的通过或故障测试，从扫描表和笔迹到arXiv 数学和密集的表格共2,937页。 而不是信任领导板，我们从Datalab 发布 27 parsers的判决中重建它，给计分员一个手制的页面，看看哪些错误推翻了判决。 然后，我们测量排名确定性，按文件类型进行切片，测试两个卡片名称是否有助于构建测试在测试中表现不同，一个模型被修正，然后在50页的样本上运行宇宙魔方和Indra OCR。 我们发现前两个解析器在统计学上是捆绑的
导入 Os
导入重置
导入 sys
导入 Json
导入时间
导入 html
导入 hipil
导入平台
导入追溯
导入子进程
导入收藏
导入urlib。请求
从 pathlib 导入路径
成果 = 步 - > 单行结果，用于总表
声称 = [#(步骤、索赔、持有)
现值 = ["1]
def 横幅( 标题):
打印("\n" + "=" * 78)
打印(标题)
打印 ("=" * 78)
def section(name):
fn) 包装：
f 运行( *a, ** kw):
现 [0] = name.split (") [0].
横幅( 名称)
t0 = 时间。 time ()
尝试：
输出 = fn(*a,**kw)
RESULTS[名] = 如果出现( 出，str) 其他"OK"。
返回出来
例外作为e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
打印 (f)\n [!] {name} 未完成：{type (e). name: {e}>
回溯.print exc(限制=3)
返回无
终于：
打印(f)\n [{time.time() - t0: 1f} s] )
返回运行
返回包装
索赔(案文，持有):
声称。 附录((CURRENT[0]，文本，bul(持有)))
持有的返回
横幅( “ 1. 安装 omni- parse- bench 1. 0.0rc1 和宇宙魔方，并给数据集修订 )
System = 平台。 system () 系统
OPB VERSION = "1.0.0rc1" # 一个预放：一个赤裸的`pip安装omni-parse-bench' 解析为0.2.0，其API不同
REPO = "数据实验室对/omni parse bench"
REV = "961c0f89a2da701f5ce53afbd5e80a469b0369a" #发布27个解析器判决的数据集
INDRA= ["indra-ocr = 0.3.2","numpy = 2.5.4","onnxruntime = 1.31.0","opencv-python-less = 5.0.0.93" (中文(简体) ).
"pypdfium2 == 5.14.0","huggingface hub = 2.2.0"# 包内自有因陀罗服务器图像的针头。
Work = path(os.environ.get ("OPB WORK") 或 ("/content/opb" if os.path.isdir ("/content") other "opb work"))). resolve ().
Work.mkdir(父母关系=True，存在 ok=True)
sub process.run([sys. executive,"-m","pip","install","-q",f"omni-parse-bench[基准测试] XOPB VERSION}],"v"]，检查=True)
如果SETE = "Linux" 和 haltil. ("Teffract") 是无：
脚本=("arab armn beng cans cher cyrl deva ethi frak geor grek gujr guru hang-vert hans hans-vert") 汉语拼音为"汉语拼音" 汉语拼音为"汉语拼音".
"hant-vert hebr jpan jpan-vert knda khmr laoo latn mlym mymr orya sinh syrc taml telu thaa thai tibt viet").split()
subprocess.run(["apt-get","-qq","update"], check=True) 页面存档备份，存于互联网档案馆。
subprocess.run(["apt-get", "-qq", "install", "-y", "tesseract-ocr", "tesseract-ocr-eng", "tesseract-ocr-osd"]
+ [f] 语录-ocr-script-{s} 用于脚本中的 s], check=True, stdout=subprocess.DEVNULL).
导入数字为 np
导入熊猫为 pd
导入 importlib.metadata 作为 md
从 huggingface hub 导入快照下载
从 omni parse bench 导入基准测试，数据、 调用、 标记降级、 vocab
从 omni parse bench. 计分器导入单位测试
print(f)"omni-parse-bench { md.version('omni-parse-bench')}, Python {plateform.python version()}, {SYSTEM} {plateform. 机器()}, "
f"{os.cpu counts()}" CPU核心").
如果系统="Linux":
print("os:", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
tess = sub process.run (['tessract, '-version'], capt output=True, text=True] (中文(简体) ).
print (“ tessract ” ), (tes. stdout 或 tes.stderr). splitlines () [0]。
TESS MAJOR = int(( ts.stdout 或 tes.stderr)). split () [1].split ("") [0]
print("test families:", {f: [t for t, g in vocab.FAMILIES.items() if g == f] for f in dict.fromkeys(vocab.FAMILIES.values())})
打印(f) 数据集 {REPO} @ {REV[:7]}；工作文件夹 {WORK} )
ResULTS ["1. 安装 omni-parse-bench 1.0.0rc1 和 Tefferact，并给数据集修正针头 = (
f"omni-parse-bench { md.version('omni-parse-bench')}, sectract {(tes.stdout 或 tess.stderr).split () [1]，数据集 @ {REV [:7]}")
我们安装了被钉在1.0.0rc1的omni-parse-bench，因为一个赤色的pip安装仍然溶解到0.2.0，一个更古老的释放，具有不同的API. 基准测试 Extra 带来解析器适配器，Hugging Face客户端，以及用于运单的Pyrow,Colab上我们通过apt添加了魔方的脚本模型。 我们还将数据集定在一个承诺上，因为寄存器会随着解析器的添加而变化。 打印家庭显示结构基准测试所基于的结构：存在，顺序，重复测试组成文本家族；table cell 组成表格家族；和排版 kind 组成布局家族。
@ section ("2. 计分器如何决定：一个手造的页面，五个测试类型，一个变化一次").
def 计分器解剖仪( ):
T=lambda i, tt, out, tags, args, st=None: {"id":i, "test type": tt, "output type": out, "开发":"作者"
"tags"：标记，"args":ags,"设置":st或QQ
准确 = lambda s: {"文本": s, "max diffs": 0}
黄金 = [
T("标题","现在","html",["标题"],{"内容"：准确("季度报告"))}
T("总计","现在","html",["eng"],{"内容"：准确("总计到期"))}
T("增长","现在","html",["eng"],{内容":{"数量":"-4.8%"})
T("amount","present","html",["eng"],{"内容":{"数字":"1,240.00"}}
T("eq ratio"),"现在","html",["math"],{"内容":{"latex":r"\alpha {1}+\β"".
T("eq half","现在","html",["math"],{"内容":{"latex":r"\frac{1}{2}\to x}}").
T("顺序","顺序","html",["eng"],{在"之前"：准确("导言：本报告涵盖第二季度"). )
"后"：准确("结论：计划坚守。 ")
T("repeat","repeat","html",["eng"],{"内容"：精确("Revenue"))},{"预期计算":2})
T("cell top","table cell","html",["表"],{"cell"：精确("1,240"),"top heading"：精确("2024")}).
T("cell left","table cell","html",["表"],{"cell"：精确("1,240"),"左头部"：精确("收入")})
T("cell price","table cell","html",["表"],{"cell"：精确("9.50"),"top heading"：精确("price")
"左传"：确切("Widget")].
T("块标题","layout kind","块",["头"])
{"内容":dict(确切的("季度报告")，位置=[[[0.10,0.04,0.90,0.09]]]]},{"kind":"heading"}).
T("块 total"),"layout kind","块",["eng"]
{"内容":dict(确切("总到期"), place=[[0.10,0.12,0.90,0.16]]]},{"kind":"text"}).
]
gold file=work / "fixture" / "gold" / "demo-p1.json" # 一个金色文件以其页面命名
gold file.parent.mkdir(父母关系=True，存在 ok=True)
gold file.write text(json.dumps({' tests': gold}，缩进=1)).
页=数据。 页面(page id="demo-p1"，文档="demo"，套件="demo",doc path=WORK/"固定"/"demo-p1.pdf"
gt path=gold file，输出=("块","html")).
测试 = 单位 tests.gold(页)
表格 a = (“表<trt><t></th>2023</th>2024</th>/trt><t><t><t>h> 收入</th><td>1,000</tdt><td>1,240</tdd></tr>)
"<tr><th> 成本</th><td>600</td><td>700</td></tr></表>").
表格 b = “<表格>tr><th>> 生产</th><th> 价格</th></tr><t><tdtd> Widget</tdtdd>9.50</td/tr></表>”
md table b = "产品价格价格价格价格价格价格
部分={标题：<h1>Q季度报告</h1>,“总数”:<p>到期总金额：1 240.00美元。</p>
"增长":"<p>年增长率为−4.8%.</p>","ratio":r"<p>Ratio: 1+持有量。</p>
"半":r"<p>Half:12x在模型中。</p>"
"入门":"<p> 导言：本报告涵盖第二季度。</p>"
"录"曰："‘p>录以千计。 第二季度收入增加。
"表 a"：表 a,"结论":"<p>结论：计划持有。</p>,"表 b"：表 b}
块 = [{'标签': "标题", "box": [0.10, 0.04, 0.90, 0.09], "文本": "季度报告"]
{"标签":"文本","box":[0.10,0.12,0.90,0.16],"文本":"到期总金额：1,240.00美元。
{"标签":"表","框":[0.10,0.40,0.90,0.60],"文本":"收入1000,1,240成本600,700"}
def pred(更改=无，顺序=无，新块=无):
p = dict(部分，**(更改或 ))
返回 {'html: "\n"。 join( p[k] for k in( 顺序或列表( 部分) )), "块"：如果新 blocks 不是其他新 blocks} 的块
base = {v.id: v.result for v in unit_tests.score(pred(), tests)}
print(f)"碱性预测：{sum(r ==========================================================================================================================================================================
互换= ["标题","总","增长","ratio","半","结论","into","收益","表 a","表 b"]
变体=[
("flip"," a type:"total dve"), pred({"total"):"<p> total dve: 1,240.00美元。</p>},()).
("flip","减号下降：4.8%"，预览({"增长"):"<p>年增长率为4.8%.</p>},())
("flip","另一个符号：α 2", pred({'ratio':r"<p>Ratio:2+ holds.</p>}),()).
("flip","介绍前的结束"，预告(命令=擦除),()).
("flip","Revenues"代表第二个"Revenue").
预览({“收益”):“<p>收益以千计。 第二季度的收入增加。</p>,()
("flip","2023年和2024年标题互换"，预告({"table a"): table a.replace ("<th>2023</th>2024</th>)
"<th>2024</th><th>2023</th>")}), ()),
("flip","road标签'margin' for 'income", pred({"table a": table a.replace ("<th> Income</th>","<th>Margin</th>"))),())
("flip","html中留下的标记下表"，预告({"table b":md table b}),()).
("flip","标题的块标签文本",red(新块=[dit(块[0]，标签="文本")]+块[1],()))
(“ control ”, r “\ frac12\rightarrow for \frac{ 1}\\2}\ pred({ 半”: r“ <p> Half: $\ frac12\rightarrow x$ in the model.</p>}), ()
(“控制”、“TOTAL DUE”和“124.00”)，预({“总计”):“<p>TOTAL DUE:124.00美元。</p>}”,()
("control","markdown table through markdown.to html"，预告({"table b":markdown.to html(md table b)}),()).
("不支持","无块，声明不支持",{"html":pred ()["html"]],("块，"))
]
预计值= 0
用于类型，名称，p，不支持的变体：
更改 = [v for v in unit tests.score(p，测试，不支持) 如果 v. results!] = base[v.id]] ).
如果仁爱的"flip":
ok=len(已更改) ===================================================================================== (============================================================================================================================================= (==================
Elif kind = "control"(控制):"(控制),"控制":
确定=未更改
其他情况：
ok = bol(已更改)和所有(v. see = "不支持" for v in change)
作为预期的。........... OK
显示= "; ". join(f)"{v.id} - > {v.results}" + (f)"({v. why})" 如果 v. sequence = "fau" else "'" v.
打印(f){kind:11s}{名称：42s}{shown}>
s = unit tests. summarize(unit tests.lines(unit tests.score(变量[0][2]，测试)，测试，页))
print(f)\\ncumary of the twitto 变体：头条{s['headline']:4.f}=".
+ ",".join(f){k}{v['core'):3.f}({v['tests'}}测试)"为k,v在s['familys'].items()中。
claim (“每个更改的变体都会翻转一个判决失败，没有控制翻转任何”, as United States len(变体))
返回 f“{as expected} of {len(vainants)}的手建变体行为符合预期(9个翻转，3个控件，1个不支持)”
计分器解剖器( )
在信任一个分数之前，我们想看看一个判定意味着什么，所以我们用手写一个黄金文件，上面有13个测试在一个发明的页面上，用单位测试进行分级预测。 分数，基准测试本身使用的函数。 一个基础预测通过一切，每个变体改变一件事。 一个字母打字，一个丢失的减号，一个方程式中不同的符号，交换标题和一个错误的块，每个翻转完全一个测试失败，并附上一个理由。 控制仍然通过：首都、千位分隔符和dfrac，因为文本是折叠的，在KaTeX制作后对方程式进行比较。
@section ("3. 数据集：16,288个测试，它们检查什么，以及每个答案是如何确定的").
def 数据集 step( ):
全球数据、页码、测试、数据数据库
shatch download(REPO, repo type="dataset")，修订版=REV, local dir=str(WORK / "dataset")
允许 patterns=['manifest.parquet],“黄金/*”,“结果.parque”
本文由 AI图鉴自动抓取并翻译，版权归原作者所有；如需查看原文请点下方链接。
In this tutorial, we implement OmniParseBench, Datalab’s open benchmark for document parsers, which grades a parser’s output with 16,288 small pass-or-fail tests across 2,937 pages, from scanned forms and handwriting to arXiv math and dense tables. Rather than trust the leaderboard, we rebuild it from the verdicts Datalab released for 27 parsers, after feeding the scorer a hand-made page to see which mistakes flip a verdict. We then measure ranking certainty, slice it by document type, test whether the two models the card names as helping build tests behave differently on tests a model corrected, and run Tesseract and Indra OCR ourselves on a 50-page sample. Along the way, we find that the top two parsers are statistically tied, the leader changes on most slices, and one launch figure doesn’t match the released files.
import os
import re
import sys
import json
import time
import html
import shutil
import platform
import traceback
import subprocess
import collections
import urllib.request
from pathlib import Path
RESULTS = {} # step -> one-line result, for the summary table
CLAIMS = [] # (step, claim, held)
CURRENT = ["1"]
def banner(title):
print("\n" + "=" * 78)
print(title)
print("=" * 78)
def section(name):
def wrap(fn):
def run(*a, **kw):
CURRENT[0] = name.split(".")[0]
banner(name)
t0 = time.time()
try:
out = fn(*a, **kw)
RESULTS[name] = out if isinstance(out, str) else "ok"
return out
except Exception as e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
traceback.print_exc(limit=3)
return None
finally:
print(f"\n[{time.time() - t0:.1f} s]")
return run
return wrap
def claim(text, held):
CLAIMS.append((CURRENT[0], text, bool(held)))
return held
banner("1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision")
SYSTEM = platform.system()
OPB_VERSION = "1.0.0rc1" # a pre-release: a bare `pip install omni-parse-bench` resolves to 0.2.0, whose API differs
REPO = "datalab-to/omni_parse_bench"
REV = "961c0f89a2da701f5ce53afbd5e80aa469b0369a" # the dataset with the published verdicts of 27 parsers
INDRA = ["indra-ocr==0.3.2", "numpy==2.5.4", "onnxruntime==1.31.0", "opencv-python-headless==5.0.0.93",
"pypdfium2==5.14.0", "huggingface_hub==2.2.0"] # the pins of the package's own Indra server image
WORK = Path(os.environ.get("OPB_WORK") or ("/content/opb" if os.path.isdir("/content") else "opb_work")).resolve()
WORK.mkdir(parents=True, exist_ok=True)
subprocess.run([sys.executable, "-m", "pip", "install", "-q", f"omni-parse-bench[benchmark]=={OPB_VERSION}", "uv"], check=True)
if SYSTEM == "Linux" and shutil.which("tesseract") is None:
scripts = ("arab armn beng cans cher cyrl deva ethi frak geor grek gujr guru hang hang-vert hans hans-vert hant "
"hant-vert hebr jpan jpan-vert knda khmr laoo latn mlym mymr orya sinh syrc taml telu thaa thai tibt viet").split()
subprocess.run(["apt-get", "-qq", "update"], check=True)
subprocess.run(["apt-get", "-qq", "install", "-y", "tesseract-ocr", "tesseract-ocr-eng", "tesseract-ocr-osd"]
+ [f"tesseract-ocr-script-{s}" for s in scripts], check=True, stdout=subprocess.DEVNULL)
import numpy as np
import pandas as pd
import importlib.metadata as md
from huggingface_hub import snapshot_download
from omni_parse_bench import benchmark, data, harness, markdown, vocab
from omni_parse_bench.scorers import unit_tests
print(f"omni-parse-bench {md.version('omni-parse-bench')}, Python {platform.python_version()}, {SYSTEM} {platform.machine()}, "
f"{os.cpu_count()} CPU cores")
if SYSTEM == "Linux":
print("os:", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
tess = subprocess.run(["tesseract", "--version"], capture_output=True, text=True)
print("tesseract:", (tess.stdout or tess.stderr).splitlines()[0])
TESS_MAJOR = int((tess.stdout or tess.stderr).split()[1].split(".")[0])
print("test families:", {f: [t for t, g in vocab.FAMILIES.items() if g == f] for f in dict.fromkeys(vocab.FAMILIES.values())})
print(f"dataset {REPO} @ {REV[:7]}; work folder {WORK}")
RESULTS["1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision"] = (
f"omni-parse-bench {md.version('omni-parse-bench')}, tesseract {(tess.stdout or tess.stderr).split()[1]}, dataset @ {REV[:7]}")
We install omni-parse-bench pinned to 1.0.0rc1, because a bare pip install still resolves to 0.2.0, an older release with a different API. The benchmark extra brings the parser adapters, the Hugging Face client, and pyarrow for the manifest, and on Colab we add Tesseract with its script models through apt. We also pin the dataset to one commit, since the repository changes as parsers are added. Printing the families shows the structure the benchmark rests on: present, order, and repeat tests form the text family; table_cell forms the tables family; and layout_kind forms the layout family.
@section("2. How the scorer decides: one hand-built page, five test types, one change at a time")
def scorer_anatomy():
T = lambda i, tt, out, tags, args, st=None: {"id": i, "test_type": tt, "output_type": out, "derivation": "authored",
"tags": tags, "args": args, "settings": st or {}}
exact = lambda s: {"text": s, "max_diffs": 0}
gold = [
T("title", "present", "html", ["heading"], {"content": exact("Quarterly Report")}),
T("total", "present", "html", ["eng"], {"content": exact("Total due")}),
T("growth", "present", "html", ["eng"], {"content": {"number": "-4.8%"}}),
T("amount", "present", "html", ["eng"], {"content": {"number": "1,240.00"}}),
T("eq_ratio", "present", "html", ["math"], {"content": {"latex": r"\alpha_{1} + \beta"}}),
T("eq_half", "present", "html", ["math"], {"content": {"latex": r"\frac{1}{2} \to x"}}),
T("order", "order", "html", ["eng"], {"before": exact("Introduction: this report covers the second quarter."),
"after": exact("Conclusion: the plan holds.")}),
T("repeat", "repeat", "html", ["eng"], {"content": exact("Revenue")}, {"expected_count": 2}),
T("cell_top", "table_cell", "html", ["table"], {"cell": exact("1,240"), "top_heading": exact("2024")}),
T("cell_left", "table_cell", "html", ["table"], {"cell": exact("1,240"), "left_heading": exact("Income")}),
T("cell_price", "table_cell", "html", ["table"], {"cell": exact("9.50"), "top_heading": exact("Price"),
"left": exact("Widget")}),
T("block_title", "layout_kind", "blocks", ["heading"],
{"content": dict(exact("Quarterly Report"), place=[[0.10, 0.04, 0.90, 0.09]])}, {"kind": "heading"}),
T("block_total", "layout_kind", "blocks", ["eng"],
{"content": dict(exact("Total due"), place=[[0.10, 0.12, 0.90, 0.16]])}, {"kind": "text"}),
]
gold_file = WORK / "fixture" / "gold" / "demo-p1.json" # a gold file is named after its page
gold_file.parent.mkdir(parents=True, exist_ok=True)
gold_file.write_text(json.dumps({"tests": gold}, indent=1))
page = data.Page(page_id="demo-p1", document="demo", suite="demo", doc_path=WORK / "fixture" / "demo-p1.pdf",
gt_path=gold_file, outputs=("blocks", "html"))
tests = unit_tests.gold(page)
table_a = ("<table><tr><th></th><th>2023</th><th>2024</th></tr><tr><th>Income</th><td>1,000</td><td>1,240</td></tr>"
"<tr><th>Costs</th><td>600</td><td>700</td></tr></table>")
table_b = "<table><tr><th>Product</th><th>Price</th></tr><tr><td>Widget</td><td>9.50</td></tr></table>"
md_table_b = "| Product | Price |\n|---|---|\n| Widget | 9.50 |"
parts = {"title": "<h1>Quarterly Report</h1>", "total": "<p>Total due: 1,240.00 USD.</p>",
"growth": "<p>Growth was -4.8% year on year.</p>", "ratio": r"<p>Ratio: 1+ holds.</p>",
"half": r"<p>Half: 12x in the model.</p>",
"intro": "<p>Introduction: this report covers the second quarter.</p>",
"revenue": "<p>Revenue is shown in thousands. Revenue grew in the second quarter.</p>",
"table_a": table_a, "conclusion": "<p>Conclusion: the plan holds.</p>", "table_b": table_b}
blocks = [{"label": "heading", "bbox": [0.10, 0.04, 0.90, 0.09], "text": "Quarterly Report"},
{"label": "text", "bbox": [0.10, 0.12, 0.90, 0.16], "text": "Total due: 1,240.00 USD."},
{"label": "table", "bbox": [0.10, 0.40, 0.90, 0.60], "text": "Income 1,000 1,240 Costs 600 700"}]
def pred(change=None, order=None, new_blocks=None):
p = dict(parts, **(change or {}))
return {"html": "\n".join(p[k] for k in (order or list(parts))), "blocks": blocks if new_blocks is None else new_blocks}
base = {v.id: v.result for v in unit_tests.score(pred(), tests)}
print(f"base prediction: {sum(r == 'pass' for r in base.values())} of {len(tests)} tests pass\n")
swapped = ["title", "total", "growth", "ratio", "half", "conclusion", "intro", "revenue", "table_a", "table_b"]
variants = [
("flip", "a typo: 'Total dve'", pred({"total": "<p>Total dve: 1,240.00 USD.</p>"}), ()),
("flip", "the minus sign dropped: 4.8%", pred({"growth": "<p>Growth was 4.8% year on year.</p>"}), ()),
("flip", "another symbol: alpha_2", pred({"ratio": r"<p>Ratio: 2+ holds.</p>"}), ()),
("flip", "conclusion before introduction", pred(order=swapped), ()),
("flip", "'Revenues' for the second 'Revenue'",
pred({"revenue": "<p>Revenue is shown in thousands. Revenues grew in the second quarter.</p>"}), ()),
("flip", "2023 and 2024 headings swapped", pred({"table_a": table_a.replace("<th>2023</th><th>2024</th>",
"<th>2024</th><th>2023</th>")}), ()),
("flip", "row label 'Margin' for 'Income'", pred({"table_a": table_a.replace("<th>Income</th>", "<th>Margin</th>")}), ()),
("flip", "a markdown table left in the html", pred({"table_b": md_table_b}), ()),
("flip", "the title's block labelled text", pred(new_blocks=[dict(blocks[0], label="text")] + blocks[1:]), ()),
("control", r"\dfrac12\rightarrow for \frac{1}{2}\to", pred({"half": r"<p>Half: $\dfrac12\rightarrow x$ in the model.</p>"}), ()),
("control", "'TOTAL DUE' and '1240.00'", pred({"total": "<p>TOTAL DUE: 1240.00 USD.</p>"}), ()),
("control", "markdown table through markdown.to_html", pred({"table_b": markdown.to_html(md_table_b)}), ()),
("unsupported", "no blocks, declared unsupported", {"html": pred()["html"]}, ("blocks",)),
]
as_expected = 0
for kind, name, p, unsupported in variants:
changed = [v for v in unit_tests.score(p, tests, unsupported) if v.result != base[v.id]]
if kind == "flip":
ok = len(changed) == 1 and changed[0].result == "fail"
elif kind == "control":
ok = not changed
else:
ok = bool(changed) and all(v.result == "unsupported" for v in changed)
as_expected += ok
shown = "; ".join(f"{v.id} -> {v.result}" + (f" ({v.why})" if v.result == "fail" else "") for v in changed) or "no verdict changes"
print(f"{kind:11s} {name:42s} {shown}")
s = unit_tests.summarize(unit_tests.lines(unit_tests.score(variants[0][2], tests), tests, page))
print(f"\nsummary of the typo variant: headline {s['headline']:.4f} = mean of "
+ ", ".join(f"{k} {v['score']:.3f} ({v['tests']} tests)" for k, v in s["families"].items()))
claim("every one-change variant flips exactly one verdict to fail, and no control flips any", as_expected == len(variants))
return f"{as_expected} of {len(variants)} hand-built variants behave as expected (9 flips, 3 controls, 1 unsupported)"
scorer_anatomy()
Before trusting a score, we want to see what a verdict means, so we write a gold file by hand with thirteen tests on an invented page and grade predictions with unit_tests.score, the function the benchmark itself uses. A base prediction passes everything, and each variant changes one thing. A one-letter typo, a lost minus sign, a different symbol in an equation, swapped headings and a mislabelled block each flip exactly one test to fail, with a reason attached. The controls still pass: capitals, a dropped thousands separator and dfrac in place of frac, because text is folded and equations are compared after KaTeX renders them.
@section("3. The dataset: 16,288 tests, what they check, and how each answer was established")
def dataset_step():
global DATA, PAGES, TESTS, CARD_TEXT
snapshot_download(REPO, repo_type="dataset", revision=REV, local_dir=str(WORK / "dataset"),
allow_patterns=["manifest.parquet", "gold/*", "results.parque
Auto-fetched and translated by AI Atlas. All rights belong to the original author.