R-ORD GOLD-ONLY not-a-function-of-the-data
european_football_2 · mini_dev_pg-00000-of-00001 from https://huggingface.co/datasets/birdsql/bird_mini_dev/resolve/f65faf4ae3b638c1fa6df1d3370c8d92c8366301/data/mini_dev_pg-00000-of-00001.json (commit f65faf4a, downloaded 2026-09-14)
Which top 4 leagues had the most games in the 2015-2016 season?
the hint the set supplies: in the 2015-2016 season refers to season = '2015/2016'; league with most games refers to League.name where MAX(COUNT(id))
This question was audited without a prediction beside it, so there is nothing to compare the gold with. The probes below read the gold alone.
SELECT t1.name FROM League AS t1 INNER JOIN Match AS t2 ON t1.id = t2.league_id WHERE t2.season = '2015/2016' GROUP BY t1.name ORDER BY COUNT(t2.id) DESC NULLS LAST LIMIT 4sha256:77990a9a27e9d00c2f3424dd164fb83148c0637b687dc6883f5c7f732b5f0604
from evidence-gold.json, 4 rows
| nametext |
|---|
| Spain LIGA BBVA |
| England Premier League |
| Italy Serie A |
| France Ligue 1 |
A smell is a mechanical reason to read this gold statement again. It is a heuristic: it does not state that the statement is wrong, and a maintainer decides.
this statement orders by a text column holding only numbers, and ordering it as a number gives a different answer, so the gold may be sorting 9.5 above 10
no ORDER BY key resolves to a text column
{
"heuristic": true,
"reason": "no ORDER BY key resolves to a text column",
"keys": [
{
"key": "count(t2.id)",
"not_applicable": "the key is an expression and not a column"
}
]
}this statement cuts its result at a LIMIT that does not decide which rows come back, so a different but equally correct statement can return other rows and score zero
{
"heuristic": true,
"cut": 4,
"offset": 0,
"distinct_kept": false,
"unbounded_sql": "SELECT t1.name, count(t2.id) AS attestql_ordering_key_0 FROM league t1 JOIN match t2 ON t1.id = t2.league_id WHERE t2.season = '2015/2016' GROUP BY t1.name ORDER BY count(t2.id) DESC NULLS LAST",
"unbounded_rows": 11,
"projected_columns": [
"name"
],
"ordering_key_columns": [
"attestql_ordering_key_0"
],
"ordering_keys": [
{
"key": "count(t2.id)",
"direction": "desc",
"nulls": "last",
"nulls_first_in_effect": false,
"returned_rows_null_in_this_key": 0,
"fires": false
}
],
"case": null
}rerun over the same rows in another physical order this statement gives another answer, so its result depends on how the rows are stored and not only on the data
from smells.json, 4 rows
| Italy Serie A |
| France Ligue 1 |
| England Premier League |
| Spain LIGA BBVA |
{
"heuristic": true,
"rule": "R-ORD",
"baseline_result_hash": "sha256:77990a9a27e9d00c2f3424dd164fb83148c0637b687dc6883f5c7f732b5f0604",
"baseline_result": {
"columns": [
{
"name": "name",
"declared_type": "text"
}
],
"row_count": 4,
"truncated": false,
"rows_shown": 4,
"rows": [
[
{
"type": "str",
"value": "Spain LIGA BBVA"
}
],
[
{
"type": "str",
"value": "England Premier League"
}
],
[
{
"type": "str",
"value": "Italy Serie A"
}
],
[
{
"type": "str",
"value": "France Ligue 1"
}
]
],
"result_hash": "sha256:77990a9a27e9d00c2f3424dd164fb83148c0637b687dc6883f5c7f732b5f0604"
},
"planner_statistics": {
"league": {
"last_analyze": "2026-09-14 12:51:02.859972+00",
"last_autoanalyze": null,
"n_mod_since_analyze": 0
},
"match": {
"last_analyze": "2026-09-14 12:51:03.86449+00",
"last_autoanalyze": null,
"n_mod_since_analyze": 0
}
},
"shuffle": {
"seed": "1",
"row_limit": 300000,
"tables": [
"league",
"match"
],
"tables_not_shuffled": [],
"tables_skipped_for_size": {
"laptimes": 400524,
"legalities": 427907,
"posthistory": 303155,
"trans": 1056320,
"yearmonth": 383282
},
"tables_not_reached_by_a_copy": {}
},
"shuffled_copies": {
"run": true,
"verdict": "not_equal",
"differs": true,
"result_hash": "sha256:8a08f18dd660eae56fd1e54ca0cbf086cbbb57a722d3df1be625f2b749d3d8c9",
"result": {
"columns": [
{
"name": "name",
"declared_type": "text"
}
],
"row_count": 4,
"truncated": false,
"rows_shown": 4,
"rows": [
[
{
"type": "str",
"value": "Italy Serie A"
}
],
[
{
"type": "str",
"value": "France Ligue 1"
}
],
[
{
"type": "str",
"value": "England Premier League"
}
],
[
{
"type": "str",
"value": "Spain LIGA BBVA"
}
]
],
"result_hash": "sha256:8a08f18dd660eae56fd1e54ca0cbf086cbbb57a722d3df1be625f2b749d3d8c9"
}
},
"plan_variant": {
"run": false,
"reason": "the plan variant was not asked for"
}
}this statement returns the same whole row more than once and never says DISTINCT, so a statement answering the same question once per row disagrees on multiplicity alone
{
"heuristic": true,
"rows": 4,
"distinct_rows": 4,
"repeated_rows": 0,
"largest_repeat": 1,
"result_bounded": false,
"distinct_stated": false,
"set_operation": false
}SELECT t1.name FROM League AS t1 INNER JOIN Match AS t2 ON t1.id = t2.league_id WHERE t2.season = '2015/2016' GROUP BY t1.name ORDER BY COUNT(t2.id) DESC NULLS LAST LIMIT 4result_hash sha256:77990a9a27e9d00c2f3424dd164fb83148c0637b687dc6883f5c7f732b5f0604 recomputed from this JSON: match
record_hash sha256:9c1d7e96b1162f457b2628b801be996f042f92724c9d981514ca213eee11da26 recomputed from this JSON: match
from evidence-gold.json, 4 rows
| nametext |
|---|
| Spain LIGA BBVA |
| England Premier League |
| Italy Serie A |
| France Ligue 1 |
re-run this statement read-only against PostgreSQL 16.15 (Debian 16.15-1.pgdg13+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 14.2.0-19) 14.2.0, 64-bit | server=172.17.0.2/32:5432 | database=bird under the session settings and over the data this record's fixture digest names, and compare the two results under R-ORD