{
  "metric_policy": "track-a-recognition-v1.1",
  "immutable_digest": "1f22aa5800ef2eb7327dcdaa9036b43481e7d27436de66d1225cbc90442e6ad1",
  "frozen_truth_sha256": "c794787c0f7fdee894eb96bc5fe630e0928da8fcc92b8badf409af0934df9996",
  "unique_samples_per_engine": 1000,
  "identical_sample_sets": true,
  "dataset": "DocTorture-1K v0.1",
  "measured_at": "2026-09-05",
  "source_leaderboard_sha256": "550d0b712e556170832c40ab17c9a34a00cecdb2d7e57e029f1093b52dff8527",
  "engines": {
    "google-native": {
      "samples": 1000,
      "recognition_quality": {
        "token_f1": 0.825429,
        "numeric_token_accuracy": 0.785071,
        "date_text_accuracy": 0.856
      },
      "serialization_sensitive_quality": {
        "cer": 0.41537,
        "wer": 0.437165
      },
      "serialization_diagnostic_exact_tokens_sequence_errors": 165,
      "empty_responses": 27
    },
    "eden-amazon": {
      "samples": 1000,
      "recognition_quality": {
        "token_f1": 0.846911,
        "numeric_token_accuracy": 0.824463,
        "date_text_accuracy": 0.871
      },
      "serialization_sensitive_quality": {
        "cer": 0.139114,
        "wer": 0.187733
      },
      "serialization_diagnostic_exact_tokens_sequence_errors": 1,
      "empty_responses": 10
    },
    "tesseract": {
      "samples": 1000,
      "recognition_quality": {
        "token_f1": 0.503137,
        "numeric_token_accuracy": 0.451565,
        "date_text_accuracy": 0.6355
      },
      "serialization_sensitive_quality": {
        "cer": 0.401099,
        "wer": 0.717636
      },
      "serialization_diagnostic_exact_tokens_sequence_errors": 0,
      "empty_responses": 12
    }
  },
  "severity_breakdown": {
    "google-native": {
      "clean": {
        "samples": 250,
        "recognition_quality": {
          "token_f1": 0.982942,
          "numeric_token_accuracy": 0.959691,
          "date_text_accuracy": 1.0
        },
        "serialization_sensitive_quality": {
          "cer": 0.31133,
          "wer": 0.295128
        }
      },
      "subtle": {
        "samples": 182,
        "recognition_quality": {
          "token_f1": 0.909699,
          "numeric_token_accuracy": 0.883519,
          "date_text_accuracy": 0.936813
        },
        "serialization_sensitive_quality": {
          "cer": 0.349412,
          "wer": 0.363225
        }
      },
      "moderate": {
        "samples": 147,
        "recognition_quality": {
          "token_f1": 0.83419,
          "numeric_token_accuracy": 0.781651,
          "date_text_accuracy": 0.863946
        },
        "serialization_sensitive_quality": {
          "cer": 0.410746,
          "wer": 0.443731
        }
      },
      "severe": {
        "samples": 421,
        "recognition_quality": {
          "token_f1": 0.692406,
          "numeric_token_accuracy": 0.640012,
          "date_text_accuracy": 0.732779
        },
        "serialization_sensitive_quality": {
          "cer": 0.50728,
          "wer": 0.551182
        }
      }
    },
    "eden-amazon": {
      "clean": {
        "samples": 250,
        "recognition_quality": {
          "token_f1": 0.980508,
          "numeric_token_accuracy": 0.960178,
          "date_text_accuracy": 1.0
        },
        "serialization_sensitive_quality": {
          "cer": 0.018492,
          "wer": 0.032192
        }
      },
      "subtle": {
        "samples": 182,
        "recognition_quality": {
          "token_f1": 0.927435,
          "numeric_token_accuracy": 0.906853,
          "date_text_accuracy": 0.961538
        },
        "serialization_sensitive_quality": {
          "cer": 0.056427,
          "wer": 0.094095
        }
      },
      "moderate": {
        "samples": 147,
        "recognition_quality": {
          "token_f1": 0.862458,
          "numeric_token_accuracy": 0.834308,
          "date_text_accuracy": 0.884354
        },
        "serialization_sensitive_quality": {
          "cer": 0.117157,
          "wer": 0.178661
        }
      },
      "severe": {
        "samples": 421,
        "recognition_quality": {
          "token_f1": 0.727338,
          "numeric_token_accuracy": 0.704816,
          "date_text_accuracy": 0.750594
        },
        "serialization_sensitive_quality": {
          "cer": 0.254155,
          "wer": 0.323745
        }
      }
    },
    "tesseract": {
      "clean": {
        "samples": 250,
        "recognition_quality": {
          "token_f1": 0.734666,
          "numeric_token_accuracy": 0.671812,
          "date_text_accuracy": 0.96
        },
        "serialization_sensitive_quality": {
          "cer": 0.123669,
          "wer": 0.307171
        }
      },
      "subtle": {
        "samples": 182,
        "recognition_quality": {
          "token_f1": 0.614163,
          "numeric_token_accuracy": 0.549999,
          "date_text_accuracy": 0.826923
        },
        "serialization_sensitive_quality": {
          "cer": 0.230954,
          "wer": 0.445312
        }
      },
      "moderate": {
        "samples": 147,
        "recognition_quality": {
          "token_f1": 0.481708,
          "numeric_token_accuracy": 0.423864,
          "date_text_accuracy": 0.561224
        },
        "serialization_sensitive_quality": {
          "cer": 0.421445,
          "wer": 0.729536
        }
      },
      "severe": {
        "samples": 421,
        "recognition_quality": {
          "token_f1": 0.325134,
          "numeric_token_accuracy": 0.287897,
          "date_text_accuracy": 0.385986
        },
        "serialization_sensitive_quality": {
          "cer": 0.632295,
          "wer": 1.074951
        }
      }
    }
  },
  "conditions": {
    "google-native": {
      "clean": {
        "samples": 250,
        "mean_token_f1": 0.982942,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "motion_blur": {
        "samples": 42,
        "mean_token_f1": 0.029839,
        "empty_responses": 17,
        "token_f1_below_085": 42
      },
      "blur": {
        "samples": 12,
        "mean_token_f1": 0.648117,
        "empty_responses": 0,
        "token_f1_below_085": 5
      },
      "perspective": {
        "samples": 100,
        "mean_token_f1": 0.841438,
        "empty_responses": 3,
        "token_f1_below_085": 32
      },
      "rotation_skew": {
        "samples": 45,
        "mean_token_f1": 0.657518,
        "empty_responses": 0,
        "token_f1_below_085": 33
      },
      "uneven_illumination": {
        "samples": 26,
        "mean_token_f1": 0.96209,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "low_light": {
        "samples": 86,
        "mean_token_f1": 0.739879,
        "empty_responses": 6,
        "token_f1_below_085": 39
      },
      "glare": {
        "samples": 50,
        "mean_token_f1": 0.896518,
        "empty_responses": 0,
        "token_f1_below_085": 8
      },
      "shadow": {
        "samples": 58,
        "mean_token_f1": 0.92834,
        "empty_responses": 0,
        "token_f1_below_085": 7
      },
      "jpeg": {
        "samples": 75,
        "mean_token_f1": 0.523811,
        "empty_responses": 13,
        "token_f1_below_085": 46
      },
      "faded": {
        "samples": 39,
        "mean_token_f1": 0.572265,
        "empty_responses": 10,
        "token_f1_below_085": 28
      },
      "noise": {
        "samples": 69,
        "mean_token_f1": 0.683917,
        "empty_responses": 6,
        "token_f1_below_085": 44
      },
      "photocopy": {
        "samples": 71,
        "mean_token_f1": 0.553324,
        "empty_responses": 10,
        "token_f1_below_085": 70
      },
      "fax": {
        "samples": 50,
        "mean_token_f1": 0.179398,
        "empty_responses": 0,
        "token_f1_below_085": 50
      },
      "thermal_fading": {
        "samples": 46,
        "mean_token_f1": 0.926662,
        "empty_responses": 0,
        "token_f1_below_085": 7
      },
      "scanner_lines": {
        "samples": 46,
        "mean_token_f1": 0.422298,
        "empty_responses": 0,
        "token_f1_below_085": 35
      },
      "coffee_stain": {
        "samples": 39,
        "mean_token_f1": 0.867085,
        "empty_responses": 0,
        "token_f1_below_085": 22
      },
      "water_stain": {
        "samples": 62,
        "mean_token_f1": 0.920428,
        "empty_responses": 0,
        "token_f1_below_085": 6
      },
      "dirt": {
        "samples": 48,
        "mean_token_f1": 0.946119,
        "empty_responses": 0,
        "token_f1_below_085": 4
      },
      "smudging": {
        "samples": 38,
        "mean_token_f1": 0.92491,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "stamp": {
        "samples": 39,
        "mean_token_f1": 0.876692,
        "empty_responses": 0,
        "token_f1_below_085": 18
      },
      "handwriting_overlay": {
        "samples": 11,
        "mean_token_f1": 0.975704,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "partial_occlusion": {
        "samples": 39,
        "mean_token_f1": 0.791323,
        "empty_responses": 0,
        "token_f1_below_085": 28
      },
      "page_cropping": {
        "samples": 70,
        "mean_token_f1": 0.846502,
        "empty_responses": 0,
        "token_f1_below_085": 41
      },
      "vignette": {
        "samples": 11,
        "mean_token_f1": 0.980605,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "color_cast": {
        "samples": 17,
        "mean_token_f1": 0.964167,
        "empty_responses": 0,
        "token_f1_below_085": 1
      },
      "exposure": {
        "samples": 11,
        "mean_token_f1": 0.902433,
        "empty_responses": 0,
        "token_f1_below_085": 2
      },
      "contrast_loss": {
        "samples": 11,
        "mean_token_f1": 0.983207,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "chromatic_aberration": {
        "samples": 11,
        "mean_token_f1": 0.429998,
        "empty_responses": 0,
        "token_f1_below_085": 8
      },
      "paper_discoloration": {
        "samples": 11,
        "mean_token_f1": 0.986704,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "dirty_scuffed": {
        "samples": 11,
        "mean_token_f1": 0.970903,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "lens_distortion": {
        "samples": 11,
        "mean_token_f1": 0.946187,
        "empty_responses": 0,
        "token_f1_below_085": 1
      }
    },
    "eden-amazon": {
      "clean": {
        "samples": 250,
        "mean_token_f1": 0.980508,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "blur": {
        "samples": 12,
        "mean_token_f1": 0.69603,
        "empty_responses": 0,
        "token_f1_below_085": 4
      },
      "perspective": {
        "samples": 100,
        "mean_token_f1": 0.861322,
        "empty_responses": 0,
        "token_f1_below_085": 27
      },
      "motion_blur": {
        "samples": 42,
        "mean_token_f1": 0.072338,
        "empty_responses": 0,
        "token_f1_below_085": 42
      },
      "rotation_skew": {
        "samples": 45,
        "mean_token_f1": 0.78258,
        "empty_responses": 0,
        "token_f1_below_085": 18
      },
      "low_light": {
        "samples": 86,
        "mean_token_f1": 0.774395,
        "empty_responses": 0,
        "token_f1_below_085": 37
      },
      "uneven_illumination": {
        "samples": 26,
        "mean_token_f1": 0.96675,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "shadow": {
        "samples": 58,
        "mean_token_f1": 0.936441,
        "empty_responses": 0,
        "token_f1_below_085": 7
      },
      "glare": {
        "samples": 50,
        "mean_token_f1": 0.906568,
        "empty_responses": 0,
        "token_f1_below_085": 7
      },
      "jpeg": {
        "samples": 75,
        "mean_token_f1": 0.553125,
        "empty_responses": 0,
        "token_f1_below_085": 44
      },
      "noise": {
        "samples": 69,
        "mean_token_f1": 0.725905,
        "empty_responses": 0,
        "token_f1_below_085": 40
      },
      "faded": {
        "samples": 39,
        "mean_token_f1": 0.659168,
        "empty_responses": 10,
        "token_f1_below_085": 22
      },
      "photocopy": {
        "samples": 71,
        "mean_token_f1": 0.725718,
        "empty_responses": 10,
        "token_f1_below_085": 42
      },
      "fax": {
        "samples": 50,
        "mean_token_f1": 0.223077,
        "empty_responses": 0,
        "token_f1_below_085": 50
      },
      "thermal_fading": {
        "samples": 46,
        "mean_token_f1": 0.933094,
        "empty_responses": 0,
        "token_f1_below_085": 5
      },
      "scanner_lines": {
        "samples": 46,
        "mean_token_f1": 0.479528,
        "empty_responses": 0,
        "token_f1_below_085": 33
      },
      "water_stain": {
        "samples": 62,
        "mean_token_f1": 0.919353,
        "empty_responses": 0,
        "token_f1_below_085": 7
      },
      "dirt": {
        "samples": 48,
        "mean_token_f1": 0.937024,
        "empty_responses": 0,
        "token_f1_below_085": 3
      },
      "smudging": {
        "samples": 38,
        "mean_token_f1": 0.921558,
        "empty_responses": 0,
        "token_f1_below_085": 2
      },
      "coffee_stain": {
        "samples": 39,
        "mean_token_f1": 0.873092,
        "empty_responses": 0,
        "token_f1_below_085": 20
      },
      "handwriting_overlay": {
        "samples": 11,
        "mean_token_f1": 0.975068,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "stamp": {
        "samples": 39,
        "mean_token_f1": 0.867242,
        "empty_responses": 0,
        "token_f1_below_085": 19
      },
      "page_cropping": {
        "samples": 70,
        "mean_token_f1": 0.858061,
        "empty_responses": 0,
        "token_f1_below_085": 34
      },
      "vignette": {
        "samples": 11,
        "mean_token_f1": 0.979791,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "color_cast": {
        "samples": 17,
        "mean_token_f1": 0.966268,
        "empty_responses": 0,
        "token_f1_below_085": 1
      },
      "partial_occlusion": {
        "samples": 39,
        "mean_token_f1": 0.784378,
        "empty_responses": 0,
        "token_f1_below_085": 29
      },
      "chromatic_aberration": {
        "samples": 11,
        "mean_token_f1": 0.564821,
        "empty_responses": 0,
        "token_f1_below_085": 8
      },
      "contrast_loss": {
        "samples": 11,
        "mean_token_f1": 0.981046,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "exposure": {
        "samples": 11,
        "mean_token_f1": 0.921546,
        "empty_responses": 0,
        "token_f1_below_085": 2
      },
      "lens_distortion": {
        "samples": 11,
        "mean_token_f1": 0.949403,
        "empty_responses": 0,
        "token_f1_below_085": 1
      },
      "paper_discoloration": {
        "samples": 11,
        "mean_token_f1": 0.982309,
        "empty_responses": 0,
        "token_f1_below_085": 0
      },
      "dirty_scuffed": {
        "samples": 11,
        "mean_token_f1": 0.978286,
        "empty_responses": 0,
        "token_f1_below_085": 0
      }
    },
    "tesseract": {
      "clean": {
        "samples": 250,
        "mean_token_f1": 0.734666,
        "empty_responses": 0,
        "token_f1_below_085": 242
      },
      "motion_blur": {
        "samples": 42,
        "mean_token_f1": 0.008431,
        "empty_responses": 2,
        "token_f1_below_085": 42
      },
      "blur": {
        "samples": 12,
        "mean_token_f1": 0.393284,
        "empty_responses": 0,
        "token_f1_below_085": 12
      },
      "perspective": {
        "samples": 100,
        "mean_token_f1": 0.313189,
        "empty_responses": 0,
        "token_f1_below_085": 100
      },
      "rotation_skew": {
        "samples": 45,
        "mean_token_f1": 0.261093,
        "empty_responses": 0,
        "token_f1_below_085": 45
      },
      "low_light": {
        "samples": 86,
        "mean_token_f1": 0.180504,
        "empty_responses": 0,
        "token_f1_below_085": 86
      },
      "uneven_illumination": {
        "samples": 26,
        "mean_token_f1": 0.631394,
        "empty_responses": 0,
        "token_f1_below_085": 25
      },
      "shadow": {
        "samples": 58,
        "mean_token_f1": 0.411121,
        "empty_responses": 0,
        "token_f1_below_085": 58
      },
      "glare": {
        "samples": 50,
        "mean_token_f1": 0.445326,
        "empty_responses": 0,
        "token_f1_below_085": 50
      },
      "jpeg": {
        "samples": 75,
        "mean_token_f1": 0.221124,
        "empty_responses": 1,
        "token_f1_below_085": 75
      },
      "faded": {
        "samples": 39,
        "mean_token_f1": 0.242022,
        "empty_responses": 10,
        "token_f1_below_085": 38
      },
      "photocopy": {
        "samples": 71,
        "mean_token_f1": 0.119129,
        "empty_responses": 10,
        "token_f1_below_085": 71
      },
      "fax": {
        "samples": 50,
        "mean_token_f1": 0.069239,
        "empty_responses": 0,
        "token_f1_below_085": 50
      },
      "scanner_lines": {
        "samples": 46,
        "mean_token_f1": 0.15978,
        "empty_responses": 0,
        "token_f1_below_085": 46
      },
      "thermal_fading": {
        "samples": 46,
        "mean_token_f1": 0.566779,
        "empty_responses": 0,
        "token_f1_below_085": 46
      },
      "coffee_stain": {
        "samples": 39,
        "mean_token_f1": 0.585616,
        "empty_responses": 0,
        "token_f1_below_085": 38
      },
      "water_stain": {
        "samples": 62,
        "mean_token_f1": 0.561188,
        "empty_responses": 0,
        "token_f1_below_085": 61
      },
      "dirt": {
        "samples": 48,
        "mean_token_f1": 0.566801,
        "empty_responses": 0,
        "token_f1_below_085": 48
      },
      "smudging": {
        "samples": 38,
        "mean_token_f1": 0.610014,
        "empty_responses": 0,
        "token_f1_below_085": 38
      },
      "handwriting_overlay": {
        "samples": 11,
        "mean_token_f1": 0.710132,
        "empty_responses": 0,
        "token_f1_below_085": 10
      },
      "noise": {
        "samples": 69,
        "mean_token_f1": 0.139032,
        "empty_responses": 0,
        "token_f1_below_085": 69
      },
      "stamp": {
        "samples": 39,
        "mean_token_f1": 0.619912,
        "empty_responses": 0,
        "token_f1_below_085": 38
      },
      "partial_occlusion": {
        "samples": 39,
        "mean_token_f1": 0.542932,
        "empty_responses": 0,
        "token_f1_below_085": 39
      },
      "page_cropping": {
        "samples": 70,
        "mean_token_f1": 0.479451,
        "empty_responses": 0,
        "token_f1_below_085": 68
      },
      "vignette": {
        "samples": 11,
        "mean_token_f1": 0.708995,
        "empty_responses": 0,
        "token_f1_below_085": 11
      },
      "color_cast": {
        "samples": 17,
        "mean_token_f1": 0.592115,
        "empty_responses": 0,
        "token_f1_below_085": 17
      },
      "exposure": {
        "samples": 11,
        "mean_token_f1": 0.566013,
        "empty_responses": 0,
        "token_f1_below_085": 11
      },
      "contrast_loss": {
        "samples": 11,
        "mean_token_f1": 0.696447,
        "empty_responses": 0,
        "token_f1_below_085": 11
      },
      "lens_distortion": {
        "samples": 11,
        "mean_token_f1": 0.651758,
        "empty_responses": 0,
        "token_f1_below_085": 11
      },
      "chromatic_aberration": {
        "samples": 11,
        "mean_token_f1": 0.082029,
        "empty_responses": 0,
        "token_f1_below_085": 11
      },
      "dirty_scuffed": {
        "samples": 11,
        "mean_token_f1": 0.696419,
        "empty_responses": 0,
        "token_f1_below_085": 11
      },
      "paper_discoloration": {
        "samples": 11,
        "mean_token_f1": 0.709544,
        "empty_responses": 0,
        "token_f1_below_085": 11
      }
    }
  },
  "limitations": [
    "Synthetic invoice benchmark; not a real-world failure-rate estimate.",
    "Severity cohorts are unpaired and uneven: clean 250, subtle 182, moderate 147, severe 421.",
    "Condition membership can overlap for mixtures; counts are not independent cohorts.",
    "ASCII-oriented recognition normalization is not a multilingual evaluation.",
    "No measurements from the newly generated v1.0 1K or 10K corpus are included."
  ],
  "composite_score": null
}