{
  "version": "2026-10-23-preliminary-v1",
  "counts": {
    "source_records_in_shard": 69907,
    "editorial_title_candidates": 39,
    "rights_reviewed_source_records": 11,
    "records_read_for_curation": 11,
    "candidate_chunks": 314,
    "attested_word_wrap_repairs": 325,
    "short_uppercase_header_lines_removed": 8,
    "boilerplate_or_page_number_lines_removed": 1,
    "retained_chunks": 37,
    "retained_source_records": 3,
    "retained_words": 8801
  },
  "rejected_chunks_by_first_reason": {
    "numeric_or_table_heavy": 14,
    "personal_narrative": 46,
    "embedded_running_header": 42,
    "manual_editorial_qa_exclusion": 19,
    "insufficient_topic_terms": 47,
    "nonprose_characters": 5,
    "broken_hyphenation": 39,
    "ocr_single_letter_noise": 28,
    "final_full_text_editorial_review": 26,
    "uppercase_or_table_noise": 1,
    "low_english_function_words": 1,
    "off_scope_or_unsafe_historical": 6,
    "table_or_ocr_symbols": 2,
    "advertising_or_lists": 1,
    "exact_normalized_duplicate": 0,
    "near_duplicate_simhash_jaccard": 0
  },
  "topic_counts": {
    "reasoning_language": 30,
    "forestry_ecology": 7
  },
  "chunks_per_source": {
    "arboretumetfruti01loudrich_33": 5,
    "18440-0-12": 30,
    "journalofforestr01lond_30": 2
  },
  "source_stats": {
    "bulletinofunit2311903unit_24": {
      "original_word_count": 6443,
      "normalization": {}
    },
    "gamefishofnorthe00roos_7": {
      "original_word_count": 7477,
      "normalization": {}
    },
    "midlandnaturalis1418midl_15": {
      "original_word_count": 7444,
      "normalization": {}
    },
    "b28095625_6": {
      "original_word_count": 6153,
      "normalization": {}
    },
    "arboretumetfruti01loudrich_33": {
      "original_word_count": 6844,
      "normalization": {}
    },
    "18440-0-12": {
      "original_word_count": 9844,
      "normalization": {
        "short_uppercase_header_lines_removed": 8,
        "boilerplate_or_page_number_lines_removed": 1
      }
    },
    "journalofforestr01lond_30": {
      "original_word_count": 6602,
      "normalization": {}
    },
    "historyofbritish02hami_11": {
      "original_word_count": 6986,
      "normalization": {}
    },
    "journalofroyalag2181roya_11": {
      "original_word_count": 6514,
      "normalization": {}
    },
    "journalofroyalag2141roya_28": {
      "original_word_count": 7263,
      "normalization": {}
    },
    "vannostrandsecl141876newy_6": {
      "original_word_count": 6883,
      "normalization": {}
    }
  },
  "source_download": {
    "url": "https://huggingface.co/datasets/PleIAs/common_corpus/resolve/307910e4c5d040d6f318e6edf2a2b97849155771/common_corpus_1/subset_100_1.parquet",
    "bytes": 429962586,
    "sha256": "4ee719a130b7f86978b08c20cc9f490309e2bae2a5c0df4f04fd427365530f07",
    "revision": "307910e4c5d040d6f318e6edf2a2b97849155771"
  },
  "deduplication": {
    "exact": "SHA256 over lowercase ASCII word tokens",
    "near": "64-bit SimHash on 5-word shingles, four 16-bit bands, Hamming distance <=3 followed by Jaccard >=0.85; heuristic, not exhaustive"
  },
  "privacy": "Whole chunks matching email, URL, phone, SSN-like, street-address or selected personal-narrative patterns are omitted. Creator fields are omitted from model-text records; historical bibliographic attribution is retained in sources.json. Historical names can remain; no guarantee of anonymization or exhaustive PII detection.",
  "quality": "Krummholz generated no new text or completions. Only documented Unicode/whitespace and attested word-wrap normalization was applied; no semantic rewrite. Upstream OCR/model corrections may remain. There is no training split or model evaluation. Historical claims may be wrong, biased or unsafe for contemporary practice.",
  "editorial_review": "Every final passage was read by the curator and independently reviewed for coherence, scope and residual OCR; selected passage keys are provided for reproducibility. This is agent-assisted editorial review, not domain-expert validation.",
  "attested_word_wrap_repairs_scope": "Count covers all candidate chunks before rejection, not only published text. Joined words must occur as complete lowercase tokens at least twice in the approved upstream text."
}