#!/usr/bin/env python3 """ make_synthetic_forestry_dataset.py Builds a 100% FICTITIOUS forestry / forest-inventory dataset styled after plot- and tree-level data from Maine and northern New England, deliberately containing missing values, duplicates and corrupted records. Output: forestry_synthetic_ME_NNE.zip (contains a folder of the same name) Requirements: Python 3.8+, standard library only. Run: python3 make_synthetic_forestry_dataset.py Preliminary Krummholz AI release; illustrative synthetic data only. See the generated README for limitations and the MIT license below. """ import csv import io import json import random import zipfile from datetime import date LICENSE_TEXT = 'MIT License\n\nCopyright (c) 2026 Hayven Jackson / Krummholz AI\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the "Software"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\n' SEED = 20240617 rng = random.Random(SEED) ZIPNAME = "forestry_synthetic_ME_NNE.zip" ROOT = "forestry_synthetic_ME_NNE" # ---------------------------------------------------------------------------- # 1. REFERENCE TABLES # ---------------------------------------------------------------------------- # SPCD values are plausible FIA-style codes but are NOT authoritative. # (code, common name, scientific name, short code, wood type) SPECIES = [ (94, "Balsam fir", "Abies balsamea", "BF", "softwood"), (12, "Black spruce", "Picea mariana", "BS", "softwood"), (97, "Red spruce", "Picea rubens", "RS", "softwood"), (95, "Northern white-cedar", "Thuja occidentalis", "NWC", "softwood"), (125, "Eastern larch", "Larix laricina", "TAM", "softwood"), (91, "Eastern white pine", "Pinus strobus", "WP", "softwood"), (105, "Red pine", "Pinus resinosa", "RP", "softwood"), (241, "Eastern hemlock", "Tsuga canadensis", "EH", "softwood"), (531, "Yellow birch", "Betula alleghaniensis", "YB", "hardwood"), (375, "Paper birch", "Betula papyrifera", "PB", "hardwood"), (833, "Sugar maple", "Acer saccharum", "SM", "hardwood"), (318, "Red maple", "Acer rubrum", "RM", "hardwood"), (802, "American beech", "Fagus grandifolia", "BE", "hardwood"), (746, "Quaking aspen", "Populus tremuloides", "QA", "hardwood"), (742, "Bigtooth aspen", "Populus grandidentata", "BA", "hardwood"), (812, "Northern red oak", "Quercus rubra", "RO", "hardwood"), (701, "White ash", "Fraxinus americana", "WA", "hardwood"), (260, "Balsam poplar", "Populus balsamifera", "BP", "hardwood"), ] BY_CODE = {s[0]: s for s in SPECIES} GEO = { "ME": {"Aroostook": (46.20, 47.30, -68.90, -67.40), "Piscataquis":(45.00, 46.10, -69.70, -68.60), "Penobscot": (44.70, 45.60, -69.20, -68.00), "Somerset": (44.90, 45.90, -70.50, -69.40), "Franklin": (44.60, 45.40, -70.80, -69.90), "Oxford": (43.90, 44.90, -71.10, -70.20), "Hancock": (44.20, 45.00, -68.70, -67.60), "Washington": (44.50, 45.40, -68.10, -67.00), "Kennebec": (44.20, 44.90, -70.10, -69.30), "York": (43.10, 43.80, -70.90, -70.40)}, "NH": {"Coos": (44.20, 45.30, -71.60, -70.90), "Grafton": (43.60, 44.40, -72.20, -71.50), "Carroll": (43.80, 44.30, -71.40, -70.90)}, "VT": {"Orleans": (44.50, 45.00, -72.50, -71.90), "Essex": (44.40, 45.10, -72.10, -71.50), "Rutland": (43.40, 43.90, -73.20, -72.70)}, } FTYPES = { "Spruce-Fir": [94, 94, 94, 12, 12, 97, 97, 125, 531, 375, 318], "Northern Hardwood": [833, 833, 531, 531, 802, 318, 701, 812, 241], "Mixedwood": [97, 94, 91, 241, 833, 531, 375, 318, 746], "White Pine-Hemlock": [91, 91, 241, 105, 318, 531, 833], "Aspen-Birch": [746, 742, 375, 531, 94, 97, 318], "Cedar-Swamp": [95, 95, 12, 125, 318, 802, 260], "Oak-Pine": [812, 91, 105, 318, 833, 802], } OWNERSHIP = ["private industrial", "private non-industrial", "state", "federal", "tribal", "conservation easement", "municipal"] DRAINAGE = ["excessively drained", "somewhat excessively drained", "well drained", "moderately well drained", "somewhat poorly drained", "poorly drained", "very poorly drained"] CREWS = ["A-1", "A-2", "B-1", "B-3", "C-2", "C-4", "D-1", "E-2"] MISSING_TOKENS = ["", "NA", "N/A", "NaN", ".", "-999", "-9999", "null", "None", " ", "not recorded"] TEXT_IN_NUMERIC = ["unknown", "TBD", "est.", "~14", "14 (est)", "12,5", "1O.5", "O.8", "n/a", "??"] MANIFEST = [] def log_defect(fname, record_id, field, dtype, original, corrupted): MANIFEST.append({ "file": fname, "record_id": str(record_id), "field": field, "defect_type": dtype, "original_value": str(original), "corrupted_value": str(corrupted), }) def jitter_date(): y = rng.randint(2015, 2023) m = rng.randint(5, 10) d = rng.randint(1, 28) return date(y, m, d) def fmt_date(d, style=0): if style == 0: return d.isoformat() if style == 1: return d.strftime("%m/%d/%Y") if style == 2: return d.strftime("%d-%b-%Y") if style == 3: return d.strftime("%B %d, %Y") if style == 4: return d.strftime("%m-%d-%y") return d.isoformat() # ---------------------------------------------------------------------------- # 2. PLOT GENERATION # ---------------------------------------------------------------------------- PLOT_HEADER = [ "plot_id", "state", "county", "latitude", "longitude", "elevation_ft", "aspect_deg", "slope_pct", "forest_type", "stand_origin", "stand_age_yr", "site_index_species", "site_index_ft_50yr", "soil_drainage", "ownership", "plot_radius_ft", "canopy_cover_pct", "basal_area_ft2_ac", "trees_per_acre", "net_volume_cuft_ac", "disturbance_code", "measurement_date", "crew", "plot_notes", ] county_pool = [(st, cty, b) for st in GEO for cty in GEO[st] for b in [GEO[st][cty]]] plots = [] for i in range(1, 43): st, cty, (la0, la1, lo0, lo1) = rng.choice(county_pool) lat = round(rng.uniform(la0, la1), 6) lon = round(rng.uniform(lo0, lo1), 6) ftype = rng.choice(list(FTYPES)) si_sp = rng.choice([97, 91, 241, 833, 94, 531]) si = int(round(rng.gauss(62, 9))) si = max(38, min(92, si)) origin = "natural" if rng.random() < 0.82 else "planted" age = int(round(rng.gauss(62, 26))) age = max(6, min(160, age)) plots.append({ "plot_id": f"{st}-{cty[:3].upper()}-{i:03d}", "state": st, "county": cty, "latitude": lat, "longitude": lon, "elevation_ft": int(round(rng.gauss(780, 420))), "aspect_deg": rng.choice([0, 45, 90, 135, 180, 225, 270, 315, rng.randint(0, 359)]), "slope_pct": max(0, int(round(rng.gauss(11, 9)))), "forest_type": ftype, "stand_origin": origin, "stand_age_yr": age, "site_index_species": BY_CODE[si_sp][3], "site_index_ft_50yr": si, "soil_drainage": rng.choice(DRAINAGE), "ownership": rng.choice(OWNERSHIP), "plot_radius_ft": rng.choice([24, 24, 24, 10, 6.8]), "canopy_cover_pct": max(0, min(100, int(round(rng.gauss(72, 18))))), "basal_area_ft2_ac": 0, "trees_per_acre": 0, "net_volume_cuft_ac": 0, "disturbance_code": rng.choices([0, 1, 2, 3, 4, 5, 6], weights=[55, 14, 12, 8, 2, 6, 3])[0], "measurement_date": fmt_date(jitter_date()), "crew": rng.choice(CREWS), "plot_notes": "", }) # ---------------------------------------------------------------------------- # 3. TREE GENERATION # ---------------------------------------------------------------------------- TREE_HEADER = [ "plot_id", "tree_number", "spcd", "species_common", "species_scientific", "wood_type", "status", "dbh_in", "total_height_ft", "height_to_live_crown_ft", "crown_ratio_pct", "crown_class", "damage_code", "decay_class", "dbh_growth_5yr_in", "basal_area_ft2", "gross_volume_cuft", "measurement_date", "crew", ] trees = [] for p in plots: ftype = p["forest_type"] n = rng.randint(6, 18) si_factor = p["site_index_ft_50yr"] / 60.0 for t in range(1, n + 1): spcd = rng.choice(FTYPES[ftype]) sp = BY_CODE[spcd] dbh = round(min(34.0, max(1.0, rng.triangular(1.5, 28.0, 9.0))), 1) height = 4.5 + 8.5 * (dbh ** 0.72) * si_factor * rng.uniform(0.84, 1.16) height = round(min(115.0, height), 1) cr = max(8, min(92, int(round(rng.gauss(46, 13))))) hlc = round(height * (1 - cr / 100.0), 1) ba = round(0.005454 * dbh ** 2, 4) vol = round(ba * height * 0.45, 3) status = "L" if rng.random() < 0.93 else "D" trees.append({ "plot_id": p["plot_id"], "tree_number": t, "spcd": spcd, "species_common": sp[1], "species_scientific": sp[2], "wood_type": sp[4], "status": status, "dbh_in": dbh, "total_height_ft": height, "height_to_live_crown_ft": hlc, "crown_ratio_pct": cr, "crown_class": rng.choices([1, 2, 3, 4, 5], weights=[8, 22, 34, 24, 12])[0], "damage_code": rng.choices([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], weights=[60, 6, 5, 6, 4, 4, 3, 6, 3, 3])[0], "decay_class": rng.choices([0, 1, 2, 3, 4, 5], weights=[62, 12, 10, 7, 5, 4])[0], "dbh_growth_5yr_in": round(max(0.0, rng.gauss(0.55, 0.35)), 2), "basal_area_ft2": ba, "gross_volume_cuft": vol, "measurement_date": p["measurement_date"], "crew": p["crew"], }) # Plot-level aggregates (computed from clean tree records) for p in plots: pt = [t for t in trees if t["plot_id"] == p["plot_id"] and t["status"] == "L"] ba_sum = sum(t["basal_area_ft2"] for t in pt) vol_sum = sum(t["gross_volume_cuft"] for t in pt) expand = 43560.0 / (3.14159265 * p["plot_radius_ft"] ** 2) p["basal_area_ft2_ac"] = round(ba_sum * expand, 1) p["trees_per_acre"] = int(round(len(pt) * expand)) p["net_volume_cuft_ac"] = round(vol_sum * expand, 0) # ---------------------------------------------------------------------------- # 4. DELIBERATE DATA-QUALITY DEFECTS # ---------------------------------------------------------------------------- NUMERIC_TREE_FIELDS = ["dbh_in", "total_height_ft", "height_to_live_crown_ft", "crown_ratio_pct", "crown_class", "damage_code", "decay_class", "dbh_growth_5yr_in", "basal_area_ft2", "gross_volume_cuft"] NUMERIC_PLOT_FIELDS = ["latitude", "longitude", "elevation_ft", "aspect_deg", "slope_pct", "stand_age_yr", "site_index_ft_50yr", "canopy_cover_pct", "basal_area_ft2_ac", "trees_per_acre", "net_volume_cuft_ac"] # --- 4a. Scattered missing values ------------------------------------------- for row in trees: if rng.random() < 0.055: f = rng.choice(NUMERIC_TREE_FIELDS + ["species_common", "crown_ratio_pct"]) tok = rng.choice(MISSING_TOKENS) log_defect("trees.csv", f'{row["plot_id"]}/T{row["tree_number"]}', f, "missing_value", row[f], tok) row[f] = tok for row in plots: if rng.random() < 0.10: f = rng.choice(NUMERIC_PLOT_FIELDS + ["soil_drainage", "plot_notes"]) tok = rng.choice(MISSING_TOKENS) log_defect("plots.csv", row["plot_id"], f, "missing_value", row[f], tok) row[f] = tok # --- 4b. Text smuggled into numeric columns --------------------------------- for row in trees: if rng.random() < 0.020: f = rng.choice(["dbh_in", "total_height_ft", "dbh_growth_5yr_in"]) tok = rng.choice(TEXT_IN_NUMERIC) log_defect("trees.csv", f'{row["plot_id"]}/T{row["tree_number"]}', f, "text_in_numeric", row[f], tok) row[f] = tok # --- 4c. Out-of-range / impossible values ----------------------------------- oob = [ ("trees.csv", trees, "dbh_in", 999.0), ("trees.csv", trees, "dbh_in", -5.0), ("trees.csv", trees, "total_height_ft", 812.0), ("trees.csv", trees, "crown_ratio_pct", 250), ("plots.csv", plots, "elevation_ft", -1200), ("plots.csv", plots, "slope_pct", 240), ("plots.csv", plots, "canopy_cover_pct", 340), ] for fname, coll, field, bad in oob: row = rng.choice(coll) rid = row["plot_id"] if fname == "plots.csv" else \ f'{row["plot_id"]}/T{row["tree_number"]}' log_defect(fname, rid, field, "out_of_range", row[field], bad) row[field] = bad # --- 4d. Unit mismatch (metres written into a *_ft column) ------------------ for _ in range(4): row = rng.choice(trees) ft = float(row["total_height_ft"]) bad = round(ft * 0.3048, 2) log_defect("trees.csv", f'{row["plot_id"]}/T{row["tree_number"]}', "total_height_ft", "unit_mismatch_ft_vs_m", ft, bad) row["total_height_ft"] = bad # --- 4e. Inconsistent species / type strings -------------------------------- for _ in range(12): row = rng.choice(trees) orig = row["species_common"] variant = rng.choice([orig.lower(), orig.upper(), f" {orig} ", orig.replace(" ", "\u00a0"), orig + " spp.", orig.replace("e", "é", 1)]) log_defect("trees.csv", f'{row["plot_id"]}/T{row["tree_number"]}', "species_common", "inconsistent_categorical", orig, variant) row["species_common"] = variant for _ in range(5): row = rng.choice(plots) orig = row["forest_type"] variant = rng.choice([orig.lower(), orig.upper(), orig.replace("-", " / "), f"{orig} ", orig.replace("-", "")]) log_defect("plots.csv", row["plot_id"], "forest_type", "inconsistent_categorical", orig, variant) row["forest_type"] = variant # --- 4f. Species code / name mismatch --------------------------------------- for _ in range(4): row = rng.choice(trees) wrong = rng.choice([s for s in SPECIES if s[0] != row["spcd"]]) log_defect("trees.csv", f'{row["plot_id"]}/T{row["tree_number"]}', "spcd", "code_name_mismatch", f'{row["spcd"]} ({row["species_common"]})', f'{wrong[0]} ({row["species_common"]})') row["spcd"] = wrong[0] # --- 4g. Date-format chaos --------------------------------------------------- for row in plots: if rng.random() < 0.35: try: d = date.fromisoformat(str(row["measurement_date"])) style = rng.choice([1, 2, 3, 4]) bad = fmt_date(d, style) log_defect("plots.csv", row["plot_id"], "measurement_date", "date_format_inconsistency", row["measurement_date"], bad) row["measurement_date"] = bad except ValueError: pass if rng.random() < 0.05: bad = rng.choice(["2019/13/45", "13/45/2019", "0000-00-00", "June 31, 2019", "31-Feb-2020"]) log_defect("plots.csv", row["plot_id"], "measurement_date", "impossible_date", row["measurement_date"], bad) row["measurement_date"] = bad # --- 4h. Swapped latitude / longitude --------------------------------------- for _ in range(3): row = rng.choice(plots) try: la, lo = float(row["latitude"]), float(row["longitude"]) except (TypeError, ValueError): continue log_defect("plots.csv", row["plot_id"], "latitude/longitude", "swapped_coordinates", f"{la},{lo}", f"{lo},{la}") row["latitude"], row["longitude"] = lo, la # --- 4i. Exact duplicate rows ------------------------------------------------ for _ in range(9): dup = dict(rng.choice(trees)) trees.append(dup) log_defect("trees.csv", f'{dup["plot_id"]}/T{dup["tree_number"]}', "(entire row)", "exact_duplicate_row", "-", "duplicated record") for _ in range(2): dup = dict(rng.choice(plots)) plots.append(dup) log_defect("plots.csv", dup["plot_id"], "(entire row)", "exact_duplicate_row", "-", "duplicated record") # --- 4j. Near-duplicates (same key, different measurement date) ------------- for _ in range(3): src = rng.choice([t for t in trees if isinstance(t["dbh_in"], float)]) near = dict(src) near["dbh_in"] = round(float(src["dbh_in"]) + rng.uniform(0.1, 0.6), 1) near["measurement_date"] = fmt_date(jitter_date()) trees.append(near) log_defect("trees.csv", f'{near["plot_id"]}/T{near["tree_number"]}', "plot_id+tree_number", "near_duplicate_key", "-", "repeated tree key with differing values") # --- 4k. Column-shifted rows (extra comma) ---------------------------------- SHIFTED = [] for _ in range(2): src = rng.choice(trees) vals = [str(src.get(h, "")) for h in TREE_HEADER] vals.insert(4, "extra_field") # pushes everything one column right SHIFTED.append(vals) log_defect("trees.csv", f'{src["plot_id"]}/T{src["tree_number"]}', "(all columns)", "column_shift", "-", "row contains one extra field") rng.shuffle(trees) # ---------------------------------------------------------------------------- # 5. SERIALISATION HELPERS # ---------------------------------------------------------------------------- def write_csv(header, rows, dup_header_at=None, blank_before=(), shifted=()): buf = io.StringIO() w = csv.writer(buf, lineterminator="\n") w.writerow(header) for i, row in enumerate(rows): if dup_header_at is not None and i == dup_header_at: w.writerow(header) if i in blank_before: buf.write("\n") w.writerow([row.get(h, "") for h in header]) for s in shifted: w.writerow(s) return buf.getvalue() plots_csv = write_csv(PLOT_HEADER, plots, dup_header_at=len(plots) // 2, blank_before=(11, 29)) trees_csv = write_csv(TREE_HEADER, trees, dup_header_at=len(trees) // 3, blank_before=(5, 41, 97), shifted=SHIFTED) # --- species reference (clean) ---------------------------------------------- sp_buf = io.StringIO() spw = csv.writer(sp_buf, lineterminator="\n") spw.writerow(["spcd", "common_name", "scientific_name", "short_code", "wood_type"]) for s in SPECIES: spw.writerow(s) # --- data dictionary (clean) ------------------------------------------------ DICT_ROWS = [ ("plots.csv", "plot_id", "string", "-", "Unique plot identifier (state-county-seq)"), ("plots.csv", "state", "string", "-", "US state abbreviation (ME, NH, VT)"), ("plots.csv", "county", "string", "-", "County name"), ("plots.csv", "latitude", "float", "decimal degrees", "Plot latitude (WGS84); some values corrupted"), ("plots.csv", "longitude", "float", "decimal degrees", "Plot longitude (WGS84); negative for western hemisphere"), ("plots.csv", "elevation_ft", "integer", "feet", "Plot elevation above sea level"), ("plots.csv", "aspect_deg", "integer", "degrees azimuth", "Slope aspect, 0 = north"), ("plots.csv", "slope_pct", "integer", "percent", "Ground slope"), ("plots.csv", "forest_type", "string", "-", "Dominant forest community type"), ("plots.csv", "stand_origin", "string", "-", "natural or planted"), ("plots.csv", "stand_age_yr", "integer", "years", "Estimated total stand age"), ("plots.csv", "site_index_species", "string", "-", "Species short code used for site index"), ("plots.csv", "site_index_ft_50yr", "integer", "feet at age 50", "Site index (base age 50)"), ("plots.csv", "soil_drainage", "string", "-", "Soil drainage class"), ("plots.csv", "ownership", "string", "-", "Land ownership category"), ("plots.csv", "plot_radius_ft", "float", "feet", "Fixed-radius plot radius"), ("plots.csv", "canopy_cover_pct", "integer", "percent", "Estimated canopy cover"), ("plots.csv", "basal_area_ft2_ac", "float", "ft^2/acre", "Live-tree basal area, expanded to per-acre"), ("plots.csv", "trees_per_acre", "integer", "trees/acre", "Live tree density, expanded to per-acre"), ("plots.csv", "net_volume_cuft_ac", "float", "ft^3/acre", "Net merchantable volume per acre"), ("plots.csv", "disturbance_code", "integer", "code", "0 none, 1 harvest, 2 wind, 3 insect, 4 fire, 5 disease, 6 ice"), ("plots.csv", "measurement_date", "date", "ISO-8601 ideal", "Field measurement date; formats inconsistent"), ("plots.csv", "crew", "string", "-", "Field crew identifier"), ("plots.csv", "plot_notes", "string", "-", "Free-text field notes"), ("trees.csv", "plot_id", "string", "-", "Foreign key to plots.csv"), ("trees.csv", "tree_number", "integer", "-", "Tree number within plot (with plot_id forms the key)"), ("trees.csv", "spcd", "integer", "code", "Species code (FIA-style, approximate)"), ("trees.csv", "species_common", "string", "-", "Common name; casing/whitespace inconsistent"), ("trees.csv", "species_scientific", "string", "-", "Scientific name"), ("trees.csv", "wood_type", "string", "-", "softwood or hardwood"), ("trees.csv", "status", "string", "-", "L = live, D = dead"), ("trees.csv", "dbh_in", "float", "inches", "Diameter at breast height (4.5 ft) outside bark"), ("trees.csv", "total_height_ft", "float", "feet", "Total tree height; some rows in metres"), ("trees.csv", "height_to_live_crown_ft", "float", "feet", "Height to base of live crown"), ("trees.csv", "crown_ratio_pct", "integer", "percent", "Live crown ratio"), ("trees.csv", "crown_class", "integer", "1-5", "1 dominant ... 5 overtopped"), ("trees.csv", "damage_code", "integer", "code", "0 none, 1 canker, 2 conk, 3 broken top, 4 dead top, 5 crook, 6 fork, 7 lean>15deg, 8 windthrow, 9 insect"), ("trees.csv", "decay_class", "integer", "0-5", "0 sound, 5 advanced decay"), ("trees.csv", "dbh_growth_5yr_in", "float", "inches", "5-year diameter increment"), ("trees.csv", "basal_area_ft2", "float", "ft^2", "Per-tree basal area = 0.005454 * dbh^2"), ("trees.csv", "gross_volume_cuft", "float", "ft^3", "Per-tree gross volume (approximate form factor)"), ("trees.csv", "measurement_date", "date", "-", "Measurement date (copied from plot)"), ("trees.csv", "crew", "string", "-", "Field crew identifier"), ] dd_buf = io.StringIO() ddw = csv.writer(dd_buf, lineterminator="\n") ddw.writerow(["file", "field", "type", "units", "description"]) ddw.writerows(DICT_ROWS) # --- soil profile JSON (nested, with its own defects) ----------------------- soil = [] for p in plots[:24]: prof = { "plot_id": p["plot_id"], "soil_series": rng.choice(["Telos", "Monson", "Berkshire", "Hermon", "Chesuncook", "Worth", "Colonel", "Lyman", "Peru", "Marlow", "Becket"]), "drainage_class": p["soil_drainage"] if isinstance(p["soil_drainage"], str) else None, "depth_to_bedrock_cm": rng.choice([None, rng.randint(35, 180)]), "horizons": [ {"horizon": "Oe", "top_cm": 0, "bottom_cm": rng.randint(2, 8), "texture": "slightly decomposed organic", "ph": round(rng.uniform(3.6, 5.2), 1)}, {"horizon": "A", "top_cm": rng.randint(2, 8), "bottom_cm": rng.randint(9, 20), "texture": "sandy loam", "ph": round(rng.uniform(4.0, 5.8), 1)}, {"horizon": "Bw", "top_cm": rng.randint(20, 30), "bottom_cm": rng.randint(45, 90), "texture": "loam", "ph": round(rng.uniform(4.6, 6.2), 1)}, {"horizon": "C", "top_cm": rng.randint(80, 95), "bottom_cm": rng.choice([None, 150]), "texture": "channery loam", "ph": round(rng.uniform(5.0, 6.8), 1)}, ], } soil.append(prof) # inject JSON defects soil.append(dict(soil[3])) # duplicate object del soil[7]["soil_series"] # missing required key soil[10]["depth_to_bedrock_cm"] = "deep" # wrong type soil[12]["horizons"][1]["ph"] = "NA" # missing inside nest soil[15]["horizons"] = soil[15]["horizons"][:2] # truncated profile soil_json = json.dumps(soil, indent=2) for idx in (3, 7, 10, 12, 15): log_defect("soil_profiles.json", f"record_index_{idx}", "*", "structural_defect", "-", "see file: duplicate / missing key / wrong type / nested NA / truncated") # --- messy free-text field notes ------------------------------------------- FIELD_NOTES = """FIELD NOTES - transcribed from crew notebooks (do not trust verbatim) ===================================================================== ME-PEN-004 6/14/19 Crew B-1 Plot relocated ~35 m NE of stored coords; GPS would not lock under canopy. Heavy balsam fir mortality, spruce budworm sign on ~40% of fir. Flagging left. ME-PIS-009 Access via old tote road, washed out at 2.1 mi. 4WD required. Slope steeper than map suggests, maybe 30-35%. Aspect reading taken twice. ME-SOM-011 09-Aug-2021 Windstorm damage from Oct 2019 still evident - approx 30% windthrow in NW quarter of plot. Some trees measured from top of blowdown (dbh estimated). ME-ARO-015 NOTE: height pole broken, heights estimated by clinometer + pacing. +/- 15%. Trees 3, 4, 7 -- dbh re-measured twice, first reading discarded (bark snag). ME-HAN-021 10/2/2022 Cedar swamp, wading. Plot center stake lost; used bearing/distance from witness tree (yellow birch, blazed). Soil auger hit refusal at ~40 cm. NH-COO-028 Crew D-1. Time constraint - only 6 trees tallied, subplot 2 skipped entirely. Recorded "TBD" for crown class on several stems. Needs remeasure. VT-ORL-033 July 2023 Plot overlaps recent select-cut (2021). Cut stumps present, ~8 stems/acre. Volume figures likely overestimate residual stand. General: - species codes entered from memory on plots 12-18, verify against list - several date fields written as "6/14" without year (assume 2019 season) - units confusion on height: some crews used metric rods - duplicate tally sheets found for plots ME-PEN-004 and ME-SOM-011 """ # --- loader example --------------------------------------------------------- LOADER = '''#!/usr/bin/env python3 """ loader_example.py -- how to read the synthetic dataset defensively. Demonstrates: tolerant CSV reading, numeric coercion, duplicate detection, range checks, and reporting of suspected data-quality problems. """ import csv import math import os from collections import Counter HERE = os.path.dirname(os.path.abspath(__file__)) MISSING = {"", "na", "n/a", "nan", ".", "-999", "-9999", "null", "none", "not recorded", "unknown", "tbd", "??", " "} TREE_NUMERIC = ["spcd", "dbh_in", "total_height_ft", "height_to_live_crown_ft", "crown_ratio_pct", "crown_class", "damage_code", "decay_class", "dbh_growth_5yr_in", "basal_area_ft2", "gross_volume_cuft"] RANGES = { "dbh_in": (0.5, 60.0), "total_height_ft": (1.0, 200.0), "crown_ratio_pct": (0, 100), "crown_class": (1, 5), "decay_class": (0, 5), "damage_code": (0, 9), } def to_number(value): """Return (float_or_None, note). Handles commas, stray units, junk.""" if value is None: return None, "null" text = str(value).strip() if text.lower() in MISSING: return None, "missing" cleaned = text.replace(",", ".") if text.count(",") == 1 and "." not in text else text cleaned = "".join(ch for ch in cleaned if ch.isdigit() or ch in ".-+") try: return float(cleaned), "ok" except ValueError: return None, "unparseable: %r" % text def read_rows(path, expected_header): """Read a CSV that may contain blank lines, repeated headers, column shifts.""" good, problems = [], [] with open(path, newline="", encoding="utf-8", errors="replace") as fh: reader = csv.reader(fh) header = next(reader, None) if header is None: return [], ["file is empty"] if header != expected_header: problems.append("unexpected header: %r" % (header[:5],)) for lineno, raw in enumerate(reader, start=2): if not raw or all(c.strip() == "" for c in raw): problems.append("line %d: blank line" % lineno) continue if raw == expected_header: problems.append("line %d: duplicated header row" % lineno) continue if len(raw) != len(expected_header): problems.append("line %d: expected %d fields, found %d (column shift?)" % (lineno, len(expected_header), len(raw))) continue good.append(dict(zip(expected_header, raw))) return good, problems def main(): trees, tree_problems = read_rows( os.path.join(HERE, "trees.csv"), ["plot_id", "tree_number", "spcd", "species_common", "species_scientific", "wood_type", "status", "dbh_in", "total_height_ft", "height_to_live_crown_ft", "crown_ratio_pct", "crown_class", "damage_code", "decay_class", "dbh_growth_5yr_in", "basal_area_ft2", "gross_volume_cuft", "measurement_date", "crew"]) print("Parsed %d tree rows" % len(trees)) print("Structural problems: %d" % len(tree_problems)) for p in tree_problems[:10]: print(" ", p) missing_counts = Counter() range_hits = Counter() key_counter = Counter() coerced = [] for row in trees: key_counter[(row["plot_id"], row["tree_number"])] += 1 for field in TREE_NUMERIC: value, note = to_number(row.get(field)) if note == "missing": missing_counts[field] += 1 elif note != "ok": missing_counts[field + " (junk)"] += 1 else: lo, hi = RANGES.get(field, (-1e9, 1e9)) if not (lo <= value <= hi): range_hits[field] += 1 row[field] = value coerced.append(row) print("\\nMissing / unparseable values by field:") for field, count in missing_counts.most_common(): print(" %-40s %d" % (field, count)) print("\\nOut-of-range values by field:") for field, count in range_hits.most_common(): print(" %-40s %d" % (field, count)) dups = {k: v for k, v in key_counter.items() if v > 1} print("\\nDuplicate (plot_id, tree_number) keys: %d" % len(dups)) for k, v in list(dups.items())[:10]: print(" ", k, "x", v) bad_coords = 0 for row in trees: pass print("\\nDone. Feed `coerced` to your model / analysis.") if __name__ == "__main__": main() ''' # --- README ---------------------------------------------------------------- README = """# Synthetic Forestry Dataset - Maine & Northern New England **Preliminary release 1.0 - ENTIRELY FICTITIOUS** Published by Krummholz AI for data-cleaning research and evaluation. Records are generated procedurally with seed 20240617; reference tables and fictional field notes are embedded in the generator. No real observations or external datasets are loaded. Geographic names and approximate bounds provide regional context, not verified plot locations. This preliminary set is not production-ready or scientifically validated. Do not use it to prescribe forestry treatments, estimate real inventories, or make operational, ecological, land-management or safety decisions. Species codes and growth/volume relationships are illustrative. The intentional missing values, duplicates, impossible measurements, inconsistent dates/units/categories, malformed CSV rows and nested JSON defects are retained. The defect manifest is a partial record of injections, not comprehensive ground truth; some structural defects are not logged and soil-profile defect indexing is imprecise. The loader is a demonstration, not a complete validator or trustworthy data-cleaning pipeline. ## What this is A synthetic, plot- and tree-level forest inventory dataset styled after field data collected in Maine, New Hampshire and Vermont. It was generated procedurally from a fixed random seed. **No real plot, landowner, crew or tree measurement is represented here.** Every identifier and value is invented. ## Why it contains defects Real forestry data is messy. This dataset intentionally includes missing values, duplicates, impossible numbers, inconsistent categorical strings, mixed date formats, unit errors and structurally broken rows so that it can be used to train or evaluate models on data cleaning, validation and robustness tasks. An optional partial log of injected defects is provided in `intended_defects_manifest.json`. **Exclude that file if you want the model to detect problems without labels.** Review and supplement it before using it as supervised training or evaluation targets. ## Files | File | Description | |---|---| | `plots.csv` | Plot-level records (~44 rows incl. duplicates) | | `trees.csv` | Tree-level records (~500 rows incl. duplicates and broken rows) | | `soil_profiles.json` | Nested soil horizon profiles for a subset of plots | | `field_notes.txt` | Messy free-text crew notes | | `species_reference.csv` | Clean species lookup table | | `data_dictionary.csv` | Clean field dictionary | | `intended_defects_manifest.json` | Partial log of injected defects (optional) | | `loader_example.py` | Dependency-free example of defensive loading | | `LICENSE.txt` | Complete MIT License | ## Defect categories present * **Missing values** - blank, `NA`, `N/A`, `NaN`, `.`, `-999`, `-9999`, `null`, `None`, `not recorded`, whitespace-only. * **Duplicates** - exact duplicate rows; near-duplicates sharing the same `plot_id` + `tree_number` key but with differing values; a duplicated plot record; a duplicated soil-profile object. * **Corrupted numerics** - letters inside numbers (`1O.5`), embedded units (`14 (est)`), comma decimals (`12,5`), free text (`unknown`, `TBD`). * **Out-of-range values** - `dbh_in` of 999 or -5, `slope_pct` of 240, canopy cover above 100%, negative elevation. * **Unit mismatches** - heights recorded in metres inside `*_ft` columns. * **Categorical inconsistency** - case changes, doubled internal spaces, non-breaking spaces, accented characters, ` spp.` suffixes, hyphen variants in `forest_type`. * **Code/name mismatch** - `spcd` disagreeing with `species_common`. * **Date problems** - six incompatible formats, plus impossible dates such as `2019/13/45` and `31-Feb-2020`. * **Coordinate errors** - latitude and longitude swapped. * **Structural breakage** - repeated header rows mid-file, blank lines, rows with an extra field (column shift). * **Nested JSON problems** - missing required key, wrong value type, `"NA"` inside a numeric field, truncated horizon list. ## Suggested tasks 1. Cleaning pipeline: normalise dates, coerce numerics, canonicalise species. 2. Duplicate detection: exact vs. near-duplicate resolution. 3. Validation rule authoring (range, referential integrity, code lists). 4. Anomaly detection / outlier flagging. 5. Text-to-SQL or table-QA over inconsistent schema. 6. Imputation of missing `dbh_in`, `total_height_ft`, `crown_ratio_pct`. 7. Provenance reasoning: reconstruct which rows are trustworthy. ## Scientific notes Values were drawn from plausible ranges for the region: mixed spruce-fir, northern hardwood, mixedwood, white pine-hemlock, aspen-birch, cedar-swamp and oak-pine communities; elevations roughly 0-2,400 ft; site index 38-92 ft base age 50; DBH 1-34 in; heights derived from a power function of DBH modulated by site index. Per-tree basal area uses 0.005454 x DBH^2. Species codes are **plausible FIA-style codes but are not authoritative** - verify against the official FIA species list before any real-world use. ## License The generator, synthetic dataset and accompanying documentation are released under the MIT License. Copyright (c) 2026 Hayven Jackson / Krummholz AI. See LICENSE.txt for the complete terms. """ # ---------------------------------------------------------------------------- # 6. PACKAGE # ---------------------------------------------------------------------------- files = { "LICENSE.txt": LICENSE_TEXT, "README.md": README, "plots.csv": plots_csv, "trees.csv": trees_csv, "soil_profiles.json": soil_json, "species_reference.csv": sp_buf.getvalue(), "data_dictionary.csv": dd_buf.getvalue(), "field_notes.txt": FIELD_NOTES, "loader_example.py": LOADER, "intended_defects_manifest.json": json.dumps( {"generator_seed": SEED, "defect_count": len(MANIFEST), "defects": MANIFEST}, indent=2), } with zipfile.ZipFile(ZIPNAME, "w", zipfile.ZIP_DEFLATED) as zf: for name, content in files.items(): zf.writestr(f"{ROOT}/{name}", content) print(f"Wrote {ZIPNAME}") print(f" files: {len(files)}") print(f" plot rows: {len(plots)} tree rows: {len(trees)}") print(f" logged defects: {len(MANIFEST)}")