import sys, os, json, time, ctypes, re, statistics, argparse, unicodedata, hashlib import pypdfium2 as pdfium import pypdfium2.raw as R SEG_GAP_EM = 3.0 SEG_GAP_LONG_EM = 1.5 SHORT_LEAD_CHARS = 3 SPLIT_GAP_EM = 0.9 SEP_MIN_ROWS = 8 SEP_MIN_SUPPORT = 0.6 SEP_MIN_SPAN_FRAC = 0.3 SEP_STRONG_ROWS = 20 WORD_GAP_EM = 0.12 LINE_OVERLAP = 0.5 BLOCK_PITCH_EM = 2.4 BLOCK_XOVERLAP = 0.3 GUTTER_MIN_ITEMS = 4 GUTTER_MIN_REGULAR_FRAC = 0.8 GUTTER_MAX_CROSSING_FRAC = 0.1 GUTTER_MIN_SPAN_FRAC = 0.2 MARGIN_FRAC = 0.12 FURNITURE_MIN_PAGES = 3 FURNITURE_STRONG_PAGES = 20 FURNITURE_LETTERLESS_PAGES = 10 STAMP_MIN_WORDS = 3 FURNITURE_Y_TOL_FRAC = 0.02 SMALL_BLOCK_LINES = 3 IMAGE_DOMINANT_FRAC = 0.5 IMAGE_REGION_MIN_FRAC = 0.05 IMAGE_REGION_TEXT_DENSITY = 0.002 LOW_TEXT_GLYPHS = 300 TIME_TOKEN = re.compile(r"^\d{1,2}:\d{2}(:\d{2})?$") ODD_TOKEN_FRAC = 0.25 ODD_TOKEN_MIN_WORDS = 30 ROMAN = re.compile(r"^(?=[ivxlcdm]+$)m{0,4}(cm|cd|d?c{0,3})(xc|xl|l?x{0,3})(ix|iv|v?i{0,3})$", re.I) PLAIN_TOKEN = re.compile(r"^[\(\"'“‘\[]?(?:[A-Za-z][a-z]*(?:[-'’][A-Za-z][a-z]*)*|[A-Z]{2,}|\d+(?:[.,:/-]\d+)*|[A-Za-z]\.|§|\$\d[\d,.]*|[ivxlc]+)[.,;:!?\)\"'”’\]]{0,3}$") DIGITS = re.compile(r"\d+") SPACE = re.compile(r"\s+") TRANS = str.maketrans({"\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"', "\u2013": "-", "\u2014": "-", "\u2010": "-", "\u2011": "-", "\u00ad": "-", "\u00a0": " "}) HYPHENS = "-\u00ad\u2010\u2011" def norm_text(s): s = unicodedata.normalize("NFKC", s).translate(TRANS).lower() return SPACE.sub(" ", s).strip() def page_images(page): out = [] def walk(count, get): for k in range(count()): obj = get(k) ty = R.FPDFPageObj_GetType(obj) if ty == R.FPDF_PAGEOBJ_IMAGE: l = ctypes.c_float(); b = ctypes.c_float(); r = ctypes.c_float(); t = ctypes.c_float() R.FPDFPageObj_GetBounds(obj, l, b, r, t) out.append((l.value, b.value, r.value, t.value)) elif ty == R.FPDF_PAGEOBJ_FORM: walk(lambda: R.FPDFFormObj_CountObjects(obj), lambda j: R.FPDFFormObj_GetObject(obj, j)) walk(lambda: R.FPDFPage_CountObjects(page), lambda j: R.FPDFPage_GetObject(page, j)) return out def page_primitives(page): tp = page.get_textpage() n = tp.count_chars() l = ctypes.c_double(); b = ctypes.c_double(); r = ctypes.c_double(); t = ctypes.c_double() lb = R.FS_RECTF() buf = ctypes.create_string_buffer(128); fl = ctypes.c_int() chars = [] for i in range(n): u = R.FPDFText_GetUnicode(tp, i) g = R.FPDFText_IsGenerated(tp, i) assert g in (0, 1), f"FPDFText_IsGenerated failed at char {i}" gen = g == 1 ok_tight = R.FPDFText_GetCharBox(tp, i, l, r, b, t) ok_loose = R.FPDFText_GetLooseCharBox(tp, i, lb) ang = R.FPDFText_GetCharAngle(tp, i) ln = R.FPDFText_GetFontInfo(tp, i, buf, 128, fl) if ln > 128: buf = ctypes.create_string_buffer(ln) R.FPDFText_GetFontInfo(tp, i, buf, ln, fl) assert gen or (ok_tight and ok_loose and ln > 0), f"PDFium returned no box/font for content char {i} ({chr(u)!r})" chars.append({"i": i, "c": chr(u), "gen": gen, "tight": (l.value, b.value, r.value, t.value), "box": (lb.left, lb.bottom, lb.right, lb.top), "angle": ang, "font": buf.value[:max(ln - 1, 0)].decode("latin1"), "ume": R.FPDFText_HasUnicodeMapError(tp, i) == 1}) return chars, page_images(page) def vover(a, b): return min(a[3], b[3]) - max(a[1], b[1]) def hover(a, b): return min(a[2], b[2]) - max(a[0], b[0]) def union(boxes): return (min(x[0] for x in boxes), min(x[1] for x in boxes), max(x[2] for x in boxes), max(x[3] for x in boxes)) def build_runs(chars): runs = [] cur = None for c in chars: if c["gen"]: continue ch = c["c"] if ord(ch) < 32: continue box = c["box"] h = box[3] - box[1] if h <= 0 and not ch.isspace(): continue if cur is not None: p = cur["chars"][-1] pb = p["box"] ph = max(pb[3] - pb[1], h, 1e-3) same = abs(c["angle"] - cur["angle"]) < 0.01 if same and cur["angle"] == 0 and h > 0: same = cur["box"] is None or vover(box, cur["box"]) >= LINE_OVERLAP * min(ph, cur["box"][3] - cur["box"][1]) same = same and box[0] >= pb[0] - 0.5 * ph and box[0] - pb[2] <= SEG_GAP_EM * ph if not same: cur = None if cur is None: cur = {"chars": [], "angle": c["angle"], "box": None} runs.append(cur) cur["chars"].append(c) if not ch.isspace(): cur["box"] = box if cur["box"] is None else union([cur["box"], box]) return [r for r in runs if r["box"] is not None] def run_words(run, rid): words = [] cur = None prev = None for c in run["chars"]: if c["c"].isspace(): cur = None prev = c continue if cur is not None and prev is not None: ph = max(prev["box"][3] - prev["box"][1], 1e-3) if c["box"][0] - prev["box"][2] > WORD_GAP_EM * ph: cur = None if cur is None: cur = {"chars": []} words.append(cur) cur["chars"].append(c) prev = c out = [] for w in words: boxes = [c["box"] for c in w["chars"]] hs = [b[3] - b[1] for b in boxes] out.append({"text": "".join(c["c"] for c in w["chars"]), "box": union(boxes), "tight": union([c["tight"] for c in w["chars"]]), "h": statistics.median(hs), "i0": w["chars"][0]["i"], "i1": w["chars"][-1]["i"], "angle": run["angle"], "run": rid, "font": w["chars"][0]["font"], "ume": any(c["ume"] for c in w["chars"])}) return out def pieces_from_words(words): pieces = [] cur = None for w in words: if cur is not None: p = cur[-1] gap = w["box"][0] - p["box"][2] if w["run"] != p["run"] or gap > SEG_GAP_EM * max(p["h"], w["h"]): cur = None if cur is None: cur = [] pieces.append(cur) cur.append(w) return pieces def column_separators(words, W, H): horiz = [w for w in words if w["angle"] == 0] if len(horiz) < SEP_MIN_ROWS: return [] hm = statistics.median([w["h"] for w in horiz]) rows = {} for w in horiz: rows.setdefault(round((w["box"][1] + w["box"][3]) / 2 / hm), []).append(w) gaps = {} for key, ws in rows.items(): ws.sort(key=lambda w: w["box"][0]) gaps[key] = [(a["box"][2], b["box"][0]) for a, b in zip(ws, ws[1:]) if b["box"][0] - a["box"][2] > SPLIT_GAP_EM * hm] step = hm / 2 x = 0.15 * W accepted = [] while x < 0.85 * W: sup = [k for k, gs in gaps.items() if any(l <= x <= r for l, r in gs)] if len(sup) >= SEP_MIN_ROWS: lo, hi = min(sup), max(sup) inrange = [k for k in rows if lo <= k <= hi] if len(sup) >= SEP_MIN_SUPPORT * len(inrange) and (hi - lo + 1) * hm >= SEP_MIN_SPAN_FRAC * H: accepted.append((x, len(sup), len(inrange), lo, hi)) x += step seps = [] for a_ in accepted: if seps and a_[0] - seps[-1][-1][0] <= step * 1.01: seps[-1].append(a_) else: seps.append([a_]) out = [] for grp in seps: best = max(grp, key=lambda a_: a_[1]) mid = grp[len(grp) // 2] out.append({"x": round((mid[0] if mid[1] == best[1] else best[0]), 1), "x_range": [round(grp[0][0], 1), round(grp[-1][0], 1)], "rows_supporting": best[1], "rows_in_span": best[2], "y_span": [round(best[3] * hm, 1), round((best[4] + 1) * hm, 1)]}) return out def split_at_separators(segs, seps, H): out = [] for s in segs: parts = [s["words"]] cy = statistics.median([(w["box"][1] + w["box"][3]) / 2 for w in s["words"]]) hs = statistics.median([w["h"] for w in s["words"]]) for sep in seps: if not (sep["y_span"][0] - hs <= cy <= sep["y_span"][1] + hs): continue nxt = [] for ws in parts: cuts = [] for k in range(1, len(ws)): a, b = ws[k - 1], ws[k] if (a["box"][0] + a["box"][2]) / 2 < sep["x"] < (b["box"][0] + b["box"][2]) / 2 and b["box"][0] - a["box"][2] >= SPLIT_GAP_EM * max(a["h"], b["h"]): cuts = [k] break if not cuts and sep["rows_supporting"] >= SEP_STRONG_ROWS and H * MARGIN_FRAC < cy < H * (1 - MARGIN_FRAC): for k, w in enumerate(ws): if w["text"].isdigit() and len(w["text"]) <= 2 and sep["x_range"][0] - w["h"] <= (w["box"][0] + w["box"][2]) / 2 <= sep["x_range"][1] + w["h"] and 0 < k < len(ws) - 1: cuts = [k, k + 1] break if not cuts: nxt.append(ws) else: prev = 0 for c in cuts: nxt.append(ws[prev:c]); prev = c nxt.append(ws[prev:]) parts = nxt for ws in parts: out.append({"words": ws}) return out def words_to_segments(words, seps, H): pieces = pieces_from_words(words) horiz = [p for p in pieces if p[0]["angle"] == 0] rot = [p for p in pieces if p[0]["angle"] != 0] for p in pieces: p_box = union([w["box"] for w in p]) p.insert(0, {"_box": p_box, "_h": statistics.median([w["h"] for w in p])}) rows = [] for p in sorted(horiz, key=lambda p: -(p[0]["_box"][1] + p[0]["_box"][3]) / 2): for row in rows: if vover(p[0]["_box"], row["box"]) >= LINE_OVERLAP * min(p[0]["_h"], row["h"]): row["pieces"].append(p) row["box"] = union([row["box"], p[0]["_box"]]) break else: rows.append({"pieces": [p], "box": p[0]["_box"], "h": p[0]["_h"]}) segs = [] for row in rows: cur = None for p in sorted(row["pieces"], key=lambda p: p[0]["_box"][0]): if cur is not None: pb = cur["words"][-1]["box"] ph = max(cur["words"][-1]["h"], p[0]["_h"]) gap = p[0]["_box"][0] - pb[2] wmin = min(pb[2] - pb[0], p[1]["box"][2] - p[1]["box"][0]) limit = SEG_GAP_EM if len("".join(w["text"] for w in cur["words"])) <= SHORT_LEAD_CHARS else SEG_GAP_LONG_EM if gap > limit * ph or gap < -min(0.5 * wmin, ph): cur = None if cur is None: cur = {"words": []} segs.append(cur) cur["words"].extend(p[1:]) segs = split_at_separators(segs, seps, H) for p in rot: segs.append({"words": p[1:]}) for s in segs: finish_segment(s) return segs def finish_segment(s): s["box"] = union([w["box"] for w in s["words"]]) s["h"] = statistics.median([w["h"] for w in s["words"]]) s["text"] = " ".join(w["text"] for w in s["words"]) s["rotated"] = s["words"][0]["angle"] != 0 def gutter_value(t): if t.isdigit() and len(t) <= 4: return int(t) if TIME_TOKEN.match(t): parts = [int(x) for x in t.split(":")] return parts[0] * 3600 + parts[1] * 60 + (parts[2] if len(parts) == 3 else 0) return None def detect_gutters(words, W, H): nums = [w for w in words if w["angle"] == 0 and gutter_value(w["text"]) is not None] others = [w for w in words if w["angle"] == 0 and gutter_value(w["text"]) is None] nums.sort(key=lambda w: (bool(TIME_TOKEN.match(w["text"])), (w["box"][0] + w["box"][2]) / 2)) bands = [] for w in nums: cx = (w["box"][0] + w["box"][2]) / 2 if bands and abs(cx - bands[-1]["cx"]) <= 1.5 * min(w["h"], bands[-1]["h"]) and bands[-1]["timed"] == bool(TIME_TOKEN.match(w["text"])): bands[-1]["items"].append(w) bands[-1]["cx"] = statistics.mean((x["box"][0] + x["box"][2]) / 2 for x in bands[-1]["items"]) bands[-1]["h"] = min(bands[-1]["h"], w["h"]) else: bands.append({"cx": cx, "h": w["h"], "items": [w], "timed": bool(TIME_TOKEN.match(w["text"]))}) gutters = [] rejected = [] for band in bands: items = sorted(band["items"], key=lambda w: -w["box"][3]) if len(items) < GUTTER_MIN_ITEMS: continue side = "left" if band["cx"] < W * 0.25 else "right" if band["cx"] > W * 0.75 else "center" cand = {"side": side, "x_center": round(band["cx"], 1), "count": len(items), "values": [w["text"] for w in items]} def reject(why): cand["reason"] = why rejected.append(cand) vals = [gutter_value(w["text"]) for w in items] timed = band["timed"] best = [0] * len(vals); prev = [-1] * len(vals) for i in range(len(vals)): best[i] = 1 for j in range(i): if (vals[j] <= vals[i] if timed else vals[j] < vals[i]) and best[j] + 1 > best[i]: best[i] = best[j] + 1; prev[i] = j k = best.index(max(best)) idx = [] while k != -1: idx.append(k); k = prev[k] inl = [items[i] for i in reversed(idx)] if len(inl) < GUTTER_MIN_ITEMS or len(inl) < GUTTER_MIN_REGULAR_FRAC * len(items): reject(f"only {len(inl)} of {len(items)} numerals form an increasing sequence downward") continue steps = [gutter_value(inl[k + 1]["text"]) - gutter_value(inl[k]["text"]) for k in range(len(inl) - 1)] step = statistics.mode(steps) if timed: step = 1 elif sum(1 for s_ in steps if s_ % step == 0) < GUTTER_MIN_REGULAR_FRAC * len(steps): reject(f"value steps {steps} are not multiples of a common step") continue span = inl[0]["box"][3] - inl[-1]["box"][1] if span < GUTTER_MIN_SPAN_FRAC * H: reject(f"band spans only {span / H:.0%} of page height") continue pitch = statistics.median([inl[k]["box"][3] - inl[k + 1]["box"][3] for k in range(len(inl) - 1)]) bx = union([w["box"] for w in inl]) hb = band["h"] chan = (bx[0] - 0.3 * hb, bx[2] + 0.3 * hb) rows = {} for o in others: key = round((o["box"][1] + o["box"][3]) / 2 / hb) rows.setdefault(key, False) if o["box"][2] > chan[0] and o["box"][0] < chan[1]: rows[key] = True crossing = sum(1 for v in rows.values() if v) if rows and crossing > GUTTER_MAX_CROSSING_FRAC * len(rows): reject(f"{crossing} of {len(rows)} text rows cross the band channel") continue if side != "center": outward = 0 for w in inl: for o in others: if vover(o["box"], w["box"]) > 0.5 * min(o["h"], w["h"]) and ((side == "left" and o["box"][2] < w["box"][0]) or (side == "right" and o["box"][0] > w["box"][2])): outward += 1 break if outward > (1 - GUTTER_MIN_REGULAR_FRAC) * len(inl): reject(f"{outward} of {len(inl)} numerals have text on their outer side") continue members = list(inl) for w in items: if w in members: continue v = gutter_value(w["text"]) prev = [m for m in members if m["box"][3] > w["box"][3]] nxt = [m for m in members if m["box"][3] < w["box"][3]] pv = gutter_value(prev[-1]["text"]) if prev else None nv = gutter_value(nxt[0]["text"]) if nxt else None ok = (pv is None or v - pv == step) and (nv is None or nv - v == step) if not timed else (pv is None or v >= pv) and (nv is None or v <= nv) if ok and (pv is not None or nv is not None): members.append(w) members.sort(key=lambda w: -w["box"][3]) cand.update({"count": len(members), "values": [w["text"] for w in members], "kind": "time" if timed else "numeral", "step": step, "median_pitch_pt": round(pitch, 2), "channel_crossing_rows": f"{crossing}/{len(rows)}", "outliers_kept": [w["text"] for w in items if w not in members], "words": members}) gutters.append(cand) return gutters, rejected def page_layout(chars, images, W, H): runs = build_runs(chars) words = [w for rid, r in enumerate(runs) for w in run_words(r, rid)] removed = [] onpage = [] clipped_runs = set() for w in words: cx = (w["box"][0] + w["box"][2]) / 2; cy = (w["box"][1] + w["box"][3]) / 2 if cx < 0 or cx > W or cy < 0 or cy > H: clipped_runs.add(w["run"]) for w in words: if w["run"] in clipped_runs: removed.append({"kind": "offpage", "text": w["text"], "box": rbox(w["box"]), "chars": [w["i0"], w["i1"]], "reason": f"text run extends outside page box {W:.0f}x{H:.0f}"}) else: onpage.append(w) gutters, rejected = detect_gutters(onpage, W, H) gutter_words = {id(w) for g in gutters for w in g["words"]} kept = [w for w in onpage if id(w) not in gutter_words] seps = column_separators(kept, W, H) segs = words_to_segments(kept, seps, H) for g in gutters: for w in g["words"]: removed.append({"kind": "line_number_gutter", "text": w["text"], "box": rbox(w["box"]), "chars": [w["i0"], w["i1"]], "reason": f"{g['side']} {g['kind']} band x={g['x_center']}: {g['count']} tokens {g['values'][0]}..{g['values'][-1]} step {g['step']} median pitch {g['median_pitch_pt']}pt, channel crossed by {g['channel_crossing_rows']} text rows"}) odd = sum(1 for w in onpage if not PLAIN_TOKEN.match(w["text"])) return {"segments": segs, "removed": removed, "gutters": [{k: v for k, v in g.items() if k != "words"} for g in gutters], "gutter_candidates_rejected": rejected, "separators": seps, "n_words": len(words), "odd_words": odd} def rbox(b): return [round(b[0], 2), round(b[1], 2), round(b[2], 2), round(b[3], 2)] def furniture_key(seg): return DIGITS.sub("#", norm_text(seg["text"])) def classify_furniture(pages): counts = {} for p in pages: H = p["H"] seen = set() for s in p["layout"]["segments"]: if s["rotated"]: continue cy = (s["box"][1] + s["box"][3]) / 2 if cy > H * (1 - MARGIN_FRAC) or cy < H * MARGIN_FRAC: k = (furniture_key(s), round(cy / H / FURNITURE_Y_TOL_FRAC)) if k not in seen: seen.add(k) counts[k] = counts.get(k, 0) + 1 strong = {} for (key, band), n in counts.items(): if n >= FURNITURE_STRONG_PAGES: strong[key] = max(n, strong.get(key, 0)) for p in pages: H = p["H"] segs = p["layout"]["segments"] for s in segs: s["class"] = "body" if s["rotated"]: s["class"] = "rotated" s["reason"] = "rotated text run, kept out of horizontal reading order" for s in segs: if s["class"] != "body": continue cy = (s["box"][1] + s["box"][3]) / 2 top = cy < H * MARGIN_FRAC bot = cy > H * (1 - MARGIN_FRAC) k = (furniture_key(s), round(cy / H / FURNITURE_Y_TOL_FRAC)) if (top or bot) and counts.get(k, 0) >= FURNITURE_STRONG_PAGES: s["class"] = "running_header" if bot else "running_footer" s["reason"] = f"same digit-masked text at same y band on {counts[k]} pages" blocks = link_blocks([s for s in segs if s["class"] == "body"]) small = {id(s) for b in blocks if len(b) <= SMALL_BLOCK_LINES for s in b} for s in segs: if s["class"] != "body" or id(s) not in small: continue cy = (s["box"][1] + s["box"][3]) / 2 top = cy < H * MARGIN_FRAC bot = cy > H * (1 - MARGIN_FRAC) key = furniture_key(s) k = (key, round(cy / H / FURNITURE_Y_TOL_FRAC)) n = counts.get(k, 0) lettered = any(ch.isalpha() for ch in key) if (top or bot) and n >= (FURNITURE_MIN_PAGES if lettered else FURNITURE_LETTERLESS_PAGES): s["class"] = "running_header" if bot else "running_footer" s["reason"] = f"same digit-masked text at same y band on {n} pages" elif (top or bot) and len(s["words"]) == 1 and (s["text"].isdigit() or ROMAN.match(s["text"])) and len(s["text"]) <= 5: s["class"] = "page_number" s["reason"] = "lone numeral in margin band" elif key in strong and len(key.split(" ")) >= STAMP_MIN_WORDS: s["class"] = "stamp_outside_margin" s["reason"] = f"digit-masked text matches a running header/footer seen in margins of {strong[key]} pages" def link_blocks(segs): segs = [s for s in segs if not s["rotated"]] n = len(segs) below = [[] for _ in range(n)] above = [[] for _ in range(n)] order = sorted(range(n), key=lambda i: -segs[i]["box"][3]) for ai in order: a = segs[ai] cands = [] for bi in order: b = segs[bi] if b["box"][3] > a["box"][1] + 0.5 * a["h"]: continue if b["box"][3] < a["box"][1] - BLOCK_PITCH_EM * a["h"]: break if hover(a["box"], b["box"]) < BLOCK_XOVERLAP * min(a["box"][2] - a["box"][0], b["box"][2] - b["box"][0]): continue if abs(b["h"] - a["h"]) > 0.6 * max(a["h"], b["h"]): continue cands.append(bi) if cands: top = max(segs[bi]["box"][3] for bi in cands) cands = [bi for bi in cands if segs[bi]["box"][3] >= top - 0.5 * a["h"]] below[ai] = cands for bi in cands: above[bi].append(ai) for bi in range(n): if above[bi]: bot = min(segs[ai]["box"][1] for ai in above[bi]) above[bi] = [ai for ai in above[bi] if segs[ai]["box"][1] <= bot + 0.5 * segs[bi]["h"]] parent = list(range(n)) def find(x): while parent[x] != x: parent[x] = parent[parent[x]] x = parent[x] return x for ai in range(n): if len(below[ai]) == 1 and len(above[below[ai][0]]) == 1 and above[below[ai][0]][0] == ai: parent[find(ai)] = find(below[ai][0]) groups = {} for i in range(n): groups.setdefault(find(i), []).append(segs[i]) out = [] for g in groups.values(): g.sort(key=lambda s: (-s["box"][3], s["box"][0])) out.append(g) return out def xy_order(blocks, h): if len(blocks) <= 1: return blocks items = [(union([s["box"] for s in b]), b) for b in blocks] def cuts(lo, hi, min_gap): iv = sorted((bx[lo], bx[hi]) for bx, _ in items) gaps = [] end = iv[0][1] for a, b in iv[1:]: if a - end > min_gap: gaps.append((end + a) / 2) end = max(end, b) return gaps ycuts = cuts(1, 3, 0.3 * h) xcuts = cuts(0, 2, 1.0 * h) bbox = union([bx for bx, _ in items]) if xcuts: parts = split(items, 0, 2, xcuts) tall = all((union([bx for bx, _ in p])[3] - union([bx for bx, _ in p])[1]) > 0.5 * (bbox[3] - bbox[1]) for p in parts) if tall or not ycuts: return [b for p in parts for b in xy_order([b for _, b in p], h)] if ycuts: parts = split(items, 1, 3, ycuts) parts.reverse() return [b for p in parts for b in xy_order([b for _, b in p], h)] return [b for _, b in sorted(items, key=lambda x: (-x[0][3], x[0][0]))] def split(items, lo, hi, cs): parts = [[] for _ in range(len(cs) + 1)] for it in items: c = (it[0][lo] + it[0][hi]) / 2 k = sum(1 for x in cs if x < c) parts[k].append(it) return [p for p in parts if p] def page_signals(p, body_glyphs, body_words): W, H = p["W"], p["H"] area = 0.0 regions = [] for im in p["images"]: l, b, r, t = max(im[0], 0), max(im[1], 0), min(im[2], W), min(im[3], H) if r > l and t > b: a_ = (r - l) * (t - b) area += a_ if a_ >= IMAGE_REGION_MIN_FRAC * W * H: inside = sum(len(w["text"]) for w in body_words if l <= (w["box"][0] + w["box"][2]) / 2 <= r and b <= (w["box"][1] + w["box"][3]) / 2 <= t) regions.append({"box": rbox((l, b, r, t)), "area_fraction": round(a_ / (W * H), 3), "body_glyphs_inside": inside, "text_covered": inside >= IMAGE_REGION_TEXT_DENSITY * a_}) frac = min(area / (W * H), 1.0) sig = [] if body_glyphs == 0: sig.append("no_body_text") if frac > IMAGE_DOMINANT_FRAC and body_glyphs < LOW_TEXT_GLYPHS: sig.append("image_dominant_low_text") elif frac > IMAGE_DOMINANT_FRAC: sig.append("image_dominant_with_text") uncovered = [r_ for r_ in regions if not r_["text_covered"]] if uncovered: sig.append("image_region_without_text") elif regions: sig.append("image_region_with_text") if p["ume"]: sig.append("unicode_map_errors") if p["glyphs"] > 50 and p["alpha"] / p["glyphs"] < 0.5: sig.append("low_alpha_ratio") nw = p["layout"]["n_words"] if nw >= ODD_TOKEN_MIN_WORDS and p["layout"]["odd_words"] / nw > ODD_TOKEN_FRAC: sig.append("irregular_tokens") if any(s["rotated"] for s in p["layout"]["segments"]): sig.append("rotated_text_present") if p["overprint"]: sig.append("overprint_present") if any(r_["kind"] == "offpage" for r_ in p["layout"]["removed"]): sig.append("offpage_text") if "image_dominant_low_text" in sig or ("no_body_text" in sig and frac > 0.2): risk = "high" elif any(x in sig for x in ("image_dominant_with_text", "image_region_without_text", "low_alpha_ratio", "unicode_map_errors", "irregular_tokens")): risk = "medium" elif regions: risk = "low" else: risk = "none" return sig, risk, round(frac, 3), regions def extract(pdf_path, out_dir, want_chars): os.makedirs(out_dir, exist_ok=False) t = {"start": time.perf_counter()} doc = pdfium.PdfDocument(pdf_path) n = len(doc) t["opened"] = time.perf_counter() pages = [] chars_f = open(os.path.join(out_dir, "chars.jsonl"), "w") if want_chars else None prim_s = 0.0 for pi in range(n): page = doc[pi] W, H = page.get_width(), page.get_height() t0 = time.perf_counter() chars, images = page_primitives(page) prim_s += time.perf_counter() - t0 if chars_f: for c in chars: chars_f.write(json.dumps({"page": pi + 1, "i": c["i"], "c": c["c"], "gen": c["gen"], "box": rbox(c["box"]), "tight": rbox(c["tight"]), "angle": round(c["angle"], 3), "font": c["font"], "ume": c["ume"]}, ensure_ascii=False) + "\n") layout = page_layout(chars, images, W, H) glyphs = [c for c in chars if not c["gen"] and not c["c"].isspace() and ord(c["c"]) >= 32] overprint = 0 segs = layout["segments"] for i, a in enumerate(segs): for b in segs[i + 1:]: if a["rotated"] or b["rotated"]: continue if vover(a["box"], b["box"]) > 0.5 * min(a["h"], b["h"]) and hover(a["box"], b["box"]) > 0.5 * min(a["h"], b["h"]): overprint += 1 pages.append({"page": pi + 1, "W": W, "H": H, "images": images, "layout": layout, "n_chars": len(chars), "glyphs": len(glyphs), "generated": sum(1 for c in chars if c["gen"]), "alpha": sum(1 for c in glyphs if c["c"].isalpha()), "ume": sum(1 for c in glyphs if c["ume"]), "overprint": overprint}) page.close() if chars_f: chars_f.close() t["layout_pass1"] = time.perf_counter() classify_furniture(pages) t["furniture"] = time.perf_counter() text_f = open(os.path.join(out_dir, "text.txt"), "w") pages_f = open(os.path.join(out_dir, "pages.jsonl"), "w") summary = {"pdf": os.path.basename(pdf_path), "pages": n, "chars_total": 0, "glyphs_total": 0, "body_glyphs_total": 0, "removed_total": {}, "ocr_risk": {"high": [], "medium": [], "low": []}, "signals": {}, "pages_with_gutters": 0} for p in pages: segs = p["layout"]["segments"] body = [s for s in segs if s["class"] == "body"] hs = [s["h"] for s in body] or [10.0] blocks = xy_order(link_blocks(body), statistics.median(hs)) body_glyphs = sum(len(w["text"]) for s in body for w in s["words"]) sig, risk, imfrac, regions = page_signals(p, body_glyphs, [w for s in body for w in s["words"]]) removed = list(p["layout"]["removed"]) for s in segs: if s["class"] != "body": removed.append({"kind": s["class"], "text": s["text"], "box": rbox(s["box"]), "chars": [s["words"][0]["i0"], s["words"][-1]["i1"]], "reason": s["reason"]}) bj = [] for bi, b in enumerate(blocks): lines = [{"text": s["text"], "box": rbox(s["box"]), "h": round(s["h"], 2), "words": [{"t": w["text"], "box": rbox(w["box"]), "chars": [w["i0"], w["i1"]]} for w in s["words"]]} for s in b] bj.append({"id": bi, "box": rbox(union([s["box"] for s in b])), "lines": lines}) rec = {"page": p["page"], "width": p["W"], "height": p["H"], "counts": {"pdfium_chars": p["n_chars"], "generated": p["generated"], "glyphs": p["glyphs"], "words": p["layout"]["n_words"], "odd_words": p["layout"]["odd_words"], "segments": len(segs), "body_segments": len(body), "body_glyphs": body_glyphs, "blocks": len(blocks), "removed": len(removed)}, "image_area_fraction": imfrac, "image_regions": regions, "signals": sig, "ocr_risk": risk, "gutters": p["layout"]["gutters"], "gutter_candidates_rejected": p["layout"]["gutter_candidates_rejected"], "column_separators": p["layout"]["separators"], "blocks": bj, "removed": removed} pages_f.write(json.dumps(rec, ensure_ascii=False) + "\n") text_f.write(f"\n===== page {p['page']} | body_glyphs={body_glyphs} blocks={len(blocks)} removed={len(removed)} ocr_risk={risk}" + (f" signals={','.join(sig)}" if sig else "") + " =====\n") if body_glyphs == 0: text_f.write("[no body text in native layer on this page]\n") for b in blocks: text_f.write("\n".join(s["text"] for s in b) + "\n\n") summary["chars_total"] += p["n_chars"]; summary["glyphs_total"] += p["glyphs"]; summary["body_glyphs_total"] += body_glyphs for r_ in removed: summary["removed_total"][r_["kind"]] = summary["removed_total"].get(r_["kind"], 0) + 1 for s_ in sig: summary["signals"][s_] = summary["signals"].get(s_, 0) + 1 if risk != "none": summary["ocr_risk"][risk].append(p["page"]) summary["pages_with_gutters"] += bool(p["layout"]["gutters"]) text_f.close(); pages_f.close() t["written"] = time.perf_counter() timing = {"open_s": t["opened"] - t["start"], "primitives_s": prim_s, "layout_pass1_s": t["layout_pass1"] - t["opened"] - prim_s, "furniture_s": t["furniture"] - t["layout_pass1"], "blocks_order_write_s": t["written"] - t["furniture"], "total_s": t["written"] - t["start"], "chars_sidecar": bool(want_chars), "pypdfium2": pdfium.PYPDFIUM_INFO.version, "pdfium": pdfium.PDFIUM_INFO.build} summary["timing"] = timing summary["source_sha256"] = hashlib.sha256(open(os.path.abspath(__file__), "rb").read()).hexdigest() summary["pdf_sha256"] = hashlib.sha256(open(pdf_path, "rb").read()).hexdigest() summary["coordinates"] = "PDF user space points, origin bottom-left; box = [left, bottom, right, top]; word/line boxes are unions of PDFium loose char boxes" summary["tuning_note"] = "thresholds developed on one legal appendix (EcoFactor v. Google, CAFC 23-1101 doc 15); no accuracy claim, no universality claim" json.dump(summary, open(os.path.join(out_dir, "summary.json"), "w"), indent=1) return summary def load_pages(out_dir): return [json.loads(l) for l in open(os.path.join(out_dir, "pages.jsonl"))] def block_variants(block): lines = block["lines"] hyph = [li for li in range(len(lines) - 1) if lines[li]["words"] and lines[li]["words"][-1]["t"][-1:] in HYPHENS and len(lines[li]["words"][-1]["t"]) > 1 and lines[li]["words"][-1]["t"][-2].islower() and lines[li + 1]["words"] and lines[li + 1]["words"][0]["t"][0].islower()] out = [] for dehyph in ([False, True] if hyph else [False]): chars = []; src = [] for li, line in enumerate(lines): for wi, w in enumerate(line["words"]): tx = w["t"] joined = dehyph and li in hyph and wi == len(line["words"]) - 1 if joined: tx = tx[:-1] for ch in tx: for k in norm_text(ch) or ch: chars.append(k); src.append((li, wi)) if not joined: chars.append(" "); src.append(None) out.append(("dehyphenated" if dehyph else "raw", "".join(chars), src)) return out def search(out_dir, query, regex, include_removed): pages = load_pages(out_dir) q = query if regex else re.escape(norm_text(query)).replace(r"\ ", r"\s+") pat = re.compile(q, re.I) hits = [] for p in pages: for b in p["blocks"]: seen = set() for vname, text, src in block_variants(b): for m in pat.finditer(text): ids = sorted({s for s in src[m.start():m.end()] if s is not None}) if not ids or (ids[0], ids[-1]) in seen: continue seen.add((ids[0], ids[-1])) per_line = {} for li, wi in ids: per_line.setdefault(li, []).append(b["lines"][li]["words"][wi]["box"]) hits.append({"page": p["page"], "block": b["id"], "variant": vname, "match": text[m.start():m.end()], "lines": [li + 1 for li in per_line], "boxes": [rbox(union(v)) for v in per_line.values()], "context": " ".join(l["text"] for l in b["lines"][max(0, ids[0][0] - 1):ids[-1][0] + 2]), "page_ocr_risk": p["ocr_risk"], "page_signals": p["signals"]}) if include_removed: for r_ in p["removed"]: if pat.search(norm_text(r_["text"])): hits.append({"page": p["page"], "removed_kind": r_["kind"], "match": r_["text"], "boxes": [r_["box"]], "reason": r_["reason"]}) return hits def overlay(pdf_path, out_dir, pno, out_png, scale=2.0): from PIL import ImageDraw p = {x["page"]: x for x in load_pages(out_dir)}[pno] doc = pdfium.PdfDocument(pdf_path) page = doc[pno - 1] img = page.render(scale=scale).to_pil().convert("RGB") H = p["height"] d = ImageDraw.Draw(img) def rect(b, col, w=2): d.rectangle([b[0] * scale, (H - b[3]) * scale, b[2] * scale, (H - b[1]) * scale], outline=col, width=w) colors = {"line_number_gutter": (220, 0, 0), "running_header": (255, 140, 0), "running_footer": (255, 140, 0), "stamp_outside_margin": (255, 140, 0), "page_number": (160, 0, 160), "rotated": (0, 160, 160), "offpage": (0, 0, 0)} for sep in p["column_separators"]: d.line([sep["x"] * scale, (H - sep["y_span"][1]) * scale, sep["x"] * scale, (H - sep["y_span"][0]) * scale], fill=(200, 0, 200), width=2) for bi, b in enumerate(p["blocks"]): for l in b["lines"]: rect(l["box"], (0, 90, 220), 1) rect(b["box"], (0, 160, 0), 3) d.text((b["box"][0] * scale - 16, (H - b["box"][3]) * scale), str(bi), fill=(0, 120, 0)) for r_ in p["removed"]: rect(r_["box"], colors[r_["kind"]], 3) img.save(out_png) return out_png def main(): ap = argparse.ArgumentParser(prog="pdfgeo", description="geometry-aware PDF text reconstruction and phrase search on PDFium character boxes") sub = ap.add_subparsers(dest="cmd", required=True) e = sub.add_parser("extract"); e.add_argument("pdf"); e.add_argument("out_dir"); e.add_argument("--chars", action="store_true", help="also write chars.jsonl (raw PDFium character provenance)") s = sub.add_parser("search"); s.add_argument("out_dir"); s.add_argument("query"); s.add_argument("--regex", action="store_true"); s.add_argument("--include-removed", action="store_true"); s.add_argument("--json", action="store_true") o = sub.add_parser("overlay"); o.add_argument("pdf"); o.add_argument("out_dir"); o.add_argument("page", type=int); o.add_argument("out_png"); o.add_argument("--scale", type=float, default=2.0) a = ap.parse_args() if a.cmd == "extract": summ = extract(a.pdf, a.out_dir, a.chars) print(json.dumps({k: summ[k] for k in ("pages", "chars_total", "glyphs_total", "body_glyphs_total", "removed_total", "signals", "timing")}, indent=1)) elif a.cmd == "search": hits = search(a.out_dir, a.query, a.regex, a.include_removed) if a.json: print(json.dumps(hits, ensure_ascii=False, indent=1)) else: for h in hits: if "removed_kind" in h: print(f"p{h['page']} [removed:{h['removed_kind']}] {h['match']!r} box={h['boxes'][0]}") else: flag = f" [ocr_risk={h['page_ocr_risk']}]" if h["page_ocr_risk"] != "none" else "" print(f"p{h['page']} b{h['block']} lines {h['lines']} {h['variant']}{flag} boxes={h['boxes']}\n {h['context']}") print(f"{len(hits)} hit(s)", file=sys.stderr) elif a.cmd == "overlay": print(overlay(a.pdf, a.out_dir, a.page, a.out_png, a.scale)) if __name__ == "__main__": main()