#!/usr/bin/env python3
"""
Workshop Document: apply redactions to a PDF and write a template with the private data removed.

Usage: redact.py <in.pdf> <redactions.json> <out.pdf>

redactions.json:
  {"redactions": [
    {"page": 1, "mode": "replace", "rects": [[x0, y0, x1, y1], ...], "replacement": "Jane Doe"},
    {"page": 2, "mode": "remove",  "rects": [[x0, y0, x1, y1]]}
  ]}

Rects are in display coordinates (see wdlib.py). A replace redaction lists one rect per word, so
a value that wraps onto a second line doesn't sweep up the words between.

- replace: the text under the rects is removed from the content stream and the replacement is
  drawn on the same baseline, in the closest base-14 font, at the same size and color. No fill.
- remove: text, image pixels and vector art under the rect are removed and the area filled black.

Afterwards annotations, links, form fields, attachments, JavaScript, metadata and hidden text are
stripped, and the file is saved with garbage collection so no orphaned objects survive. Finally
the output is searched (text layer, OCR of flat pages, and the raw objects) for every original
value; if any is still there the output is deleted and the run fails.

Prints {"status": "ok", "pages": N} or {"status": "error", "message": "..", "fields": [i, ..]}
where fields are indexes into the redactions list. A failed final check adds
"leftovers": [{"field": i, "page": n, "bbox": [..]}, ..], the boxes (display coordinates) of the
words where each value still appears. Original values are never printed.
"""

import json
import os
import re
import sys

import pymupdf

import wdlib

# Shortest original value the final check looks for. Shorter ones ("1", "of") would match
# unrelated text all over the page.
CHECK_MIN_CHARS = 3
RAW_CHECK_MIN_CHARS = 5


def norm(s):
    return re.sub(r"\s+", " ", s).strip().lower()


def is_noise(word):
    """A word with no letter or digit, such as OCR of a dotted leader line ("......", "-—-"). It
    holds no data, and a value made of them matches every leader on every scanned page."""
    return not re.search(r"[^\W_]", word)


def check_value(value):
    """The original value as the final check searches for it: its noise words dropped."""
    return " ".join(w for w in norm(value).split(" ") if not is_noise(w))


def center_in(bbox, rects):
    p = pymupdf.Point((bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2)
    return any(r.contains(p) for r in rects)


def grow(rect, by):
    return pymupdf.Rect(rect.x0 - by, rect.y0 - by, rect.x1 + by, rect.y1 + by)


def words_in(page, textpage, rects):
    return [w[4] for w in page.get_text("words", textpage=textpage, sort=True) if center_in(w[:4], rects)]


def base14(font, flags):
    """Closest base-14 font. The original is usually an embedded subset that lacks the glyphs
    the replacement needs, so it can't be reused."""
    name = font.lower()
    bold = bool(flags & pymupdf.TEXT_FONT_BOLD) or "bold" in name or "black" in name or "heavy" in name
    italic = bool(flags & pymupdf.TEXT_FONT_ITALIC) or "italic" in name or "oblique" in name
    if flags & pymupdf.TEXT_FONT_MONOSPACED or "courier" in name or "mono" in name:
        family = "co"
    elif flags & pymupdf.TEXT_FONT_SERIFED or "times" in name or "serif" in name and "sans" not in name:
        family = "ti"
    else:
        family = "he"
    styles = {
        "he": ["helv", "heit", "hebo", "hebi"],
        "ti": ["tiro", "tiit", "tibo", "tibi"],
        "co": ["cour", "coit", "cobo", "cobi"],
    }
    return styles[family][(2 if bold else 0) + (1 if italic else 0)]


def text_angle(direction):
    """insert_text() rotation for a line direction; only right angles are supported."""
    dx, dy = direction
    if abs(dx) >= abs(dy):
        return 0 if dx >= 0 else 180
    return 90 if dy < 0 else 270


def line_band(chars, size, origin, angle):
    """A thin band through the middle of the glyphs on one line. MuPDF removes every glyph whose
    box touches a redaction rect, so the full word box would also clip the lines above and below."""
    x0 = min(c["bbox"][0] for c in chars)
    y0 = min(c["bbox"][1] for c in chars)
    x1 = max(c["bbox"][2] for c in chars)
    y1 = max(c["bbox"][3] for c in chars)
    bx, by = origin
    inset = min(0.2, size * 0.02)
    if angle == 0:
        return pymupdf.Rect(x0 + inset, by - 0.6 * size, x1 - inset, by - 0.15 * size)
    if angle == 180:
        return pymupdf.Rect(x0 + inset, by + 0.15 * size, x1 - inset, by + 0.6 * size)
    if angle == 90:
        return pymupdf.Rect(bx - 0.6 * size, y0 + inset, bx - 0.15 * size, y1 - inset)
    return pymupdf.Rect(bx + 0.15 * size, y0 + inset, bx + 0.6 * size, y1 - inset)


def plan_replace(page, rawdict, rects):
    """Group the characters under the rects by line, with the style each line is drawn in."""
    lines = []
    for block in rawdict["blocks"]:
        for line in block.get("lines", []):
            chars = []
            style = None
            for span in line["spans"]:
                picked = [c for c in span["chars"] if c["c"].strip() and center_in(c["bbox"], rects)]
                if picked:
                    chars.extend(picked)
                    style = style or span
            if not chars:
                continue
            angle = text_angle(line["dir"])
            origin = chars[0]["origin"]
            lines.append({
                "band": line_band(chars, style["size"], origin, angle),
                "origin": origin,
                "angle": angle,
                "length": (max(c["bbox"][2] for c in chars) - min(c["bbox"][0] for c in chars)) if angle in (0, 180)
                          else (max(c["bbox"][3] for c in chars) - min(c["bbox"][1] for c in chars)),
                "size": style["size"],
                "font": base14(style["font"], style["flags"]),
                "color": pymupdf.sRGB_to_pdf(style["color"]),
            })
    return lines


def split_over_lines(text, lines):
    """Share the replacement's words out over the original's lines, filling each but the last."""
    if len(lines) == 1:
        return [text]
    words = text.split()
    parts = []
    for i, line in enumerate(lines):
        if i == len(lines) - 1:
            parts.append(" ".join(words))
            break
        taken = []
        while words:
            trial = " ".join(taken + [words[0]])
            if taken and pymupdf.get_text_length(trial, line["font"], line["size"]) > line["length"]:
                break
            taken.append(words.pop(0))
        parts.append(" ".join(taken))
    return parts


def draw_replacement(page, line, text):
    if not text:
        return
    size = line["size"]
    width = pymupdf.get_text_length(text, line["font"], size)
    if line["length"] > 0 and width > line["length"] * 1.02:
        size = max(size * line["length"] / width, size * 0.5)
    page.insert_text(line["origin"], text, fontsize=size, fontname=line["font"],
                     color=line["color"], rotate=line["angle"])


def strip(doc):
    for page in doc:
        for widget in list(page.widgets()):
            page.delete_widget(widget)
        for annot in list(page.annots()):
            page.delete_annot(annot)
        for link in page.get_links():
            page.delete_link(link)
    doc.scrub(
        attached_files=True,
        clean_pages=True,
        embedded_files=True,
        hidden_text=True,
        javascript=True,
        metadata=True,
        redactions=False,
        remove_links=True,
        reset_fields=True,
        reset_responses=True,
        thumbnails=True,
        xml_metadata=True,
    )
    for name in doc.embfile_names():
        doc.embfile_del(name)
    doc.set_toc([])
    doc.set_metadata({})
    doc.del_xml_metadata()


def raw_objects(doc):
    """Every object and decompressed stream except images, as bytes."""
    for xref in range(1, doc.xref_length()):
        try:
            obj = doc.xref_object(xref, compressed=False)
        except Exception:
            continue
        yield obj.encode("latin-1", errors="ignore")
        if doc.xref_is_stream(xref) and "/Image" not in obj:
            try:
                yield doc.xref_stream(xref) or b""
            except Exception:
                continue


def page_words(page, kind):
    """The output page's words in display coordinates: its text layer, plus OCR on flat pages."""
    found = wdlib.words(page, page.get_textpage())
    if kind == wdlib.FLAT:
        found += wdlib.words(page, wdlib.ocr_textpage_forced(page))
    return found


def find_in_words(pattern, words):
    """Boxes of the words a match covers. The words are searched joined by single spaces, the
    same normalisation as norm(), so a value split over words or lines is still found."""
    text, starts = "", []
    for w in words:
        starts.append(len(text))
        text += norm(w["text"]) + " "
    hits = []
    for m in pattern.finditer(text):
        hits.append([w["bbox"] for w, at in zip(words, starts) if at < m.end() and at + len(norm(w["text"])) > m.start()])
    return hits


def leftovers(path, kinds, originals):
    """For each redaction whose original value still appears in the output, where: a list of
    {"page": n, "bbox": [..]} (one per word), or {"page": None} when it's only in the raw objects."""
    doc = pymupdf.open(path)
    pages = [[w for w in page_words(page, kinds[page.number]) if not is_noise(w["text"])] for page in doc]
    raw = list(raw_objects(doc))
    doc.close()

    found = {}
    for i, value in originals.items():
        needle = check_value(value)
        if len(re.sub(r"[\W_]", "", needle)) < CHECK_MIN_CHARS:
            continue
        pattern = re.compile(r"(?<!\w)" + re.escape(needle) + r"(?!\w)")
        hits = [{"page": n + 1, "bbox": bbox} for n, words in enumerate(pages)
                for match in find_in_words(pattern, words) for bbox in match]

        if not hits and len(value.strip()) >= RAW_CHECK_MIN_CHARS:
            forms = [value.strip().encode("latin-1", errors="ignore"), value.strip().encode("utf-16-be")]
            if any(f and f in chunk for f in forms for chunk in raw):
                hits = [{"page": None}]
        if hits:
            found[i] = hits
    return found


def main():
    if len(sys.argv) != 4:
        raise wdlib.UserError("usage: redact.py <in.pdf> <redactions.json> <out.pdf>")
    src, spec_path, out = sys.argv[1:]

    with open(spec_path) as f:
        redactions = json.load(f)["redactions"]

    doc = wdlib.open_pdf(src)
    kinds = [wdlib.page_kind(page) for page in doc]

    by_page = {}
    for i, r in enumerate(redactions):
        n = int(r["page"]) - 1
        if not 0 <= n < doc.page_count:
            raise wdlib.UserError(f"Redaction {i + 1} is on a page the PDF doesn't have.")
        by_page.setdefault(n, []).append((i, r))

    # Plan every page from its untouched state, then change it.
    originals = {}
    replacements = []
    plans = {}
    bad = []
    for n, items in by_page.items():
        page = doc[n]
        textpage = wdlib.textpage_for(page, kinds[n])
        rawdict = page.get_text("rawdict", textpage=textpage) if kinds[n] == wdlib.DIGITAL else None
        removes, replaces = [], []
        for i, r in items:
            rects = [wdlib.to_page(page, rect) for rect in r["rects"]]
            if r["mode"] == "replace":
                lines = plan_replace(page, rawdict, [grow(rect, 0.5) for rect in rects]) if rawdict else []
                if not lines:
                    bad.append(i)
                    continue
                originals[i] = " ".join(words_in(page, textpage, [grow(rect, 0.5) for rect in rects]))
                replaces.append((lines, r["replacement"]))
                replacements.append(norm(r["replacement"]))
            else:
                originals[i] = " ".join(words_in(page, textpage, rects))
                removes.extend(rects)
        plans[n] = (removes, replaces)

    if bad:
        raise_fields("No text was found under some replace fields. Use a rectangle for those instead.", bad)

    for n, (removes, replaces) in plans.items():
        page = doc[n]
        if removes:
            for rect in removes:
                page.add_redact_annot(rect, fill=(0, 0, 0))
            page.apply_redactions(images=pymupdf.PDF_REDACT_IMAGE_PIXELS,
                                  graphics=pymupdf.PDF_REDACT_LINE_ART_REMOVE_IF_COVERED,
                                  text=pymupdf.PDF_REDACT_TEXT_REMOVE)
        if replaces:
            for lines, _ in replaces:
                for line in lines:
                    page.add_redact_annot(line["band"], fill=False)
            page.apply_redactions(images=pymupdf.PDF_REDACT_IMAGE_NONE,
                                  graphics=pymupdf.PDF_REDACT_LINE_ART_NONE,
                                  text=pymupdf.PDF_REDACT_TEXT_REMOVE)
            for lines, text in replaces:
                for line, part in zip(lines, split_over_lines(text, lines)):
                    draw_replacement(page, line, part)

    strip(doc)
    doc.save(out, garbage=4, deflate=True, clean=True)
    doc.close()

    # A value the user typed as some replacement can't be told apart from a leftover.
    checked = {i: v for i, v in originals.items() if not any(norm(v) in rep for rep in replacements)}
    found = leftovers(out, kinds, checked)
    if found:
        os.remove(out)
        pages = sorted({h["page"] for hits in found.values() for h in hits if h["page"]})
        where = f" on page {', '.join(map(str, pages))} (outlined in red)" if pages else " in the file's data"
        wdlib.fail("Some of the removed text still appears" + where +
                   ". Redact every place it appears, then save again.",
                   fields=sorted(found),
                   leftovers=[{"field": i, **h} for i, hits in sorted(found.items()) for h in hits if h["page"]])

    print(json.dumps({"status": "ok", "pages": len(kinds)}))


def raise_fields(message, fields):
    wdlib.fail(message, fields=fields)


if __name__ == "__main__":
    wdlib.run(main)
