#!/usr/bin/env python3
"""Build a Ghidra project for the Kenwood TH-D74 firmware and name what we know.

The TH-D74 runs an OMAP-L138 (ARM926EJ-S, ARMv5TE) and its FIRMWARE section is
linked at 0xC0000000, the SoC's DDR window. Ghidra cannot infer any of that from
a raw image, and the image carries no symbols, so this script does three things:

  1. imports firmware.bin with the right language at the right base address,
  2. marks up the anchors documented at <https://kk4vcz.com/posts/th-d74-firmware/>
     (exception vectors, the localized string tables, the two CAT command tables)
     so that auto-analysis has somewhere to start,
  3. after analysis, names the statically linked libpng 1.6.8 functions by the
     error messages only they can print.

Nothing here is a symbol table recovered from the vendor. Every name is an
inference from evidence in the image, and the plate comment on each named
function records which evidence.

Usage:
    python3 build_project.py --firmware /path/to/firmware.bin \\
        --project-dir ~/clones/tools/thd74/ghidra --png-map png_msgmap.json

Re-running against an existing project reuses the imported program and redoes
the markup, so it is safe to iterate.
"""

import argparse
import json
import re
import string
import struct
import sys
from pathlib import Path

BASE = 0xC0000000              # DDR window the FIRMWARE section is linked for
RAM_TOP = 0xC4000000           # 64 MB of DDR, generous but harmless
PERIPH_BASE = 0x01C00000       # OMAP-L138 system/peripheral region (SYSCFG at 0x01C14000)
PERIPH_SIZE = 0x00400000

# Exception vectors: constant-pool slot -> handler name. The first eight words of
# the image are "ldr pc, [pc, #imm]" and the pool they read sits at +0x24.
VECTORS = [
    (0x24, "reset_handler", "Reset: vector 0xC0000000 loads PC from this slot"),
    (0x28, "undefined_instruction_handler", "Undefined instruction vector"),
    (0x2C, "svc_handler", "Supervisor call (SWI) vector"),
    (0x30, "prefetch_abort_handler", "Prefetch abort vector"),
    (0x34, "data_abort_handler", "Data abort vector"),
    (0x38, "irq_handler", "IRQ vector"),
    (0x3C, "fiq_handler", "FIQ vector"),
]

# Anchors from the kk4vcz write-up, all verified against this image before use.
LOCALIZE_STRING = 0xC0000238
STRING_TABLE = 0xC000026C
STRING_WIDTH = 0x14
STRING_STRIDE = 0x28
CAT_TABLES = [(0xC002A828, "cat"), (0xC0064EDC, "cat_svc")]

# Helpers named by what they do, since the image has no symbols for them.
HELPERS = [
    (0xC000FD58, "is_secondary_language",
     "Returns 1 when the second (Japanese) column of a localized string table\n"
     "should be used, 0 for the first (English) column."),
    (0xC00105DE, "copy_block",
     "Block copy: 16 bytes at a time through 0xC001ACE8, then the remainder.\n"
     "Named by behaviour; this is almost certainly memcpy/memmove."),
]

# Version strings of the statically linked third-party code. Both libraries are
# from late 2013, which dates the vendor's tree.
VERSION_STRINGS = [
    (b"1.6.8", "libpng_version_string"),
    (b"1.2.8", "zlib_version_string"),
]

NAME_CHARS = set((string.ascii_uppercase + string.digits).encode())
PNG_IN_MSG = re.compile(r"(png_[a-z0-9_]+)\(\)")
SLUG_BAD = re.compile(r"[^A-Za-z0-9]+")


def log(msg):
    print(f"[thd74] {msg}", flush=True)


def gaddr(flat, value):
    """toAddr() via a string: 0xC0000000 does not fit a Java int."""
    return flat.toAddr(f"0x{value:08x}")


# --------------------------------------------------------------------------- #
# Everything below the import runs inside the JVM pyghidra starts for us.
# --------------------------------------------------------------------------- #

def build(args):
    import pyghidra

    pyghidra.start(install_dir=args.ghidra)

    from ghidra.app.cmd.disassemble import ArmDisassembleCommand
    from ghidra.program.model.data import (ArrayDataType, CharDataType, DataUtilities,
                                           StructureDataType, PointerDataType)
    from ghidra.program.model.symbol import SourceType
    from ghidra.program.flatapi import FlatProgramAPI
    from ghidra.util.task import ConsoleTaskMonitor, TaskMonitor

    monitor = ConsoleTaskMonitor()
    image = Path(args.firmware).read_bytes()
    png_map = json.loads(Path(args.png_map).read_text()) if args.png_map else {}

    project_dir = Path(args.project_dir).expanduser()
    project_dir.mkdir(parents=True, exist_ok=True)
    project = pyghidra.open_project(project_dir, args.project_name, create=True)

    consumer = None
    try:
        program, consumer = _load(pyghidra, project, args, monitor)
        flat = FlatProgramAPI(program)
        ctx = {
            "flat": flat, "program": program, "monitor": monitor, "image": image,
            "ArmDisassembleCommand": ArmDisassembleCommand, "SourceType": SourceType,
            "ArrayDataType": ArrayDataType, "CharDataType": CharDataType,
            "StructureDataType": StructureDataType, "PointerDataType": PointerDataType,
            "DataUtilities": DataUtilities,
        }

        with pyghidra.transaction(program):
            program.setName(args.program_name)
            _memory_blocks(ctx)
            _vectors(ctx)
            _localized_strings(ctx)
            _cat_tables(ctx)
            _helpers(ctx)
            _versions(ctx)

        if args.analyze:
            log("running auto-analysis (several minutes on 2.5 MB of ARM code)")
            pyghidra.analyze(program, monitor)
            log("analysis finished")

        with pyghidra.transaction(program):
            named = _libpng(ctx, png_map)
            _source_file_hints(ctx)
        log(f"libpng functions named: {named}")

        # analysis leaves the console monitor in a cancelled state, and save()
        # refuses to run against a cancelled monitor.
        monitor.clearCanceled()
        program.save("thd74 markup", TaskMonitor.DUMMY)
        log(f"saved {args.project_name} in {project_dir}")
    finally:
        if consumer is not None:
            program.release(consumer)
        project.close()


def _load(pyghidra, project, args, monitor):
    """Import firmware.bin as raw ARM at BASE, or reuse it if already imported."""
    from ghidra.app.util.importer import ProgramLoader
    from ghidra.app.util.opinion import BinaryLoader

    name = args.program_name
    existing = project.getProjectData().getRootFolder().getFile(name)
    if existing is not None:
        log(f"reusing already-imported {name}")
        return pyghidra.consume_program(project, f"/{name}")

    log(f"importing {args.firmware} as {args.language} at 0x{BASE:08X}")
    builder = (ProgramLoader.builder()
               .source(str(args.firmware))
               .project(project)
               .projectFolderPath("/")
               .name(name)
               .language(args.language)
               .loaders(BinaryLoader.class_)
               .addLoaderArg("-loader-baseAddr", f"0x{BASE:X}")
               .monitor(monitor))
    results = builder.load()
    results.save(monitor)
    return results.getPrimaryDomainObject(), None


# --------------------------------------------------------------------------- #
# markup passes
# --------------------------------------------------------------------------- #

def _memory_blocks(ctx):
    """Name the loaded block and give the rest of DDR somewhere to point."""
    program, flat = ctx["program"], ctx["flat"]
    mem = program.getMemory()
    block = mem.getBlock(gaddr(flat, BASE))
    block.setName("FIRMWARE")
    block.setRead(True)
    block.setWrite(False)
    block.setExecute(True)
    block.setComment("FIRMWARE section of the TH-D74 flash image, flash offset "
                     "0x00200000, linked for the OMAP-L138 DDR window at 0xC0000000.")
    end = BASE + len(ctx["image"])
    log(f"FIRMWARE block 0x{BASE:08X}-0x{end - 1:08X}")

    for name, start, size, comment in (
        ("RAM", end, RAM_TOP - end,
         "Rest of the DDR window: .bss, heap and stacks live here. Size is an "
         "assumption, not a datasheet fact."),
        ("PERIPH", PERIPH_BASE, PERIPH_SIZE,
         "OMAP-L138 system/peripheral registers (SYSCFG REVID at 0x01C14000, "
         "TI SPRS586J table 4-1)."),
    ):
        if mem.getBlock(gaddr(flat, start)) is not None:
            continue
        blk = mem.createUninitializedBlock(name, gaddr(flat, start), size, False)
        blk.setRead(True)
        blk.setWrite(True)
        blk.setExecute(name == "RAM")
        blk.setComment(comment)
        log(f"{name} block 0x{start:08X}-0x{start + size - 1:08X}")
    flat.createLabel(gaddr(flat, 0x01C14000), "SYSCFG_REVID", True)


def _vectors(ctx):
    """Disassemble the ARM vector table and make functions of its handlers."""
    flat, image = ctx["flat"], ctx["image"]
    arm(ctx, BASE, 0x20)
    flat.setPlateComment(gaddr(flat, BASE),
                         "ARM exception vector table. Each entry is "
                         "'ldr pc, [pc, #imm]' reading the pool at 0xC0000024.")
    flat.addEntryPoint(gaddr(flat, BASE))

    for slot, name, comment in VECTORS:
        slot_addr = gaddr(flat, BASE + slot)
        target = struct.unpack_from("<I", image, slot)[0]
        flat.clearListing(slot_addr, slot_addr.add(3))
        flat.createData(slot_addr, ctx["PointerDataType"]())
        flat.createLabel(slot_addr, f"vector_{name}", True)
        flat.setEOLComment(slot_addr, comment)
        func(ctx, target, name, comment, thumb=bool(target & 1))
    flat.addEntryPoint(gaddr(flat, struct.unpack_from("<I", image, 0x24)[0]))
    log(f"vector table: {len(VECTORS)} handlers")


def _localized_strings(ctx):
    """Type the localized string tables and label every entry with its text."""
    flat, image = ctx["flat"], ctx["image"]
    tables = _find_string_tables(image)

    for n, (fn_addr, tbl_addr, count) in enumerate(tables):
        primary = fn_addr == LOCALIZE_STRING
        fname = "localize_string" if primary else f"localize_string_{n}"
        func(ctx, fn_addr | 1, fname,
             f"Copies {STRING_WIDTH:#x} bytes of entry r1 from the table at "
             f"0x{tbl_addr:08X} into r0. Entries are {STRING_STRIDE:#x} bytes: "
             f"column 0 English, column 1 (+{STRING_WIDTH:#x}) Japanese.",
             thumb=True)
        if count == 0:
            continue
        label = "localized_strings" if primary else f"localized_strings_{n}"
        _type_string_table(ctx, tbl_addr, count, label)
        log(f"{fname}: {count} entries at 0x{tbl_addr:08X}")


def _find_string_tables(image):
    """Locate localize_string and its clones: prologue, then the adr r1 it uses.

    Each helper is 'push {r3,r4,r5,lr}; movs r4,r0; movs r5,r1' followed within a
    few instructions by 'adr r1, <table>'. The table runs until the next helper.
    """
    prologue = re.compile(rb"\x38\xb5\x04\x00\x0d\x00")
    stride_calc = b"\xab\x00\x5b\x19\xdb\x00"   # r3 = r5 * 40
    found = []
    for m in prologue.finditer(image):
        off = m.start()
        window = image[off:off + 0x40]
        if stride_calc not in window:
            continue
        tbl = None
        for o in range(off, off + 0x40, 2):
            if image[o + 1] == 0xA1:             # adr r1, pc+imm
                tbl = ((o + 4) & ~3) + image[o] * 4
                break
        if tbl is None:
            continue
        found.append((off, tbl))

    tables = []
    for i, (off, tbl) in enumerate(found):
        limit = found[i + 1][0] if i + 1 < len(found) else len(image)
        count = 0
        while tbl + (count + 1) * STRING_STRIDE <= limit:
            entry = tbl + count * STRING_STRIDE
            if not _plausible_entry(image, entry):
                break
            count += 1
        tables.append((BASE + off, BASE + tbl, count))
    return tables


def _plausible_entry(image, off):
    for half in (off, off + STRING_WIDTH):
        chunk = image[half:half + STRING_WIDTH]
        if not all(c == 0 or 0x20 <= c <= 0xFF for c in chunk):
            return False
    return True


def _type_string_table(ctx, addr, count, label):
    flat, image = ctx["flat"], ctx["image"]
    char = ctx["CharDataType"]()
    entry = ctx["StructureDataType"](f"localized_string", 0)
    entry.add(ctx["ArrayDataType"](char, STRING_WIDTH, 1), "en", "first column")
    entry.add(ctx["ArrayDataType"](char, STRING_WIDTH, 1), "ja", "second column")
    array = ctx["ArrayDataType"](entry, count, STRING_STRIDE)
    start = gaddr(flat, addr)
    flat.clearListing(start, start.add(count * STRING_STRIDE - 1))
    flat.createData(start, array)
    flat.createLabel(start, label, True)

    off = addr - BASE
    for i in range(count):
        text = image[off + i * STRING_STRIDE:off + i * STRING_STRIDE + STRING_WIDTH]
        text = text.split(b"\x00")[0].decode("latin-1").strip()
        slug = SLUG_BAD.sub("_", text).strip("_")[:24]
        ea = gaddr(flat, addr + i * STRING_STRIDE)
        flat.createLabel(ea, f"{label}_{i:03d}" + (f"_{slug}" if slug else ""), False)
        if text:
            flat.setEOLComment(ea, text)


def _cat_tables(ctx):
    """Name every CAT command handler after the two-letter command it serves."""
    flat, image = ctx["flat"], ctx["image"]
    seen = {}
    for table, prefix in CAT_TABLES:
        entries = _cat_entries(image, table - BASE)
        label = "cat_command_table" if prefix == "cat" else "cat_service_command_table"
        flat.createLabel(gaddr(flat, table), label, True)
        flat.setPlateComment(gaddr(flat, table),
                             f"CAT command dispatch table: {len(entries)} entries of "
                             "{ handler pointer, 4-byte command name }.")
        for off, handler, name in entries:
            ea = gaddr(flat, BASE + off)
            flat.clearListing(ea, ea.add(7))
            flat.createData(ea, ctx["PointerDataType"]())
            flat.createLabel(ea, f"{label}_{name}", False)
            target = handler & ~1
            if target in seen:
                flat.createLabel(gaddr(flat, target), f"{prefix}_{name}", False)
                continue
            seen[target] = name
            func(ctx, handler, f"{prefix}_{name}",
                 f"Handler for CAT command '{name}' ({label} entry at "
                 f"0x{BASE + off:08X}).", thumb=bool(handler & 1))
        log(f"{label}: {len(entries)} handlers at 0x{table:08X}")


def _cat_entries(image, start):
    def ok(off):
        if off < 0 or off + 8 > len(image):
            return False
        handler = struct.unpack_from("<I", image, off)[0]
        if not BASE <= (handler & ~1) < BASE + len(image):
            return False
        raw = image[off + 4:off + 8]
        name = raw.split(b"\x00")[0]
        return (1 <= len(name) <= 4 and all(c in NAME_CHARS for c in name)
                and raw[len(name):].strip(b"\x00") == b"")

    lo = start
    while ok(lo - 8):
        lo -= 8
    out = []
    off = lo
    while ok(off):
        handler = struct.unpack_from("<I", image, off)[0]
        name = image[off + 4:off + 8].split(b"\x00")[0].decode()
        out.append((off, handler, name))
        off += 8
    return out


def _helpers(ctx):
    for addr, name, comment in HELPERS:
        func(ctx, addr | 1, name, comment, thumb=True)
    log(f"helpers named: {len(HELPERS)}")


def _versions(ctx):
    """Label the libpng and zlib version strings, which date the vendor tree."""
    flat, image = ctx["flat"], ctx["image"]
    for needle, name in VERSION_STRINGS:
        off, n = image.find(needle + b"\x00"), 0
        while off >= 0:
            addr = gaddr(flat, BASE + off)
            flat.createLabel(addr, name if n == 0 else f"{name}_{n}", False)
            flat.setEOLComment(addr, needle.decode())
            off, n = image.find(needle + b"\x00", off + 1), n + 1
        log(f"{name}: {n} copies")


def _libpng(ctx, png_map):
    """Name statically linked libpng functions by their unique error messages."""
    flat, image = ctx["flat"], ctx["image"]
    named, conflicts = {}, []

    for message, fname in sorted(png_map.items()):
        needle = message.encode("latin-1", "ignore") + b"\x00"
        if image.count(needle) != 1:
            continue
        addr = gaddr(flat, BASE + image.find(needle))
        m = PNG_IN_MSG.search(message)
        if fname == "PNG_FUNCTION" or not fname.startswith("png"):
            if not m:
                continue
            fname = m.group(1)
        try:
            flat.createAsciiString(addr)
        except Exception:
            pass
        for function in _referring_functions(ctx, addr):
            at = function.getEntryPoint()
            if at in named and named[at] != fname:
                conflicts.append((at, named[at], fname))
                continue
            named[at] = fname

    used = {}
    for at, fname in named.items():
        function = flat.getFunctionAt(at)
        if function is None:
            continue
        unique = fname if fname not in used else f"{fname}_{used[fname]}"
        used[fname] = used.get(fname, 0) + 1
        if function.getName().startswith(("FUN_", "thunk_FUN_")):
            function.setName(unique, ctx["SourceType"].ANALYSIS)
        else:
            flat.createLabel(at, unique, False)
        flat.setPlateComment(at, f"libpng 1.6.8 {fname}(), identified by an error "
                                 "message only this function prints.")
    for at, first, second in conflicts:
        flat.setPlateComment(at, f"Two libpng candidates here: {first} and {second}. "
                                 "Inlined, or the messages were pooled.")
    if conflicts:
        log(f"libpng conflicts left commented, not named: {len(conflicts)}")
    return len(used)


def _referring_functions(ctx, addr, depth=2):
    """Functions that reach addr, following one hop through a literal pool."""
    flat = ctx["flat"]
    out, seen = [], set()
    frontier = [addr]
    for _ in range(depth):
        next_frontier = []
        for target in frontier:
            for ref in flat.getReferencesTo(target):
                src = ref.getFromAddress()
                if src in seen:
                    continue
                seen.add(src)
                function = flat.getFunctionContaining(src)
                if function is not None:
                    out.append(function)
                else:
                    next_frontier.append(src)
        frontier = next_frontier
    return out


def _source_file_hints(ctx):
    """Bookmark the handful of vendor source-file names left in the image."""
    flat, image = ctx["flat"], ctx["image"]
    hits = 0
    for m in re.finditer(rb"[A-Za-z][A-Za-z0-9_]{2,30}\.c\x00", image):
        addr = gaddr(flat, BASE + m.start())
        name = m.group()[:-1].decode()
        flat.createLabel(addr, f"srcfile_{SLUG_BAD.sub('_', name)}", False)
        flat.createBookmark(addr, "THD74", f"vendor source file name: {name}")
        hits += 1
    log(f"vendor source-file strings bookmarked: {hits}")


# --------------------------------------------------------------------------- #
# small wrappers over the flat API
# --------------------------------------------------------------------------- #

def arm(ctx, addr, length=0, thumb=False):
    """Disassemble at addr in ARM or Thumb state, setting TMode accordingly."""
    flat = ctx["flat"]
    start = gaddr(flat, addr)
    cmd = ctx["ArmDisassembleCommand"](start, None, thumb)
    cmd.applyTo(ctx["program"], ctx["monitor"])
    return start


def func(ctx, addr, name, comment=None, thumb=False):
    """Disassemble, create and name a function, Thumb bit tolerated in addr."""
    flat = ctx["flat"]
    entry = gaddr(flat, addr & ~1)
    arm(ctx, addr & ~1, thumb=thumb)
    function = flat.getFunctionAt(entry)
    if function is None:
        function = flat.createFunction(entry, name)
    if function is None:
        flat.createLabel(entry, name, True)
        if comment:
            flat.setPlateComment(entry, comment + "\n(Could not create a function here.)")
        return None
    function.setName(name, ctx["SourceType"].USER_DEFINED)
    if comment:
        flat.setPlateComment(entry, comment)
    return function


def main(argv=None):
    here = Path(__file__).resolve().parent
    p = argparse.ArgumentParser(description=__doc__,
                                formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("--firmware", required=True,
                   help="flattened FIRMWARE section (see srec2bin.py)")
    p.add_argument("--project-dir", required=True, help="where to put the Ghidra project")
    p.add_argument("--project-name", default="THD74")
    p.add_argument("--program-name", default="FIRMWARE_V1.10.000")
    p.add_argument("--language", default="ARM:LE:32:v5t",
                   help="ARM926EJ-S is ARMv5TE; use ARM:LE:32:v7 to test the "
                        "Thumb-2 reading of the image")
    p.add_argument("--ghidra", default="/opt/ghidra", help="Ghidra installation directory")
    p.add_argument("--png-map", default=str(here / "png_msgmap.json"),
                   help="message -> libpng function map from png_msgmap.py")
    p.add_argument("--no-analyze", dest="analyze", action="store_false",
                   help="skip auto-analysis (markup only, for quick iteration)")
    args = p.parse_args(argv)
    if not Path(args.firmware).is_file():
        p.error(f"no such firmware image: {args.firmware}")
    build(args)
    return 0


if __name__ == "__main__":
    sys.exit(main())
