rust/library/compiler-builtins/etc/update-api-list.py

#!/usr/bin/env python3
"""Create a text file listing all public API. This can be used to ensure that all
functions are covered by our macros.

This file additionally does tidy-esque checks that all functions are listed where
needed, or that lists are sorted.
"""

import difflib
import json
import re
import subprocess as sp
import sys
from dataclasses import dataclass
from glob import glob
from pathlib import Path
from typing import Any, Callable, TypeAlias

SELF_PATH = Path(__file__)
ETC_DIR = SELF_PATH.parent
ROOT_DIR = ETC_DIR.parent

# These files do not trigger a retest.
IGNORED_SOURCES = ["libm/src/libm_helper.rs", "libm/src/math/support/float_traits.rs"]

IndexTy: TypeAlias = dict[str, dict[str, Any]]
"""Type of the `index` item in rustdoc's JSON output"""


def eprint(*args, **kwargs):
    """Print to stderr."""
    print(*args, file=sys.stderr, **kwargs)


@dataclass
class Crate:
    """Representation of public interfaces and function defintion locations in
    `libm`.
    """

    public_functions: list[str]
    """List of all public functions."""
    defs: dict[str, list[str]]
    """Map from `name->[source files]` to find all places that define a public
    function. We track this to know which tests need to be rerun when specific files
    get updated.
    """
    types: dict[str, str]
    """Map from `name->type`."""

    def __init__(self) -> None:
        self.public_functions = []
        self.defs = {}
        self.types = {}

        j = self.get_rustdoc_json()
        index: IndexTy = j["index"]
        self._init_function_list(index)
        self._init_defs(index)
        self._init_types()

    @staticmethod
    def get_rustdoc_json() -> dict[Any, Any]:
        """Get rustdoc's JSON output for the `libm` crate."""

        j = sp.check_output(
            [
                "rustdoc",
                "libm/src/lib.rs",
                "--edition=2021",
                "--document-private-items",
                "--output-format=json",
                "--cfg=f16_enabled",
                "--cfg=f128_enabled",
                "-Zunstable-options",
                "-o-",
            ],
            cwd=ROOT_DIR,
            text=True,
        )
        j = json.loads(j)
        return j

    def _init_function_list(self, index: IndexTy) -> None:
        """Get a list of public functions from rustdoc JSON output.

        Note that this only finds functions that are reexported in `lib.rs`, this will
        need to be adjusted if we need to account for functions that are defined there, or
        glob reexports in other locations.
        """
        # Filter out items that are not public
        public = [i for i in index.values() if i["visibility"] == "public"]

        # Collect a list of source IDs for reexported items in `lib.rs` or `mod math`.
        use = (i for i in public if "use" in i["inner"])
        use = (
            i
            for i in use
            if i["span"]["filename"] in ["libm/src/math/mod.rs", "libm/src/lib.rs"]
        )
        reexported_ids = [item["inner"]["use"]["id"] for item in use]

        # Collect a list of reexported items that are functions
        for id in reexported_ids:
            srcitem = index.get(str(id))
            # External crate
            if srcitem is None:
                continue

            # Skip if not a function
            if "function" not in srcitem["inner"]:
                continue

            self.public_functions.append(srcitem["name"])
        self.public_functions.sort()

    def _init_defs(self, index: IndexTy) -> None:
        defs = {name: set() for name in self.public_functions}
        funcs = (i for i in index.values() if "function" in i["inner"])
        funcs = (f for f in funcs if f["name"] in self.public_functions)
        for func in funcs:
            defs[func["name"]].add(func["span"]["filename"])

        # A lot of the `arch` module is often configured out so doesn't show up in docs. Use
        # string matching as a fallback.
        for fname in glob(
            "libm/src/math/arch/**/*.rs", root_dir=ROOT_DIR, recursive=True
        ):
            contents = (ROOT_DIR.joinpath(fname)).read_text()

            for name in self.public_functions:
                if f"fn {name}" in contents:
                    defs[name].add(fname)

        for name, sources in defs.items():
            base_sources = defs[base_name(name)[0]]
            for src in (s for s in base_sources if "generic" in s):
                sources.add(src)

            for src in IGNORED_SOURCES:
                sources.discard(src)

        # Sort the set
        self.defs = {k: sorted(v) for (k, v) in defs.items()}

    def _init_types(self) -> None:
        self.types = {name: base_name(name)[1] for name in self.public_functions}

    def write_function_list(self, check: bool) -> None:
        """Collect the list of public functions to a simple text file."""
        output = "# autogenerated by update-api-list.py\n"
        for name in self.public_functions:
            output += f"{name}\n"

        out_file = ETC_DIR.joinpath("function-list.txt")

        if check:
            with open(out_file, "r") as f:
                current = f.read()
            diff_and_exit(current, output, "function list")
        else:
            with open(out_file, "w") as f:
                f.write(output)

    def write_function_defs(self, check: bool) -> None:
        """Collect the list of information about public functions to a JSON file ."""
        comment = (
            "Autogenerated by update-api-list.py. "
            "List of files that define a function with a given name. "
            "This file is checked in to make it obvious if refactoring breaks things"
        )

        d = {"__comment": comment}
        d |= {
            name: {"sources": self.defs[name], "type": self.types[name]}
            for name in self.public_functions
        }

        out_file = ETC_DIR.joinpath("function-definitions.json")
        output = json.dumps(d, indent=4) + "\n"

        if check:
            with open(out_file, "r") as f:
                current = f.read()
            diff_and_exit(current, output, "source list")
        else:
            with open(out_file, "w") as f:
                f.write(output)

    def tidy_lists(self) -> None:
        """In each file, check annotations indicating blocks of code should be sorted or should
        include all public API.
        """

        flist = sp.check_output(["git", "ls-files"], cwd=ROOT_DIR, text=True)

        for path in flist.splitlines():
            fpath = ROOT_DIR.joinpath(path)
            if fpath.is_dir() or fpath == SELF_PATH:
                continue

            lines = fpath.read_text().splitlines()

            validate_delimited_block(
                fpath,
                lines,
                "verify-sorted-start",
                "verify-sorted-end",
                ensure_sorted,
            )

            validate_delimited_block(
                fpath,
                lines,
                "verify-apilist-start",
                "verify-apilist-end",
                lambda p, n, lines: self.ensure_contains_api(p, n, lines),
            )

    def ensure_contains_api(self, fpath: Path, line_num: int, lines: list[str]):
        """Given a list of strings, ensure that each public function we have is named
        somewhere.
        """
        not_found = []
        for func in self.public_functions:
            # The function name may be on its own or somewhere in a snake case string.
            pat = re.compile(rf"(\b|_){func}(\b|_)")
            found = next((line for line in lines if pat.search(line)), None)

            if found is None:
                not_found.append(func)

        if len(not_found) == 0:
            return

        relpath = fpath.relative_to(ROOT_DIR)
        eprint(f"functions not found at {relpath}:{line_num}: {not_found}")
        exit(1)


def validate_delimited_block(
    fpath: Path,
    lines: list[str],
    start: str,
    end: str,
    validate: Callable[[Path, int, list[str]], None],
) -> None:
    """Identify blocks of code wrapped within `start` and `end`, collect their contents
    to a list of strings, and call `validate` for each of those lists.
    """
    relpath = fpath.relative_to(ROOT_DIR)
    block_lines = []
    block_start_line: None | int = None
    for line_num, line in enumerate(lines):
        line_num += 1

        if start in line:
            block_start_line = line_num
            continue

        if end in line:
            if block_start_line is None:
                eprint(f"`{end}` without `{start}` at {relpath}:{line_num}")
                exit(1)

            validate(fpath, block_start_line, block_lines)
            block_lines = []
            block_start_line = None
            continue

        if block_start_line is not None:
            block_lines.append(line)

    if block_start_line is not None:
        eprint(f"`{start}` without `{end}` at {relpath}:{block_start_line}")
        exit(1)


def ensure_sorted(fpath: Path, block_start_line: int, lines: list[str]) -> None:
    """Ensure that a list of lines is sorted, otherwise print a diff and exit."""
    relpath = fpath.relative_to(ROOT_DIR)
    diff_and_exit(
        "\n".join(lines),
        "\n".join(sorted(lines)),
        f"sorted block at {relpath}:{block_start_line}",
    )


def diff_and_exit(actual: str, expected: str, name: str):
    """If the two strings are different, print a diff between them and then exit
    with an error.
    """
    if actual == expected:
        print(f"{name} output matches expected; success")
        return

    a = [f"{line}\n" for line in actual.splitlines()]
    b = [f"{line}\n" for line in expected.splitlines()]

    diff = difflib.unified_diff(a, b, "actual", "expected")
    sys.stdout.writelines(diff)
    print(f"mismatched {name}")
    exit(1)


def base_name(name: str) -> tuple[str, str]:
    """Return the basename and type from a full function name. Keep in sync with Rust's
    `fn base_name`.
    """
    known_mappings = [
        ("erff", ("erf", "f32")),
        ("erf", ("erf", "f64")),
        ("modff", ("modf", "f32")),
        ("modf", ("modf", "f64")),
        ("lgammaf_r", ("lgamma_r", "f32")),
        ("lgamma_r", ("lgamma_r", "f64")),
    ]

    found = next((base for (full, base) in known_mappings if full == name), None)
    if found is not None:
        return found

    if name.endswith("f"):
        return (name.rstrip("f"), "f32")

    if name.endswith("f16"):
        return (name.rstrip("f16"), "f16")

    if name.endswith("f128"):
        return (name.rstrip("f128"), "f128")

    return (name, "f64")


def ensure_updated_list(check: bool) -> None:
    """Runner to update the function list and JSON, or check that it is already up
    to date.
    """
    crate = Crate()
    crate.write_function_list(check)
    crate.write_function_defs(check)

    crate.tidy_lists()


def main():
    """By default overwrite the file. If `--check` is passed, print a diff instead and
    error if the files are different.
    """
    match sys.argv:
        case [_]:
            ensure_updated_list(False)
        case [_, "--check"]:
            ensure_updated_list(True)
        case _:
            print("unrecognized arguments")
            exit(1)


if __name__ == "__main__":
    main()