Add more detailed definition output for update-api-list.py

Update the script to produce, in addition to the simple text list, a
JSON file listing routine names, the types they work with, and the
source files that contain a function with the routine name. This gets
consumed by another script and will be used to determine which extensive
CI jobs to run.
This commit is contained in:
Trevor Gross 2025-01-02 10:21:35 +00:00 committed by Trevor Gross
parent f453f5e5b5
commit 7c04b1916a
2 changed files with 940 additions and 60 deletions

View file

@ -0,0 +1,764 @@
{
"__comment": "Autogenerated by update-api-list.py. List of files that define a function with a given name. This file is checked in to make it obvious if refactoring breaks things",
"acos": {
"sources": [
"src/libm_helper.rs",
"src/math/acos.rs"
],
"type": "f64"
},
"acosf": {
"sources": [
"src/math/acosf.rs"
],
"type": "f32"
},
"acosh": {
"sources": [
"src/libm_helper.rs",
"src/math/acosh.rs"
],
"type": "f64"
},
"acoshf": {
"sources": [
"src/math/acoshf.rs"
],
"type": "f32"
},
"asin": {
"sources": [
"src/libm_helper.rs",
"src/math/asin.rs"
],
"type": "f64"
},
"asinf": {
"sources": [
"src/math/asinf.rs"
],
"type": "f32"
},
"asinh": {
"sources": [
"src/libm_helper.rs",
"src/math/asinh.rs"
],
"type": "f64"
},
"asinhf": {
"sources": [
"src/math/asinhf.rs"
],
"type": "f32"
},
"atan": {
"sources": [
"src/libm_helper.rs",
"src/math/atan.rs"
],
"type": "f64"
},
"atan2": {
"sources": [
"src/libm_helper.rs",
"src/math/atan2.rs"
],
"type": "f64"
},
"atan2f": {
"sources": [
"src/math/atan2f.rs"
],
"type": "f32"
},
"atanf": {
"sources": [
"src/math/atanf.rs"
],
"type": "f32"
},
"atanh": {
"sources": [
"src/libm_helper.rs",
"src/math/atanh.rs"
],
"type": "f64"
},
"atanhf": {
"sources": [
"src/math/atanhf.rs"
],
"type": "f32"
},
"cbrt": {
"sources": [
"src/libm_helper.rs",
"src/math/cbrt.rs"
],
"type": "f64"
},
"cbrtf": {
"sources": [
"src/math/cbrtf.rs"
],
"type": "f32"
},
"ceil": {
"sources": [
"src/libm_helper.rs",
"src/math/arch/i586.rs",
"src/math/arch/intrinsics.rs",
"src/math/ceil.rs"
],
"type": "f64"
},
"ceilf": {
"sources": [
"src/math/arch/intrinsics.rs",
"src/math/ceilf.rs"
],
"type": "f32"
},
"copysign": {
"sources": [
"src/libm_helper.rs",
"src/math/copysign.rs",
"src/math/generic/copysign.rs",
"src/math/support/float_traits.rs"
],
"type": "f64"
},
"copysignf": {
"sources": [
"src/math/copysignf.rs",
"src/math/generic/copysign.rs"
],
"type": "f32"
},
"cos": {
"sources": [
"src/libm_helper.rs",
"src/math/cos.rs"
],
"type": "f64"
},
"cosf": {
"sources": [
"src/math/cosf.rs"
],
"type": "f32"
},
"cosh": {
"sources": [
"src/libm_helper.rs",
"src/math/cosh.rs"
],
"type": "f64"
},
"coshf": {
"sources": [
"src/math/coshf.rs"
],
"type": "f32"
},
"erf": {
"sources": [
"src/libm_helper.rs",
"src/math/erf.rs"
],
"type": "f64"
},
"erfc": {
"sources": [
"src/libm_helper.rs",
"src/math/erf.rs"
],
"type": "f64"
},
"erfcf": {
"sources": [
"src/math/erff.rs"
],
"type": "f32"
},
"erff": {
"sources": [
"src/math/erff.rs"
],
"type": "f32"
},
"exp": {
"sources": [
"src/libm_helper.rs",
"src/math/exp.rs",
"src/math/support/float_traits.rs"
],
"type": "f64"
},
"exp10": {
"sources": [
"src/libm_helper.rs",
"src/math/exp10.rs"
],
"type": "f64"
},
"exp10f": {
"sources": [
"src/math/exp10f.rs"
],
"type": "f32"
},
"exp2": {
"sources": [
"src/libm_helper.rs",
"src/math/exp2.rs"
],
"type": "f64"
},
"exp2f": {
"sources": [
"src/math/exp2f.rs"
],
"type": "f32"
},
"expf": {
"sources": [
"src/math/expf.rs"
],
"type": "f32"
},
"expm1": {
"sources": [
"src/libm_helper.rs",
"src/math/expm1.rs"
],
"type": "f64"
},
"expm1f": {
"sources": [
"src/math/expm1f.rs"
],
"type": "f32"
},
"fabs": {
"sources": [
"src/libm_helper.rs",
"src/math/arch/intrinsics.rs",
"src/math/fabs.rs",
"src/math/generic/fabs.rs"
],
"type": "f64"
},
"fabsf": {
"sources": [
"src/math/arch/intrinsics.rs",
"src/math/fabsf.rs",
"src/math/generic/fabs.rs"
],
"type": "f32"
},
"fdim": {
"sources": [
"src/libm_helper.rs",
"src/math/fdim.rs"
],
"type": "f64"
},
"fdimf": {
"sources": [
"src/math/fdimf.rs"
],
"type": "f32"
},
"floor": {
"sources": [
"src/libm_helper.rs",
"src/math/arch/i586.rs",
"src/math/arch/intrinsics.rs",
"src/math/floor.rs"
],
"type": "f64"
},
"floorf": {
"sources": [
"src/math/arch/intrinsics.rs",
"src/math/floorf.rs"
],
"type": "f32"
},
"fma": {
"sources": [
"src/libm_helper.rs",
"src/math/fma.rs"
],
"type": "f64"
},
"fmaf": {
"sources": [
"src/math/fmaf.rs"
],
"type": "f32"
},
"fmax": {
"sources": [
"src/libm_helper.rs",
"src/math/fmax.rs"
],
"type": "f64"
},
"fmaxf": {
"sources": [
"src/math/fmaxf.rs"
],
"type": "f32"
},
"fmin": {
"sources": [
"src/libm_helper.rs",
"src/math/fmin.rs"
],
"type": "f64"
},
"fminf": {
"sources": [
"src/math/fminf.rs"
],
"type": "f32"
},
"fmod": {
"sources": [
"src/libm_helper.rs",
"src/math/fmod.rs"
],
"type": "f64"
},
"fmodf": {
"sources": [
"src/math/fmodf.rs"
],
"type": "f32"
},
"frexp": {
"sources": [
"src/libm_helper.rs",
"src/math/frexp.rs"
],
"type": "f64"
},
"frexpf": {
"sources": [
"src/math/frexpf.rs"
],
"type": "f32"
},
"hypot": {
"sources": [
"src/libm_helper.rs",
"src/math/hypot.rs"
],
"type": "f64"
},
"hypotf": {
"sources": [
"src/math/hypotf.rs"
],
"type": "f32"
},
"ilogb": {
"sources": [
"src/libm_helper.rs",
"src/math/ilogb.rs"
],
"type": "f64"
},
"ilogbf": {
"sources": [
"src/math/ilogbf.rs"
],
"type": "f32"
},
"j0": {
"sources": [
"src/libm_helper.rs",
"src/math/j0.rs"
],
"type": "f64"
},
"j0f": {
"sources": [
"src/math/j0f.rs"
],
"type": "f32"
},
"j1": {
"sources": [
"src/libm_helper.rs",
"src/math/j1.rs"
],
"type": "f64"
},
"j1f": {
"sources": [
"src/math/j1f.rs"
],
"type": "f32"
},
"jn": {
"sources": [
"src/libm_helper.rs",
"src/math/jn.rs"
],
"type": "f64"
},
"jnf": {
"sources": [
"src/math/jnf.rs"
],
"type": "f32"
},
"ldexp": {
"sources": [
"src/libm_helper.rs",
"src/math/ldexp.rs"
],
"type": "f64"
},
"ldexpf": {
"sources": [
"src/math/ldexpf.rs"
],
"type": "f32"
},
"lgamma": {
"sources": [
"src/libm_helper.rs",
"src/math/lgamma.rs"
],
"type": "f64"
},
"lgamma_r": {
"sources": [
"src/libm_helper.rs",
"src/math/lgamma_r.rs"
],
"type": "f64"
},
"lgammaf": {
"sources": [
"src/math/lgammaf.rs"
],
"type": "f32"
},
"lgammaf_r": {
"sources": [
"src/math/lgammaf_r.rs"
],
"type": "f32"
},
"log": {
"sources": [
"src/libm_helper.rs",
"src/math/log.rs"
],
"type": "f64"
},
"log10": {
"sources": [
"src/libm_helper.rs",
"src/math/log10.rs"
],
"type": "f64"
},
"log10f": {
"sources": [
"src/math/log10f.rs"
],
"type": "f32"
},
"log1p": {
"sources": [
"src/libm_helper.rs",
"src/math/log1p.rs"
],
"type": "f64"
},
"log1pf": {
"sources": [
"src/math/log1pf.rs"
],
"type": "f32"
},
"log2": {
"sources": [
"src/libm_helper.rs",
"src/math/log2.rs"
],
"type": "f64"
},
"log2f": {
"sources": [
"src/math/log2f.rs"
],
"type": "f32"
},
"logf": {
"sources": [
"src/math/logf.rs"
],
"type": "f32"
},
"modf": {
"sources": [
"src/libm_helper.rs",
"src/math/modf.rs"
],
"type": "f64"
},
"modff": {
"sources": [
"src/math/modff.rs"
],
"type": "f32"
},
"nextafter": {
"sources": [
"src/libm_helper.rs",
"src/math/nextafter.rs"
],
"type": "f64"
},
"nextafterf": {
"sources": [
"src/math/nextafterf.rs"
],
"type": "f32"
},
"pow": {
"sources": [
"src/libm_helper.rs",
"src/math/pow.rs"
],
"type": "f64"
},
"powf": {
"sources": [
"src/math/powf.rs"
],
"type": "f32"
},
"remainder": {
"sources": [
"src/libm_helper.rs",
"src/math/remainder.rs"
],
"type": "f64"
},
"remainderf": {
"sources": [
"src/math/remainderf.rs"
],
"type": "f32"
},
"remquo": {
"sources": [
"src/libm_helper.rs",
"src/math/remquo.rs"
],
"type": "f64"
},
"remquof": {
"sources": [
"src/math/remquof.rs"
],
"type": "f32"
},
"rint": {
"sources": [
"src/libm_helper.rs",
"src/math/rint.rs"
],
"type": "f64"
},
"rintf": {
"sources": [
"src/math/rintf.rs"
],
"type": "f32"
},
"round": {
"sources": [
"src/libm_helper.rs",
"src/math/round.rs"
],
"type": "f64"
},
"roundf": {
"sources": [
"src/math/roundf.rs"
],
"type": "f32"
},
"scalbn": {
"sources": [
"src/libm_helper.rs",
"src/math/scalbn.rs"
],
"type": "f64"
},
"scalbnf": {
"sources": [
"src/math/scalbnf.rs"
],
"type": "f32"
},
"sin": {
"sources": [
"src/libm_helper.rs",
"src/math/sin.rs"
],
"type": "f64"
},
"sincos": {
"sources": [
"src/libm_helper.rs",
"src/math/sincos.rs"
],
"type": "f64"
},
"sincosf": {
"sources": [
"src/math/sincosf.rs"
],
"type": "f32"
},
"sinf": {
"sources": [
"src/math/sinf.rs"
],
"type": "f32"
},
"sinh": {
"sources": [
"src/libm_helper.rs",
"src/math/sinh.rs"
],
"type": "f64"
},
"sinhf": {
"sources": [
"src/math/sinhf.rs"
],
"type": "f32"
},
"sqrt": {
"sources": [
"src/libm_helper.rs",
"src/math/arch/i686.rs",
"src/math/arch/intrinsics.rs",
"src/math/sqrt.rs"
],
"type": "f64"
},
"sqrtf": {
"sources": [
"src/math/arch/i686.rs",
"src/math/arch/intrinsics.rs",
"src/math/sqrtf.rs"
],
"type": "f32"
},
"tan": {
"sources": [
"src/libm_helper.rs",
"src/math/tan.rs"
],
"type": "f64"
},
"tanf": {
"sources": [
"src/math/tanf.rs"
],
"type": "f32"
},
"tanh": {
"sources": [
"src/libm_helper.rs",
"src/math/tanh.rs"
],
"type": "f64"
},
"tanhf": {
"sources": [
"src/math/tanhf.rs"
],
"type": "f32"
},
"tgamma": {
"sources": [
"src/libm_helper.rs",
"src/math/tgamma.rs"
],
"type": "f64"
},
"tgammaf": {
"sources": [
"src/math/tgammaf.rs"
],
"type": "f32"
},
"trunc": {
"sources": [
"src/libm_helper.rs",
"src/math/arch/intrinsics.rs",
"src/math/trunc.rs"
],
"type": "f64"
},
"truncf": {
"sources": [
"src/math/arch/intrinsics.rs",
"src/math/truncf.rs"
],
"type": "f32"
},
"y0": {
"sources": [
"src/libm_helper.rs",
"src/math/j0.rs"
],
"type": "f64"
},
"y0f": {
"sources": [
"src/math/j0f.rs"
],
"type": "f32"
},
"y1": {
"sources": [
"src/libm_helper.rs",
"src/math/j1.rs"
],
"type": "f64"
},
"y1f": {
"sources": [
"src/math/j1f.rs"
],
"type": "f32"
},
"yn": {
"sources": [
"src/libm_helper.rs",
"src/math/jn.rs"
],
"type": "f64"
},
"ynf": {
"sources": [
"src/math/jnf.rs"
],
"type": "f32"
}
}

View file

@ -3,68 +3,166 @@
functions are covered by our macros.
"""
import difflib
import json
import subprocess as sp
import sys
import difflib
from dataclasses import dataclass
from glob import glob
from pathlib import Path
from typing import Any
from typing import Any, TypeAlias
ETC_DIR = Path(__file__).parent
def get_rustdoc_json() -> dict[Any, Any]:
"""Get rustdoc's JSON output for the `libm` crate."""
librs_path = ETC_DIR.joinpath("../src/lib.rs")
j = sp.check_output(
[
"rustdoc",
librs_path,
"--edition=2021",
"--output-format=json",
"-Zunstable-options",
"-o-",
],
text=True,
)
j = json.loads(j)
return j
IndexTy: TypeAlias = dict[str, dict[str, Any]]
"""Type of the `index` item in rustdoc's JSON output"""
def list_public_functions() -> list[str]:
"""Get a list of public functions from rustdoc JSON output.
Note that this only finds functions that are reexported in `lib.rs`, this will
need to be adjusted if we need to account for functions that are defined there.
@dataclass
class Crate:
"""Representation of public interfaces and function defintion locations in
`libm`.
"""
names = []
index: dict[str, dict[str, Any]] = get_rustdoc_json()["index"]
for item in index.values():
# Find public items
if item["visibility"] != "public":
continue
# Find only reexports
if "use" not in item["inner"].keys():
continue
public_functions: list[str]
"""List of all public functions."""
defs: dict[str, list[str]]
"""Map from `name->[source files]` to find all places that define a public
function. We track this to know which tests need to be rerun when specific files
get updated.
"""
types: dict[str, str]
"""Map from `name->type`."""
# Locate the item that is reexported
id = item["inner"]["use"]["id"]
srcitem = index.get(str(id))
def __init__(self) -> None:
self.public_functions = []
self.defs = {}
self.types = {}
# External crate
if srcitem is None:
continue
j = self.get_rustdoc_json()
index: IndexTy = j["index"]
self._init_function_list(index)
self._init_defs(index)
self._init_types()
# Skip if not a function
if "function" not in srcitem["inner"].keys():
continue
@staticmethod
def get_rustdoc_json() -> dict[Any, Any]:
"""Get rustdoc's JSON output for the `libm` crate."""
names.append(srcitem["name"])
j = sp.check_output(
[
"rustdoc",
"src/lib.rs",
"--edition=2021",
"--document-private-items",
"--output-format=json",
"-Zunstable-options",
"-o-",
],
cwd=ETC_DIR.parent,
text=True,
)
j = json.loads(j)
return j
names.sort()
return names
def _init_function_list(self, index: IndexTy) -> None:
"""Get a list of public functions from rustdoc JSON output.
Note that this only finds functions that are reexported in `lib.rs`, this will
need to be adjusted if we need to account for functions that are defined there, or
glob reexports in other locations.
"""
# Filter out items that are not public
public = [i for i in index.values() if i["visibility"] == "public"]
# Collect a list of source IDs for reexported items in `lib.rs` or `mod math`.
use = (i for i in public if "use" in i["inner"])
use = (
i for i in use if i["span"]["filename"] in ["src/math/mod.rs", "src/lib.rs"]
)
reexported_ids = [item["inner"]["use"]["id"] for item in use]
# Collect a list of reexported items that are functions
for id in reexported_ids:
srcitem = index.get(str(id))
# External crate
if srcitem is None:
continue
# Skip if not a function
if "function" not in srcitem["inner"]:
continue
self.public_functions.append(srcitem["name"])
self.public_functions.sort()
def _init_defs(self, index: IndexTy) -> None:
defs = {name: set() for name in self.public_functions}
funcs = (i for i in index.values() if "function" in i["inner"])
funcs = (f for f in funcs if f["name"] in self.public_functions)
for func in funcs:
defs[func["name"]].add(func["span"]["filename"])
# A lot of the `arch` module is often configured out so doesn't show up in docs. Use
# string matching as a fallback.
for fname in glob("src/math/arch/**.rs", root_dir=ETC_DIR.parent):
contents = Path(fname).read_text()
for name in self.public_functions:
if f"fn {name}" in contents:
defs[name].add(fname)
for name, sources in defs.items():
base_sources = defs[base_name(name)[0]]
for src in (s for s in base_sources if "generic" in s):
sources.add(src)
# Sort the set
self.defs = {k: sorted(v) for (k, v) in defs.items()}
def _init_types(self) -> None:
self.types = {name: base_name(name)[1] for name in self.public_functions}
def write_function_list(self, check: bool) -> None:
"""Collect the list of public functions to a simple text file."""
output = "# autogenerated by update-api-list.py\n"
for name in self.public_functions:
output += f"{name}\n"
out_file = ETC_DIR.joinpath("function-list.txt")
if check:
with open(out_file, "r") as f:
current = f.read()
diff_and_exit(current, output)
else:
with open(out_file, "w") as f:
f.write(output)
def write_function_defs(self, check: bool) -> None:
"""Collect the list of information about public functions to a JSON file ."""
comment = (
"Autogenerated by update-api-list.py. "
"List of files that define a function with a given name. "
"This file is checked in to make it obvious if refactoring breaks things"
)
d = {"__comment": comment}
d |= {
name: {"sources": self.defs[name], "type": self.types[name]}
for name in self.public_functions
}
out_file = ETC_DIR.joinpath("function-definitions.json")
output = json.dumps(d, indent=4) + "\n"
if check:
with open(out_file, "r") as f:
current = f.read()
diff_and_exit(current, output)
else:
with open(out_file, "w") as f:
f.write(output)
def diff_and_exit(actual: str, expected: str):
@ -84,6 +182,35 @@ def diff_and_exit(actual: str, expected: str):
exit(1)
def base_name(name: str) -> tuple[str, str]:
"""Return the basename and type from a full function name. Keep in sync with Rust's
`fn base_name`.
"""
known_mappings = [
("erff", ("erf", "f32")),
("erf", ("erf", "f64")),
("modff", ("modf", "f32")),
("modf", ("modf", "f64")),
("lgammaf_r", ("lgamma_r", "f32")),
("lgamma_r", ("lgamma_r", "f64")),
]
found = next((base for (full, base) in known_mappings if full == name), None)
if found is not None:
return found
if name.endswith("f"):
return (name.rstrip("f"), "f32")
if name.endswith("f16"):
return (name.rstrip("f16"), "f16")
if name.endswith("f128"):
return (name.rstrip("f128"), "f128")
return (name, "f64")
def main():
"""By default overwrite the file. If `--check` is passed, print a diff instead and
error if the files are different.
@ -97,20 +224,9 @@ def main():
print("unrecognized arguments")
exit(1)
names = list_public_functions()
output = "# autogenerated by update-api-list.py\n"
for name in names:
output += f"{name}\n"
out_file = ETC_DIR.joinpath("function-list.txt")
if check:
with open(out_file, "r") as f:
current = f.read()
diff_and_exit(current, output)
else:
with open(out_file, "w") as f:
f.write(output)
crate = Crate()
crate.write_function_list(check)
crate.write_function_defs(check)
if __name__ == "__main__":