Auto merge of #79539 - aDotInTheVoid:json-mvp, r=jyn514

Rustdoc: JSON backend experimental impl, with new tests.

Based on #75114 by `@P1n3appl3`

The first commit is all of #75114, but squased to 1 commit, as that was much easier to rebase onto master.

The git history is a mess, but I think I'll edit it after review, so it's obvious whats new.

## Still to do

- [ ] Update docs.
- [ ] Add bless option to tests.
- [ ] Add test option for multiple files in same crate.
- [ ] Decide if the tests should check for json to be equal or subset.
- [ ] Go through the rest of the review for the original pr. (This is open because the test system is done(ish), but stuff like [not using a hashmap](https://github.com/rust-lang/rust/pull/75114#discussion_r519474420) and [using `CRATE_DEF_INDEX` ](https://github.com/rust-lang/rust/pull/75114#discussion_r519470764) hasn't)

I'm also sure how many of these we need to do before landing on nightly, as it would be nice to get this in tree, so it isn't effected by churn like #79125, #79041, #79061

r? `@jyn514`
This commit is contained in:
bors 2020-12-02 23:18:43 +00:00
commit 7dc1e852d4
14 changed files with 2192 additions and 21 deletions

View file

@ -0,0 +1,187 @@
#!/usr/bin/env python
# This test ensures that every ID in the produced json actually resolves to an item either in
# `index` or `paths`. It DOES NOT check that the structure of the produced json is actually in
# any way correct, for example an empty map would pass.
import sys
import json
crate = json.load(open(sys.argv[1]))
def get_local_item(item_id):
if item_id in crate["index"]:
return crate["index"][item_id]
print("Missing local ID:", item_id)
sys.exit(1)
# local IDs have to be in `index`, external ones can sometimes be in `index` but otherwise have
# to be in `paths`
def valid_id(item_id):
return item_id in crate["index"] or item_id[0] != "0" and item_id in crate["paths"]
def check_generics(generics):
for param in generics["params"]:
check_generic_param(param)
for where_predicate in generics["where_predicates"]:
if "bound_predicate" in where_predicate:
pred = where_predicate["bound_predicate"]
check_type(pred["ty"])
for bound in pred["bounds"]:
check_generic_bound(bound)
elif "region_predicate" in where_predicate:
pred = where_predicate["region_predicate"]
for bound in pred["bounds"]:
check_generic_bound(bound)
elif "eq_predicate" in where_predicate:
pred = where_predicate["eq_predicate"]
check_type(pred["rhs"])
check_type(pred["lhs"])
def check_generic_param(param):
if "type" in param["kind"]:
ty = param["kind"]["type"]
if ty["default"]:
check_type(ty["default"])
for bound in ty["bounds"]:
check_generic_bound(bound)
elif "const" in param["kind"]:
check_type(param["kind"]["const"])
def check_generic_bound(bound):
if "trait_bound" in bound:
for param in bound["trait_bound"]["generic_params"]:
check_generic_param(param)
check_type(bound["trait_bound"]["trait"])
def check_decl(decl):
for (_name, ty) in decl["inputs"]:
check_type(ty)
if decl["output"]:
check_type(decl["output"])
def check_type(ty):
if ty["kind"] == "resolved_path":
for bound in ty["inner"]["param_names"]:
check_generic_bound(bound)
args = ty["inner"]["args"]
if args:
if "angle_bracketed" in args:
for arg in args["angle_bracketed"]["args"]:
if "type" in arg:
check_type(arg["type"])
elif "const" in arg:
check_type(arg["const"]["type"])
for binding in args["angle_bracketed"]["bindings"]:
if "equality" in binding["binding"]:
check_type(binding["binding"]["equality"])
elif "constraint" in binding["binding"]:
for bound in binding["binding"]["constraint"]:
check_generic_bound(bound)
elif "parenthesized" in args:
for ty in args["parenthesized"]["inputs"]:
check_type(ty)
if args["parenthesized"]["output"]:
check_type(args["parenthesized"]["output"])
if not valid_id(ty["inner"]["id"]):
print("Type contained an invalid ID:", ty["inner"]["id"])
sys.exit(1)
elif ty["kind"] == "tuple":
for ty in ty["inner"]:
check_type(ty)
elif ty["kind"] == "slice":
check_type(ty["inner"])
elif ty["kind"] == "impl_trait":
for bound in ty["inner"]:
check_generic_bound(bound)
elif ty["kind"] in ("raw_pointer", "borrowed_ref", "array"):
check_type(ty["inner"]["type"])
elif ty["kind"] == "function_pointer":
for param in ty["inner"]["generic_params"]:
check_generic_param(param)
check_decl(ty["inner"]["inner"])
elif ty["kind"] == "qualified_path":
check_type(ty["inner"]["self_type"])
check_type(ty["inner"]["trait"])
work_list = set([crate["root"]])
visited = work_list.copy()
while work_list:
current = work_list.pop()
visited.add(current)
item = get_local_item(current)
# check intradoc links
for (_name, link) in item["links"].items():
if not valid_id(link):
print("Intra-doc link contains invalid ID:", link)
# check all fields that reference types such as generics as well as nested items
# (modules, structs, traits, and enums)
if item["kind"] == "module":
work_list |= set(item["inner"]["items"]) - visited
elif item["kind"] == "struct":
check_generics(item["inner"]["generics"])
work_list |= (
set(item["inner"]["fields"]) | set(item["inner"]["impls"])
) - visited
elif item["kind"] == "struct_field":
check_type(item["inner"])
elif item["kind"] == "enum":
check_generics(item["inner"]["generics"])
work_list |= (
set(item["inner"]["variants"]) | set(item["inner"]["impls"])
) - visited
elif item["kind"] == "variant":
if item["inner"]["variant_kind"] == "tuple":
for ty in item["inner"]["variant_inner"]:
check_type(ty)
elif item["inner"]["variant_kind"] == "struct":
work_list |= set(item["inner"]["variant_inner"]) - visited
elif item["kind"] in ("function", "method"):
check_generics(item["inner"]["generics"])
check_decl(item["inner"]["decl"])
elif item["kind"] in ("static", "constant", "assoc_const"):
check_type(item["inner"]["type"])
elif item["kind"] == "typedef":
check_type(item["inner"]["type"])
check_generics(item["inner"]["generics"])
elif item["kind"] == "opaque_ty":
check_generics(item["inner"]["generics"])
for bound in item["inner"]["bounds"]:
check_generic_bound(bound)
elif item["kind"] == "trait_alias":
check_generics(item["inner"]["params"])
for bound in item["inner"]["bounds"]:
check_generic_bound(bound)
elif item["kind"] == "trait":
check_generics(item["inner"]["generics"])
for bound in item["inner"]["bounds"]:
check_generic_bound(bound)
work_list |= (
set(item["inner"]["items"]) | set(item["inner"]["implementors"])
) - visited
elif item["kind"] == "impl":
check_generics(item["inner"]["generics"])
if item["inner"]["trait"]:
check_type(item["inner"]["trait"])
if item["inner"]["blanket_impl"]:
check_type(item["inner"]["blanket_impl"])
check_type(item["inner"]["for"])
for assoc_item in item["inner"]["items"]:
if not valid_id(assoc_item):
print("Impl block referenced a missing ID:", assoc_item)
sys.exit(1)
elif item["kind"] == "assoc_type":
for bound in item["inner"]["bounds"]:
check_generic_bound(bound)
if item["inner"]["default"]:
check_type(item["inner"]["default"])

View file

@ -0,0 +1,129 @@
#!/usr/bin/env python
# This script can check that an expected json blob is a subset of what actually gets produced.
# The comparison is independent of the value of IDs (which are unstable) and instead uses their
# relative ordering to check them against eachother by looking them up in their respective blob's
# `index` or `paths` mappings. To add a new test run `rustdoc --output-format json -o . yourtest.rs`
# and then create `yourtest.expected` by stripping unnecessary details from `yourtest.json`. If
# you're on windows, replace `\` with `/`.
import copy
import sys
import json
import types
# Used instead of the string ids when used as references.
# Not used as keys in `index` or `paths`
class ID(str):
pass
class SubsetException(Exception):
def __init__(self, msg, trace):
self.msg = msg
self.trace = msg
super().__init__("{}: {}".format(trace, msg))
def check_subset(expected_main, actual_main, base_dir):
expected_index = expected_main["index"]
expected_paths = expected_main["paths"]
actual_index = actual_main["index"]
actual_paths = actual_main["paths"]
already_checked = set()
def _check_subset(expected, actual, trace):
expected_type = type(expected)
actual_type = type(actual)
if actual_type is str:
actual = normalize(actual).replace(base_dir, "$TEST_BASE_DIR")
if expected_type is not actual_type:
raise SubsetException(
"expected type `{}`, got `{}`".format(expected_type, actual_type), trace
)
if expected_type in (int, bool, str) and expected != actual:
raise SubsetException("expected `{}`, got: `{}`".format(expected, actual), trace)
if expected_type is dict:
for key in expected:
if key not in actual:
raise SubsetException(
"Key `{}` not found in output".format(key), trace
)
new_trace = copy.deepcopy(trace)
new_trace.append(key)
_check_subset(expected[key], actual[key], new_trace)
elif expected_type is list:
expected_elements = len(expected)
actual_elements = len(actual)
if expected_elements != actual_elements:
raise SubsetException(
"Found {} items, expected {}".format(
expected_elements, actual_elements
),
trace,
)
for expected, actual in zip(expected, actual):
new_trace = copy.deepcopy(trace)
new_trace.append(expected)
_check_subset(expected, actual, new_trace)
elif expected_type is ID and expected not in already_checked:
already_checked.add(expected)
_check_subset(
expected_index.get(expected, {}), actual_index.get(actual, {}), trace
)
_check_subset(
expected_paths.get(expected, {}), actual_paths.get(actual, {}), trace
)
_check_subset(expected_main["root"], actual_main["root"], [])
def rustdoc_object_hook(obj):
# No need to convert paths, index and external_crates keys to ids, since
# they are the target of resolution, and never a source itself.
if "id" in obj and obj["id"]:
obj["id"] = ID(obj["id"])
if "root" in obj:
obj["root"] = ID(obj["root"])
if "items" in obj:
obj["items"] = [ID(id) for id in obj["items"]]
if "variants" in obj:
obj["variants"] = [ID(id) for id in obj["variants"]]
if "fields" in obj:
obj["fields"] = [ID(id) for id in obj["fields"]]
if "impls" in obj:
obj["impls"] = [ID(id) for id in obj["impls"]]
if "implementors" in obj:
obj["implementors"] = [ID(id) for id in obj["implementors"]]
if "links" in obj:
obj["links"] = {s: ID(id) for s, id in obj["links"]}
if "variant_kind" in obj and obj["variant_kind"] == "struct":
obj["variant_inner"] = [ID(id) for id in obj["variant_inner"]]
return obj
def main(expected_fpath, actual_fpath, base_dir):
print(
"checking that {} is a logical subset of {}".format(
expected_fpath, actual_fpath
)
)
with open(expected_fpath) as expected_file:
expected_main = json.load(expected_file, object_hook=rustdoc_object_hook)
with open(actual_fpath) as actual_file:
actual_main = json.load(actual_file, object_hook=rustdoc_object_hook)
check_subset(expected_main, actual_main, base_dir)
print("all checks passed")
def normalize(s):
return s.replace('\\', '/')
if __name__ == "__main__":
if len(sys.argv) < 4:
print("Usage: `compare.py expected.json actual.json test-dir`")
else:
main(sys.argv[1], sys.argv[2], normalize(sys.argv[3]))

View file

@ -0,0 +1,456 @@
{
"root": "0:0",
"version": null,
"includes_private": false,
"index": {
"0:9": {
"crate_id": 0,
"name": "Unit",
"source": {
"filename": "$TEST_BASE_DIR/structs.rs",
"begin": [
7,
0
],
"end": [
7,
16
]
},
"visibility": "public",
"docs": "",
"links": {},
"attrs": [],
"deprecation": null,
"kind": "struct",
"inner": {
"struct_type": "unit",
"generics": {
"params": [],
"where_predicates": []
},
"fields_stripped": false,
"fields": []
}
},
"0:8": {
"crate_id": 0,
"name": "1",
"source": {
"filename": "$TEST_BASE_DIR/structs.rs",
"begin": [
5,
22
],
"end": [
5,
28
]
},
"visibility": "default",
"docs": "",
"links": {},
"attrs": [],
"deprecation": null,
"kind": "struct_field",
"inner": {
"kind": "resolved_path",
"inner": {
"name": "String",
"id": "5:5035",
"args": {
"angle_bracketed": {
"args": [],
"bindings": []
}
},
"param_names": []
}
}
},
"0:18": {
"crate_id": 0,
"name": "stuff",
"source": {
"filename": "$TEST_BASE_DIR/structs.rs",
"begin": [
15,
4
],
"end": [
15,
17
]
},
"visibility": "default",
"docs": "",
"links": {},
"attrs": [],
"deprecation": null,
"kind": "struct_field",
"inner": {
"kind": "resolved_path",
"inner": {
"name": "Vec",
"id": "5:4322",
"args": {
"angle_bracketed": {
"args": [
{
"type": {
"kind": "generic",
"inner": "T"
}
}
],
"bindings": []
}
},
"param_names": []
}
}
},
"0:11": {
"crate_id": 0,
"name": "WithPrimitives",
"source": {
"filename": "$TEST_BASE_DIR/structs.rs",
"begin": [
9,
0
],
"end": [
12,
1
]
},
"visibility": "public",
"docs": "",
"links": {},
"attrs": [],
"deprecation": null,
"kind": "struct",
"inner": {
"struct_type": "plain",
"generics": {
"params": [
{
"name": "'a",
"kind": "lifetime"
}
],
"where_predicates": []
},
"fields_stripped": true
}
},
"0:14": {
"crate_id": 0,
"name": "s",
"source": {
"filename": "$TEST_BASE_DIR/structs.rs",
"begin": [
11,
4
],
"end": [
11,
14
]
},
"visibility": "default",
"docs": "",
"links": {},
"attrs": [],
"deprecation": null,
"kind": "struct_field",
"inner": {
"kind": "borrowed_ref",
"inner": {
"lifetime": "'a",
"mutable": false,
"type": {
"kind": "primitive",
"inner": "str"
}
}
}
},
"0:19": {
"crate_id": 0,
"name": "things",
"source": {
"filename": "$TEST_BASE_DIR/structs.rs",
"begin": [
16,
4
],
"end": [
16,
25
]
},
"visibility": "default",
"docs": "",
"links": {},
"attrs": [],
"deprecation": null,
"kind": "struct_field",
"inner": {
"kind": "resolved_path",
"inner": {
"name": "HashMap",
"id": "1:6600",
"args": {
"angle_bracketed": {
"args": [
{
"type": {
"kind": "generic",
"inner": "U"
}
},
{
"type": {
"kind": "generic",
"inner": "U"
}
}
],
"bindings": []
}
},
"param_names": []
}
}
},
"0:15": {
"crate_id": 0,
"name": "WithGenerics",
"source": {
"filename": "$TEST_BASE_DIR/structs.rs",
"begin": [
14,
0
],
"end": [
17,
1
]
},
"visibility": "public",
"docs": "",
"links": {},
"attrs": [],
"deprecation": null,
"kind": "struct",
"inner": {
"struct_type": "plain",
"generics": {
"params": [
{
"name": "T",
"kind": {
"type": {
"bounds": [],
"default": null
}
}
},
{
"name": "U",
"kind": {
"type": {
"bounds": [],
"default": null
}
}
}
],
"where_predicates": []
},
"fields_stripped": true
}
},
"0:0": {
"crate_id": 0,
"name": "structs",
"source": {
"filename": "$TEST_BASE_DIR/structs.rs",
"begin": [
1,
0
],
"end": [
17,
1
]
},
"visibility": "public",
"docs": "",
"links": {},
"attrs": [],
"deprecation": null,
"kind": "module",
"inner": {
"is_crate": true,
"items": [
"0:4",
"0:5",
"0:9",
"0:11",
"0:15"
]
}
},
"0:13": {
"crate_id": 0,
"name": "num",
"source": {
"filename": "$TEST_BASE_DIR/structs.rs",
"begin": [
10,
4
],
"end": [
10,
12
]
},
"visibility": "default",
"docs": "",
"links": {},
"attrs": [],
"deprecation": null,
"kind": "struct_field",
"inner": {
"kind": "primitive",
"inner": "u32"
}
},
"0:5": {
"crate_id": 0,
"name": "Tuple",
"source": {
"filename": "$TEST_BASE_DIR/structs.rs",
"begin": [
5,
0
],
"end": [
5,
30
]
},
"visibility": "public",
"docs": "",
"links": {},
"attrs": [],
"deprecation": null,
"kind": "struct",
"inner": {
"struct_type": "tuple",
"generics": {
"params": [],
"where_predicates": []
},
"fields_stripped": true
}
},
"0:4": {
"crate_id": 0,
"name": "PlainEmpty",
"source": {
"filename": "$TEST_BASE_DIR/structs.rs",
"begin": [
3,
0
],
"end": [
3,
24
]
},
"visibility": "public",
"docs": "",
"links": {},
"attrs": [],
"deprecation": null,
"kind": "struct",
"inner": {
"struct_type": "plain",
"generics": {
"params": [],
"where_predicates": []
},
"fields_stripped": false,
"fields": []
}
},
"0:7": {
"crate_id": 0,
"name": "0",
"source": {
"filename": "$TEST_BASE_DIR/structs.rs",
"begin": [
5,
17
],
"end": [
5,
20
]
},
"visibility": "default",
"docs": "",
"links": {},
"attrs": [],
"deprecation": null,
"kind": "struct_field",
"inner": {
"kind": "primitive",
"inner": "u32"
}
}
},
"paths": {
"5:4322": {
"crate_id": 5,
"path": [
"alloc",
"vec",
"Vec"
],
"kind": "struct"
},
"5:5035": {
"crate_id": 5,
"path": [
"alloc",
"string",
"String"
],
"kind": "struct"
},
"1:6600": {
"crate_id": 1,
"path": [
"std",
"collections",
"hash",
"map",
"HashMap"
],
"kind": "struct"
}
},
"external_crates": {
"1": {
"name": "std"
},
"5": {
"name": "alloc"
}
},
"format_version": 1
}

View file

@ -0,0 +1,17 @@
use std::collections::HashMap;
pub struct PlainEmpty {}
pub struct Tuple(u32, String);
pub struct Unit;
pub struct WithPrimitives<'a> {
num: u32,
s: &'a str,
}
pub struct WithGenerics<T, U> {
stuff: Vec<T>,
things: HashMap<U, U>,
}