diff --git a/bin/axiomcode b/bin/axiomcode index c3948dfa..99f3f668 100755 --- a/bin/axiomcode +++ b/bin/axiomcode @@ -295,6 +295,13 @@ case "$cmd" in echo "▶ parsing $src → IR" "$0" parser "$src" "$ir" ${popts[@]+"${popts[@]}"} > "$int/parse.log" 2>&1 || { echo "❌ parser failed — see $int/parse.log" >&2; tail -5 "$int/parse.log" >&2; exit 1; } head -1 "$int/parse.log" + # WHAT AN INSTALLED PYTHON LIBRARY HANDS BACK to the objects the client gives it (graph/python/libsum): read from + # the project's environment, cached by package and version, staged beside the client IR. Never fatal -- without + # it the engine sees no library hand-backs, exactly as before. AXIOMCODE_LIB_CALLBACKS=0 turns it off. + if [ -d "$ir/python" ] && [ "${AXIOMCODE_LIB_CALLBACKS:-1}" != 0 ]; then + python3 "$ROOT/graph/python/libsum/libsum.py" --src "$src" --out "$ir/python" > "$int/libsum.log" 2>&1 \ + && tail -1 "$int/libsum.log" || echo "⚠️ library callback summaries failed (see $int/libsum.log); continuing without them" >&2 + fi # which languages the IR holds — one folder each — and which of them have an engine found=(); for d in "$ir"/*/; do l="$(basename "$d")"; [ -d "$ROOT/graph/$l/engine" ] && found+=("$l"); done [ ${#found[@]} -gt 0 ] || { echo "❌ no language with an engine in the IR (folders: $(ls "$ir" | tr '\n' ' '))" >&2; exit 1; } diff --git a/graph/bundle/SCHEMA.md b/graph/bundle/SCHEMA.md index 2c17b391..8a5c923b 100644 --- a/graph/bundle/SCHEMA.md +++ b/graph/bundle/SCHEMA.md @@ -442,7 +442,7 @@ One row per place a call is written (or, for a synthesised edge, the construct t - **typescript** — end_line / end_column come from the expression row; the call-site row itself records only the start. - **javascript** — caller_id is the parser's enclosing method, or the module initializer for top-level code. end_line / end_column come from the expression row. `require()` is a module edge, not a call site. - **typescript** — PROPERTY_READ and PROPERTY_WRITE rows are accessor invocations with no written call: the site is the property-access expression that runs the getter or setter, positioned from the expressions table, and callee_name is NULL because nothing was written; the accessor's name is on the callee's methods row. Filter them out with kind NOT IN (…) when counting calls. -- **python** — PROPERTY_READ, CONTEXT_MANAGER, ITERATION_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls. +- **python** — PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls. - **python** — The id is an EXPRESSION hash for a written call; a DECORATOR hash (PY_DECORATOR_…) for DECORATOR_APPLICATION and DECORATOR_* sites, positioned at the decorator line; and the class's TYPE hash for METACLASS_CREATION, positioned at the class declaration. ### `call_edges` @@ -467,6 +467,7 @@ THE GRAPH. One row per (site, resolved target). A site with N possible targets h | `new` | csharp | An object creation. The target is the constructed type's constructor, and it is never dispatched. | | `ctor_delegate` | csharp | `: this(...)` or `: base(...)`. No name is written, so the target is structural. | | `primary_ctor_base` | csharp | SYNTHESISED. A primary constructor's base invocation, written in the heritage clause: `class D(int a) : B(a)`. There is no call syntax anywhere in the body. FromExpr is the heritage type reference. | +| `implicit_base_ctor` | csharp | SYNTHESISED. The base class's parameterless constructor that a constructor with no `: base(...)` / `: this(...)` runs before its body, or that the implicit constructor of a class declaring none runs at its `new`. FromExpr is the heritage type reference, or the `new` site. | | `delegate` | csharp | A call through a delegate value: `handler(x)` or `handler.Invoke(x)`. | | `operator` | csharp | A user-defined operator invoked by operator syntax. | | `conversion` | csharp | A user-defined conversion. An implicit one has no syntax at the call site. | @@ -531,9 +532,11 @@ THE GRAPH. One row per (site, resolved target). A site with N possible targets h | `DECORATOR_*` | python | Applying an unparenthesised decorator; the suffix is the parser's decorator kind: BARE, ATTRIBUTE, SUBSCRIPT, EXPRESSION (and CALL/ATTRIBUTE_CALL when the factory expression is not itself a call site). The site is the decorator hash. | | `METACLASS_CREATION` | python | A class statement invokes its metaclass's `__new__` / `__init__` at import time, whether the metaclass is written on the statement (`class X(metaclass=M)`) or inherited from a base, and the nearest base's `__init_subclass__`. No written call; the site is the class's type hash. | | `PROPERTY_READ` | python | Reading `obj.attr` where `attr` is a `@property` runs the getter; reading `Cls.attr` where the METACLASS defines `attr` as a property runs that getter. No written call; the site is the attribute-access expression. | +| `PROPERTY_WRITE` | python | Assigning `obj.attr = v` where `attr` is a `@property` with a setter runs the setter; `del obj.attr` runs its deleter. No written call; the site is the attribute-access expression. | | `CONTEXT_MANAGER` | python | `with expr:` runs `__enter__` / `__exit__` (or the async pair). No written call; the site is the context-manager expression. | | `ITERATION_PROTOCOL` | python | `for x in expr:` (and comprehensions) runs `__iter__` / `__next__` (or the async pair). No written call; the site is the iterated expression. | | `SUBSCRIPT_PROTOCOL` | python | `x[k]` runs `__getitem__` (and `x[k] = v` / `del x[k]` the setter and deleter) of the receiver's class. No written call; the site is the subscript expression. Its own kind so it is never counted as a written call. | +| `BUILTIN_PROTOCOL` | python | `repr(x)`, `str(x)`, `len(x)`, `hash(x)`, `bool(x)`, `iter(x)`, `next(x)`, `abs(x)`, `format(x)` and `reversed(x)` run the matching dunder of the argument's class (str falls back to `__repr__`, bool to `__len__`). The written call is to the builtin; this edge is the dunder it runs. The site is the argument expression. Its own kind so it is never counted as a written call. | **`call_edges.tier` values** diff --git a/graph/bundle/schema.ts b/graph/bundle/schema.ts index 2e9af322..306817ac 100644 --- a/graph/bundle/schema.ts +++ b/graph/bundle/schema.ts @@ -358,6 +358,7 @@ export const VOCAB: readonly VocabSpec[] = [ { table: 'call_edges', column: 'kind', value: 'new', languages: C, meaning: 'An object creation. The target is the constructed type\'s constructor, and it is never dispatched.' }, { table: 'call_edges', column: 'kind', value: 'ctor_delegate', languages: C, meaning: '`: this(...)` or `: base(...)`. No name is written, so the target is structural.' }, { table: 'call_edges', column: 'kind', value: 'primary_ctor_base', languages: C, meaning: 'SYNTHESISED. A primary constructor\'s base invocation, written in the heritage clause: `class D(int a) : B(a)`. There is no call syntax anywhere in the body. FromExpr is the heritage type reference.' }, + { table: 'call_edges', column: 'kind', value: 'implicit_base_ctor', languages: C, meaning: 'SYNTHESISED. The base class\'s parameterless constructor that a constructor with no `: base(...)` / `: this(...)` runs before its body, or that the implicit constructor of a class declaring none runs at its `new`. FromExpr is the heritage type reference, or the `new` site.' }, { table: 'call_edges', column: 'kind', value: 'delegate', languages: C, meaning: 'A call through a delegate value: `handler(x)` or `handler.Invoke(x)`.' }, { table: 'call_edges', column: 'kind', value: 'operator', languages: C, meaning: 'A user-defined operator invoked by operator syntax.' }, { table: 'call_edges', column: 'kind', value: 'conversion', languages: C, meaning: 'A user-defined conversion. An implicit one has no syntax at the call site.' }, @@ -667,9 +668,11 @@ export const VOCAB: readonly VocabSpec[] = [ { table: 'call_edges', column: 'kind', value: 'DECORATOR_*', languages: P, meaning: 'Applying an unparenthesised decorator; the suffix is the parser\'s decorator kind: BARE, ATTRIBUTE, SUBSCRIPT, EXPRESSION (and CALL/ATTRIBUTE_CALL when the factory expression is not itself a call site). The site is the decorator hash.' }, { table: 'call_edges', column: 'kind', value: 'METACLASS_CREATION', languages: P, meaning: 'A class statement invokes its metaclass\'s `__new__` / `__init__` at import time, whether the metaclass is written on the statement (`class X(metaclass=M)`) or inherited from a base, and the nearest base\'s `__init_subclass__`. No written call; the site is the class\'s type hash.' }, { table: 'call_edges', column: 'kind', value: 'PROPERTY_READ', languages: P, meaning: 'Reading `obj.attr` where `attr` is a `@property` runs the getter; reading `Cls.attr` where the METACLASS defines `attr` as a property runs that getter. No written call; the site is the attribute-access expression.' }, + { table: 'call_edges', column: 'kind', value: 'PROPERTY_WRITE', languages: P, meaning: 'Assigning `obj.attr = v` where `attr` is a `@property` with a setter runs the setter; `del obj.attr` runs its deleter. No written call; the site is the attribute-access expression.' }, { table: 'call_edges', column: 'kind', value: 'CONTEXT_MANAGER', languages: P, meaning: '`with expr:` runs `__enter__` / `__exit__` (or the async pair). No written call; the site is the context-manager expression.' }, { table: 'call_edges', column: 'kind', value: 'ITERATION_PROTOCOL', languages: P, meaning: '`for x in expr:` (and comprehensions) runs `__iter__` / `__next__` (or the async pair). No written call; the site is the iterated expression.' }, { table: 'call_edges', column: 'kind', value: 'SUBSCRIPT_PROTOCOL', languages: P, meaning: '`x[k]` runs `__getitem__` (and `x[k] = v` / `del x[k]` the setter and deleter) of the receiver\'s class. No written call; the site is the subscript expression. Its own kind so it is never counted as a written call.' }, + { table: 'call_edges', column: 'kind', value: 'BUILTIN_PROTOCOL', languages: P, meaning: '`repr(x)`, `str(x)`, `len(x)`, `hash(x)`, `bool(x)`, `iter(x)`, `next(x)`, `abs(x)`, `format(x)` and `reversed(x)` run the matching dunder of the argument\'s class (str falls back to `__repr__`, bool to `__len__`). The written call is to the builtin; this edge is the dunder it runs. The site is the argument expression. Its own kind so it is never counted as a written call.' }, // entry_points.reason { table: 'entry_points', column: 'reason', value: 'main', languages: ['java', 'csharp'], meaning: 'A static `main`. C#: a static `Main`, or the method top-level statements compile to.' }, @@ -748,7 +751,7 @@ export const NOTES: readonly NoteSpec[] = [ { language: 'typescript', table: 'overrides', note: 'EMPTY — this table is Java-shaped. The TypeScript dispatch envelope is in dispatch_candidates, with basis `nominal` or `structural`.' }, { language: 'typescript', table: 'type_instantiated', note: 'Every row has how = `new`. Not restricted to client provenance: a type the library constructs is still a type that exists at run time, and dropping it would narrow the envelope unsoundly.' }, { language: 'typescript', table: 'call_sites', note: 'PROPERTY_READ and PROPERTY_WRITE rows are accessor invocations with no written call: the site is the property-access expression that runs the getter or setter, positioned from the expressions table, and callee_name is NULL because nothing was written; the accessor\'s name is on the callee\'s methods row. Filter them out with kind NOT IN (…) when counting calls.' }, - { language: 'python', table: 'call_sites', note: 'PROPERTY_READ, CONTEXT_MANAGER, ITERATION_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls.' }, + { language: 'python', table: 'call_sites', note: 'PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, BUILTIN_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls.' }, { language: 'python', table: 'call_sites', note: 'The id is an EXPRESSION hash for a written call; a DECORATOR hash (PY_DECORATOR_…) for DECORATOR_APPLICATION and DECORATOR_* sites, positioned at the decorator line; and the class\'s TYPE hash for METACLASS_CREATION, positioned at the class declaration.' }, { language: 'python', table: 'call_edges', note: 'A `boundary_lib` edge may point at a builtin (callee_provenance builtin, callee_label `builtin:NAME`) or at an unstaged import path (callee_provenance external) — neither has a methods row.' }, { language: 'java', table: 'call_edges', note: 'A `boundary_lib` edge with callee_provenance external names a method of an ancestor type no staged IR declares (callee_label `external:.`, no methods row). A site whose receiver is declared as such a type is multi_inferred even with one client override: the platform method itself, and the platform\'s own subclasses, are the other possible targets. Stage the library to replace the label with the real method.' }, diff --git a/graph/csharp/engine/call-edge-generation/call_chain.dl b/graph/csharp/engine/call-edge-generation/call_chain.dl index 69647979..c0e3de06 100644 --- a/graph/csharp/engine/call-edge-generation/call_chain.dl +++ b/graph/csharp/engine/call-edge-generation/call_chain.dl @@ -286,6 +286,52 @@ call_chain_edge(tr, caller, "-", ctor, "lib", "boundary_lib", "primary_ctor_base method_decl("lib", _, _, _, ctor), type_ctor("client", gk, caller). +// ── SYNTHESISED EDGES: THE IMPLICIT `base()` ──────────────────────────────── +// A constructor that writes neither `: base(..)` nor `: this(..)` runs its base +// class's parameterless constructor before its own body, and a class that declares +// no constructor at all gets one that does the same. Neither is written anywhere, so +// without these edges a base constructor that every derived class runs has no caller. +// +// implicit_base_ctor(TypeGroup, Ctor): the constructor `base()` runs for a type -- +// its base class's constructor that takes no argument, or, where the base declares +// no constructor either, the one the base's own implicit constructor runs. +implicit_base_ctor(gk, ctor) :- + type_extends("client", gk, bgk), + type_ctor("client", bgk, ctor), ctor_accepts_argc("client", ctor, "0"). +implicit_base_ctor(gk, ctor) :- + type_extends("client", gk, bgk), + !type_ctor("client", bgk, _), type_group("client", _, bgk), + implicit_base_ctor(bgk, ctor). + +// A constructor that hands off with `: this(..)` or `: base(..)`, or a primary +// constructor whose heritage passes arguments: the base constructor it runs is the +// one written there, already an edge. +ctor_has_initializer(m) :- call_is_base_ctor("client", e), expr_ultimate_method("client", e, m). +ctor_has_initializer(m) :- call_is_this_ctor("client", e), expr_ultimate_method("client", e, m). +ctor_has_initializer(m) :- + heritage_has_ctor_args("client", t, _), type_group("client", t, gk), type_ctor("client", gk, m). + +// The FromExpr is the base class's entry in the heritage clause, as for a primary +// constructor's base invocation: the closest thing to where the call is written. +implicit_base_site(gk, tr) :- + type_extends("client", gk, bgk), + heritage_resolves("client", gk, pos, bgk), + heritage_of_entity("client", gk, pos, _, _, tr), tr != "". + +// (1) A declared constructor with no initializer. +call_chain_edge(tr, m, "-", ctor, "client", "known_edge", "implicit_base_ctor") :- + type_ctor("client", gk, m), method_decl("client", _, _, _, m), + !ctor_has_initializer(m), + implicit_base_ctor(gk, ctor), + implicit_base_site(gk, tr). + +// (2) `new T()` where T declares no constructor: the implicit one runs T's base's. +// The edge sits on the `new` site itself, beside its known_implicit_ctor row. +call_chain_edge(e, caller, "-", ctor, "client", "known_edge", "implicit_base_ctor") :- + call_ctor_implicit("client", e, gk), + implicit_base_ctor(gk, ctor), + call_from_expr("client", e, caller). + // ── ACCESSOR EDGES: A PROPERTY READ IS A CALL ─────────────────────────────── // `x.Name` invokes get_Name, `a[i]` an indexer accessor, `e += h` an add accessor. // These have no cs_call_site row -- the parser cannot know whether `x.Name` is a diff --git a/graph/csharp/engine/containment/type-nesting.dl b/graph/csharp/engine/containment/type-nesting.dl index e584db41..8ac8b94d 100644 --- a/graph/csharp/engine/containment/type-nesting.dl +++ b/graph/csharp/engine/containment/type-nesting.dl @@ -149,3 +149,27 @@ module_in_scope(prov, mod, outer) :- // The GLOBAL namespace ("") is always in scope. module_in_scope(prov, mod, "") :- module_decl(prov, _, _, _, mod). + +// module_ns_chain(Prov, ModuleHash, Namespace) -- the namespaces a name written in this +// file is read from: the file's own namespaces and every dotted prefix of them, as +// text. A prefix that holds no type of its own (`Acme` over `Acme.Shop`) still +// qualifies a name written below it, so this is not ns_encloses, which needs one. +module_ns_chain(prov, mod, ns) :- module_namespace_own(prov, mod, ns), ns != "". +module_ns_chain(prov, mod, p) :- module_namespace_own(prov, mod, inner), ns_prefix(prov, inner, p). + +// ── A USING WRITTEN INSIDE A NAMESPACE ───────────────────────────────────── +// `namespace Shop.Tests; using Results;` imports Shop.Results. C# reads the name in +// a using directive from the namespace the directive sits in, innermost outward, +// exactly as it reads any other name there. Taken as written it names a namespace +// that does not exist, and every type the file names through it reads as a library +// type. +// +// The IR does not record whether a using sits inside the namespace or above it. +// The language decides it instead: a using whose name is no namespace at all only +// compiles inside a namespace that has it as a child. So the relative reading is +// taken only when the name as written names nothing, and only for a child that +// exists -- a file whose namespaces have no such child gains nothing. +module_in_scope(prov, mod, full) :- + using_namespace(prov, mod, n), !ns_exists(prov, n), + module_ns_chain(prov, mod, e), full = cat(e, cat(".", n)), + ns_exists(prov, full). diff --git a/graph/csharp/engine/expression-resolution/expr-type.dl b/graph/csharp/engine/expression-resolution/expr-type.dl index 5c962379..fe5e35e7 100644 --- a/graph/csharp/engine/expression-resolution/expr-type.dl +++ b/graph/csharp/engine/expression-resolution/expr-type.dl @@ -459,6 +459,25 @@ expr_type(prov, e, gk) :- expr_type(prov, q, qgk), member_lookup(prov, qgk, n, "field", f), field_type(prov, f, gk). +// A STATIC member read through its TYPE NAME: `Context.Current.Factory.Create()`. +// The qualifier `Context` names a type, not a value, so it has no expr_type and the +// two clauses above cannot start from it: the getter of `Current` is an edge +// (properties.dl reads the type name), and these give its RESULT a type, so that +// `.Factory` and every call after it resolve. ref_names_type already refuses a name +// that also binds a value, which is the language's member-over-type rule. +expr_type(prov, e, gk) :- + expr_kind(prov, e, "MEMBER_ACCESS"), + expr_qualifier_child(prov, e, q), expr_member_name_child(prov, e, nameExpr), + expr_written_name(prov, nameExpr, n), + ref_names_type(prov, q, qgk), + member_lookup(prov, qgk, n, "property", p), property_type(prov, p, gk). +expr_type(prov, e, gk) :- + expr_kind(prov, e, "MEMBER_ACCESS"), + expr_qualifier_child(prov, e, q), expr_member_name_child(prov, e, nameExpr), + expr_written_name(prov, nameExpr, n), + ref_names_type(prov, q, qgk), + member_lookup(prov, qgk, n, "field", f), field_type(prov, f, gk). + // ── A GENERIC ARGUMENT, SUBSTITUTED FOR THE PARAMETER IT FILLS ────────────── // `IWrap w; w.Value.Flag` -- `Value` is declared `T`, and without binding // T to Settings the access resolves and the type of the RESULT is the type parameter, diff --git a/graph/csharp/engine/projections/expressions.dl b/graph/csharp/engine/projections/expressions.dl index 46e28c76..8a82e1ba 100644 --- a/graph/csharp/engine/projections/expressions.dl +++ b/graph/csharp/engine/projections/expressions.dl @@ -117,6 +117,15 @@ expr_written_name("lib", e, n) :- lib_cs_expression(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, n, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, e), n != "". +// ── expr_type_argc(Prov, ExprHash, TypeArgumentCount) ────────────────────── +// The type arguments written on a name: `Cache` is the name `Cache` at arity 1. +// A generic name used as a receiver or qualifier is resolved at this arity, since +// `Cache` and `Cache` are two types. +expr_type_argc("client", e, tac) :- + cs_expression(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, tac, _, _, _, _, _, _, _, _, _, _, _, _, e). +expr_type_argc("lib", e, tac) :- + lib_cs_expression(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, tac, _, _, _, _, _, _, _, _, _, _, _, _, e). + // ── expr_literal(Prov, ExprHash, LiteralKind, LiteralValue) ───────────────── expr_literal("client", e, lk, lv) :- cs_expression(_, _, _, _, _, _, _, _, _, _, lk, lv, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, e), diff --git a/graph/csharp/engine/projections/variables.dl b/graph/csharp/engine/projections/variables.dl index 27be6bf5..841c1f86 100644 --- a/graph/csharp/engine/projections/variables.dl +++ b/graph/csharp/engine/projections/variables.dl @@ -61,6 +61,10 @@ var_initializer("lib", v, e) :- // The initializer IS a value of the variable's type. var_value_initializer(prov, v, e) :- var_initializer(prov, v, e), var_decl(prov, _, "LOCAL", _, v). var_value_initializer(prov, v, e) :- var_initializer(prov, v, e), var_decl(prov, _, "CONST", _, v). +// A `using` resource -- `using (var w = new Writer())` and `using var w = ...;` -- is +// the disposable itself, so its initializer is a value of its type exactly as a +// local's is. Without it every call on such a resource has an untyped receiver. +var_value_initializer(prov, v, e) :- var_initializer(prov, v, e), var_decl(prov, _, "USING", _, v). // The initializer is a COLLECTION and the variable takes its element type. var_iterated_initializer(prov, v, e) :- var_initializer(prov, v, e), var_decl(prov, _, "FOREACH", _, v). diff --git a/graph/csharp/engine/resolution/extensions.dl b/graph/csharp/engine/resolution/extensions.dl index 23516414..5a09820c 100644 --- a/graph/csharp/engine/resolution/extensions.dl +++ b/graph/csharp/engine/resolution/extensions.dl @@ -118,6 +118,25 @@ call_recv_type_name(prov, e, tn) :- call_recv_type_name(prov, e, tn) :- call_receiver_expr(prov, e, r), ref_denotes(prov, r, "LAMBDA_PARAMETER", p), param_type_name(prov, p, tn, _). +// AN IMPLICITLY TYPED LAMBDA PARAMETER takes its type from the delegate it is +// converted to (lambda-parameters.dl): `Configure(s => s.AddRules())` with +// `Configure(Action c)` makes `s` an IServiceCollection. Where +// that type is in source, param_type already says so; where it is unstaged, the +// NAME is all there is, and without it an extension written for +// `this IServiceCollection` cannot match the lambda that calls it. +call_recv_type_name(prov, e, tn) :- + call_receiver_expr(prov, e, r), + ref_denotes(prov, r, "LAMBDA_PARAMETER", p), + lambda_param_arg_ref(prov, p, ar), type_ref_name(prov, ar, tn), tn != "". +// AN IMPLICITLY TYPED LOCAL made by `new T(..)` holds a T, resolved or not: +// `var services = new ServiceCollection(); services.AddRules();`. The name is the +// one written at the creation (external-types.dl labels the boundary from the same +// fact); without it the extension rungs have nothing to compare. +call_recv_type_name(prov, e, tn) :- + call_receiver_expr(prov, e, r), + ref_denotes(prov, r, "LOCAL_VARIABLE", v), var_is_implicit(prov, v), + var_value_initializer(prov, v, init), expr_kind(prov, init, "OBJECT_CREATION"), + call_callee_name(prov, init, tn), tn != "". // A receiver that is itself a call: the callee's declared return type name. The ROOT // reference's name where there is one (`List`, not `List`), which is how every // other receiver is named; the written text only where no reference was recorded. diff --git a/graph/csharp/engine/resolution/local-binding.dl b/graph/csharp/engine/resolution/local-binding.dl index f2ec9440..e7bc391f 100644 --- a/graph/csharp/engine/resolution/local-binding.dl +++ b/graph/csharp/engine/resolution/local-binding.dl @@ -220,7 +220,8 @@ ref_unbound(prov, e, n) :- !ref_names_type(prov, e, _). // A reference that names a TYPE rather than a value: the receiver of a static call. -// Resolved through type_name_resolves in the file's scope. +// Resolved through type_name_resolves in the file's scope, at the arity written on +// it: `Cache.Get()` names Cache`1, not a non-generic Cache. // // NOT WHERE THE NAME ALSO BINDS TO A VALUE. C# allows a member and a type to share a // name (the language's own "Color Color" rule) and prefers the MEMBER in an @@ -235,10 +236,10 @@ ref_binds_value(prov, e) :- ref_denotes(prov, e, _, _). ref_binds_value(prov, e) :- local_binds(prov, e, _). ref_names_type(prov, e, gk) :- - expr_written_name(prov, e, n), + expr_written_name(prov, e, n), expr_type_argc(prov, e, ar), expr_ultimate_module(prov, e, mod), !ref_binds_value(prov, e), - type_name_resolves(prov, mod, n, "0", gk). + type_name_resolves(prov, mod, n, ar, gk). // ── IMPLICIT `this` ───────────────────────────────────────────────────────── // An unqualified instance member reference has an implicit `this` receiver whose diff --git a/graph/csharp/engine/resolution/type-resolution.dl b/graph/csharp/engine/resolution/type-resolution.dl index fb90ddc4..f4d1b803 100644 --- a/graph/csharp/engine/resolution/type-resolution.dl +++ b/graph/csharp/engine/resolution/type-resolution.dl @@ -89,6 +89,26 @@ type_name_cand(prov, mod, qn, ar, gk, 1) :- type_decl(prov, _, qn, ar, _, t), type_group(prov, t, gk), !type_is_top_level(prov, gk). +// ── A QUALIFIED NAME READ FROM ITS NAMESPACE ──────────────────────────────── +// `Reflection.Info.Create(..)` written inside namespace Shop names +// Shop.Reflection.Info: the first segment of a qualified name is a +// simple name, looked up in the enclosing namespaces like any other. Matched only +// in full, such a receiver stays untyped and every call through it unresolved. +// +// Keyed from the DECLARATION side, so nothing is enumerated per written name: each +// type's qualified name is split at each dot into the namespace before it and the +// dotted tail after it, and the tail is a candidate in every file whose namespace +// chain holds that prefix. A tail with no dot is a simple name, which ranks 2-5 +// already answer. +type_qn_tail(prov, qn, p, s) :- + type_decl(prov, _, qn, _, _, _), + i = range(1, strlen(qn)), substr(qn, i, 1) = ".", + p = substr(qn, 0, i), s = substr(qn, i + 1, strlen(qn) - i - 1), + contains(".", s). +type_name_cand(prov, mod, s, ar, gk, 1) :- + type_qn_tail(prov, qn, p, s), module_ns_chain(prov, mod, p), + type_decl(prov, _, qn, ar, _, t), type_group(prov, t, gk). + // ── A CLIENT FILE NAMING A STAGED LIBRARY TYPE ────────────────────────────── // Every clause above is SINGLE-PROVENANCE: it joins a module and a declaration // under one `prov`, so a client module and a library namespace are never brought @@ -211,10 +231,10 @@ type_ref_demand(prov, mod, n, ar) :- // a type name and produced `external:s.Describe` -- an external target named after a // local variable, which is a wrong answer rather than a missing one. The !ref_denotes // term is what makes the guard "unbound by anyone". -type_ref_demand(prov, mod, n, "0") :- +type_ref_demand(prov, mod, n, ar) :- call_receiver_expr(prov, e, r), expr_ref_unknown(prov, r), - expr_written_name(prov, r, n), + expr_written_name(prov, r, n), expr_type_argc(prov, r, ar), !ref_denotes(prov, r, _, _), call_module(prov, e, mod). diff --git a/graph/csharp/souffle/decls_all.dl b/graph/csharp/souffle/decls_all.dl index b1faa46b..89a6ff36 100644 --- a/graph/csharp/souffle/decls_all.dl +++ b/graph/csharp/souffle/decls_all.dl @@ -65,9 +65,9 @@ .decl call_callee_name(c0:symbol,c1:symbol,c2:symbol) .decl call_caller_unknown(c0:symbol,c1:symbol) .decl call_chain_edge(c0:symbol,c1:symbol,c2:symbol,c3:symbol,c4:symbol,c5:symbol,c6:symbol) -.decl call_class_kind(c0:symbol) .decl call_chain_summary(c0:symbol,c1:number) .decl call_class(c0:symbol,c1:symbol,c2:symbol) +.decl call_class_kind(c0:symbol) .decl call_context(c0:symbol,c1:symbol,c2:symbol,c3:symbol,c4:symbol) .decl call_ctor_implicit(c0:symbol,c1:symbol,c2:symbol) .decl call_ctor_target(c0:symbol,c1:symbol,c2:symbol) @@ -265,6 +265,7 @@ .decl ctl_route_c(c0:symbol,c1:symbol,c2:symbol) .decl ctl_route_tok(c0:symbol,c1:symbol,c2:symbol) .decl ctor_accepts_argc(c0:symbol,c1:symbol,c2:symbol) +.decl ctor_has_initializer(c0:symbol) .decl ctor_implicit_type(c0:symbol,c1:symbol) .decl decl_return_param(c0:symbol,c1:symbol,c2:symbol,c3:symbol) .decl delegate_arg_expr(c0:symbol,c1:symbol) @@ -359,6 +360,7 @@ .decl expr_target_any(c0:symbol,c1:symbol,c2:symbol) .decl expr_target_type(c0:symbol,c1:symbol,c2:symbol) .decl expr_type(c0:symbol,c1:symbol,c2:symbol) +.decl expr_type_argc(c0:symbol,c1:symbol,c2:symbol) .decl expr_type_owner(c0:symbol,c1:symbol,c2:symbol) .decl expr_type_via_ref(c0:symbol,c1:symbol,c2:symbol) .decl expr_ultimate_method(c0:symbol,c1:symbol,c2:symbol) @@ -487,8 +489,10 @@ .decl implements_member(c0:symbol,c1:symbol,c2:symbol) .decl implements_params_conflict(c0:symbol,c1:symbol) .decl implements_shape(c0:symbol,c1:symbol,c2:symbol) +.decl implicit_base_ctor(c0:symbol,c1:symbol) .decl implicit_base_group(c0:symbol,c1:symbol) .decl implicit_base_name(c0:symbol) +.decl implicit_base_site(c0:symbol,c1:symbol) .decl implicit_new_untargeted(c0:symbol,c1:symbol,c2:symbol) .decl indexer_access(c0:symbol,c1:symbol,c2:symbol) .decl indexer_getter(c0:symbol,c1:symbol,c2:symbol) @@ -663,6 +667,7 @@ .decl module_lang(c0:symbol,c1:symbol,c2:symbol,c3:symbol) .decl module_namespace_own(c0:symbol,c1:symbol,c2:symbol) .decl module_namespace_style(c0:symbol,c1:symbol,c2:symbol,c3:symbol) +.decl module_ns_chain(c0:symbol,c1:symbol,c2:symbol) .decl module_parse_errors(c0:symbol,c1:symbol,c2:symbol,c3:symbol) .decl module_regime(c0:symbol,c1:symbol,c2:symbol,c3:symbol) .decl module_target(c0:symbol,c1:symbol,c2:symbol,c3:symbol) @@ -948,6 +953,7 @@ .decl type_parent(c0:symbol,c1:symbol,c2:symbol) .decl type_placement(c0:symbol,c1:symbol,c2:symbol) .decl type_primary_ctor(c0:symbol,c1:symbol,c2:symbol) +.decl type_qn_tail(c0:symbol,c1:symbol,c2:symbol,c3:symbol) .decl type_ref(c0:symbol,c1:symbol,c2:symbol,c3:symbol,c4:symbol,c5:symbol,c6:symbol) .decl type_ref_arg(c0:symbol,c1:symbol,c2:symbol,c3:symbol) .decl type_ref_argc(c0:symbol,c1:symbol,c2:symbol) diff --git a/graph/javascript/engine/call-edge-generation/calls.dl b/graph/javascript/engine/call-edge-generation/calls.dl index 39dbe789..5745b29d 100644 --- a/graph/javascript/engine/call-edge-generation/calls.dl +++ b/graph/javascript/engine/call-edge-generation/calls.dl @@ -170,6 +170,29 @@ call_chain_edge(ce, caller, "-", m, "client", "multi_inferred", kind) :- wrapper_call_target(ce, m), !call_over_cap(ce), call_from(ce, caller), invocation_site(ce, kind). variable_reassigned(v) :- expr_kind(_, "ASSIGNMENT", _, a), expr_child(_, a, "ASSIGNMENT_TARGET", _, tgt), expr_binding(_, v, tgt). +// ── a call of what a PROJECT wrapper returned runs what that site was handed ── +// `export default promisable(eachItem, 3)`, `const add = curry2(function add(a, b) {…})`: +// the wrapper returns a closure that calls its parameter, and every export of the module +// is that closure. The closure's call of the parameter is read context-insensitively, so +// it held every function any of the wrapper's sites passed (past the fan cap: nothing), +// and a test calling the export reached the closure and stopped there. The site is +// already a value of its own, ("wrap", site), beside the closure (value-flow.dl), and a +// call through it takes its RESULT from that site's argument alone; the same site says +// what the call RUNS: the function it was handed at the forwarded position, what the +// wrapper writes into that parameter, or what a wrapped argument itself runs +// (`memoize(once(f))`). One of a set, beside the edge to the closure the call keeps, and +// under the same fan cap: a callee that may hold more wrapped values than --dispatch-cap +// (a parameter every wrapped export is passed to) runs one of them, and naming all of +// them says nothing. +wrap_runs(s, g) :- wrap_site_arg(s, _, arg), expr_value(arg, "func", g). +wrap_runs(s, g) :- wrap_site_arg(s, _, arg), expr_value(arg, "wrap", s2), wrap_runs(s2, g). +wrap_runs(s, g) :- wrap_site(s, h, pos), wrapper_param_written(h, pos, o), expr_value(o, "func", g). +wrapped_call_target(ce, g) :- callee_value(ce, "wrap", s), wrap_runs(s, g), method_prov(g, "client"), + !call_site(_, "FUNCTION_CALL_BIND", _, _, _, _, ce, _, _). +wrapped_target_count(ce, n) :- wrapped_call_target(ce, _), n = count : { wrapped_call_target(ce, _) }. +wrapped_over_cap(ce) :- wrapped_target_count(ce, n), dispatch_cap(c), n > to_number(c). +call_chain_edge(ce, caller, "-", g, "client", "multi_inferred", kind) :- + wrapped_call_target(ce, g), !call_over_cap(ce), !wrapped_over_cap(ce), call_from(ce, caller), invocation_site(ce, kind). // `(c ? a : b)()`, `(0, cb)()`, `make()()`: the callee is computed by an expression. // `new Function(s)()` / `Function(s)()` runs code built at run time. unresolved_value_callee(ce, "dynamic_code") :- value_callee_unresolved(ce, c), callee_builds_code(c). diff --git a/graph/javascript/engine/resolution/fan-cap.dl b/graph/javascript/engine/resolution/fan-cap.dl index af6bdd93..ad69acdb 100644 --- a/graph/javascript/engine/resolution/fan-cap.dl +++ b/graph/javascript/engine/resolution/fan-cap.dl @@ -80,14 +80,48 @@ direct_export_func(mod, n, m) :- export_decl(_, n, _, _, _, _, _, se, mod, _), s // shares its name with hundreds of unrelated sites and must not be capped by them, // while a module-level `each` or `merge` called under its own name from everywhere // is exactly the generic utility the cap exists for. +// +// The PROGRAM's arguments are decided by the program's sites alone. A suite calls the +// API it tests over and over, and counted with the program those calls made exactly the +// functions under test "generic utilities": their parameters went untracked, so a +// callback the program hands a constructor and stores on the instance (`this.decode = +// options.decode`) was lost, and the program's own call through that field resolved only +// to the fallback beside it. So there are two verdicts: +// method_is_hot every site counts (as before): decides whether a TEST's +// arguments flow in +// method_is_hot_in_program test files' sites do not count: decides whether the +// PROGRAM's arguments flow in +// Nothing that flowed before stops flowing (a test handing the program a plugin it calls +// back still reaches it). A function hot only because of its tests now receives what the +// program passes it and still nothing from the tests: their callbacks would be the widest +// values it holds, and with them the call through the stored field would be over the +// dispatch cap and answer nothing. param_flow_capped_at(n) :- dispatch_cap(s), n = to_number(s). direct_call_count(m, n) :- direct_call_site(_, m), n = count : { direct_call_site(_, m) }. method_is_hot(m, n) :- direct_call_count(m, n), param_flow_capped_at(c), n > c. name_call_count(cn, n) :- call_site(_, _, cn, _, _, _, _, _, _), cn != "", n = count : { call_site(_, _, cn, _, _, _, _, _, _) }. +program_direct_site(ce, m) :- direct_call_site(ce, m), !site_in_test_module(ce). +program_direct_count(m, n) :- program_direct_site(_, m), n = count : { program_direct_site(_, m) }. +method_is_hot_in_program(m, n) :- program_direct_count(m, n), param_flow_capped_at(c), n > c. +// The NAME count is the overcount for module-level functions. A test's member call that +// only shares the name (`lib.add(1, 2)` on an export that is a different function, a +// curried closure) stays in it, as before; a test's bare call or `new` of the name +// (`build(...)`, `new lib.Codec(...)`) denotes the function itself and leaves it. +test_names_itself(ce) :- site_in_test_module(ce), call_site(_, ck, _, _, _, _, ce, _, _), call_kind_is_callee_form(ck). +test_names_itself(ce) :- site_in_test_module(ce), call_site(_, "CONSTRUCTOR_CALL", _, _, _, _, ce, _, _). +program_name_site(m, ce) :- module_level_function(m, cn), call_site(_, _, cn, _, _, _, ce, _, _), !test_names_itself(ce). +program_name_count(m, n) :- program_name_site(m, _), n = count : { program_name_site(m, _) }. +// ── test_module / site_in_test_module ────────────── +// By path, as runners find their files: a `test`/`tests`/`__tests__`/`spec(s)` directory, +// or a `.test.` / `.spec.` file. +test_module(mod) :- module_file("client", fp, mod), + match("(.*/)?(tests?|__tests__|specs?)/.*|.*[.](test|spec)[.][cm]?[jt]sx?", fp). +site_in_test_module(ce) :- expr_owner(_, _, mod, ce), test_module(mod). module_level_function(m, n) :- method_decl(_, n, "FUNCTION_DECLARATION", _, _, "", mod, m), method_scopes(_, _, _, init, m), module_init(_, init, mod). module_level_function(m, n) :- var_decl(_, n, _, _, _, mod, v), function_binding_value(v, m), method_decl(_, _, _, _, _, "", mod, m), method_scopes(_, _, _, init, m), module_init(_, init, mod). method_is_hot(m, n) :- module_level_function(m, cn), name_call_count(cn, n), param_flow_capped_at(c), n > c. +method_is_hot_in_program(m, n) :- program_name_count(m, n), param_flow_capped_at(c), n > c. // ── param_fan_capped(ParamHash, SiteCount) — countable ────────────────────── -param_fan_capped(p, n) :- method_is_hot(m, n), param_decl(_, _, _, m, p). +param_fan_capped(p, n) :- method_is_hot_in_program(m, n), param_decl(_, _, _, m, p). diff --git a/graph/javascript/engine/resolution/value-flow.dl b/graph/javascript/engine/resolution/value-flow.dl index 2c79e52b..e78863e2 100644 --- a/graph/javascript/engine/resolution/value-flow.dl +++ b/graph/javascript/engine/resolution/value-flow.dl @@ -264,8 +264,15 @@ param_value(p, "inst", t) :- param_type_name(_, _, tr, p), type_ref_type(tr, t). // values on every export, a 20-minute solve — because a generic library is exactly // the shape whose parameters unify everything. What a library does with a callback it // is handed is a MODEL (resolution/frameworks.dl), stated per library, not inferred. -param_value(p, k, i) :- call_arg(ce, pos, arg), expr_resolves_to_method(ce, m), !method_is_hot(m, _), - method_decl("client", _, _, _, _, _, _, m), +// The program's sites pass their arguments unless the PROGRAM calls the function from +// more places than the cap; a test file's sites only while every site together stays +// under it (fan-cap.dl). +param_value(p, k, i) :- call_arg(ce, pos, arg), expr_resolves_to_method(ce, m), !site_in_test_module(ce), + !method_is_hot_in_program(m, _), method_decl("client", _, _, _, _, _, _, m), + param_decl(_, _, ppos, m, p), to_number(ppos) = pos, !param_is_rest(_, p), + expr_value(arg, k, i). +param_value(p, k, i) :- call_arg(ce, pos, arg), expr_resolves_to_method(ce, m), site_in_test_module(ce), + !method_is_hot(m, _), method_decl("client", _, _, _, _, _, _, m), param_decl(_, _, ppos, m, p), to_number(ppos) = pos, !param_is_rest(_, p), expr_value(arg, k, i). // THE ONE LIBRARY EXCEPTION: a UMD wrapper (#710). `(function (root, factory) { @@ -283,8 +290,12 @@ param_value(p, "func", f) :- call_site(_, "IIFE_CALL", _, _, _, _, ce, _, _), ca expr_value(arg, "func", f). // A rest parameter IS an array; the arguments at and past its position are its elements. param_value(p, "arr", p) :- param_is_rest(_, p). -elem_value(p, k, i) :- call_arg(ce, pos, arg), expr_resolves_to_method(ce, m), !method_is_hot(m, _), - method_decl("client", _, _, _, _, _, _, m), +elem_value(p, k, i) :- call_arg(ce, pos, arg), expr_resolves_to_method(ce, m), !site_in_test_module(ce), + !method_is_hot_in_program(m, _), method_decl("client", _, _, _, _, _, _, m), + param_decl(_, _, ppos, m, p), param_is_rest(_, p), to_number(ppos) <= pos, + expr_value(arg, k, i). +elem_value(p, k, i) :- call_arg(ce, pos, arg), expr_resolves_to_method(ce, m), site_in_test_module(ce), + !method_is_hot(m, _), method_decl("client", _, _, _, _, _, _, m), param_decl(_, _, ppos, m, p), param_is_rest(_, p), to_number(ppos) <= pos, expr_value(arg, k, i). // A default value: `cb = () => {}`. The parser gives the parameter the default's diff --git a/graph/javascript/souffle/decls_all.dl b/graph/javascript/souffle/decls_all.dl index b7b56647..bc489cb6 100644 --- a/graph/javascript/souffle/decls_all.dl +++ b/graph/javascript/souffle/decls_all.dl @@ -112,6 +112,10 @@ .decl wrapper_holder_call(c0:symbol, c1:symbol) .decl wrapper_runs(c0:symbol, c1:symbol) .decl wrapper_call_target(c0:symbol, c1:symbol) +.decl wrap_runs(c0:symbol, c1:symbol) +.decl wrapped_call_target(c0:symbol, c1:symbol) +.decl wrapped_target_count(c0:symbol, c1:number) +.decl wrapped_over_cap(c0:symbol) .decl live_export_variable(c0:symbol, c1:symbol) .decl this_type_open(c0:symbol) @@ -462,6 +466,14 @@ .decl param_fan_capped(c0:symbol, c1:number) .decl import_binding_is_default_like(c0:symbol) .decl name_call_count(c0:symbol, c1:number) +.decl test_module(c0:symbol) +.decl site_in_test_module(c0:symbol) +.decl program_direct_site(c0:symbol, c1:symbol) +.decl program_direct_count(c0:symbol, c1:number) +.decl method_is_hot_in_program(c0:symbol, c1:number) +.decl test_names_itself(c0:symbol) +.decl program_name_site(c0:symbol, c1:symbol) +.decl program_name_count(c0:symbol, c1:number) .decl module_level_function(c0:symbol, c1:symbol) .decl ambient_collection(c0:symbol) .decl coll_method_call(c0:symbol, c1:symbol, c2:symbol) diff --git a/graph/pipeline/run-souffle.sh b/graph/pipeline/run-souffle.sh index b0e9fd5b..0347b7a5 100755 --- a/graph/pipeline/run-souffle.sh +++ b/graph/pipeline/run-souffle.sh @@ -34,7 +34,7 @@ JDK_DEPTH=1 # max JDK-hop depth engine-ii expands. FORCED (always applied). De # hop 1; deeper JDK expansion only traces internal plumbing (the explosion source). LIB_DEPTH="" # max external-lib-hop depth. OPTIONAL — empty = UNCAPPED (dive deep into the client's # real deps, no sink catalog there). Set with --lib-depth to bound a runaway library. -DISPATCH_CAP="${DISPATCH_CAP:-20}" # fan-width cap on virtual dispatch. DEFAULT 20 (measured): +DISPATCH_CAP="${DISPATCH_CAP:-}" # fan-width cap on virtual dispatch. DEFAULT 20, off for Python (below) (measured): # on cassandra 6.0-alpha2 it lifts change-impact precision d2 0.782->0.920 and # d3 0.569->0.753 at ZERO recall cost against the must-have bytecode set (recall # 1.000/0.996/0.988 unchanged), and halves fabricated false positives. Cap 10 is @@ -81,6 +81,9 @@ PKG="$(cd "$SRC/.." && pwd)" # the package root: package.json, node_modules, p . "$SRC/pipeline/compile-lock.sh" # Rules are PER-LANGUAGE and live under graph//; the executor itself is shared. LANG_ARG="${LANG_ARG:-java}" +# Python's cap is off by default: a Python site over the cap is dropped outright, and for test selection a missed +# test costs more than an extra one. Every other language keeps 20. An explicit --dispatch-cap still wins. +if [ -z "$DISPATCH_CAP" ]; then [ "$LANG_ARG" = python ] && DISPATCH_CAP=off || DISPATCH_CAP=20; fi ENG="$SRC/$LANG_ARG/engine"; ENG2="$SRC/$LANG_ARG/engine-ii"; DL="$SRC/$LANG_ARG/souffle"; TPL="$SRC/$LANG_ARG/templates" [ -d "$ENG" ] || { echo "no rule set for --language=$LANG_ARG (looked in $ENG)" >&2; exit 1; } # shellcheck source=souffle-include.sh @@ -201,6 +204,7 @@ write_program(){ _SU=() t+="#include \"$LANG_ARG/souffle/decls_base.dl\"$nl#include \"$LANG_ARG/souffle/decls_all.dl\"$nl" map_rels_su "$TPL/client-ir.map" || return 1 + if [ -f "$TPL/client-extra.map" ]; then map_rels_su "$TPL/client-extra.map" || return 1; fi map_rels_su "$TPL/lib.map" sig || return 1 for r in $LIB_BODY; do _SU+=("$r"); done _SU+=(jdk_max_depth lib_max_depth taint_gating dispatch_cap dispatch_closed_world) @@ -254,7 +258,8 @@ write_program(){ # lost on either side shows up as a difference. Every .input and .output line must be exactly # the expected one, each exactly once, with nothing extra. Exit status only: no pipe to lose. verify_program(){ - awk -v cmap="$TPL/client-ir.map" -v lmap="$TPL/lib.map" -v man="$DL/export_manifest.tsv" \ + local xmap=""; [ -f "$TPL/client-extra.map" ] && xmap="$TPL/client-extra.map" + awk -v cmap="$TPL/client-ir.map" -v xmap="$xmap" -v lmap="$TPL/lib.map" -v man="$DL/export_manifest.tsv" \ -v libsig=" $LIB_SIG " -v extra="$LIB_BODY jdk_max_depth lib_max_depth taint_gating dispatch_cap dispatch_closed_world" ' function want(line) { if (!(line in need)) { need[line] = 1; n++ } } function inp(r) { want(".input " r "(IO=file, filename=\"" r ".facts\", delimiter=\"\\t\", rfc4180=true)") } @@ -269,7 +274,7 @@ verify_program(){ close(path) } BEGIN { - maprels(cmap, 0); maprels(lmap, 1) + maprels(cmap, 0); maprels(lmap, 1); if (xmap != "") maprels(xmap, 0) m = split(extra, e, " "); for (i = 1; i <= m; i++) if (e[i] != "") inp(e[i]) while ((rc = (getline l < man)) > 0) { sub(/^\t+/, "", l); k = index(l, "\t"); p = l; f = "" @@ -813,7 +818,7 @@ while IFS=$'\t' read -r rel csv; do if [ -f "$CLIENT/$csv.csv" ]; then awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$CLIENT/$csv.csv" > "$FACTS/$rel.facts" else : > "$FACTS/$rel.facts"; fi CLIENT_INPUTS="$CLIENT_INPUTS$rel"$'\n' -done < <(read_map "$TPL/client-ir.map") +done < <(read_map "$TPL/client-ir.map"; if [ -f "$TPL/client-extra.map" ]; then read_map "$TPL/client-extra.map"; fi) # --- LIB: signature relations, concatenated across every module of every root --- # CACHED. This concatenation reads the ENTIRE library IR (the JDK alone is 2.0 GB in, diff --git a/graph/python/engine/call-edge-generation/call_chain.dl b/graph/python/engine/call-edge-generation/call_chain.dl index b1c334c5..c527a151 100644 --- a/graph/python/engine/call-edge-generation/call_chain.dl +++ b/graph/python/engine/call-edge-generation/call_chain.dl @@ -156,6 +156,40 @@ method_returns_method("client", m, r) :- expr_call_candidate(site, g), method_returns_method("client", g, r). +// A function returning a member it LOOKED UP BY NAME: `def get_visitor(self, node): return +// getattr(self, f"visit_{type(node).__name__}", None)`, called as `f = self.get_visitor(node); +// f(node)`. The lookup is resolved where it is written (name-resolution.dl's getattr clauses), +// and the method hands that set back exactly as it would hand back a bare name. +method_returns_method("client", m, r) :- + method_return_value_expr("client", m, e), + call_of_expr(e, site), call_name(site, "getattr"), + expr_denotes_method("client", e, r). + +// ── a call that hands back one of its arguments (py_returns_arg, resolution/builtins.dl) ── +// Matched through the import that binds the callee, never by the bare name: +// `functools.update_wrapper(...)` / `t.cast(...)` where the receiver is the name an `import` +// bound, or `update_wrapper(...)` bound by `from functools import update_wrapper`. A local +// function that happens to share the name does neither. +py_passthrough_arg(site, pos) :- + call_name(site, fn), call_receiver_object(site, obj), + expr_binding("client", rb, ctx, obj), ctx != "STORE", binding_lookup("client", rb, rb2), + import_binding("client", rb2, i), import_decl("client", k, mod, _, i), + (k = "MODULE_IMPORT" ; k = "MODULE_IMPORT_ALIAS"), // `import typing as t` is the alias kind + py_returns_arg(path, pos), cat(mod, cat(".", fn)) = path. +py_passthrough_arg(site, pos) :- + call_callee_is_value(site), call_callee_expr(site, callee), + expr_binding("client", rb, ctx, callee), ctx != "STORE", binding_lookup("client", rb, rb2), + import_binding("client", rb2, i), import_decl("client", _, path, _, i), + py_returns_arg(path, pos). +// the expression a value really is, through any number of such calls +py_passthrough_root(e, e) :- method_return_value_expr("client", _, e). +py_passthrough_root(e, x) :- + py_passthrough_root(e, c), call_of_expr(c, site), py_passthrough_arg(site, pos), + call_arg(site, pos, x). +method_returns_method("client", m, r) :- + method_return_value_expr("client", m, e), py_passthrough_root(e, x), x != e, + expr_names_method("client", x, r). + // ── decorator_hits_lib(SiteKey, LibMethodHash) ─────────────────────────────── // `@abstractmethod` names `abc.abstractmethod`, which HAS Python source and is in the // staged stdlib IR — it is a library boundary, not a blind spot. A BARE decorator has no @@ -320,6 +354,25 @@ iter_protocol_edge(src, caller, m) :- expr_type("client", src, t), iter_protocol_target(t, m), expr_ultimate_method("client", src, caller). +// ── property_write_edge(WriteExprHash, CallerMethodHash, AccessorMethodHash) ── +// The store and delete halves of the property protocol: `obj.x = v` calls x's setter, +// `del obj.x` its deleter (type_property_accessor, resolution/attribute-lookup.dl). Same +// shape as a read, keyed on the access's own name context. +property_write_edge(e, caller, m) :- + expr_node("client", "ATTRIBUTE_ACCESS", _, n, e), + expr_name_context("client", "STORE", e), + expr_parent("client", e, "ATTRIBUTE_OBJECT", _, obj), + expr_type("client", obj, t), + type_property_accessor("client", t, n, "PROPERTY_SETTER", m), + expr_ultimate_method("client", e, caller). +property_write_edge(e, caller, m) :- + expr_node("client", "ATTRIBUTE_ACCESS", _, n, e), + expr_name_context("client", "DEL", e), + expr_parent("client", e, "ATTRIBUTE_OBJECT", _, obj), + expr_type("client", obj, t), + type_property_accessor("client", t, n, "PROPERTY_DELETER", m), + expr_ultimate_method("client", e, caller). + // ── property_read_edge(ReadExprHash, CallerMethodHash, GetterMethodHash) ───── property_read_edge(e, caller, getter) :- expr_node("client", "ATTRIBUTE_ACCESS", _, n, e), @@ -629,6 +682,33 @@ subscript_protocol_edge(sub, caller, m) :- mro_lookup("client", t, "__class_getitem__", m), expr_ultimate_method("client", sub, caller). +// ── BUILTIN PROTOCOL — `repr(x)` runs type(x).__repr__ ───────────────────────── +// A builtin function that exists to call one dunder of its argument: repr, str, len, +// hash, bool, iter, next, abs, format, reversed. The written call resolves to the builtin +// (C code, boundary_lib), and the dunder it runs -- a client method, often the one a test +// exists to check -- had no caller: `assert repr(v) == "<...>"` reached nothing of v's. +// Only the BARE builtin name (bound to nothing in the client, so it is the builtin), with +// exactly one argument (`str(b, "utf-8")` decodes and runs no __str__), on an argument the +// engine can type. str() falls back to __repr__ and bool() to __len__ when the class +// defines no __str__ / __bool__ of its own, as CPython's type slots do. The builtin names +// and their dunders are catalogued in resolution/builtins.dl (builtin_protocol_slot). +builtin_protocol_arg(e, fn, x) :- + call_of_expr(e, site), call_callee_is_value(site), call_callee_expr(site, callee), + expr_binding("client", b, _, callee), binding_lookup_unresolved("client", b, fn, _), + builtin_protocol_slot(fn, _), + call_arg(site, "0", x), !call_arg(site, "1", _), !call_kwarg(site, _, _). +// The edge's site is the ARGUMENT expression, as an iteration's is the iterated one: the +// call expression is already the written call to the builtin. +builtin_protocol_edge(x, caller, m) :- + builtin_protocol_arg(_, fn, x), builtin_protocol_slot(fn, d), + expr_type("client", x, t), mro_lookup("client", t, d, m), + expr_ultimate_method("client", x, caller). +builtin_protocol_edge(x, caller, m) :- + builtin_protocol_arg(_, fn, x), builtin_protocol_fallback(fn, d, d2), + expr_type("client", x, t), !type_has_member(t, d), mro_lookup("client", t, d2, m), + expr_ultimate_method("client", x, caller). +type_has_member(t, d) :- builtin_protocol_fallback(_, d, _), mro_lookup("client", t, d, _). + // ── THE THREE PROTOCOL EDGES ARE TIERED BY TARGET COUNT, like every other edge ── // // A property read, a context-manager entry and an iteration all reach their target @@ -668,9 +748,11 @@ subscript_protocol_edge(sub, caller, m) :- // from the graph rather than being re-tiered. A dropped edge is worse than a mislabelled // one, and the golden caught it. protocol_edge(e, d, caller, m) :- property_read_edge(e, caller, m), method_decl(_, d, _, _, _, m). +protocol_edge(e, d, caller, m) :- property_write_edge(e, caller, m), method_decl(_, d, _, _, _, m). protocol_edge(e, d, caller, m) :- with_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m). protocol_edge(e, d, caller, m) :- iter_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m). protocol_edge(e, d, caller, m) :- subscript_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m). +protocol_edge(e, d, caller, m) :- builtin_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m). // Aggregate in its own stratum, mirroring site_client_target_count above. protocol_edge_target_count(e, d, n) :- @@ -731,6 +813,12 @@ call_chain_edge(e, caller, "-", getter, "client", cls, "PROPERTY_READ") :- property_read_edge(e, caller, getter), method_decl(_, d, _, _, _, getter), protocol_edge_class(e, d, cls). +// PROPERTY WRITE — `obj.x = v` runs x's setter and `del obj.x` its deleter. Likewise no +// call site, and its own kind so it is never counted as a written call. +call_chain_edge(e, caller, "-", m, "client", cls, "PROPERTY_WRITE") :- + property_write_edge(e, caller, m), method_decl(_, d, _, _, _, m), + protocol_edge_class(e, d, cls). + // CONTEXT MANAGER — the same shape: an edge with no call site, its own kind so it can // never be mistaken for a written call. call_chain_edge(cm, caller, "-", m, "client", cls, "CONTEXT_MANAGER") :- @@ -749,6 +837,12 @@ call_chain_edge(sub, caller, "-", m, "client", cls, "SUBSCRIPT_PROTOCOL") :- subscript_protocol_edge(sub, caller, m), method_decl(_, d, _, _, _, m), protocol_edge_class(sub, d, cls). +// BUILTIN PROTOCOL — likewise. Its own kind: the written call is to the builtin, and +// this edge is the dunder that builtin runs, never a second written call. +call_chain_edge(x, caller, "-", m, "client", cls, "BUILTIN_PROTOCOL") :- + builtin_protocol_edge(x, caller, m), method_decl(_, d, _, _, _, m), + protocol_edge_class(x, d, cls). + // A site whose caller could not be determined AT ALL would vanish from every rule // above. That must be impossible (the parser guarantees a non-empty owner), so it is // asserted rather than assumed: any such site is emitted with caller "-" so the diff --git a/graph/python/engine/config-resolution/knobs.dl b/graph/python/engine/config-resolution/knobs.dl index 19231f11..27c24871 100644 --- a/graph/python/engine/config-resolution/knobs.dl +++ b/graph/python/engine/config-resolution/knobs.dl @@ -165,6 +165,14 @@ py_parametrize_deco("parametrize"). py_parametrize_argnames_kw("argnames"). py_parametrize_indirect_kw("indirect"). py_parametrize_indirect_all("True"). +// The VALUES the runner hands in: the decorator's second argument (or `argvalues=`), a +// fixture's `params=`, a row wrapped as `pytest.param(v, ..., id=...)`, and, inside a fixture +// declared with params, the attribute `request.param` it reads the current one from. +py_parametrize_argvalues_kw("argvalues"). +py_fixture_params_kw("params"). +py_param_row_wrapper("param"). +py_fixture_request_param("request"). +py_request_param_attr("param"). // ── py_argnames_sep / py_argnames_lead: how "a, b" separates argument names ── // The runner splits the string on "," and strips each piece. Souffle cannot split, so a diff --git a/graph/python/engine/expression-resolution/expr-type.dl b/graph/python/engine/expression-resolution/expr-type.dl index d46b6eca..ed530619 100644 --- a/graph/python/engine/expression-resolution/expr-type.dl +++ b/graph/python/engine/expression-resolution/expr-type.dl @@ -585,6 +585,15 @@ binding_declared_nominal(p, bind, d) :- binding_declared_ref(p, bind, r), type_ref_resolved(p, d, r), !type_is_structural(p, d). +// `x: Optional[Order] = ...` / `x: Order | None = ...` IS an Order (or None) — the rule the +// parameter, return and field annotations already have (resolution/annotations.dl, +// "Optional[X] IS X"), which the local annotation alone was missing: a local written +// with the commonest modern spelling stayed untyped while the same annotation on a +// parameter resolved. +binding_declared_nominal(p, bind, t) :- + type_ref_owner(p, bind, "BINDING", r), + type_ref(p, k, "VARIABLE_ANNOTATION", _, _, r), annotation_optional_kind(k), + type_ref_element(p, r, t), !type_is_structural(p, t). // The FK is resolved on only 14 of 479 variable annotations, so the name-based lookup // beside it is what carries this -- the same two clauses the parameter path uses, with // the same kind restriction. A SUBSCRIPT or a UNION names no single type and is served by diff --git a/graph/python/engine/framework-behavior/dispatch.dl b/graph/python/engine/framework-behavior/dispatch.dl index 0ede5dbf..f7277835 100644 --- a/graph/python/engine/framework-behavior/dispatch.dl +++ b/graph/python/engine/framework-behavior/dispatch.dl @@ -326,7 +326,7 @@ py_argnames_sep_at(h, s, i) :- py_parametrize_argnames(h, s), py_argnames_sep(se // A whole token: led by the string's start, a separator or a space, and followed by its // end, a separator or a space, so `server` is not found inside `server_url`. .decl py_argnames_found(h:symbol, name:symbol, i:number) -py_argnames_found(h, p, i) :- py_fixture_value_param(h, p), py_parametrize_argnames(h, s), +py_argnames_found(h, p, i) :- py_argnames_demand(h, p), py_parametrize_argnames(h, s), strlen(s) > strlen(p), i = range(0, strlen(s) - strlen(p) + 1), substr(s, i, strlen(p)) = p. .decl py_argnames_led(h:symbol, name:symbol, i:number) py_argnames_led(h, p, 0) :- py_argnames_found(h, p, 0). @@ -346,10 +346,15 @@ py_value_row_kind("LIST"). py_argnames_pos(h, p, k) :- py_argnames_name_at(h, p, i), py_parametrize_argnames(h, s), k = count : { py_argnames_sep_at(h, s, j), j < i }. // ... or, for argnames written as a list or tuple of strings, its element position there. -py_argnames_pos(h, p, k) :- py_fixture_value_param(h, p), +py_argnames_pos(h, p, k) :- py_argnames_demand(h, p), decorator_expr("client", de, h), call_arg(de, "0", a), expr_parent("client", a, "ELEMENT", ks, el), expr_node("client", "LITERAL", _, p, el), k = to_number(ks). +// the argument names whose position is asked for: one a getfixturevalue call reads, and +// every argument the decorator supplies directly (its values type the parameter, below) +.decl py_argnames_demand(h:symbol, p:symbol) +py_argnames_demand(h, p) :- py_fixture_value_param(h, p). +py_argnames_demand(h, p) :- py_parametrize_direct(req, p), py_parametrize_on(req, h). .decl py_fixture_value_param(h:symbol, p:symbol) py_fixture_value_param(h, p) :- py_fixture_value_call(req, e), call_arg(e, "0", a), expr_node("client", "NAME_REFERENCE", _, p, a), @@ -362,6 +367,70 @@ py_fixture_request(req, name) :- py_fixture_value_call(req, e), expr_parent("client", tup, "ELEMENT", to_string(k), el), expr_node("client", "LITERAL", _, name, el), name != "". +// ── PARAMETRIZE AND FIXTURE PARAMS: the VALUES the runner hands an argument ── +// @pytest.mark.parametrize("field_cls", [String, Integer]) -> field_cls is one of the classes +// @pytest.mark.parametrize("obj, n", [(Box(), 1), pytest.param(Bag(), 2)]) +// @pytest.fixture(params=(Converter, BaseConverter)) +// def converter_cls(request): return request.param -> one of the classes +// The runner calls the test once per row with the row's values in place of the argument, +// so each value is an argument reaching the parameter exactly as one written at a call +// site is (resolution/value-flow.dl types it). Only values WRITTEN in the decorator, or in +// a display a single-write name in this module (or one it imports the name from) holds; +// a value list built by a call is not read. +.decl py_pz_rows(h:symbol, rows:symbol) +py_pz_rows(h, a) :- py_parametrize_on(_, h), decorator_expr("client", de, h), call_arg(de, "1", a). +py_pz_rows(h, a) :- py_parametrize_on(_, h), decorator_expr("client", de, h), call_kwarg(de, k, a), py_parametrize_argvalues_kw(k). +py_pz_rows(h, a) :- py_fixture_deco_on(_, h), decorator_expr("client", de, h), call_kwarg(de, k, a), py_fixture_params_kw(k). +// the display a rows expression is: written in place, or held by a single-write name +.decl py_pz_display(rows:symbol, d:symbol) +py_pz_display(a, a) :- py_pz_rows(_, a), py_value_row_kind(k), expr_node("client", k, _, _, a). +py_pz_display(a, d) :- py_pz_rows(_, a), + expr_binding("client", b, ctx, a), ctx != "STORE", binding_lookup("client", b, b2), + py_pz_name_display(b2, d). +.decl py_pz_name_display(b:symbol, d:symbol) +py_pz_name_display(b, d) :- binding_rebinding("client", "1", _, _, b), + expr_binding("client", b, "STORE", tgt), assign_pair("client", tgt, d), + py_value_row_kind(k), expr_node("client", k, _, _, d). +// `from tests.base import ALL_FIELDS`: the module-level binding of that name in the module +py_pz_name_display(b, d) :- import_binding("client", b, i), + import_alias("client", orig, _, i), orig != "", import_relative("client", _, pkg, i), pkg != "", + module_decl("client", _, pkg, _, mod), + binding_decl("client", orig, _, "MODULE_LEVEL", _, mb), binding_owner("client", mod, _, mb), + binding_rebinding("client", "1", _, _, mb), + expr_binding("client", mb, "STORE", tgt), assign_pair("client", tgt, d), + py_value_row_kind(k), expr_node("client", k, _, _, d). +// a row, with `pytest.param(v, ..., id=...)` read as its positional values +.decl py_pz_row(h:symbol, row:symbol) +py_pz_row(h, r) :- py_pz_rows(h, a), py_pz_display(a, d), expr_parent("client", d, "ELEMENT", _, r). +.decl py_pz_param_call(row:symbol) +py_pz_param_call(r) :- py_pz_row(_, r), call_of_expr(r, site), call_name(site, w), py_param_row_wrapper(w). +// the value at one argument's position in a row: the whole row (one argument name), else +// the row's k-th element +.decl py_pz_single(h:symbol) +py_pz_single(h) :- py_parametrize_argnames(h, s), !py_argnames_sep_at(h, s, _). +py_pz_single(h) :- py_fixture_deco_on(_, h). +.decl py_pz_value(h:symbol, k:number, v:symbol) +py_pz_value(h, 0, r) :- py_pz_single(h), py_pz_row(h, r), !py_pz_param_call(r). +py_pz_value(h, 0, v) :- py_pz_single(h), py_pz_row(h, r), py_pz_param_call(r), call_arg(r, "0", v). +py_pz_value(h, k, v) :- !py_pz_single(h), py_pz_row(h, r), !py_pz_param_call(r), + py_value_row_kind(rk), expr_node("client", rk, _, _, r), + expr_parent("client", r, "ELEMENT", ks, v), k = to_number(ks). +py_pz_value(h, k, v) :- !py_pz_single(h), py_pz_row(h, r), py_pz_param_call(r), + call_arg(r, ks, v), k = to_number(ks). +// the parameter each value reaches: a test's directly parametrized argument ... +.decl py_pz_param_value(ph:symbol, v:symbol) +py_pz_param_value(ph, v) :- py_parametrize_direct(req, name), py_parametrize_on(req, h), + !py_pz_single(h), py_argnames_pos(h, name, k), py_pz_value(h, k, v), + param_decl("client", name, _, _, req, ph). +py_pz_param_value(ph, v) :- py_parametrize_direct(req, name), py_parametrize_on(req, h), + py_parametrize_argname(req, h, name), py_pz_single(h), py_pz_value(h, 0, v), + param_decl("client", name, _, _, req, ph). +// ... and, in a fixture declared with params=, what `request.param` evaluates to +.decl py_pz_request_param(e:symbol, v:symbol) +py_pz_request_param(e, v) :- py_fixture_deco_on(f, h), py_pz_value(h, 0, v), + py_fixture_request_param(rq), param_decl("client", rq, _, _, f, ph), + py_request_param_attr(pa), attr_access("client", e, pa, obj), expr_names_param("client", obj, ph). + // (0) a fixture a CLASS declares serves the tests of that class and of its subclasses, // and nothing else; for them it wins over the module's fixture of the name. Read as a // plain same-module fixture it was also handed to a sibling class's tests, and the @@ -412,6 +481,53 @@ py_fixture_requested(m) :- py_fixture_injection(_, m, _). framework_edge(from, to, "fixture_injection", name, "by_name") :- py_fixture_injection(from, to, name). +// ── 2a. THE SAME LOOKUP, FROM SYNTAX ALONE ────────────────────────────────────── +// resolution/value-flow.dl types a parameter by the fixture serving it, so it needs the +// runner's lookup INSIDE the resolution fixpoint. py_fixture_injection cannot be read +// there: a fixture a conftest star-imports is found through module_member_method, which +// is resolution, and the nearest-conftest MAX over it would then be a cyclic aggregate +// souffle refuses to stratify. This chain keeps the runner's order (class, own module, +// nearest conftest by its declaring file) and leaves out the two clauses that need +// resolution — a star-imported fixture and a pytest_plugins one. Those still reach their +// tests through the injection edge; they only go untyped. +.decl py_fixs_in_file(fix:symbol, file:symbol) +py_fixs_in_file(fix, p) :- py_fixture_decl(fix, _), method_file("client", p, fix). +.decl py_fixs_same_module(req:symbol, name:symbol, fix:symbol) +py_fixs_same_module(req, name, fix) :- py_fixture_request(req, name), + py_fixture_decl(fix, name), fix != req, !py_fixture_class(fix, _), + !py_fixture_class_shadowed(req, name), + method_file("client", p, req), py_fixs_in_file(fix, p). +.decl py_fixs_conftest_cand(req:symbol, name:symbol, fix:symbol, depth:number) +py_fixs_conftest_cand(req, name, fix, dl) :- py_fixture_request(req, name), + py_fixture_decl(fix, name), fix != req, !py_fixture_class(fix, _), + py_fixs_in_file(fix, cp), py_fixture_scope_file(c), + strlen(cp) >= strlen(c), substr(cp, strlen(cp) - strlen(c), strlen(c)) = c, + d = substr(cp, 0, strlen(cp) - strlen(c)), + method_file("client", p, req), + strlen(p) > strlen(d), substr(p, 0, strlen(d)) = d, dl = strlen(d). +.decl py_fixs_nearest(req:symbol, name:symbol, depth:number) +py_fixs_nearest(req, name, m) :- py_fixs_conftest_cand(req, name, _, _), + m = max dl : { py_fixs_conftest_cand(req, name, _, dl) }. +.decl py_fixs_injection(from:symbol, to:symbol, name:symbol) +py_fixs_injection(req, fix, name) :- py_fixture_in_class(req, name, fix). +py_fixs_injection(req, fix, name) :- py_fixs_same_module(req, name, fix). +py_fixs_injection(req, fix, name) :- py_fixs_conftest_cand(req, name, fix, dl), + !py_fixture_class_shadowed(req, name), !py_fixs_same_module(req, name, _), + py_fixs_nearest(req, name, dl). + +// ── 2b. FIXTURE VALUE (what the runner hands the parameter) ──────────────────── +// py_fixture_value_type(Fixture, Type) — the value a fixture hands over: its return or +// yield value's type, or its declared return. Consumed by resolution/value-flow.dl, where +// the runner's call `test(fixture_value)` is one more argument reaching a parameter. +.decl py_fixture_value_type(fix:symbol, t:symbol) +py_fixture_value_type(fix, t) :- py_fixture_decl(fix, _), + method_return_value_expr("client", fix, e), expr_type("client", e, t). +py_fixture_value_type(fix, t) :- py_fixture_decl(fix, _), + method_declared_return_type("client", fix, d), declared_dispatch("client", d, t). +py_fixture_value_type(fix, t) :- py_fixture_decl(fix, _), + expr_node("client", "YIELD", _, _, y), expr_ultimate_method("client", y, fix), + expr_parent("client", y, "YIELD_VALUE", _, v), expr_type("client", v, t). + // ───────────────────────────────────────────────────────────────────────────── // 3. URL DISPATCH (route table -> view) // ───────────────────────────────────────────────────────────────────────────── diff --git a/graph/python/engine/framework-behavior/library-callbacks.dl b/graph/python/engine/framework-behavior/library-callbacks.dl new file mode 100644 index 00000000..e17d9043 --- /dev/null +++ b/graph/python/engine/framework-behavior/library-callbacks.dl @@ -0,0 +1,368 @@ +// ============================================================================ +// FRAMEWORK-BEHAVIOR · LIBRARY CALLBACKS (the hop a dependency makes back into the client) +// +// ── THE GAP ───────────────────────────────────────────────────────────────── +// The engine reads a dependency's signatures, never its bodies. So when a library calls +// BACK into client code, the call graph stops at the library frame: a test client's +// `client.get("/")` runs the `open()` a client subclass overrides and then the +// application object it was built around (`app(environ, start_response)`); a CLI test +// runner's `runner.invoke(cli)` runs `cli.main()`, which runs the command's callback; an +// `ExitStack.enter_context(cm)` runs `cm.__enter__`. Measured on a 16-repository Python +// oracle, a library frame is the first broken hop behind most test-impact misses on the +// web-framework subject: the test calls the library, the library calls the code under +// change, and the graph has only the first half. +// +// ── WHERE THE KNOWLEDGE COMES FROM ────────────────────────────────────────── +// graph/python/libsum/libsum.py reads the installed library's SOURCE once (cached by the +// installed distributions and the client's imports, never solved here) and stages these tables +// beside the client IR, for the library classes and functions the client can name: +// py_lib_callback(Callee, Root, Pos, Path, Member, Via, Type) +// calling the library callable Callee hands back `Member` on the object that arrived +// as its parameter Root (written by the client at position Pos, -1 = the receiver +// `self`), after following the attribute Path ("" = the object itself); Via is the +// library function where that invocation is written; Type the class the library declares that +// object to be, "" when it declares none +// py_lib_ancestor(Class, Ancestor) a library class's library ancestors +// py_lib_field_init(Class, Field, Param, Pos) +// constructing Class stores the argument Param on the new object as Field +// py_lib_alias(Alias, Canonical) +// a name a package re-exports +// py_lib_decorates(Decorator, Form, Class, Field) +// `@Decorator` (DIRECT) or `@Decorator(...)` (FACTORY) turns a def into a Class whose Field +// holds it; Class "" when the def itself comes back +// No row for a library written in C: a declared blind spot, never a guess. +// +// ── WHAT IT EMITS, AND WHY NOT AS A CALL ──────────────────────────────────── +// lib_callback_edge(Caller, Target, Callee, Member, Via, How) +// Caller the client method containing the call into the library +// Target the client method the library is summarised to call back +// Callee the library callable the client called (canonical name) +// How receiver Target is a member of the object the call was made on +// (a client subclass overriding what the library's own code calls) +// receiver_field ... of an object the library holds in a field of it +// argument ... of an object (or the callable itself) passed as an argument +// argument_field ... of an object held in a field of an argument +// The contract of this phase holds: it only ADDS a relation. The hop is a library's +// behaviour summarised from its source, joined to the client by a member NAME on a type +// the client handed over; a resolved call is stronger knowledge, so call_chain_edge is +// untouched and a consumer opts in (impact walks it as its own hop, `library callback`). +// No rule here names a library or a framework: every name comes from the staged tables. +// ============================================================================ + +// ── the dotted path a client expression spells through an unlinked import ─── +// `Client` after `from webkit.testing import Client`, `clikit.command` after `import clikit`: the import's path, +// then each attribute read off it. Only an import that linked to nothing in the client starts one. +.decl lcb_path_of(e:symbol, p:symbol) +lcb_path_of(e, p) :- + expr_binding("client", b, ctx, e), ctx != "STORE", + binding_lookup("client", b, b2), import_unlinked(b2, p). +lcb_path_of(e, cat(p, cat(".", n))) :- + expr_node("client", "ATTRIBUTE_ACCESS", _, n, e), n != "", + expr_parent("client", e, "ATTRIBUTE_OBJECT", _, obj), lcb_path_of(obj, p), strlen(p) < 200. + +// ── its canonical name, following the package's re-exports ────────────────── +.decl lcb_named(p:symbol) +lcb_named(p) :- lcb_path_of(_, p). +.decl lcb_canon(p:symbol, c:symbol) +lcb_canon(p, c) :- lcb_named(p), py_lib_alias(p, c). +lcb_canon(p, p) :- lcb_named(p), !py_lib_alias(p, _). +.decl lcb_expr_names(e:symbol, c:symbol) +lcb_expr_names(e, c) :- lcb_path_of(e, p), lcb_canon(p, c). + +// ── a client class whose base is a library class ───────────────────────────── +// Written `class AppClient(Client)` with `Client` imported from a dependency that was not +// staged: the base resolves to nothing and type_base_unresolved records it by text. Here it +// is named by the import the text binds to, and inherited through client subclasses. +.decl lcb_direct_ext_base(t:symbol, c:symbol) +lcb_direct_ext_base(t, c) :- + type_base_slot("client", t, _, "NAME", _, _, bh), base_expr("client", bh, e), lcb_expr_names(e, c). +lcb_direct_ext_base(t, c) :- + type_base_slot("client", t, _, "SUBSCRIPT", _, _, bh), base_expr("client", bh, e), + expr_parent("client", e, "SUBSCRIPT_OBJECT", _, obj), lcb_expr_names(obj, c). +.decl lcb_ext_class(t:symbol, c:symbol) +lcb_ext_class(t, c) :- lcb_direct_ext_base(t, c). +lcb_ext_class(t, c) :- type_ancestor("client", t, a), lcb_direct_ext_base(a, c). + +// a client type and every client subclass of it: the object a library calls back may be a +// subclass of the type the call site was written against +.decl lcb_self_type(t:symbol, s:symbol) +lcb_self_type(t, t) :- lcb_ext_class(t, _). +lcb_self_type(t, s) :- lcb_ext_class(t, _), type_ancestor("client", s, t). + +.decl lcb_has_rows(c:symbol) +lcb_has_rows(c) :- py_lib_callback(c, _, _, _, _, _, _). + +// ── lcb_entry(Site, Callee, SelfType) — a client call into a summarised library callable ── +// SelfType is the client type the library's `self` is, or "" when the receiver is not a +// client object. +.decl lcb_entry(site:symbol, callee:symbol, selfT:symbol) +// (a) a member the client type inherits from its library base and does not define itself: +// `client.get("/")` on an AppClient-like subclass, `self.get(...)` inside one +lcb_entry(site, q, t) :- + call_receiver_object(site, obj), expr_type("client", obj, t), + call_name(site, n), n != "", + !expr_call_candidate(site, _), + lcb_ext_class(t, c), q = cat(c, cat(".", n)), lcb_has_rows(q). +// (b) super().n(...) whose MRO leaves the client for a library base +lcb_entry(site, q, a) :- + call_recv_kind(site, "SUPER"), call_super_anchor(site, _, a), + call_name(site, n), n != "", + !expr_call_candidate(site, _), + lcb_ext_class(a, c), q = cat(c, cat(".", n)), lcb_has_rows(q). +// (c) a library function or class called by a name the client imported +lcb_entry(site, c, "") :- + call_callee_is_value(site), call_callee_expr(site, callee), + lcb_expr_names(callee, c), lcb_has_rows(c). +// (d) a member of a library object the client built: `runner = CliRunner(); runner.invoke(cli)`, or one a +// fixture or a helper returned. Its `self` is a library object, so it has no client type; what the client +// gave its constructor is reached through the allocation (lcb_lobj_entry below). +lcb_entry(site, q, "") :- lcb_lobj_entry(site, q, _). +// (e) constructing a client class whose constructor is the library base's: its __init__ may +// call members the client overrides +lcb_entry(site, c, t) :- + call_constructs_type(site, t), lcb_ext_class(t, c), + !type_constructor("client", t, _), lcb_has_rows(c). + +// (f) a hand-back to a member the client object does not define but INHERITS from a library class: the +// library calls `cli.main()` on the command it was given, `main` is the library base's own, and that +// method's hand-backs apply to the same object (its make_context override). Same site, same caller. +lcb_entry(site, q2, t) :- + lcb_arg(site, q, r, a), py_lib_callback(q, r, _, "", n, _, _), + expr_type("client", a, t), !mro_lookup("client", t, n, _), + lcb_ext_class(t, c), q2 = cat(c, cat(".", n)), lcb_has_rows(q2). +lcb_entry(site, q2, t) :- + lcb_entry(site, q, t), t != "", py_lib_callback(q, "self", _, "", n, _, _), + !mro_lookup("client", t, n, _), + lcb_ext_class(t, c), q2 = cat(c, cat(".", n)), lcb_has_rows(q2). + +// ── lcb_ext_instance(Expr, Class, Alloc) — an expression holding an object of a library class ── +// Alloc is the call site that constructed it. Followed through a single-binding local, a client function's +// return value, and the fixture a test parameter is filled from. +.decl lcb_ext_instance(e:symbol, c:symbol, alloc:symbol) +lcb_ext_instance(e, c, site) :- + call_of_expr(e, site), call_callee_is_value(site), call_callee_expr(site, callee), + lcb_expr_names(callee, c), lcb_is_class(c). +lcb_ext_instance(e, c, al) :- + expr_binding("client", bref, ctx, e), ctx != "STORE", + binding_lookup("client", bref, bind), + binding_rebinding("client", "1", _, _, bind), + expr_binding("client", bind, "STORE", tgt), + assign_pair("client", tgt, val), + lcb_ext_instance(val, c, al). +lcb_ext_instance(e, c, al) :- + call_of_expr(e, site), expr_call_candidate(site, m), + method_return_value_expr("client", m, r), lcb_ext_instance(r, c, al). +lcb_ext_instance(e, c, al) :- + expr_names_param("client", e, ph), param_decl("client", pn, _, _, req, ph), + py_fixs_injection(req, fix, pn), method_return_value_expr("client", fix, r), lcb_ext_instance(r, c, al). +lcb_ext_instance(e, c, al) :- + expr_names_param("client", e, ph), param_decl("client", pn, _, _, req, ph), + py_fixs_injection(req, fix, pn), + expr_node("client", "YIELD", _, _, y), expr_ultimate_method("client", y, fix), + expr_parent("client", y, "YIELD_VALUE", _, v), lcb_ext_instance(v, c, al). +.decl lcb_is_class(c:symbol) +lcb_is_class(c) :- py_lib_field_init(c, _, _, _). +lcb_is_class(c) :- py_lib_callback(c, "self", _, _, _, _, _). + +// ── LIBRARY OBJECTS THE CLIENT HOLDS ──────────────────────────────────────── +// lcb_lobj(Expr, Class, Obj): Expr holds a library object of Class, identified by Obj -- the call site that +// constructed it, or the client function a library decorator wrapped into it (`@command() def greet()` makes +// `greet` a Command whose `callback` is the def). Its fields are what the client gave it. +.decl lcb_lobj(e:symbol, c:symbol, obj:symbol) +lcb_lobj(e, c, al) :- lcb_ext_instance(e, c, al). +lcb_lobj(e, c, m) :- lcb_names_def(e, m), lcb_deco_obj(m, c, _). + +// the def a name denotes, decorated or not (the decorated name's VALUE is what the decorator returned) +.decl lcb_names_def(e:symbol, m:symbol) +lcb_names_def(e, m) :- expr_binding("client", bref, ctx, e), ctx != "STORE", binding_lookup("client", bref, b), + method_declaring_binding("client", b, m). +lcb_names_def(e, m) :- expr_binding("client", bref, ctx, e), ctx != "STORE", binding_lookup("client", bref, b), + import_binds_method("client", b, m). +lcb_names_def(e, m) :- expr_attr_names_method("client", e, m). + +// the library callable a decorator expression applies: `@q` (direct) or `@q(...)` (factory) +.decl lcb_deco_callable(d:symbol, q:symbol, form:symbol) +lcb_deco_callable(d, q, "DIRECT") :- decorator_expr("client", e, d), !call_of_expr(e, _), lcb_expr_names(e, q). +lcb_deco_callable(d, q, "FACTORY") :- decorator_expr("client", e, d), call_of_expr(e, site), + call_callee_expr(site, callee), lcb_expr_names(callee, q). +lcb_deco_callable(d, q, "FACTORY") :- decorator_expr("client", e, d), call_of_expr(e, site), + call_receiver_object(site, obj), expr_type("client", obj, t), call_name(site, n), !mro_lookup("client", t, n, _), + lcb_ext_class(t, c), q = cat(c, cat(".", n)). +// a decorator that hands the function straight back (registers it, records an option on it) +.decl lcb_deco_identity(d:symbol) +lcb_deco_identity(d) :- lcb_deco_callable(d, q, form), py_lib_decorates(q, form, "", ""). +lcb_deco_identity(d) :- decorator_builtin("client", bk, d), decorator_transparent(bk). +// the def m enters application stage k unchanged: every decorator applied before it handed it straight back +.decl lcb_deco_blocked(m:symbol, k:number) +lcb_deco_blocked(m, k) :- deco_at(m, k, _), deco_at(m, k2, d2), k2 < k, !lcb_deco_identity(d2), !deco_stage(m, k2 + 1, m). +.decl lcb_deco_obj(m:symbol, c:symbol, f:symbol) +lcb_deco_obj(m, c, f) :- deco_at(m, k, d), !lcb_deco_blocked(m, k), lcb_deco_callable(d, q, form), + py_lib_decorates(q, form, c, f), c != "". + +// ── entries on a library object, and what its fields hand back ────────────── +.decl lcb_lobj_entry(site:symbol, callee:symbol, obj:symbol) +lcb_lobj_entry(site, q, o) :- + call_receiver_object(site, recv), lcb_lobj(recv, c, o), + call_name(site, n), n != "", + !expr_call_candidate(site, _), + q = cat(c, cat(".", n)), lcb_has_rows(q). +// a library object handed to a library callable that calls one of the object's own (library) members +lcb_lobj_entry(site, q2, o) :- + lcb_arg(site, q, r, a), py_lib_callback(q, r, _, "", n, _, _), lcb_lobj(a, c, o), + q2 = cat(c, cat(".", n)), lcb_has_rows(q2). + +// what a library object's field holds that offers member n: the argument its construction was given, or the +// function a decorator wrapped (called as `__call__`) +.decl lcb_lobj_field_target(o:symbol, f:symbol, n:symbol, m:symbol) +lcb_lobj_field_target(al, f, n, m) :- lcb_alloc_field(al, f, a), lcb_member_of_expr(a, n, m). +lcb_lobj_field_target(m, f, "__call__", m) :- lcb_deco_obj(m, _, f), lcb_field_wanted(m, f). +.decl lcb_field_wanted(o:symbol, f:symbol) +lcb_field_wanted(o, f) :- lcb_lobj_entry(_, q, o), py_lib_callback(q, "self", _, f, _, _, _), f != "". +lcb_field_wanted(o, f) :- lcb_arg(site, q, r, a), py_lib_callback(q, r, _, f, _, _, _), f != "", lcb_lobj(a, _, o), + lcb_entry(site, q, _). + +// what the constructor of a library object stored in a field, as the client wrote it at the allocation +.decl lcb_alloc_field(al:symbol, f:symbol, a:symbol) +lcb_alloc_field(al, f, a) :- + lcb_field_wanted(al, f), call_callee_expr(al, callee), lcb_expr_names(callee, c), + py_lib_field_init(c, f, _, pos), call_arg(al, ps, a), to_number(ps) = to_number(pos). +lcb_alloc_field(al, f, a) :- + lcb_field_wanted(al, f), call_callee_expr(al, callee), lcb_expr_names(callee, c), + py_lib_field_init(c, f, pn, _), call_kwarg(al, pn, a). + +// ── the argument the client wrote for a summarised parameter ───────────────── +.decl lcb_arg(site:symbol, callee:symbol, root:symbol, a:symbol) +lcb_arg(site, q, r, a) :- + lcb_entry(site, q, _), py_lib_callback(q, r, pos, _, _, _, _), r != "self", + to_number(pos) >= 0, call_arg(site, ps, a), to_number(ps) = to_number(pos). +lcb_arg(site, q, r, a) :- + lcb_entry(site, q, _), py_lib_callback(q, r, _, _, _, _, _), r != "self", + call_kwarg(site, r, a). + +// ── what a library-held field of a client object holds ─────────────────────── +// lcb_field_holds(T, Field, Expr): an object of client type T was built with Expr as the +// argument the library constructor stores in Field (the construction of T or of a client +// subclass, or T's own super().__init__(...) call) +.decl lcb_ctor_store(t:symbol, f:symbol, param:symbol, pos:symbol) +lcb_ctor_store(t, f, pn, pos) :- lcb_ext_class(t, c), py_lib_field_init(c, f, pn, pos). +.decl lcb_field_holds(t:symbol, f:symbol, e:symbol) +lcb_field_holds(t, f, a) :- + lcb_ctor_store(t, f, _, pos), lcb_self_type(t, s), call_constructs_type(site, s), + call_arg(site, ps, a), to_number(ps) = to_number(pos). +lcb_field_holds(t, f, a) :- + lcb_ctor_store(t, f, pn, _), lcb_self_type(t, s), call_constructs_type(site, s), + call_kwarg(site, pn, a). +lcb_field_holds(t, f, a) :- + lcb_ctor_store(t, f, _, pos), call_recv_kind(site, "SUPER"), call_super_anchor(site, _, t), + call_name(site, "__init__"), call_arg(site, ps, a), to_number(ps) = to_number(pos). +lcb_field_holds(t, f, a) :- + lcb_ctor_store(t, f, pn, _), call_recv_kind(site, "SUPER"), call_super_anchor(site, _, t), + call_name(site, "__init__"), call_kwarg(site, pn, a). + +// ── the client member an object expression offers under a name ─────────────── +// the type the expression has, or a client subclass of it; `__call__` on an expression that +// IS a function or a bound method is that function +.decl lcb_member_of_expr(e:symbol, n:symbol, m:symbol) +lcb_member_of_expr(e, n, m) :- + lcb_wanted(e, n), expr_type("client", e, t), lcb_sub(t, s), mro_lookup("client", s, n, m), + method_owner("client", _, m). +lcb_member_of_expr(e, "__call__", m) :- lcb_wanted(e, "__call__"), expr_denotes_method("client", e, m). +lcb_member_of_expr(e, "__call__", m) :- lcb_wanted(e, "__call__"), expr_names_method("client", e, m). +.decl lcb_sub(t:symbol, s:symbol) +lcb_sub(t, t) :- lcb_wanted(e, _), expr_type("client", e, t). +lcb_sub(t, s) :- lcb_wanted(e, _), expr_type("client", e, t), type_ancestor("client", s, t). + +// the member of a client object's own field, as the client wrote it (`self.app = app`) +.decl lcb_member_of_field(t:symbol, f:symbol, n:symbol, m:symbol) +lcb_member_of_field(t, f, n, m) :- + lcb_wanted_field(t, f, n), field_decl("client", f, owner, _, fh), + (owner = t ; type_ancestor("client", t, owner)), + field_type("client", fh, x), mro_lookup("client", x, n, m), method_owner("client", _, m). + +// demand: only the (expression, member) pairs a summary asks about are looked up +.decl lcb_wanted(e:symbol, n:symbol) +lcb_wanted(a, n) :- lcb_arg(site, q, r, a), py_lib_callback(q, r, _, "", n, _, _), lcb_entry(site, q, _). +lcb_wanted(e, n) :- lcb_entry(_, q, t), t != "", py_lib_callback(q, "self", _, f, n, _, _), f != "", + lcb_field_holds(t, f, e). +lcb_wanted(a, n) :- lcb_alloc_field(al, f, a), lcb_lobj_entry(_, q, al), py_lib_callback(q, "self", _, f, n, _, _). +lcb_wanted(e, n) :- lcb_alloc_field(al, f, e), lcb_lobj(a, _, al), lcb_arg(_, q, r, a), py_lib_callback(q, r, _, f, n, _, _). +lcb_wanted(e, n) :- lcb_arg(site, q, r, a), py_lib_callback(q, r, _, f, n, _, _), f != "", lcb_entry(site, q, _), + expr_type("client", a, t), lcb_field_holds(t, f, e). +.decl lcb_wanted_field(t:symbol, f:symbol, n:symbol) +lcb_wanted_field(s, f, n) :- lcb_entry(_, q, t), t != "", py_lib_callback(q, "self", _, f, n, _, _), f != "", + !contains(".", f), lcb_self_type(t, s). +lcb_wanted_field(t, f, n) :- lcb_arg(site, q, r, a), lcb_entry(site, q, _), py_lib_callback(q, r, _, f, n, _, _), f != "", + !contains(".", f), expr_type("client", a, t). + +// ── the edges ─────────────────────────────────────────────────────────────── +// lcb_hit(Site, Caller, Target, Callee, Root, Path, Member, Via, How): one summary row joined to the client object +// the call actually hands over. +.decl lcb_hit(site:symbol, caller:symbol, m:symbol, q:symbol, r:symbol, f:symbol, n:symbol, via:symbol, how:symbol) +// the receiver itself: a member the client type (or a subclass) defines over the library's +lcb_hit(site, caller, m, q, "self", "", n, via, "receiver") :- + lcb_entry(site, q, t), t != "", site_caller(site, caller), + py_lib_callback(q, "self", _, "", n, via, _), + lcb_self_type(t, s), mro_lookup("client", s, n, m), method_owner("client", _, m). +// a field of the receiver the library constructor filled from a client argument +lcb_hit(site, caller, m, q, "self", f, n, via, "receiver_field") :- + lcb_entry(site, q, t), t != "", site_caller(site, caller), + py_lib_callback(q, "self", _, f, n, via, _), f != "", + lcb_field_holds(t, f, e), lcb_member_of_expr(e, n, m). +// a field of a library object the client holds (built by it, or made by a library decorator around its def) +lcb_hit(site, caller, m, q, "self", f, n, via, "receiver_field") :- + lcb_lobj_entry(site, q, o), site_caller(site, caller), + py_lib_callback(q, "self", _, f, n, via, _), f != "", + lcb_lobj_field_target(o, f, n, m). +lcb_hit(site, caller, m, q, r, f, n, via, "argument_field") :- + lcb_arg(site, q, r, a), site_caller(site, caller), lcb_entry(site, q, _), + py_lib_callback(q, r, _, f, n, via, _), f != "", + lcb_lobj(a, _, o), lcb_lobj_field_target(o, f, n, m). +// a field of the receiver the client itself assigned +lcb_hit(site, caller, m, q, "self", f, n, via, "receiver_field") :- + lcb_entry(site, q, t), t != "", site_caller(site, caller), + py_lib_callback(q, "self", _, f, n, via, _), f != "", + lcb_self_type(t, s), lcb_member_of_field(s, f, n, m). +// an argument, or the callable passed as one +lcb_hit(site, caller, m, q, r, "", n, via, "argument") :- + lcb_arg(site, q, r, a), site_caller(site, caller), + py_lib_callback(q, r, _, "", n, via, _), + lcb_member_of_expr(a, n, m). +// a field of an argument +lcb_hit(site, caller, m, q, r, f, n, via, "argument_field") :- + lcb_arg(site, q, r, a), site_caller(site, caller), + py_lib_callback(q, r, _, f, n, via, _), f != "", + expr_type("client", a, t), lcb_field_holds(t, f, e), lcb_member_of_expr(e, n, m). +lcb_hit(site, caller, m, q, r, f, n, via, "argument_field") :- + lcb_arg(site, q, r, a), site_caller(site, caller), + py_lib_callback(q, r, _, f, n, via, _), f != "", + expr_type("client", a, t), lcb_member_of_field(t, f, n, m). + +// ── WHEN THE OBJECT CANNOT BE TOLD: THE TYPE THE LIBRARY DECLARES ──────────── +// A row whose object the client side could not name (a loader passed through **options, a command a helper +// returns typed only as the library's base) is still a hand-back to SOME object of the type the library declares +// there. The client classes that can be that object are its client subclasses; the one whose member runs is the one +// that defines it. A class-hierarchy answer, used only for a row no client object answered at that site. +.decl lcb_row_hit(site:symbol, q:symbol, r:symbol, f:symbol, n:symbol) +lcb_row_hit(site, q, r, f, n) :- lcb_hit(site, _, _, q, r, f, n, _, _). +.decl lcb_cha_wanted(ty:symbol, n:symbol) +lcb_cha_wanted(ty, n) :- lcb_entry(_, q, _), py_lib_callback(q, r, _, f, n, _, ty), ty != "", (r != "self" ; f != ""). +.decl lcb_cha(ty:symbol, n:symbol, m:symbol) +lcb_cha(ty, n, m) :- lcb_cha_wanted(ty, n), lcb_ext_class(s, ty), mro_lookup("client", s, n, m), method_owner("client", _, m). +lcb_cha(ty, n, m) :- lcb_cha_wanted(ty, n), lcb_ext_class(s, c), py_lib_ancestor(c, ty), + mro_lookup("client", s, n, m), method_owner("client", _, m). + +// lib_callback_edge(Caller, Target, Callee, Member, Via, How): the client method Caller calls the library +// callable Callee, which is summarised (from the library's source) to invoke Member -- written in the library +// function Via -- on an object that resolves to the client method Target. How: receiver | receiver_field | +// argument | argument_field | declared_type. A library's behaviour joined by a member name, not a resolved call. +lib_callback_edge(caller, m, q, n, via, how) :- lcb_hit(_, caller, m, q, _, _, n, via, how). +lib_callback_edge(caller, m, q, n, via, "declared_type") :- + lcb_entry(site, q, _), site_caller(site, caller), + py_lib_callback(q, r, _, f, n, via, ty), ty != "", (r != "self" ; f != ""), + !lcb_row_hit(site, q, r, f, n), lcb_cha(ty, n, m). + +// lib_callback_site(Site, Callee): a client call site the engine left unresolved that enters the library callable +// Callee, which the library's own source declares (a member the client type inherits from its library base, a +// super() call past the client classes, a member of a library object the client built). Not a call to any +// same-named client method, so a consumer must not guess one by name. +lib_callback_site(site, q) :- lcb_entry(site, q, _), q != "". diff --git a/graph/python/engine/resolution/annotations.dl b/graph/python/engine/resolution/annotations.dl index 0d671aa9..2b9087a4 100644 --- a/graph/python/engine/resolution/annotations.dl +++ b/graph/python/engine/resolution/annotations.dl @@ -199,6 +199,25 @@ type_ref_element(p, parentRef, t) :- annotation_owner_module(p, owner, ok, mod), type_name_in_module(p, mod, tn, t). +// ── union_operand(Prov, UnionRefHash, OperandRefHash) — every operand of a union ── +// `|` is LEFT-associative, so `A | B | None` is `(A | B) | None`: A and B are not +// children of the annotation but grandchildren, under a nested UNION_PEP604, and the +// depth-1 clauses above saw only that nested union (which names no type) and None. Every +// rule reading "Optional[X] IS X" therefore lost the members of any union with three or +// more operands, on a parameter, a return, a field or a local alike. +union_operand(p, r, c) :- + type_ref(p, k, _, _, _, r), annotation_optional_kind(k), type_ref_nesting(p, r, _, _, c). +union_operand(p, r, c) :- + union_operand(p, r, u), type_ref(p, "UNION_PEP604", _, _, _, u), type_ref_nesting(p, u, _, _, c). +// the operands BELOW depth 1 (depth 1 is the clauses above), resolved, and by name +type_ref_element(p, r, t) :- + union_operand(p, r, c), type_ref_nesting(p, _, _, d, c), d != "1", type_ref_resolved(p, t, c). +type_ref_element(p, r, t) :- + union_operand(p, r, c), type_ref_nesting(p, _, _, d, c), d != "1", + type_ref(p, _, "GENERIC_ARGUMENT", tn, _, c), tn != "", + type_ref_owner(p, owner, ok, r), annotation_owner_module(p, owner, ok, mod), + type_name_in_module(p, mod, tn, t). + // ── annotation_owner_module(Prov, OwnerHash, OwnerKind, ModuleHash) ────────── // The module an annotation was written in, whatever kind of declaration owns it. annotation_owner_module(p, ph, "METHOD_PARAM", mod) :- diff --git a/graph/python/engine/resolution/attribute-lookup.dl b/graph/python/engine/resolution/attribute-lookup.dl index 3c032d13..7b44fe2d 100644 --- a/graph/python/engine/resolution/attribute-lookup.dl +++ b/graph/python/engine/resolution/attribute-lookup.dl @@ -317,6 +317,15 @@ type_call_target(p, t, m) :- mro_lookup(p, t, "__call__", m). type_property_getter(p, t, n, m) :- mro_lookup(p, t, n, m), method_kind(p, "PROPERTY_GETTER", _, m). +// ── type_property_accessor(Prov, TypeHash, Name, Kind, MethodHash) ─────────── +// `@x.setter` and `@x.deleter` are the other two halves: `obj.x = v` runs the setter and +// `del obj.x` the deleter, and like the getter neither has a call site. They share the +// getter's name and binding, so they are found on the class that WINS the name in the +// receiver's MRO, by their method kind, rather than through mro_lookup's one answer. +type_property_accessor(p, t, n, k, m) :- + mro_winner(p, t, n, c), method_owner(p, c, m), method_decl(p, n, _, _, _, m), + method_kind(p, k, _, m), (k = "PROPERTY_SETTER" ; k = "PROPERTY_DELETER"). + // ── A USER-WRITTEN DATA DESCRIPTOR IS THE SAME PROTOCOL (issue #326) ───────── // `@property` IS a data descriptor; the decorator is sugar over `__get__`/`__set__`. So // the clause above reads one spelling of the protocol and a class-level diff --git a/graph/python/engine/resolution/builtins.dl b/graph/python/engine/resolution/builtins.dl index ca1e14d1..f530f72d 100644 --- a/graph/python/engine/resolution/builtins.dl +++ b/graph/python/engine/resolution/builtins.dl @@ -182,6 +182,18 @@ py_copy_module("copy"). py_copy_function("copy"). py_copy_function("deepcopy"). +// ── py_returns_arg(DottedName, Position) — a library call that hands back an argument ── +// Each returns, unchanged, the argument at Position (documented behaviour, not inference): +// functools.update_wrapper(wrapper, wrapped) -> wrapper (copies __name__/__doc__ onto it) +// typing.cast(T, value) -> value (a no-op at run time) +// A decorator written `return update_wrapper(wrapper, f)`, or the typed spelling +// `return t.cast(F, update_wrapper(wrapper, f))`, is therefore the same shape as one +// returning `wrapper` under @functools.wraps. Without these every method it decorates was +// decorator_replaced_target, and no call through the attribute reached anything +// (call-edge-generation/call_chain.dl). Neither module is staged in a client-only run. +py_returns_arg("functools.update_wrapper", "0"). +py_returns_arg("typing.cast", "1"). + // ── py_builtin_dynamic(Name) — the escape hatches ──────────────────────────── // These do not merely lack Python source; they make the PROGRAM unanalysable at that // point. A call to getattr/eval/exec means the engine cannot know what runs, and the @@ -358,3 +370,30 @@ class_definition_keyword("metaclass"). // (expression-resolution/expr-type.dl, clause a'), where the convention is the only // signal there is: a lambda has no annotation and no `def` to classify. instance_receiver_name("self"). + +// ── builtin_protocol_slot(Builtin, Dunder) / builtin_protocol_fallback ─────── +// The one-dunder builtins: `repr(x)` runs type(x).__repr__, and so on. str() falls back to +// __repr__ and bool() to __len__ when the class has no slot of its own. Read by +// call-edge-generation/call_chain.dl's BUILTIN PROTOCOL edges. +builtin_protocol_slot("repr", "__repr__"). +builtin_protocol_slot("str", "__str__"). +builtin_protocol_slot("len", "__len__"). +builtin_protocol_slot("hash", "__hash__"). +builtin_protocol_slot("bool", "__bool__"). +builtin_protocol_slot("iter", "__iter__"). +builtin_protocol_slot("next", "__next__"). +builtin_protocol_slot("abs", "__abs__"). +builtin_protocol_slot("format", "__format__"). +builtin_protocol_slot("reversed", "__reversed__"). +builtin_protocol_fallback("str", "__str__", "__repr__"). +builtin_protocol_fallback("bool", "__bool__", "__len__"). + +// ── dict_view_name / dict_view_slot / dict_items_view — a mapping's views ──── +// `keys()` yields the key (generic position 0), `values()` the value (position 1), and +// `items()` a (key, value) pair. Read by resolution/iteration.dl. +dict_view_name("keys"). +dict_view_name("values"). +dict_view_name("items"). +dict_view_slot("keys", "0"). +dict_view_slot("values", "1"). +dict_items_view("items"). diff --git a/graph/python/engine/resolution/dispatch.dl b/graph/python/engine/resolution/dispatch.dl index f1f50d21..ed56e8cc 100644 --- a/graph/python/engine/resolution/dispatch.dl +++ b/graph/python/engine/resolution/dispatch.dl @@ -142,7 +142,31 @@ call_candidate_count(site, n) :- // ── call_dispatch_over_cap(SiteHash) ───────────────────────────────────────── call_dispatch_over_cap(site) :- - call_candidate_count(site, n), dispatch_capped_at(cap), n > cap. + call_candidate_count(site, n), dispatch_capped_at(cap), n > cap, + !call_computed_name_family(site). + +// ── call_computed_name_family(SiteHash) — a fan the PROGRAM wrote down ─────── +// The cap guards against a receiver type inferred too wide, where most of the fan is the +// engine's imprecision. A callee looked up by a name with a written prefix -- +// `getattr(self, f"visit_{...}")`, directly, through the local it was stored in, or +// through a method that returns it -- is the program's own dispatch table: every member +// named with the prefix is a target by construction, and a visitor with sixty visit_* +// methods is sixty targets, not sixty guesses. Refusing it dropped the visitor's whole +// body from every test that drives it. +method_returns_computed_name(m) :- + method_return_value_expr("client", m, e), getattr_computed_name(e, _, _). +computed_name_value(e) :- getattr_computed_name(e, _, _). +computed_name_value(e) :- + call_of_expr(e, s), expr_call_candidate(s, m), method_returns_computed_name(m). +call_computed_name_family(site) :- + call_callee_is_value(site), call_callee_expr(site, callee), computed_name_value(callee). +call_computed_name_family(site) :- + call_callee_is_value(site), call_callee_expr(site, callee), + expr_binding("client", b, ctx, callee), ctx != "STORE", + binding_lookup("client", b, b2), + expr_binding("client", b2, "STORE", tgt), + assign_pair("client", tgt, val), + computed_name_value(val). // ── expr_resolves_to_method(SiteHash, MethodHash) ──────────────────────────── // THE DISAMBIGUATED CALLEE SET the edge layer consumes. Same relation name as Java's, diff --git a/graph/python/engine/resolution/iteration.dl b/graph/python/engine/resolution/iteration.dl index 7a3e3279..76d50692 100644 --- a/graph/python/engine/resolution/iteration.dl +++ b/graph/python/engine/resolution/iteration.dl @@ -98,6 +98,18 @@ with_target_declared(p, tgt, ct) :- method_decl(p, _, _, _, ret, m), self_return_name(ret). +// The same, WRITTEN rather than declared: an __enter__ with no return annotation whose +// return value is `self` (`def __enter__(self): return self`). The protocol hands the +// target what __enter__ returned, and that is the context manager itself. +with_target_declared(p, tgt, ct) :- + with_pair(p, tgt, cm), + expr_root_context(p, crc, cm), context_manager_enter_of(crc, en), + expr_type(p, cm, ct), + mro_lookup(p, ct, en, m), + !method_declared_return_type(p, m, _), + method_return_value_expr(p, m, e), + expr_node(p, "SELF_REFERENCE", _, _, e). + // AN __enter__ RETURN IS A DECLARED TYPE, not an exact one, so it goes out through // declared_dispatch and is registered with call_recv_declared in dispatch.dl. A declared // type escaping that gate has produced a confident-wrong answer four times in this engine. @@ -281,6 +293,79 @@ binding_element_type(p, b, t) :- expr_binding(p, b, _, tgt), element_type_of(p, src, t). +// ── A MAPPING'S VIEWS: `.values()`, `.keys()`, `.items()` ───────────────────── +// self.dump_fields: dict[str, Field] = {} +// for name, field_obj in self.dump_fields.items(): field_obj.serialize(...) +// Every clause above types the container itself; a view is a CALL on it, and none of them +// looked through one, so the commonest way to walk a mapping typed nothing -- and an +// `items()` loop binds a TUPLE target, which no clause bound at all. A mapping annotation +// names its key at generic position 0 and its value at position 1, so the slots are +// read positionally rather than through type_ref_element (which merges both, the safe +// direction for iterating the mapping itself, and wrong for a view that yields one side). +// view_slot names which side each view yields; `items()` yields (key, value) pairs. +// dict_view_slot / dict_items_view are CPython names, catalogued in resolution/builtins.dl. + +// the annotation a mapping-valued expression carries: a field (through self or through +// any typed object), a parameter, or a callee's declared return +mapping_annotation_of(src, r) :- + expr_attr_of_self("client", src, n, owner), + type_attr_field("client", owner, n, f), + type_ref_owner("client", f, "FIELD", r), + type_ref("client", k, "FIELD_TYPE", _, _, r), annotation_container_kind(k). +mapping_annotation_of(src, r) :- + attr_access("client", src, n, obj), + expr_type("client", obj, ot), + type_attr_field("client", ot, n, f), + type_ref_owner("client", f, "FIELD", r), + type_ref("client", k, "FIELD_TYPE", _, _, r), annotation_container_kind(k). +mapping_annotation_of(src, r) :- + expr_names_param("client", src, ph), + type_ref_owner("client", ph, "METHOD_PARAM", r), + type_ref("client", k, "METHOD_PARAM", _, _, r), annotation_container_kind(k). +mapping_annotation_of(src, r) :- + call_of_expr(src, site), expr_call_candidate(site, m), + type_ref_owner("client", m, "METHOD", r), + type_ref("client", k, "METHOD_RETURN", _, _, r), annotation_container_kind(k). +// a single-write local holding one of those +mapping_annotation_of(src, r) :- + expr_binding("client", b, ctx, src), ctx != "STORE", + binding_lookup("client", b, b2), + binding_rebinding("client", "1", _, _, b2), + expr_binding("client", b2, "STORE", tgt), + assign_pair("client", tgt, val), + mapping_annotation_of(val, r). + +// the type at one generic position of an annotation (resolved, or by name in its module) +type_ref_slot(r, pos, t) :- + type_ref_nesting("client", r, pos, "1", child), + type_ref("client", _, "GENERIC_ARGUMENT", _, _, child), + type_ref_resolved("client", t, child). +type_ref_slot(r, pos, t) :- + type_ref_nesting("client", r, pos, "1", child), + type_ref("client", _, "GENERIC_ARGUMENT", tn, _, child), tn != "", + type_ref_owner("client", owner, ok, r), + annotation_owner_module("client", owner, ok, mod), + type_name_in_module("client", mod, tn, t). + +// `m.keys()` / `m.values()` / `m.items()` written with no argument +dict_view_call(src, recv, view) :- + call_of_expr(src, site), call_name(site, view), dict_view_name(view), + call_receiver_object(site, recv), !call_arg(site, _, _). +// the element a `keys()` / `values()` view yields +element_type_of("client", src, t) :- + dict_view_call(src, recv, view), dict_view_slot(view, pos), + mapping_annotation_of(recv, r), r != "", type_ref_slot(r, pos, t). +// an `items()` pair's two slots, and the names a TUPLE loop target binds to them +dict_item_slot_type(src, pos, t) :- + dict_items_view(view), dict_view_call(src, recv, view), dict_view_slot(_, pos), + mapping_annotation_of(recv, r), type_ref_slot(r, pos, t). +binding_element_type("client", b, t) :- + iter_pair("client", tgt, src), + expr_node("client", "TUPLE", _, _, tgt), + expr_parent("client", tgt, "ELEMENT", pos, el), + expr_binding("client", b, _, el), + dict_item_slot_type(src, pos, t). + // ── iteration_untyped(Prov, TargetExprHash, IterableExprHash) ──────────────── // A loop whose container the engine could not type. Emitted so the cost is countable // rather than absorbed into a general "receiver untyped". diff --git a/graph/python/engine/resolution/name-resolution.dl b/graph/python/engine/resolution/name-resolution.dl index 30888484..b9345eb1 100644 --- a/graph/python/engine/resolution/name-resolution.dl +++ b/graph/python/engine/resolution/name-resolution.dl @@ -1008,18 +1008,41 @@ computed_name_prefix(nameArg, pre) :- expr_operator("client", op, nameArg), py_str_concat_operator(op), expr_parent("client", nameArg, "OPERAND_LEFT", _, l), expr_node("client", "LITERAL", _, pre, l), pre != "". +// The same prefix written as an f-string, `getattr(self, f"visit_{type(node).__name__}")`: +// the parser hands over an f-string's constant lead (the text before its first `{`) as its +// literal value, and every string the f-string can produce starts with it. +computed_name_prefix(nameArg, pre) :- + expr_node("client", "FSTRING", _, pre, nameArg), pre != "". getattr_computed_name(e, obj, pre) :- call_of_expr(e, site), call_name(site, "getattr"), call_arg(site, "0", obj), call_arg(site, "1", nameArg), computed_name_prefix(nameArg, pre). -expr_denotes_method("client", e, m) :- +// The receiver's type, and where the receiver is `self`, every CONSTRUCTED subclass of it too: +// a visitor base names `visit_*` members it does not define itself, and self is whichever +// subclass was built -- the same self-dispatch bound (RTA) dispatch.dl puts on `self.m()`. +getattr_computed_type(e, pre, t) :- getattr_computed_name(e, obj, pre), - expr_type("client", obj, t), + expr_type("client", obj, t). +getattr_computed_type(e, pre, s) :- + getattr_computed_name(e, obj, pre), + expr_node("client", "SELF_REFERENCE", _, _, obj), + expr_type("client", obj, k), + type_subclass("client", k, s), + type_instantiated_reachable("client", s). +getattr_computed_member(e, m) :- + getattr_computed_type(e, pre, t), mro_lookup("client", t, n, m), strlen(n) > strlen(pre), substr(n, 0, strlen(pre)) = pre. +// a member a decorator replaced is the callable the decorator returned (a wrapper) +getattr_computed_member(e, m) :- + getattr_computed_type(e, pre, t), + mro_lookup_decorated("client", t, n, m), + strlen(n) > strlen(pre), + substr(n, 0, strlen(pre)) = pre. +expr_denotes_method("client", e, m) :- getattr_computed_member(e, m). // ── getattr(obj, name, default) IS AT LEAST the default ────────────────────── // h = getattr(record, "highlighter", self.highlighter) diff --git a/graph/python/engine/resolution/value-flow.dl b/graph/python/engine/resolution/value-flow.dl index 14ef74b5..6fd224af 100644 --- a/graph/python/engine/resolution/value-flow.dl +++ b/graph/python/engine/resolution/value-flow.dl @@ -97,6 +97,52 @@ param_arg_type(ph, t) :- param_decl("client", kw, _, _, m, ph), expr_type("client", a, t). +// ── a parameter the TEST RUNNER fills ───────────────────────────────────────── +// pytest calls `test_commit(session=)`: the call is the runner's, +// so no call site spells it, but it is an argument reaching a parameter all the same, and +// the parameter is typed exactly as one passed at a call site would be. Which fixture +// serves the parameter is framework-behavior/dispatch.dl's py_fixs_injection (2a), the +// runner's own lookup (same class, same module, nearest conftest, a pytest_plugins +// module), so this is as deterministic as a name the LEGB walk resolves. +// MEASURED across ten public suites: 1,529 method calls on such parameters were +// ambiguous_unknown, and everything derived from them (`tx = session.begin(); tx.commit()`) +// with them; a suite whose fixtures carry annotations resolved its own. +param_arg_type(ph, t) :- + py_fixs_injection(req, fix, pn), + param_decl("client", pn, _, _, req, ph), + py_fixture_value_type(fix, t). + +// ── a parameter's DEFAULT VALUE reaches it ──────────────────────────────────── +// `def getter(val, field_obj=field_obj): return field_obj.deserialize(val)` -- the closure +// idiom that pins a loop variable -- is called with one argument, so the only value +// field_obj ever holds is its default, and no call site passes one. The default is an +// argument the def statement writes once, evaluated where the def runs: one more value +// reaching the parameter, alongside whatever the call sites pass. +param_arg_type(ph, t) :- param_default_expr("client", e, ph), expr_type("client", e, t). +param_arg_class(ph, t) :- param_default_expr("client", e, ph), expr_type_class_object("client", e, t). +param_arg_method(ph, m) :- param_default_expr("client", e, ph), expr_denotes_method("client", e, m). + +// ── a parameter the runner fills from a PARAMETRIZE row, or from FIXTURE PARAMS ── +// `@pytest.mark.parametrize("cls", [String, Integer])` calls the test once per value, so +// each value reaches `cls` as an argument would: an instance types it, a class object +// makes it that class, a function makes it callable as that function. Inside a fixture +// declared `params=[...]`, `request.param` is each of those values in turn. +// framework-behavior/dispatch.dl reads the rows (py_pz_param_value, py_pz_request_param). +param_arg_type(ph, t) :- py_pz_param_value(ph, v), expr_type("client", v, t). +param_arg_class(ph, t) :- py_pz_param_value(ph, v), expr_type_class_object("client", v, t). +param_arg_method(ph, m) :- py_pz_param_value(ph, v), expr_denotes_method("client", v, m). +expr_type("client", e, t) :- py_pz_request_param(e, v), expr_type("client", v, t). +expr_type_class_object("client", e, t) :- py_pz_request_param(e, v), expr_type_class_object("client", v, t). +expr_denotes_method("client", e, m) :- py_pz_request_param(e, v), expr_denotes_method("client", v, m). +// a fixture that hands back a CLASS (`return request.param` over classes) makes the +// requesting parameter that class, as a returned instance makes it an instance +py_fixture_value_class(fix, t) :- py_fixture_decl(fix, _), + method_return_value_expr("client", fix, e), expr_type_class_object("client", e, t). +param_arg_class(ph, t) :- + py_fixs_injection(req, fix, pn), + param_decl("client", pn, _, _, req, ph), + py_fixture_value_class(fix, t). + // ── param_arg_method(ParamHash, MethodHash) — a CALLABLE reaching a parameter ── // `Delegator(target)` puts the FUNCTION `target` into the parameter `fn`. Java would // need a functional interface for this; in Python it is an ordinary assignment. diff --git a/graph/python/libsum/libsum.py b/graph/python/libsum/libsum.py new file mode 100644 index 00000000..e910d123 --- /dev/null +++ b/graph/python/libsum/libsum.py @@ -0,0 +1,1090 @@ +#!/usr/bin/env python3 +"""libsum.py -- what an installed Python library does with the objects a client hands it. + + libsum.py --src --out [--site [,...]] [--cache ] + +THE GAP THIS CLOSES. The engine reads a dependency's signatures, never its bodies, so a hop the +LIBRARY makes back into client code is invisible: a test client's `get()` calls the `open()` a +client subclass overrides, a CLI runner calls `main()` on the command it was handed, a test client +built around an application calls `app(environ, start_response)` on it. Every one of those is a +library frame between two client frames, and the call graph stops at the first. + +WHAT A SUMMARY SAYS. One row per hand-back a library callable is known to make, rooted at one of +its parameters (or at `self`, the object it was called on): + + py-lib-callback.csv callee root pos path member via type + callee the library callable, canonical dotted name: `pkg.mod.func`, `pkg.mod.Class.method` + (one row per method a class HAS, its inherited ones included), `pkg.mod.Class` for + constructing it + root the parameter the object arrives in, or `self` + pos where the CLIENT writes that argument: the index among the positional arguments of + the call as written (an instance method's `self` is not written), -1 for `self` + path the attribute chain the library follows from that object first ("" for the object + itself, `application` for `self.application`), at most two long + member the method the library invokes on what it reached: a name it calls, or the dunder a + syntax runs (`__call__` for `x()`, `__enter__` for `with x`, `__iter__` for `for`...) + via the library function in which that invocation is written + type the class the LIBRARY declares that object to be (a parameter annotation, a field's), "" when it + declares none: when the client's own object cannot be told, a client subclass of it is the answer + py-lib-field-init.csv class field param pos + constructing `class` stores the argument `param` (written at `pos`) on the new object as + `field`, so a later `self.` hand-back reaches what the client passed + py-lib-ancestor.csv class ancestor + a library class's library ancestors, for the classes the client can name + py-lib-alias.csv alias canonical + a name a package re-exports (`clikit.Command` is `clikit.core.Command`) + py-lib-decorates.csv decorator form class field + applying the library callable as a decorator -- `@decorator` (DIRECT) or `@decorator(...)` (FACTORY) -- + to a function f yields a new `class` whose `field` holds f; class "" when f itself comes back + +HOW, AND WHY NOT IN THE SOLVE. Library source is just Python, so the facts are DERIVED from it by a +small flow-insensitive pass over each function body (which parameter-rooted value meets which call or +protocol syntax), composed to a fixpoint across the library's own calls: `Client.get` calls +`self.open`, `open` calls `run_app(self.application, ...)`, `run_app` calls `app(...)`, so +`Client.get` hands back `__call__` on `self.application`. Nothing is solved over library bodies at +index time: the pass reads only the packages the client imports (and what they import), is cached by +package and version, and writes three small tables. A library written in C has no body to read and +yields nothing; that is a declared blind spot, not a guess. + +WHERE THE LIBRARY IS. The project's own virtual environment (`.venv`, `venv`, `env` in the repository, +else $VIRTUAL_ENV), its stdlib through pyvenv.cfg, or AXIOMCODE_PY_SITE (a comma list of directories) +when set. No environment found means no summaries: the engine then behaves exactly as before. +""" +import argparse, ast, csv, hashlib, json, os, re, sys + +VERSION = '1' +MAXPATH = 2 +DUNDER_BUILTIN = {'len': '__len__', 'str': '__str__', 'repr': '__repr__', 'iter': '__iter__', 'next': '__next__', + 'hash': '__hash__', 'bool': '__bool__', 'format': '__format__', 'reversed': '__reversed__', + 'abs': '__abs__', 'aiter': '__aiter__', 'anext': '__anext__'} +SKIP_DIRS = {'__pycache__', 'tests', 'test', 'testing_support', 'docs', 'examples', 'benchmarks'} + + +# ───────────────────────────────────────────────────────────────────────────── +# where the library is +# ───────────────────────────────────────────────────────────────────────────── +def find_sites(src): + if os.environ.get('AXIOMCODE_PY_SITE'): + return [d for d in os.environ['AXIOMCODE_PY_SITE'].split(',') if os.path.isdir(d)], None + envs = [os.path.join(src, n) for n in ('.venv', 'venv', 'env', '.env')] + if os.environ.get('VIRTUAL_ENV'): + envs.append(os.environ['VIRTUAL_ENV']) + for e in envs: + cfg = os.path.join(e, 'pyvenv.cfg') + if not os.path.isfile(cfg): + continue + sites = [] + for root in (os.path.join(e, 'lib'), os.path.join(e, 'Lib')): + if not os.path.isdir(root): + continue + if os.path.isdir(os.path.join(root, 'site-packages')): + sites.append(os.path.join(root, 'site-packages')) + for d in sorted(os.listdir(root)): + sp = os.path.join(root, d, 'site-packages') + if d.startswith('python') and os.path.isdir(sp): + sites.append(sp) + stdlib = None + try: + kv = dict(l.split('=', 1) for l in open(cfg, encoding='utf-8') if '=' in l) + kv = {k.strip(): v.strip() for k, v in kv.items()} + home, ver = kv.get('home', ''), (kv.get('version_info') or kv.get('version') or '') + mm = '.'.join(ver.split('.')[:2]) + for cand in (os.path.join(os.path.dirname(home), 'lib', f'python{mm}'), os.path.join(home, 'Lib'), + os.path.join(os.path.dirname(home), 'Lib')): + if mm and os.path.isfile(os.path.join(cand, 'contextlib.py')): + stdlib = cand; break + except Exception: + pass + if sites: + return sites, stdlib + return [], None + + +# ───────────────────────────────────────────────────────────────────────────── +# which modules: the client's imports, then what those import, transitively +# ───────────────────────────────────────────────────────────────────────────── +IMPORT_RE = re.compile(r'^\s*(?:from\s+([A-Za-z_][\w.]*)\s+import|import\s+([A-Za-z_][\w.]*(?:\s*,\s*[A-Za-z_][\w.]*)*))', re.M) + + +def client_imports(src, ir): + """every dotted path the client imports: `from a.b import C` gives a.b.C, `import a.b` gives a.b. From the + parser's import table when the IR is there, else from the source""" + paths = set() + p = os.path.join(ir, 'all-python-imports.csv') if ir else '' + if p and os.path.isfile(p): + try: + with open(p, newline='', encoding='utf-8') as fh: + r = csv.reader(fh, delimiter='\t') + head = next(r) + if 'importedPath' in head: + col = head.index('importedPath') + for row in r: + if len(row) > col and row[col]: + paths.add(row[col]) + return paths + except Exception: + pass + skip = {'.venv', 'venv', 'env', '.env', '.git', 'node_modules', '.tox', '__pycache__', '.axiomcode'} + for d, ds, fs in os.walk(src): + ds[:] = [x for x in ds if x not in skip] + for f in fs: + if not f.endswith('.py'): + continue + try: + tree = ast.parse(open(os.path.join(d, f), encoding='utf-8', errors='replace').read()) + except Exception: + continue + for n in ast.walk(tree): + if isinstance(n, ast.Import): + paths.update(a.name for a in n.names) + elif isinstance(n, ast.ImportFrom) and n.module and not n.level: + paths.update(f'{n.module}.{a.name}' for a in n.names if a.name != '*') + paths.add(n.module) + return paths + + +class Package: + """one importable top-level name: a directory package or a single module file""" + def __init__(self, name, path, base): + self.name, self.path, self.base = name, path, base + + def files(self): + if os.path.isfile(self.path): + yield self.name, self.path + return + for d, ds, fs in os.walk(self.path): + ds[:] = sorted(x for x in ds if x not in SKIP_DIRS and not x.startswith('.')) + for f in sorted(fs): + if not f.endswith('.py'): + continue + rel = os.path.relpath(os.path.join(d, f), self.base)[:-3].replace(os.sep, '.') + if rel.endswith('.__init__'): + rel = rel[:-9] + yield rel, os.path.join(d, f) + + +def locate(name, sites, stdlib): + for base in list(sites) + ([stdlib] if stdlib else []): + p = os.path.join(base, name) + if os.path.isdir(p) and (os.path.isfile(os.path.join(p, '__init__.py')) or base != stdlib): + return Package(name, p, base) + if os.path.isfile(p + '.py'): + return Package(name, p + '.py', base) + return None + + +# ───────────────────────────────────────────────────────────────────────────── +# one module: its namespace, its classes, and the events in each function body +# ───────────────────────────────────────────────────────────────────────────── +class Fn: + __slots__ = ('q', 'params', 'pos', 'kind', 'events', 'cls', 'stores', 'ptypes', 'rets', 'local') + + def __init__(self, q, cls): + self.q, self.cls, self.ptypes, self.rets, self.local = q, cls, {}, [], False + self.params, self.pos, self.kind = [], {}, 'function' + self.events = [] # ('use', val, member) | ('call', callee, selfval, {param: [vals]}, [posvals], {kw: [vals]}) + self.stores = [] # (field, val) written on self + + +class Cls: + __slots__ = ('q', 'bases', 'methods', 'mod', 'ftypes') + + def __init__(self, q, mod): + self.q, self.mod, self.bases, self.methods, self.ftypes = q, mod, [], {}, {} + + +def resolve_rel(mod, is_pkg, level, name): + parts = mod.split('.') + if not is_pkg: + parts = parts[:-1] + if level > 1: + parts = parts[:len(parts) - (level - 1)] + return '.'.join([p for p in parts if p] + ([name] if name else [])) + + +class Module: + def __init__(self, name, path, is_pkg): + self.name, self.path, self.is_pkg = name, path, is_pkg + self.ns = {} # local name -> qualified name (a def, a class, an imported thing) + self.fns, self.classes = {}, {} + self.imports = set() + + def load(self): + try: + tree = ast.parse(open(self.path, encoding='utf-8', errors='replace').read()) + except Exception: + return False + self.tree = tree + for st in tree.body: + self._ns_stmt(st) + return True + + def _ns_stmt(self, st): + if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + self.ns[st.name] = f'{self.name}.{st.name}' + elif isinstance(st, ast.ImportFrom): + base = resolve_rel(self.name, self.is_pkg, st.level, st.module or '') if st.level else (st.module or '') + if base: + self.imports.add(base) + for a in st.names: + if a.name != '*': + self.ns[a.asname or a.name] = f'{base}.{a.name}' if base else a.name + elif isinstance(st, ast.Import): + for a in st.names: + self.imports.add(a.name) + if a.asname: + self.ns[a.asname] = a.name + else: + self.ns.setdefault(a.name.split('.')[0], a.name.split('.')[0]) + elif isinstance(st, ast.Assign) and len(st.targets) == 1 and isinstance(st.targets[0], ast.Name): + v = dotted(st.value) + if v and v.split('.')[0] in self.ns: + self.ns.setdefault(st.targets[0].id, self.ns[v.split('.')[0]] + v[len(v.split('.')[0]):]) + elif isinstance(st, (ast.If, ast.Try)): + for b in [st.body, getattr(st, 'orelse', []), getattr(st, 'finalbody', [])] + [h.body for h in getattr(st, 'handlers', [])]: + for s in b: + self._ns_stmt(s) + + def qual(self, n): + return self.ns.get(n) + + def analyse(self): + for st in self.tree.body: + if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef)): + self._fn(st, None) + elif isinstance(st, ast.ClassDef): + c = Cls(f'{self.name}.{st.name}', self) + c.bases = [b for b in (self.qual_expr(x) for x in st.bases) if b] + self.classes[c.q] = c + for s in st.body: + if isinstance(s, ast.AnnAssign) and isinstance(s.target, ast.Name): + t = self.ann_qual(s.annotation) + if t: + c.ftypes[s.target.id] = t + for s in st.body: + if isinstance(s, (ast.FunctionDef, ast.AsyncFunctionDef)): + f = self._fn(s, c) + c.methods[s.name] = f.q + + def ann_qual(self, ann): + """the class an annotation names: `T`, `pkg.T`, `"T"`, `Optional[T]`, `T | None`""" + if isinstance(ann, ast.Constant) and isinstance(ann.value, str): + try: + ann = ast.parse(ann.value, mode='eval').body + except SyntaxError: + return None + if isinstance(ann, ast.BinOp): + return self.ann_qual(ann.left) + if isinstance(ann, ast.Subscript) and (dotted(ann.value) or '').split('.')[-1] == 'Optional': + return self.ann_qual(ann.slice) + return self.qual_expr(ann) if ann is not None else None + + def qual_expr(self, e): + d = dotted(e.value if isinstance(e, ast.Subscript) else e) + if not d: + return None + head, _, rest = d.partition('.') + q = self.qual(head) + return (q + ('.' + rest if rest else '')) if q else None + + def _fn(self, node, cls): + q = f'{cls.q}.{node.name}' if cls else f'{self.name}.{node.name}' + f = self.signature(node, q, cls) + self.fns[q] = f + Body(self, f, f.params[0] if f.kind in ('method', 'classmethod', 'property') and f.params else None, cls).run(node) + return f + + def signature(self, node, q, cls): + f = Fn(q, cls.q if cls else None) + decos = {dotted(d.func if isinstance(d, ast.Call) else d) for d in node.decorator_list} + a = node.args + names = [x.arg for x in a.posonlyargs + a.args] + if cls is not None: + if 'staticmethod' in decos: + f.kind = 'static' + elif 'classmethod' in decos: + f.kind = 'classmethod' + elif 'property' in decos or any(d and d.endswith(('.setter', '.getter', 'cached_property')) for d in decos): + f.kind = 'property' + else: + f.kind = 'method' + f.params = names + [x.arg for x in a.kwonlyargs] + skip = 1 if f.kind in ('method', 'classmethod', 'property') and names else 0 + for i, n in enumerate(names): + f.pos[n] = i - skip if i >= skip else -1 + for x in a.kwonlyargs: + f.pos[x.arg] = -2 # keyword only: matched by name + for x in a.posonlyargs + a.args + a.kwonlyargs: + if x.annotation is not None: + t = self.ann_qual(x.annotation) + if t: + f.ptypes[x.arg] = t + return f + + +def dotted(e): + if isinstance(e, ast.Name): + return e.id + if isinstance(e, ast.Attribute): + d = dotted(e.value) + return f'{d}.{e.attr}' if d else None + return None + + +class Body: + """a flow-insensitive pass over one function body (nested defs and lambdas included: a closure the + library builds around a parameter is how a decorator or a callback wrapper hands it back)""" + def __init__(self, mod, fn, selfname, cls): + self.mod, self.fn, self.selfname, self.cls = mod, fn, selfname, cls + self.env = {} + self.params = set(fn.params) + + def run(self, node): + for _ in range(2): # twice: a local assigned below its first use + self.fn.events = []; self.fn.stores = []; self.fn.rets = [] + for st in node.body: + self.stmt(st) + return self + + # values: ('p', root, path) | ('ref', qname) | ('new', class, argvals) | ('self', ...) is ('p', 'self', ()) + def val(self, e): + if isinstance(e, ast.Name): + if e.id in self.env: + return self.env[e.id] + if e.id in self.params: + return [('p', 'self' if e.id == self.selfname else e.id, ())] + q = self.mod.qual(e.id) + return [('ref', q)] if q else [] + if isinstance(e, ast.Attribute): + out = [] + for v in self.val(e.value): + if v[0] == 'p' and len(v[2]) < MAXPATH: + out.append(('p', v[1], v[2] + (e.attr,))) + elif v[0] == 'ref': + out.append(('ref', f'{v[1]}.{e.attr}')) + elif v[0] == 'new': + out.append(('field', v, e.attr)) + return out + if isinstance(e, ast.Call): + return self.call(e) + if isinstance(e, (ast.IfExp,)): + self.val(e.test) + return self.val(e.body) + self.val(e.orelse) + if isinstance(e, ast.BoolOp): + out = [] + for x in e.values: + out += self.val(x) + return out + if isinstance(e, ast.NamedExpr): + v = self.val(e.value) + if isinstance(e.target, ast.Name): + self.env[e.target.id] = v + return v + if isinstance(e, ast.Await): + return self.val(e.value) + if isinstance(e, ast.Subscript): + for v in self.val(e.value): + self.use(v, '__getitem__') + self.val(e.slice) + return [] + if isinstance(e, ast.Compare): + self.val(e.left) + for op, c in zip(e.ops, e.comparators): + cv = self.val(c) + if isinstance(op, (ast.In, ast.NotIn)): + for v in cv: + self.use(v, '__contains__') + return [] + if isinstance(e, ast.JoinedStr): + for x in e.values: + if isinstance(x, ast.FormattedValue): + m = {115: '__str__', 114: '__repr__', 97: '__repr__'}.get(x.conversion, '__format__') + for v in self.val(x.value): + self.use(v, m) + return [] + if isinstance(e, ast.Lambda): + self.nested(e.body if isinstance(e.body, list) else [ast.Expr(e.body)], e.args) + return [] + if isinstance(e, (ast.ListComp, ast.SetComp, ast.GeneratorExp, ast.DictComp)): + for g in e.generators: + for v in self.val(g.iter): + self.use(v, '__iter__') + for c in g.ifs: + self.val(c) + for x in ([e.key, e.value] if isinstance(e, ast.DictComp) else [e.elt]): + self.val(x) + return [] + for ch in ast.iter_child_nodes(e): + if isinstance(ch, ast.expr): + self.val(ch) + return [] + + def use(self, v, member): + if member == '__call__' and v == ('p', 'self', ()) and self.fn.kind == 'classmethod': + member = '__init__' # cls(...) constructs the class the method was called on + if v[0] == 'p' or v[0] == 'field': + # what the library itself declares the object to be, where it says: a parameter's annotation, a field's + hint = None + if v[0] == 'p': + base = self.cls.q if v[1] == 'self' and self.cls else self.fn.ptypes.get(v[1]) + if base and not v[2]: + hint = ('t', base) + elif base and len(v[2]) == 1: + hint = ('f', base, v[2][0]) + self.fn.events.append(('use', v, member, hint)) + + def call(self, e): + f = e.func + posv = [self.val(a) if not isinstance(a, ast.Starred) else None for a in e.args] + kwv = {k.arg: self.val(k.value) for k in e.keywords if k.arg} + for k in e.keywords: + if not k.arg: + self.val(k.value) + if isinstance(f, ast.Name) and f.id in DUNDER_BUILTIN and f.id not in self.env and not self.mod.qual(f.id) and posv and posv[0]: + for v in posv[0]: + self.use(v, DUNDER_BUILTIN[f.id]) + return [] + if isinstance(f, ast.Name) and f.id == 'super' and not e.args: + return [('super',)] + fd = dotted(f) or '' + if fd.split('.')[-1] == 'cast' and len(posv) == 2 and posv[1]: + return posv[1] # typing.cast(T, x) is x + if fd == 'type' and len(posv) == 1 and posv[0]: + return posv[0] # type(x).__enter__ is looked up on x's class: x's member + if isinstance(f, ast.Attribute): + out = [] + for rv in self.val(f.value): + if rv[0] in ('p', 'field'): + self.use(rv, f.attr) + self.fn.events.append(('call', ('method', rv, f.attr), posv, kwv)) + elif rv[0] == 'super': + self.fn.events.append(('call', ('super', self.cls.q if self.cls else None, f.attr), posv, kwv)) + elif rv[0] == 'new': + self.fn.events.append(('call', ('method', rv, f.attr), posv, kwv)) + elif rv[0] == 'ref': + self.fn.events.append(('call', ('ref', f'{rv[1]}.{f.attr}'), posv, kwv)) + return out + out = [] + for cv in self.val(f): + if cv[0] in ('p', 'field'): + self.use(cv, '__call__') + if cv[0] == 'p' and cv[2]: + # a member read first and called later (`_enter = type(cm).__enter__; _enter(cm)`) + self.use(('p', cv[1], cv[2][:-1]), cv[2][-1]) + elif cv[0] == 'ref': + self.fn.events.append(('call', ('ref', cv[1]), posv, kwv)) + out.append(('new', cv[1], self.args(posv, kwv))) + elif cv[0] == 'closure': + self.fn.events.append(('call', ('ref', cv[1]), posv, kwv)) + out.append(('res', cv[1], self.args(posv, kwv))) + elif cv[0] in ('new', 'res', 'res2'): + out.append(('res2', cv, self.args(posv, kwv))) # calling what a call returned: `command(...)(f)` + return out + + @staticmethod + def args(posv, kwv): + return (tuple(tuple(p) if p is not None else None for p in posv), tuple(sorted((k, tuple(v)) for k, v in kwv.items()))) + + def nested(self, body, args): + saved = set(self.params) + for a in args.posonlyargs + args.args + args.kwonlyargs + [x for x in (args.vararg, args.kwarg) if x]: + self.params.discard(a.arg); self.env.pop(a.arg, None) + for st in body: + self.stmt(st) + self.params = saved + + def assign(self, t, vals): + if isinstance(t, ast.Name): + # flow-insensitive: a parameter rebound keeps what it arrived as (`callback = other.callback`) + base = self.env.get(t.id) or ([('p', 'self' if t.id == self.selfname else t.id, ())] if t.id in self.params else []) + self.env[t.id] = list(dict.fromkeys(base + vals)) + elif isinstance(t, ast.Attribute): + for ov in self.val(t.value): + if ov == ('p', 'self', ()): + for v in vals: + self.fn.stores.append((t.attr, v)) + if self.cls and v[0] == 'p' and not v[2] and v[1] in self.fn.ptypes: + self.cls.ftypes.setdefault(t.attr, self.fn.ptypes[v[1]]) + elif isinstance(t, (ast.Tuple, ast.List)): + for x in t.elts: + self.assign(x, []) + elif isinstance(t, ast.Subscript): + for v in self.val(t.value): + self.use(v, '__setitem__') + + def stmt(self, st): + if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef)): + self.nested(st.body, st.args) + # and as a callable of its own, so what it RETURNS is known: a decorator factory returns this closure, + # and the closure returns the object it wrapped the decorated function in + sq = f'{self.fn.q}..{st.name}' + sub = self.mod.signature(st, sq, None) + sub.local = True + self.mod.fns[sq] = sub + b = Body(self.mod, sub, None, None) + b.env = {k: list(v) for k, v in self.env.items() if k not in set(sub.params)} + b.run(st) + self.env[st.name] = [('closure', sq)] + return + if isinstance(st, ast.Return): + if st.value is not None: + self.fn.rets.extend(self.val(st.value)) + return + if isinstance(st, ast.ClassDef): + return + if isinstance(st, ast.Assign): + v = self.val(st.value) + for t in st.targets: + self.assign(t, v) + return + if isinstance(st, ast.AnnAssign) and isinstance(st.target, ast.Attribute) and self.cls \ + and isinstance(st.target.value, ast.Name) and st.target.value.id == self.selfname: + t = self.mod.ann_qual(st.annotation) + if t: + self.cls.ftypes[st.target.attr] = t + if isinstance(st, (ast.AnnAssign, ast.AugAssign)): + if st.value is not None: + v = self.val(st.value) + self.assign(st.target, v if isinstance(st, ast.AnnAssign) else []) + return + if isinstance(st, (ast.For, ast.AsyncFor)): + for v in self.val(st.iter): + self.use(v, '__aiter__' if isinstance(st, ast.AsyncFor) else '__iter__') + self.assign(st.target, []) + if isinstance(st, (ast.With, ast.AsyncWith)): + a = isinstance(st, ast.AsyncWith) + for it in st.items: + vs = self.val(it.context_expr) + for v in vs: + self.use(v, '__aenter__' if a else '__enter__'); self.use(v, '__aexit__' if a else '__exit__') + if it.optional_vars is not None: + self.assign(it.optional_vars, []) + if isinstance(st, ast.Delete): + for t in st.targets: + if isinstance(t, ast.Subscript): + for v in self.val(t.value): + self.use(v, '__delitem__') + return + for fld, ch in ast.iter_fields(st): + if fld in ('target', 'targets', 'iter', 'items') and isinstance(st, (ast.For, ast.AsyncFor, ast.With, ast.AsyncWith)): + continue + for c in (ch if isinstance(ch, list) else [ch]): + if isinstance(c, ast.stmt): + self.stmt(c) + elif isinstance(c, ast.expr): + self.val(c) + elif isinstance(c, ast.excepthandler): + for s in c.body: + self.stmt(s) + elif isinstance(c, ast.match_case) if hasattr(ast, 'match_case') else False: + for s in c.body: + self.stmt(s) + + +# ───────────────────────────────────────────────────────────────────────────── +# the program: every analysed module, then the fixpoint across the library's own calls +# ───────────────────────────────────────────────────────────────────────────── +class Program: + def __init__(self): + self.mods, self.fns, self.classes = {}, {}, {} + + def add(self, m): + self.mods[m.name] = m + self.fns.update(m.fns); self.classes.update(m.classes) + + def canon(self, q, seen=None): + """follow a module re-export to the declaration: clikit.Command -> clikit.core.Command""" + if q in self.fns or q in self.classes: + return q + seen = seen or set() + if q in seen or not q: + return None + seen.add(q) + parts = q.split('.') + for i in range(len(parts) - 1, 0, -1): + m = self.mods.get('.'.join(parts[:i])) + if m is None: + continue + nxt = m.qual(parts[i]) + if nxt and nxt != '.'.join(parts[:i + 1]): + return self.canon('.'.join([nxt] + parts[i + 1:]), seen) + if i + 1 < len(parts): # Class.method + c = self.canon('.'.join(parts[:i + 1]), seen) + if c in self.classes: + r = self.lookup(c, parts[i + 1]) + return r if len(parts) == i + 2 else None + return None + return None + + def type_of(self, hint): + """the canonical library class a use hint names: ('t', T) the object's own declared type, ('f', B, f) the + declared type of field f on B (annotated in the class, or assigned from an annotated parameter)""" + if not hint: + return '' + if hint[0] == 't': + c = self.canon(hint[1]) + return c if c in self.classes else '' + b = self.canon(hint[1]) + if b not in self.classes: + return '' + for k in self.mro(b): + t = self.classes[k].ftypes.get(hint[2]) + if t: + c = self.canon(t) or self.canon(f'{self.classes[k].mod.name}.{t}') + return c if c in self.classes else '' + return '' + + def mro(self, c, seen=None): + seen = seen if seen is not None else set() + if c in seen: + return [] + seen.add(c) + out = [c] + k = self.classes.get(c) + for b in (k.bases if k else []): + bc = self.canon(b) + if bc in self.classes: + for x in self.mro(bc, seen): + if x not in out: + out.append(x) + return out + + def lookup(self, c, name): + for k in self.mro(c): + m = self.classes[k].methods.get(name) + if m: + return m + return None + + def solve(self): + S = {q: set() for q in self.fns} # (root, path, member, via) + ST = {c: set() for c in self.classes} # class -> (field, init param) + for f in self.fns.values(): + for ev in f.events: + if ev[0] == 'use' and ev[1][0] == 'p' and (ev[1][1] == 'self' or ev[1][1] in f.pos): + S[f.q].add((ev[1][1], ev[1][2], ev[2], f.q, self.type_of(ev[3]))) + if f.cls and f.q.endswith('.__init__'): + for fld, v in f.stores: + if v[0] == 'p' and v[2] == () and v[1] in f.pos: + ST[f.cls].add((fld, v[1])) + + def init_of(c): + return self.lookup(c, '__init__') + + def stores(c): + out = set() + for k in self.mro(c): + out |= ST.get(k, set()) + return out + + def bind(callee, posv, kwv): + """callee parameter -> the caller's values""" + g = self.fns.get(callee) + if not g: + return {} + out = {} + byp = {i: n for n, i in g.pos.items() if i >= 0} + for i, vs in enumerate(posv): + if vs is not None and i in byp: + out.setdefault(byp[i], []).extend(vs) + for k, vs in kwv.items(): + if k in g.pos: + out.setdefault(k, []).extend(vs) + return out + + def lift(v, path, depth=0): + """a value reached in the caller, followed down `path`: param-rooted values only""" + if v[0] == 'p': + p = v[2] + tuple(path) + return [(v[1], p)] if len(p) <= MAXPATH else [] + if v[0] == 'field' and depth < 3: + return [x for w in field_vals(v[1], v[2]) for x in lift(w, path, depth + 1)] + if v[0] == 'new' and path and depth < 3: + return [x for w in field_vals(v, path[0]) for x in lift(w, path[1:], depth + 1)] + return [] + + def field_vals(newv, fld): + c = self.canon(newv[1]) + if c not in self.classes: + return [] + ini = init_of(c) + if not ini: + return [] + posv = [list(x) if x is not None else None for x in newv[2][0]] + kwv = {k: list(v) for k, v in newv[2][1]} + b = bind(ini, posv, kwv) + return [w for f, pn in stores(c) if f == fld for w in b.get(pn, [])] + + changed, rounds = True, 0 + while changed and rounds < 12: + changed, rounds = False, rounds + 1 + for f in self.fns.values(): + acc = S[f.q]; n0 = len(acc) + for ev in f.events: + if ev[0] == 'use' and ev[1][0] == 'field': + for root, path in lift(ev[1], ()): + if root == 'self' or root in f.pos: + acc.add((root, path, ev[2], f.q, '')) + continue + if ev[0] != 'call': + continue + tgt, posv, kwv = ev[1], ev[2], ev[3] + targets = [] # (callee, selfvals) + if tgt[0] == 'ref': + q = self.canon(tgt[1]) + if q in self.fns: + targets.append((q, [])) + elif q in self.classes and init_of(q): + targets.append((init_of(q), [])) + elif tgt[0] == 'method': + rv, name = tgt[1], tgt[2] + if rv == ('p', 'self', ()) and f.cls: + m = self.lookup(f.cls, name) + if m: + targets.append((m, [rv])) + elif rv[0] == 'p' and rv[2] == () and rv[1] in f.ptypes: + c = self.canon(f.ptypes[rv[1]]) + m = self.lookup(c, name) if c in self.classes else None + if m: + targets.append((m, [rv])) + elif rv[0] == 'p' and len(rv[2]) == 1: + # `self.loader.load(...)`: the field's declared type says whose `load` runs + base = f.cls if rv[1] == 'self' else f.ptypes.get(rv[1]) + c = self.type_of(('f', base, rv[2][0])) if base else '' + m = self.lookup(c, name) if c else None + if m: + targets.append((m, [rv])) + elif rv[0] == 'new': + c = self.canon(rv[1]) + if c in self.classes and self.lookup(c, name): + targets.append((self.lookup(c, name), [rv])) + elif tgt[0] == 'super' and tgt[1]: + for k in self.mro(tgt[1])[1:]: + m = self.classes[k].methods.get(tgt[2]) + if m: + targets.append((m, [('p', 'self', ())])); break + for callee, selfv in targets: + b = bind(callee, posv, kwv) + for (r, p, mem, via, ty) in list(S.get(callee, ())): + src = selfv if r == 'self' else b.get(r, []) + for v in src: + for root, path in lift(v, p): + if root == 'self' or root in f.pos: + acc.add((root, path, mem, via, ty)) + if callee.endswith('.__init__'): + g = self.fns[callee] + # super().__init__(x) / Base.__init__(self, x): what the base stores, this class stores + if f.cls and f.q.endswith('.__init__') and (selfv == [('p', 'self', ())] or tgt[0] == 'super'): + for fld, pn in stores(g.cls): + for v in b.get(pn, []): + if v[0] == 'p' and v[2] == () and v[1] in f.pos and (fld, v[1]) not in ST[f.cls]: + ST[f.cls].add((fld, v[1])); changed = True + if len(acc) != n0: + changed = True + self.S, self.ST = S, ST + return rounds + + def returns(self): + """R[f]: what f's return value is, as far as a decorator needs it -- + ('obj', C, field, param) a new C whose `field` holds f's parameter `param` + ('closure', D) the nested function D (a decorator factory's decorator) + ('param', param) f's own parameter, unchanged (an identity / registering decorator)""" + R = {q: set() for q in self.fns} + + def bindv(g, args): + posv = [list(x) if x is not None else None for x in args[0]] + return self._bind(g, posv, {k: list(v) for k, v in args[1]}) + + def own(f, w): + return w[0] == 'p' and w[2] == () and w[1] in f.pos and w[1] != 'self' + + def apply(f, g, args, out): + b = bindv(g, args) + for d in list(R.get(g, ())): + if d[0] == 'closure': + out.add(d) + elif d[0] in ('obj', 'param'): + for w in b.get(d[-1], []): + if own(f, w): + out.add(d[:-1] + (w[1],)) + elif w[0] == 'closure' and d[0] == 'param': + out.add(w) + + def resolve(f, v, out, depth=0): + if depth > 4: + return + if v[0] == 'closure': + out.add(v) + elif own(f, v): + out.add(('param', v[1])) + elif v[0] == 'new': + c = self.canon(v[1]) + if c in self.classes: + ini = self.lookup(c, '__init__') + if ini: + b = bindv(ini, v[2]) + for fld, pn in self._stores(c): + for w in b.get(pn, []): + if own(f, w): + out.add(('obj', c, fld, w[1])) + elif c in self.fns: + apply(f, c, v[2], out) + elif v[0] == 'res': + apply(f, v[1], v[2], out) + elif v[0] == 'res2': + inner = set() + resolve(f, v[1], inner, depth + 1) + for d in inner: + if d[0] == 'closure': + apply(f, d[1], v[2], out) + + for _ in range(8): + changed = False + for f in self.fns.values(): + acc = R[f.q]; n0 = len(acc) + for v in f.rets: + resolve(f, v, acc) + changed |= len(acc) != n0 + if not changed: + break + self.R = R + + def _bind(self, callee, posv, kwv): + g = self.fns.get(callee) + if not g: + return {} + out = {} + byp = {i: n for n, i in g.pos.items() if i >= 0} + for i, vs in enumerate(posv): + if vs is not None and i in byp: + out.setdefault(byp[i], []).extend(vs) + for k, vs in kwv.items(): + if k in g.pos: + out.setdefault(k, []).extend(vs) + return out + + def _stores(self, c): + out = set() + for k in self.mro(c): + out |= self.ST.get(k, set()) + return out + + def decorates(self, g): + """what applying the callable g as a decorator to a function f yields: (form, class, field) -- DIRECT for + `@g`, FACTORY for `@g(...)`; class '' when f itself comes back (a registering decorator)""" + f = self.fns[g] + first = lambda h: next((n for n, i in self.fns[h].pos.items() if i == 0), None) + out = set() + p0 = first(g) + for d in self.R.get(g, ()): + if d[0] == 'obj' and d[3] == p0: + out.add(('DIRECT', d[1], d[2])) + elif d[0] == 'param' and d[1] == p0: + out.add(('DIRECT', '', '')) + elif d[0] == 'closure': + q0 = first(d[1]) + for e in self.R.get(d[1], ()): + if e[0] == 'obj' and e[3] == q0: + out.add(('FACTORY', e[1], e[2])) + elif e[0] == 'param' and e[1] == q0: + out.add(('FACTORY', '', '')) + return out + + def named(self, imports, lib_pkgs): + """the library classes and functions the client can name: what it imports, what a module it imports + (and that module's direct submodules) declares, and the class a decorator it imports wraps a function + in. Summaries are emitted for these only -- the solve reads what the client can reach, not the venv.""" + out = set() + def add(q): + c = self.canon(q) + if c and c.split('.')[0] in lib_pkgs and '' not in c: + out.add(c) + for p in imports: + add(p) + if p in self.mods: + for m in [p] + [x for x in self.mods if x.startswith(p + '.') and '.' not in x[len(p) + 1:]]: + for local in self.mods[m].ns: + add(f'{m}.{local}') + for q in list(out): + if q in self.fns: + for _, c, _ in self.decorates(q): + if c: + out.add(c) + elif q in self.classes: + for n in {n for k in self.mro(q) for n in self.classes[k].methods}: + m = self.lookup(q, n) + if m: + for _, c, _ in self.decorates(m): + if c: + out.add(c) + return out + + def rows(self, keep): + """flattened rows: every method a class HAS (inherited ones included) and every constructor""" + cb, fi, al, de, an = set(), set(), set(), set(), set() + + def emit(callee_name, fq, kind): + g = self.fns[fq] + for (r, p, mem, via, ty) in self.S.get(fq, ()): + if r == 'self': + pos = -1 + else: + pos = g.pos.get(r, -3) + if pos == -3 or (pos == -1 and kind != 'function'): + continue + cb.add((callee_name, r, pos, '.'.join(p), mem, via, ty)) + for q, g in self.fns.items(): + if g.cls is None and not g.local and keep(q): + emit(q, q, 'function') + for form, c, fld in self.decorates(q): + de.add((q, form, c, fld)) + for c in self.classes: + if not keep(c): + continue + names = set() + for k in self.mro(c): + names |= set(self.classes[k].methods) + for n in names: + m = self.lookup(c, n) + if m and self.fns[m].kind != 'property': + emit(f'{c}.{n}', m, 'method' if self.fns[m].kind != 'static' else 'function') + for form, k, fld in self.decorates(m): + de.add((f'{c}.{n}', form, k, fld)) + for a in self.mro(c)[1:]: + an.add((c, a)) + ini = self.lookup(c, '__init__') + if ini: + emit(c, ini, 'ctor') + g = self.fns[ini] + for fld, pn in set().union(*[self.ST.get(k, set()) for k in self.mro(c)]): + p = g.pos.get(pn, -3) + if p != -3: + fi.add((c, fld, pn, p)) + if p == 0: + de.add((c, 'DIRECT', c, fld)) # `@C def f`: C(f) holds f + for mname, m in self.mods.items(): + for local, q in m.ns.items(): + a = f'{mname}.{local}' + if a == q: + continue + c = self.canon(a) + if c and c != a and keep(c): + al.add((a, c)) + return cb, fi, al, de, an + + +def write(path, head, rows): + tmp = path + '.tmp' + with open(tmp, 'w', newline='', encoding='utf-8') as fh: + w = csv.writer(fh, delimiter='\t', lineterminator='\n') + w.writerow(head) + for r in sorted(rows, key=lambda x: tuple(str(c) for c in x)): + w.writerow(r) + os.replace(tmp, path) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--src', required=True); ap.add_argument('--out', required=True) + ap.add_argument('--ir', default=None); ap.add_argument('--stats', action='store_true') + A = ap.parse_args() + os.makedirs(A.out, exist_ok=True) + heads = (('py-lib-callback.csv', ('callee', 'root', 'pos', 'path', 'member', 'via', 'type')), + ('py-lib-field-init.csv', ('class', 'field', 'param', 'pos')), + ('py-lib-alias.csv', ('alias', 'canonical')), + ('py-lib-decorates.csv', ('decorator', 'form', 'class', 'field')), + ('py-lib-ancestor.csv', ('class', 'ancestor'))) + sites, stdlib = find_sites(os.path.abspath(A.src)) + if not sites: + for f, h in heads: + write(os.path.join(A.out, f), h, []) + print('▶ library callbacks: no Python environment found in the repository (.venv, venv, $VIRTUAL_ENV, ' + 'AXIOMCODE_PY_SITE); none summarised') + return + imports = client_imports(A.src, A.ir or A.out) + # CACHED BY WHAT DECIDES THE ANSWER: this file, the client's import paths, and the installed distributions + # (name and version from each *.dist-info) of every environment read. An edit that adds no import -- every + # refresh but a few -- reuses the tables instead of reading the library again. + h = hashlib.sha256() + h.update(open(os.path.abspath(__file__), 'rb').read()) + h.update('\n'.join(sorted(imports)).encode()) + for d in sites + ([stdlib] if stdlib else []): + h.update(d.encode()) + try: + h.update('\n'.join(sorted(x for x in os.listdir(d) if x.endswith(('.dist-info', '.egg-info', '.pth')))).encode()) + except OSError: + pass + cache = os.path.join(os.environ.get('XDG_CACHE_HOME') or os.path.join(os.path.expanduser('~'), '.cache'), + 'axiomcode', 'libsum', h.hexdigest()[:32]) + if all(os.path.isfile(os.path.join(cache, f)) for f, _ in heads): + import shutil + for f, _ in heads: + shutil.copyfile(os.path.join(cache, f), os.path.join(A.out, f)) + try: + os.utime(cache) + print(open(os.path.join(cache, 'summary.txt'), encoding='utf-8').read().strip() + ' (cached)') + except OSError: + print('▶ library callbacks: reused the cached summaries') + return + roots = {p.split('.')[0] for p in imports} + prog, todo, seen, client_pkgs = Program(), sorted(roots), set(), set() + # the client's own package installed in editable mode is not a library: its top-level names in the source tree + for n in os.listdir(A.src): + if os.path.isfile(os.path.join(A.src, n, '__init__.py')) or n.endswith('.py'): + client_pkgs.add(n[:-3] if n.endswith('.py') else n) + for sub in ('src', 'lib'): + d = os.path.join(A.src, sub) + if os.path.isdir(d): + for n in os.listdir(d): + if os.path.isfile(os.path.join(d, n, '__init__.py')): + client_pkgs.add(n) + nfiles = 0 + while todo: + r = todo.pop() + if r in seen or r in client_pkgs: + continue + seen.add(r) + pkg = locate(r, sites, stdlib) + if not pkg: + continue + for modname, path in pkg.files(): + m = Module(modname, path, path.endswith('__init__.py')) + if not m.load(): + continue + nfiles += 1 + m.analyse() + del m.tree + prog.add(m) + for imp in m.imports: + t = imp.split('.')[0] + if t and t not in seen: + todo.append(t) + rounds = prog.solve() + prog.returns() + lib_pkgs = {p for p in seen if p not in client_pkgs} + named = prog.named(imports, lib_pkgs) + keep = lambda q: q in named + cb, fi, al, de, an = prog.rows(keep) + for (f, h), rows in zip(heads, (cb, fi, al, de, an)): + write(os.path.join(A.out, f), h, rows) + line = (f'▶ library callbacks: {len(cb)} hand-backs, {len(fi)} constructor stores, {len(al)} re-exports, ' + f'{len(de)} decorators, from {nfiles} library modules ({len(lib_pkgs)} packages, {rounds} rounds)') + print(line) + try: + import shutil + tmp = cache + f'.tmp{os.getpid()}' + os.makedirs(tmp, exist_ok=True) + for f, _ in heads: + shutil.copyfile(os.path.join(A.out, f), os.path.join(tmp, f)) + open(os.path.join(tmp, 'summary.txt'), 'w', encoding='utf-8').write(line + '\n') + os.replace(tmp, cache) + root = os.path.dirname(cache) # keep the 64 most recently used + old = sorted((os.path.getmtime(os.path.join(root, x)), x) for x in os.listdir(root) if '.tmp' not in x) + for _, x in old[:-64]: + shutil.rmtree(os.path.join(root, x), ignore_errors=True) + except OSError: + pass + + +if __name__ == '__main__': + main() diff --git a/graph/python/souffle/decls_all.dl b/graph/python/souffle/decls_all.dl index d6b069c6..c221749a 100644 --- a/graph/python/souffle/decls_all.dl +++ b/graph/python/souffle/decls_all.dl @@ -52,6 +52,11 @@ .decl py_fixture_value_method(v:symbol) .decl py_parametrize_deco(v:symbol) .decl py_parametrize_argnames_kw(v:symbol) +.decl py_parametrize_argvalues_kw(v:symbol) +.decl py_fixture_params_kw(v:symbol) +.decl py_param_row_wrapper(v:symbol) +.decl py_fixture_request_param(v:symbol) +.decl py_request_param_attr(v:symbol) .decl py_parametrize_indirect_kw(v:symbol) .decl py_parametrize_indirect_all(v:symbol) .decl py_argnames_sep(v:symbol) @@ -238,6 +243,7 @@ .decl element_lib_type_of(c0:symbol,c1:symbol) .decl binding_element_lib_type(c0:symbol,c1:symbol) .decl param_declared_type_by_parser(c0:symbol,c1:symbol,c2:symbol) +.decl union_operand(c0:symbol,c1:symbol,c2:symbol) .decl type_ref_element(c0:symbol,c1:symbol,c2:symbol) .decl annotation_owner_module(c0:symbol,c1:symbol,c2:symbol,c3:symbol) .decl annotation_container_kind(c0:symbol) @@ -313,6 +319,9 @@ .decl py_path_join_function(c0:symbol) .decl py_path_join_operator(c0:symbol) .decl py_copy_function(c0:symbol) +.decl py_returns_arg(c0:symbol,c1:symbol) +.decl py_passthrough_arg(c0:symbol,c1:symbol) +.decl py_passthrough_root(c0:symbol,c1:symbol) .decl dict_lookup_method(c0:symbol) .decl builtin_target(c0:symbol,c1:symbol) .decl builtin_object_init_target(c0:symbol) @@ -526,10 +535,14 @@ .decl field_holds_method(c0:symbol,c1:symbol,c2:symbol) .decl type_attr_method_value(c0:symbol,c1:symbol,c2:symbol,c3:symbol) .decl type_attr_class_value(c0:symbol,c1:symbol,c2:symbol,c3:symbol) +.decl dict_view_name(c0:symbol) +.decl dict_items_view(c0:symbol) .decl computed_name_prefix(c0:symbol,c1:symbol) .decl expr_operator(c0:symbol,c1:symbol,c2:symbol) .decl local_holds_untyped_return(c0:symbol,c1:symbol) .decl getattr_computed_name(c0:symbol,c1:symbol,c2:symbol) +.decl getattr_computed_type(c0:symbol,c1:symbol,c2:symbol) +.decl getattr_computed_member(c0:symbol,c1:symbol) .decl method_is_async(c0:symbol,c1:symbol) .decl await_operand(c0:symbol,c1:symbol,c2:symbol) .decl subscript_index_is_slice(c0:symbol,c1:symbol) @@ -594,6 +607,8 @@ .decl decorator_hits_lib(c0:symbol,c1:symbol) .decl decorator_hits_builtin(c0:symbol,c1:symbol) .decl property_read_edge(c0:symbol,c1:symbol,c2:symbol) +.decl property_write_edge(c0:symbol,c1:symbol,c2:symbol) +.decl type_property_accessor(c0:symbol,c1:symbol,c2:symbol,c3:symbol,c4:symbol) .decl with_protocol_edge(c0:symbol,c1:symbol,c2:symbol) .decl iteration_protocol_sync(c0:symbol) .decl iteration_protocol_of(c0:symbol,c1:symbol) @@ -662,3 +677,25 @@ .decl binding_declared_call_protocol(c0:symbol,c1:symbol,c2:symbol) .decl binding_assigned_lambda(c0:symbol,c1:symbol,c2:symbol) .decl instance_attr_method_value(c0:symbol,c1:symbol,c2:symbol,c3:symbol) +.decl dict_view_slot(c0:symbol,c1:symbol) +.decl mapping_annotation_of(c0:symbol,c1:symbol) +.decl type_ref_slot(c0:symbol,c1:symbol,c2:symbol) +.decl dict_view_call(c0:symbol,c1:symbol,c2:symbol) +.decl dict_item_slot_type(c0:symbol,c1:symbol,c2:symbol) +.decl method_returns_computed_name(c0:symbol) +.decl computed_name_value(c0:symbol) +.decl call_computed_name_family(c0:symbol) +.decl py_fixture_value_class(c0:symbol,c1:symbol) +.decl builtin_protocol_slot(c0:symbol,c1:symbol) +.decl builtin_protocol_fallback(c0:symbol,c1:symbol,c2:symbol) +.decl builtin_protocol_arg(c0:symbol,c1:symbol,c2:symbol) +.decl builtin_protocol_edge(c0:symbol,c1:symbol,c2:symbol) +.decl type_has_member(c0:symbol,c1:symbol) +// staged beside the client IR by graph/python/libsum/libsum.py (framework-behavior/library-callbacks.dl) +.decl py_lib_callback(callee:symbol, root:symbol, pos:symbol, path:symbol, member:symbol, via:symbol, type:symbol) +.decl py_lib_ancestor(cls:symbol, ancestor:symbol) +.decl py_lib_field_init(cls:symbol, field:symbol, param:symbol, pos:symbol) +.decl py_lib_alias(alias:symbol, canon:symbol) +.decl lib_callback_edge(c0:symbol,c1:symbol,c2:symbol,c3:symbol,c4:symbol,c5:symbol) +.decl lib_callback_site(c0:symbol,c1:symbol) +.decl py_lib_decorates(decorator:symbol, form:symbol, cls:symbol, field:symbol) diff --git a/graph/python/souffle/export_manifest.tsv b/graph/python/souffle/export_manifest.tsv index 758e6d9e..19aaa3d2 100644 --- a/graph/python/souffle/export_manifest.tsv +++ b/graph/python/souffle/export_manifest.tsv @@ -91,3 +91,5 @@ remote_unserved remote-unserved.csv remote_unsent remote-unsent.csv framework_edge framework-edge.csv framework_unjoined framework-unjoined.csv +lib_callback_edge lib-callback-edge.csv +lib_callback_site lib-callback-site.csv diff --git a/graph/python/templates/client-extra.map b/graph/python/templates/client-extra.map new file mode 100644 index 00000000..18f5076f --- /dev/null +++ b/graph/python/templates/client-extra.map @@ -0,0 +1,12 @@ +# Soufflé input-relation -> CSV basename for facts staged BESIDE the client IR that are not parser facts. +# Same format and staging as client-ir.map (absent file = empty relation, so every project compiles one program), +# but outside the frozen parser schema: no decls_base.dl row, no lib.map counterpart (a library's own summary of +# its dependencies is not staged), declared in decls_all.dl instead. +# +# The library callback summaries graph/python/libsum/libsum.py writes from the project's installed dependencies +# (framework-behavior/library-callbacks.dl). Absent = empty = no library hand-backs, exactly as before. +py_lib_callback py-lib-callback +py_lib_field_init py-lib-field-init +py_lib_alias py-lib-alias +py_lib_decorates py-lib-decorates +py_lib_ancestor py-lib-ancestor diff --git a/graph/test/javascript/expected/64-memoize-wrapper-result.edges b/graph/test/javascript/expected/64-memoize-wrapper-result.edges index c3d46e25..46a76a11 100644 --- a/graph/test/javascript/expected/64-memoize-wrapper-result.edges +++ b/graph/test/javascript/expected/64-memoize-wrapper-result.edges @@ -1,6 +1,8 @@ app.js:11:3 FUNCTION_CALL getAlpha -> known_edge memoize.js:5:10 +app.js:11:3 FUNCTION_CALL getAlpha -> multi_inferred app.js:7:26 app.js:11:3 METHOD_CALL getAlpha().getHooks -> known_edge alpha.js:2:3 getHooks app.js:12:3 FUNCTION_CALL getBeta -> known_edge memoize.js:5:10 +app.js:12:3 FUNCTION_CALL getBeta -> multi_inferred app.js:8:25 app.js:12:3 METHOD_CALL getBeta().getHooks -> known_edge beta.js:2:3 getHooks app.js:16:20 FUNCTION_CALL once -> callback_registered app.js:16:25 app.js:16:20 FUNCTION_CALL once -> known_edge once.js:2:1 once @@ -15,12 +17,16 @@ app.js:19:19 FUNCTION_CALL lazy -> callback_registered app.js:19:24 app.js:19:19 FUNCTION_CALL lazy -> known_edge once.js:14:1 lazy app.js:19:30 CONSTRUCTOR_CALL AlphaPlugin -> implicit_constructor - app.js:22:3 FUNCTION_CALL firstAlpha -> known_edge once.js:5:10 +app.js:22:3 FUNCTION_CALL firstAlpha -> multi_inferred app.js:16:25 app.js:22:3 METHOD_CALL firstAlpha().run -> known_edge alpha.js:3:3 run app.js:23:3 FUNCTION_CALL makeBeta -> known_edge once.js:15:10 +app.js:23:3 FUNCTION_CALL makeBeta -> multi_inferred app.js:17:23 app.js:23:3 METHOD_CALL makeBeta().run -> known_edge beta.js:3:3 run app.js:24:3 FUNCTION_CALL firstBeta -> known_edge once.js:5:10 +app.js:24:3 FUNCTION_CALL firstBeta -> multi_inferred app.js:18:24 app.js:24:3 METHOD_CALL firstBeta().run -> known_edge beta.js:3:3 run app.js:25:3 FUNCTION_CALL makeAlpha -> known_edge once.js:15:10 +app.js:25:3 FUNCTION_CALL makeAlpha -> multi_inferred app.js:19:24 app.js:25:3 METHOD_CALL makeAlpha().run -> known_edge alpha.js:3:3 run app.js:29:18 FUNCTION_CALL constant -> known_edge once.js:18:1 constant app.js:29:27 CONSTRUCTOR_CALL BetaPlugin -> implicit_constructor - @@ -57,10 +63,14 @@ declared.js:27:16 FUNCTION_CALL pickShared -> callback_registered declared.js declared.js:27:16 FUNCTION_CALL pickShared -> known_edge declared.js:20:1 pickShared declared.js:27:33 CONSTRUCTOR_CALL BetaPlugin -> implicit_constructor - declared.js:30:3 FUNCTION_CALL declAlpha -> known_edge declared.js:9:3 inner +declared.js:30:3 FUNCTION_CALL declAlpha -> multi_inferred declared.js:22:28 declared.js:30:3 METHOD_CALL declAlpha().run -> known_edge alpha.js:3:3 run declared.js:31:3 FUNCTION_CALL declBeta -> known_edge declared.js:9:3 inner +declared.js:31:3 FUNCTION_CALL declBeta -> multi_inferred declared.js:23:27 declared.js:31:3 METHOD_CALL declBeta().run -> known_edge beta.js:3:3 run declared.js:32:3 FUNCTION_CALL declOfGetter -> known_edge declared.js:9:3 inner +declared.js:32:3 FUNCTION_CALL declOfGetter -> multi_inferred declared.js:24:26 +declared.js:32:3 FUNCTION_CALL declOfGetter -> multi_inferred memoize.js:5:10 declared.js:32:3 METHOD_CALL declOfGetter().run -> known_edge alpha.js:3:3 run declared.js:33:3 FUNCTION_CALL tappedDecl -> known_edge declared.js:15:3 inner declared.js:33:3 METHOD_CALL tappedDecl().markOnly -> known_edge declared.js:13:14 markOnly @@ -202,24 +212,33 @@ shapes.js:45:27 FUNCTION_CALL memoize -> callback_registered shapes.js:45:35 shapes.js:45:27 FUNCTION_CALL memoize -> known_edge memoize.js:2:17 shapes.js:45:41 CONSTRUCTOR_CALL BetaPlugin -> implicit_constructor - shapes.js:48:3 FUNCTION_CALL getFb -> known_edge shapes.js:15:10 +shapes.js:48:3 FUNCTION_CALL getFb -> multi_inferred shapes.js:36:28 shapes.js:48:3 METHOD_CALL getFb().run -> known_edge alpha.js:3:3 run shapes.js:49:3 FUNCTION_CALL getFb -> known_edge shapes.js:15:10 +shapes.js:49:3 FUNCTION_CALL getFb -> multi_inferred shapes.js:36:28 shapes.js:49:3 METHOD_CALL getFb().fallbackOnly -> known_edge shapes.js:8:18 fallbackOnly shapes.js:50:3 FUNCTION_CALL getOr -> known_edge shapes.js:22:10 +shapes.js:50:3 FUNCTION_CALL getOr -> multi_inferred shapes.js:38:22 shapes.js:50:3 METHOD_CALL getOr().run -> ambient_terminal - shapes.js:50:3 METHOD_CALL getOr().run -> multi_inferred beta.js:3:3 run shapes.js:51:3 FUNCTION_CALL getOr -> known_edge shapes.js:22:10 +shapes.js:51:3 FUNCTION_CALL getOr -> multi_inferred shapes.js:38:22 shapes.js:51:3 METHOD_CALL getOr().mockOnly -> ambient_terminal - shapes.js:51:3 METHOD_CALL getOr().mockOnly -> multi_inferred shapes.js:9:14 mockOnly shapes.js:52:3 FUNCTION_CALL getD -> known_edge shapes.js:29:10 +shapes.js:52:3 FUNCTION_CALL getD -> multi_inferred shapes.js:39:26 shapes.js:52:3 METHOD_CALL getD().run -> known_edge alpha.js:3:3 run shapes.js:53:3 FUNCTION_CALL getD -> known_edge shapes.js:29:10 +shapes.js:53:3 FUNCTION_CALL getD -> multi_inferred shapes.js:39:26 shapes.js:53:3 METHOD_CALL getD().defaultsOnly -> known_edge shapes.js:25:20 defaultsOnly shapes.js:54:3 FUNCTION_CALL getDefault -> known_edge shapes.js:33:44 shapes.js:54:3 METHOD_CALL getDefault().defaultOnly -> known_edge shapes.js:10:19 defaultOnly shapes.js:55:3 FUNCTION_CALL getSpread -> known_edge shapes.js:33:44 shapes.js:55:3 METHOD_CALL getSpread().run -> ambiguous_unknown - shapes.js:56:3 FUNCTION_CALL warm -> known_edge shapes.js:34:32 +shapes.js:56:3 FUNCTION_CALL warm -> multi_inferred shapes.js:43:20 shapes.js:56:3 METHOD_CALL warm().run -> known_edge alpha.js:3:3 run shapes.js:57:3 FUNCTION_CALL getNested -> known_edge memoize.js:5:10 +shapes.js:57:3 FUNCTION_CALL getNested -> multi_inferred memoize.js:5:10 +shapes.js:57:3 FUNCTION_CALL getNested -> multi_inferred shapes.js:45:35 shapes.js:57:3 METHOD_CALL getNested().run -> known_edge beta.js:3:3 run diff --git a/graph/test/javascript/expected/65-wrapper-param-reassigned.edges b/graph/test/javascript/expected/65-wrapper-param-reassigned.edges index d56c695d..8373c4e3 100644 --- a/graph/test/javascript/expected/65-wrapper-param-reassigned.edges +++ b/graph/test/javascript/expected/65-wrapper-param-reassigned.edges @@ -2,51 +2,66 @@ app.js:10:23 CONSTRUCTOR_CALL A -> implicit_constructor - app.js:10:3 FUNCTION_CALL lazyOrDefault -> callback_registered app.js:10:17 app.js:10:3 FUNCTION_CALL lazyOrDefault -> known_edge wrap.js:17:1 lazyOrDefault app.js:10:3 FUNCTION_CALL lazyOrDefault(() => new A()) -> known_edge wrap.js:19:10 +app.js:10:3 FUNCTION_CALL lazyOrDefault(() => new A()) -> multi_inferred app.js:10:17 +app.js:10:3 FUNCTION_CALL lazyOrDefault(() => new A()) -> multi_inferred wrap.js:18:15 app.js:10:3 METHOD_CALL lazyOrDefault(() => new A())().aOnly -> known_edge wrap.js:2:11 aOnly app.js:11:3 FUNCTION_CALL lazyNullish -> known_edge wrap.js:22:1 lazyNullish app.js:11:3 FUNCTION_CALL lazyNullish() -> known_edge wrap.js:24:10 +app.js:11:3 FUNCTION_CALL lazyNullish() -> multi_inferred wrap.js:23:10 app.js:11:3 METHOD_CALL lazyNullish()().dfltOnly -> known_edge wrap.js:4:14 dfltOnly app.js:15:15 CONSTRUCTOR_CALL A -> implicit_constructor - app.js:15:3 FUNCTION_CALL plain -> callback_registered app.js:15:9 app.js:15:3 FUNCTION_CALL plain -> known_edge wrap.js:27:1 plain app.js:15:3 FUNCTION_CALL plain(() => new A()) -> known_edge wrap.js:28:10 +app.js:15:3 FUNCTION_CALL plain(() => new A()) -> multi_inferred app.js:15:9 app.js:15:3 METHOD_CALL plain(() => new A())().aOnly -> known_edge wrap.js:2:11 aOnly app.js:16:15 CONSTRUCTOR_CALL A -> implicit_constructor - app.js:16:3 FUNCTION_CALL plain -> callback_registered app.js:16:9 app.js:16:3 FUNCTION_CALL plain -> known_edge wrap.js:27:1 plain app.js:16:3 FUNCTION_CALL plain(() => new A()) -> known_edge wrap.js:28:10 +app.js:16:3 FUNCTION_CALL plain(() => new A()) -> multi_inferred app.js:16:9 app.js:16:3 METHOD_CALL plain(() => new A())().replacedOnly -> ambiguous_unknown - app.js:17:22 CONSTRUCTOR_CALL A -> implicit_constructor - app.js:17:3 FUNCTION_CALL otherWritten -> callback_registered app.js:17:16 app.js:17:3 FUNCTION_CALL otherWritten -> known_edge wrap.js:31:1 otherWritten app.js:17:3 FUNCTION_CALL otherWritten(() => new A(), null) -> known_edge wrap.js:33:10 +app.js:17:3 FUNCTION_CALL otherWritten(() => new A(), null) -> multi_inferred app.js:17:16 app.js:17:3 METHOD_CALL otherWritten(() => new A(), null)().aOnly -> known_edge wrap.js:2:11 aOnly app.js:18:22 CONSTRUCTOR_CALL A -> implicit_constructor - app.js:18:3 FUNCTION_CALL otherWritten -> callback_registered app.js:18:16 app.js:18:3 FUNCTION_CALL otherWritten -> known_edge wrap.js:31:1 otherWritten app.js:18:3 FUNCTION_CALL otherWritten(() => new A(), null) -> known_edge wrap.js:33:10 +app.js:18:3 FUNCTION_CALL otherWritten(() => new A(), null) -> multi_inferred app.js:18:16 app.js:18:3 METHOD_CALL otherWritten(() => new A(), null)().replacedOnly -> ambiguous_unknown - app.js:4:14 FUNCTION_CALL swapped -> callback_registered app.js:4:22 app.js:4:14 FUNCTION_CALL swapped -> known_edge wrap.js:7:1 swapped app.js:4:28 CONSTRUCTOR_CALL A -> implicit_constructor - app.js:5:3 FUNCTION_CALL sw -> known_edge wrap.js:9:10 +app.js:5:3 FUNCTION_CALL sw -> multi_inferred wrap.js:8:8 app.js:5:3 METHOD_CALL sw().replacedOnly -> known_edge wrap.js:3:18 replacedOnly app.js:6:3 FUNCTION_CALL sw -> known_edge wrap.js:9:10 +app.js:6:3 FUNCTION_CALL sw -> multi_inferred wrap.js:8:8 app.js:6:3 METHOD_CALL sw().aOnly -> ambiguous_unknown - app.js:7:3 FUNCTION_CALL lazyGuard -> known_edge wrap.js:12:1 lazyGuard app.js:7:3 FUNCTION_CALL lazyGuard(undefined) -> known_edge wrap.js:14:10 +app.js:7:3 FUNCTION_CALL lazyGuard(undefined) -> multi_inferred wrap.js:13:38 app.js:7:3 METHOD_CALL lazyGuard(undefined)().replacedOnly -> known_edge wrap.js:3:18 replacedOnly app.js:8:19 CONSTRUCTOR_CALL A -> implicit_constructor - app.js:8:3 FUNCTION_CALL lazyGuard -> callback_registered app.js:8:13 app.js:8:3 FUNCTION_CALL lazyGuard -> known_edge wrap.js:12:1 lazyGuard app.js:8:3 FUNCTION_CALL lazyGuard(() => new A()) -> known_edge wrap.js:14:10 +app.js:8:3 FUNCTION_CALL lazyGuard(() => new A()) -> multi_inferred app.js:8:13 +app.js:8:3 FUNCTION_CALL lazyGuard(() => new A()) -> multi_inferred wrap.js:13:38 app.js:8:3 METHOD_CALL lazyGuard(() => new A())().aOnly -> known_edge wrap.js:2:11 aOnly app.js:9:3 FUNCTION_CALL lazyOrDefault -> known_edge wrap.js:17:1 lazyOrDefault app.js:9:3 FUNCTION_CALL lazyOrDefault() -> known_edge wrap.js:19:10 +app.js:9:3 FUNCTION_CALL lazyOrDefault() -> multi_inferred wrap.js:18:15 app.js:9:3 METHOD_CALL lazyOrDefault()().dfltOnly -> known_edge wrap.js:4:14 dfltOnly many.js:11:3 FUNCTION_CALL s21 -> known_edge wrap.js:9:10 +many.js:11:3 FUNCTION_CALL s21 -> multi_inferred wrap.js:8:8 many.js:11:3 METHOD_CALL s21().replacedOnly -> known_edge wrap.js:3:18 replacedOnly many.js:12:3 FUNCTION_CALL s21 -> known_edge wrap.js:9:10 +many.js:12:3 FUNCTION_CALL s21 -> multi_inferred wrap.js:8:8 many.js:12:3 METHOD_CALL s21().aOnly -> ambiguous_unknown - many.js:3:13 FUNCTION_CALL swapped -> callback_registered many.js:3:21 many.js:3:13 FUNCTION_CALL swapped -> known_edge wrap.js:7:1 swapped diff --git a/graph/test/python/expected/33-url-view-forms.edges b/graph/test/python/expected/33-url-view-forms.edges index f5bb9f3f..c0500600 100644 --- a/graph/test/python/expected/33-url-view-forms.edges +++ b/graph/test/python/expected/33-url-view-forms.edges @@ -1,9 +1,11 @@ ambiguous_unknown DECORATOR_APPLICATION shop.decorators.audited -> - ambiguous_unknown DECORATOR_BARE shop.views. -> - ambiguous_unknown METHOD_CALL shop.urls. -> - -ambiguous_unknown SIMPLE_CALL catalog.urls. -> - -ambiguous_unknown SIMPLE_CALL shop.urls. -> - boundary_lib DECORATOR_CALL shop.decorators.audited -> external:functools.wraps +boundary_lib SIMPLE_CALL catalog.urls. -> external:path +boundary_lib SIMPLE_CALL shop.urls. -> external:include +boundary_lib SIMPLE_CALL shop.urls. -> external:path +boundary_lib SIMPLE_CALL shop.urls. -> external:re_path known_edge DECORATOR_BARE shop.views. -> shop.decorators.audited known_edge SIMPLE_CALL shop.decorators.audited..wrapper -> shop.views.order_history known_edge SIMPLE_CALL shop.views.BaseWidgetView.get -> shop.views._render diff --git a/graph/test/python/expected/33-url-view-forms.tiers b/graph/test/python/expected/33-url-view-forms.tiers index 2713fd47..f05de8d3 100644 --- a/graph/test/python/expected/33-url-view-forms.tiers +++ b/graph/test/python/expected/33-url-view-forms.tiers @@ -1,8 +1,8 @@ distinct call sites emitted: 51 --- by tier: edge ROWS, and the distinct SITES they cover --- - 32 rows 32 sites ambiguous_unknown - 1 rows 1 sites boundary_lib + 6 rows 6 sites ambiguous_unknown + 27 rows 27 sites boundary_lib 18 rows 18 sites known_edge --- edge rows by call kind --- @@ -14,13 +14,13 @@ distinct call sites emitted: 51 --- unresolved reasons --- 1 decorator_factory_result_untyped - 27 no_rule + 1 no_rule 4 unmodelled_decorator --- the engine's own conservation ledger --- 51 _total_sites - 32 ambiguous_unknown - 1 boundary_lib + 6 ambiguous_unknown + 27 boundary_lib 18 known_edge --- reconciling rows against the conserved site count --- diff --git a/graph/test/python/expected/37-model-signals.edges b/graph/test/python/expected/37-model-signals.edges index a50f46f3..67531829 100644 --- a/graph/test/python/expected/37-model-signals.edges +++ b/graph/test/python/expected/37-model-signals.edges @@ -1,6 +1,4 @@ ambiguous_unknown DECORATOR_APPLICATION models. -> - -ambiguous_unknown METHOD_CALL models.Invoice. -> - -ambiguous_unknown METHOD_CALL models.Order. -> - ambiguous_unknown METHOD_CALL views.create_order -> - ambiguous_unknown METHOD_CALL views.drop_invoice -> - ambiguous_unknown METHOD_CALL views.drop_order -> - @@ -9,4 +7,6 @@ ambiguous_unknown METHOD_CALL views.save_draft -> - ambiguous_unknown METHOD_CALL views.touch_invoice -> - boundary_lib DECORATOR_CALL models. -> external:receiver boundary_lib METHOD_CALL models. -> external:post_save.connect +boundary_lib METHOD_CALL models.Invoice. -> external:models.IntegerField +boundary_lib METHOD_CALL models.Order. -> external:models.CharField known_edge METHOD_CALL views.save_cache -> views.Cache.save diff --git a/graph/test/python/expected/37-model-signals.tiers b/graph/test/python/expected/37-model-signals.tiers index a7e08a5c..e87fd3fa 100644 --- a/graph/test/python/expected/37-model-signals.tiers +++ b/graph/test/python/expected/37-model-signals.tiers @@ -1,8 +1,8 @@ distinct call sites emitted: 14 --- by tier: edge ROWS, and the distinct SITES they cover --- - 10 rows 10 sites ambiguous_unknown - 3 rows 3 sites boundary_lib + 8 rows 8 sites ambiguous_unknown + 5 rows 5 sites boundary_lib 1 rows 1 sites known_edge --- edge rows by call kind --- @@ -14,13 +14,12 @@ distinct call sites emitted: 14 2 decorator_factory_result_untyped 4 escape_hatch 1 untyped_receiver:class_attribute_absent - 2 untyped_receiver:local_untyped 1 untyped_receiver:parameter --- the engine's own conservation ledger --- 14 _total_sites - 10 ambiguous_unknown - 3 boundary_lib + 8 ambiguous_unknown + 5 boundary_lib 1 known_edge --- reconciling rows against the conserved site count --- diff --git a/graph/test/python/expected/38-app-wiring.edges b/graph/test/python/expected/38-app-wiring.edges index 89dad32f..39d93d28 100644 --- a/graph/test/python/expected/38-app-wiring.edges +++ b/graph/test/python/expected/38-app-wiring.edges @@ -3,8 +3,6 @@ ambiguous_unknown DECORATOR_ATTRIBUTE main. -> - ambiguous_unknown DECORATOR_BARE main. -> - ambiguous_unknown SIMPLE_CALL main.Timing.dispatch -> - ambiguous_unknown SIMPLE_CALL main.add_header -> - -ambiguous_unknown SIMPLE_CALL shop.jobs.nightly -> - -ambiguous_unknown SIMPLE_CALL shop.jobs.unknown -> - boundary_lib DECORATOR_CALL main. -> external:APIRouter.api_route boundary_lib DECORATOR_CALL main. -> external:FastAPI.exception_handler boundary_lib DECORATOR_CALL main. -> external:FastAPI.middleware @@ -17,6 +15,8 @@ boundary_lib SIMPLE_CALL main. -> external:FastAPI boundary_lib SIMPLE_CALL main. -> external:Route boundary_lib SIMPLE_CALL main. -> external:Starlette boundary_lib SIMPLE_CALL main. -> external:WebSocketRoute +boundary_lib SIMPLE_CALL shop.jobs.nightly -> external:call_command +boundary_lib SIMPLE_CALL shop.jobs.unknown -> external:call_command known_edge METHOD_CALL main. -> main.Registry.add_route known_edge SIMPLE_CALL main. -> main.configure known_edge SIMPLE_CALL shop.management.commands.close_orders.Command.handle -> shop.orders.close_all diff --git a/graph/test/python/expected/38-app-wiring.tiers b/graph/test/python/expected/38-app-wiring.tiers index 4df06d8a..8496a4b9 100644 --- a/graph/test/python/expected/38-app-wiring.tiers +++ b/graph/test/python/expected/38-app-wiring.tiers @@ -1,8 +1,8 @@ distinct call sites emitted: 26 --- by tier: edge ROWS, and the distinct SITES they cover --- - 10 rows 10 sites ambiguous_unknown - 13 rows 13 sites boundary_lib + 8 rows 8 sites ambiguous_unknown + 15 rows 15 sites boundary_lib 3 rows 3 sites known_edge --- edge rows by call kind --- @@ -16,13 +16,12 @@ distinct call sites emitted: 26 --- unresolved reasons --- 2 callee_is_parameter 4 decorator_factory_result_untyped - 2 no_rule 2 unmodelled_decorator --- the engine's own conservation ledger --- 26 _total_sites - 10 ambiguous_unknown - 13 boundary_lib + 8 ambiguous_unknown + 15 boundary_lib 3 known_edge --- reconciling rows against the conserved site count --- diff --git a/graph/test/python/tools/literal_gate.py b/graph/test/python/tools/literal_gate.py index db44b130..0e7af6f7 100644 --- a/graph/test/python/tools/literal_gate.py +++ b/graph/test/python/tools/literal_gate.py @@ -83,7 +83,12 @@ # unresolved_value_callee's second column is `binding`: WHAT the callee of an # unresolved call is ("parameter", "loop_variable", "field", ...), the same words # the JavaScript rules write. Written into the output and never joined on. - "unresolved_value_callee(") + "unresolved_value_callee(", + # lib_callback_edge's How column (receiver / receiver_field / argument / argument_field) says + # where on the call the handed-back object arrived. Written into the output by + # framework-behavior/library-callbacks.dl and never joined on; every name that DOES decide a + # library hand-back comes from the tables graph/python/libsum derives from the library's source. + "lib_callback_edge(", "lcb_hit(") GROUND_FACT = re.compile(r'^[a-z_]+\((?:\s*"[^"]*"\s*,?)+\)\.\s*$') diff --git a/graph/test/python/tools/protocol-fan-test.sh b/graph/test/python/tools/protocol-fan-test.sh index 009607d6..b6c9b283 100755 --- a/graph/test/python/tools/protocol-fan-test.sh +++ b/graph/test/python/tools/protocol-fan-test.sh @@ -64,7 +64,7 @@ PYEOF node "$PARSER" "$W/src" protofan false "$W/ir" > "$W/parse.log" 2>&1 || { echo " FAIL protocol-fan: the parser failed on the fixture"; sed 's/^/ /' "$W/parse.log" | tail -3 echo "protocol-fan: FAILED (1 check)"; exit 1; } -bash "$ROOT/graph/pipeline/run-souffle.sh" --debug --language python --client-ir "$W/ir" \ +bash "$ROOT/graph/pipeline/run-souffle.sh" --debug --language python --dispatch-cap 20 --client-ir "$W/ir" \ --library "$EMPTY_LIB" --intermediate "$W/int" --output "$W/out" > "$W/solve.log" 2>&1 || { echo " FAIL protocol-fan: the solve failed"; tail -3 "$W/solve.log" | sed 's/^/ /' echo "protocol-fan: FAILED (1 check)"; exit 1; } diff --git a/graph/test/python/torture/client/f43_def_rebind.py b/graph/test/python/torture/client/f43_def_rebind.py index 107989af..3a454e7e 100644 --- a/graph/test/python/torture/client/f43_def_rebind.py +++ b/graph/test/python/torture/client/f43_def_rebind.py @@ -89,10 +89,10 @@ def call_branched() -> str: def read_property() -> int: """The getter survives — the control that keeps this rule off property pairs. - EXPECT: miss — `h.value = 5` invokes the SETTER, and the engine emits a - PROPERTY_READ edge for a property read and nothing at all for a property WRITE. - That is a pre-existing gap this fixture happens to expose, not something #383 - changed; the getter edge on the next line is what this family is asserting. + `h.value = 5` invokes the SETTER, and the engine now emits a PROPERTY_WRITE + edge for it beside the PROPERTY_READ for the read; both halves of the pair + are live, which is exactly what this control exists to keep true. The getter + edge on the next line is what this family is asserting. """ h = Holder() h.value = 5 diff --git a/graph/test/python/torture/expected/coverage.txt b/graph/test/python/torture/expected/coverage.txt index 06b88e32..daf7e130 100644 --- a/graph/test/python/torture/expected/coverage.txt +++ b/graph/test/python/torture/expected/coverage.txt @@ -1,4 +1,4 @@ -=== per-family coverage (tier-4, 471 scored sites) === +=== per-family coverage (tier-4, 474 scored sites) === f01 inheritance & MRO links= 10 found= 10 (100.0%) missed= 0 wide= 0 WRONG= 0 f02 callables & closures links= 10 found= 9 (90.0%) missed= 1 wide= 0 WRONG= 0 f03 generics links= 14 found= 14 (100.0%) missed= 0 wide= 0 WRONG= 0 @@ -38,16 +38,16 @@ f40 data descriptor links= 14 found= 14 (100.0%) missed= 0 wide= 0 WRONG= 0 f41 class attribute absent links= 3 found= 3 (100.0%) missed= 0 wide= 0 WRONG= 0 f42 f42 links= 6 found= 6 (100.0%) missed= 0 wide= 0 WRONG= 0 - f43 f43 links= 7 found= 7 (100.0%) missed= 0 wide= 1 WRONG= 0 + f43 f43 links= 10 found= 10 (100.0%) missed= 0 wide= 1 WRONG= 0 nestmod.py nestmod.py links= 1 found= 1 (100.0%) missed= 0 wide= 0 WRONG= 0 pkgmod relative imports (subpackage) links= 3 found= 3 (100.0%) missed= 0 wide= 0 WRONG= 0 - TOTAL links=434 found=428 (98.6%) missed= 6 wide=37 WRONG= 0 - recall 428/434 = 98.6% of the links that actually ran + TOTAL links=437 found=431 (98.6%) missed= 6 wide=37 WRONG= 0 + recall 431/437 = 98.6% of the links that actually ran wide 37 a member of a SOUND SET that did not run on this pass WRONG 0 a single target asserted as certain that never ran - EXPECTED-MISS cases: {'FOUND': 20, 'MISSED': 24} (a CONCRETE here means a known blind spot closed) + EXPECTED-MISS cases: {'FOUND': 18, 'MISSED': 23} (a CONCRETE here means a known blind spot closed) --- non-concrete sites --- f02_callables.py:31 MISSED true=[('lib', 'callables.py', 22)] engine=[] diff --git a/graph/test/python/torture/expected/torture.edges b/graph/test/python/torture/expected/torture.edges index 4d85cc1a..557d3208 100644 --- a/graph/test/python/torture/expected/torture.edges +++ b/graph/test/python/torture/expected/torture.edges @@ -635,6 +635,7 @@ known_edge PROPERTY_READ f40_data_descriptor.via_data_descriptor -> f40_data_des known_edge PROPERTY_READ f40_data_descriptor.via_delete_descriptor -> f40_data_descriptor.Deletable.__get__ known_edge PROPERTY_READ f40_data_descriptor.via_property -> f40_data_descriptor.Prop.slot known_edge PROPERTY_READ f43_def_rebind.read_property -> f43_def_rebind.Holder.value +known_edge PROPERTY_WRITE f43_def_rebind.read_property -> f43_def_rebind.Holder.value known_edge SELF_CALL f02_callables.HoldsCallables.run -> f02_callables.LocalCallable.__call__ known_edge SELF_CALL f28_await.Registry.into_a_field -> f28_await.Registry.build known_edge SELF_CALL f28_await.Registry.via_self_receiver -> f28_await.Registry.build diff --git a/graph/test/python/torture/expected/torture.oracle b/graph/test/python/torture/expected/torture.oracle index 95fd8d90..4267008a 100644 --- a/graph/test/python/torture/expected/torture.oracle +++ b/graph/test/python/torture/expected/torture.oracle @@ -1 +1 @@ -oracle=634 engine=624 agree=587 missing=47 extra=37 +oracle=634 engine=625 agree=588 missing=46 extra=37 diff --git a/graph/test/typescript/expected/74-jsx-component-forms.edges b/graph/test/typescript/expected/74-jsx-component-forms.edges index 8a9a7700..1215ed2b 100644 --- a/graph/test/typescript/expected/74-jsx-component-forms.edges +++ b/graph/test/typescript/expected/74-jsx-component-forms.edges @@ -10,6 +10,7 @@ callback_registered FUNCTION_CALL app#() @L4 -> app#() callback_registered FUNCTION_CALL parts#() @L33 -> parts#UserCard({ id: string }) callback_registered FUNCTION_CALL parts#() @L41 -> parts#FieldImpl({ label: string },unknown) callback_registered FUNCTION_CALL parts#() @L73 -> parts#buildDefault() +callback_registered JSX_COMPONENT_CALL app#App() @L13 -> Panel#render() intrinsic_terminal DYNAMIC_IMPORT_CALL app#() @L4 -> - known_edge FUNCTION_CALL app#useFactory() @L23 -> parts#()@L22 known_edge FUNCTION_CALL parts#FieldImpl({ label: string },unknown) @L39 -> parts#trackClick(string) diff --git a/graph/test/typescript/expected/80-vue-definecomponent-jsx.edges b/graph/test/typescript/expected/80-vue-definecomponent-jsx.edges index 332f0630..15c58497 100644 --- a/graph/test/typescript/expected/80-vue-definecomponent-jsx.edges +++ b/graph/test/typescript/expected/80-vue-definecomponent-jsx.edges @@ -14,6 +14,7 @@ ambiguous_unknown FUNCTION_CALL parts#render() @L35 -> - ambiguous_unknown FUNCTION_CALL reg#() @L18 -> - ambiguous_unknown JSX_COMPONENT_CALL app#() @L20 -> - callback_registered FUNCTION_CALL parts#() @L40 -> parts#(?) +callback_registered JSX_COMPONENT_CALL app#() @L? -> app#onPing() known_edge FUNCTION_CALL app#onPing() @L12 -> app#helper() known_edge FUNCTION_CALL button#() @L10 -> button#leafButton() known_edge FUNCTION_CALL card#render() @L12 -> card#leafCard() diff --git a/graph/test/typescript/expected/80-vue-definecomponent-jsx.lib.edges b/graph/test/typescript/expected/80-vue-definecomponent-jsx.lib.edges index 40aed3c8..92c54f4d 100644 --- a/graph/test/typescript/expected/80-vue-definecomponent-jsx.lib.edges +++ b/graph/test/typescript/expected/80-vue-definecomponent-jsx.lib.edges @@ -14,6 +14,7 @@ boundary_lib FUNCTION_CALL parts#() @L42 -> vue#defineNuxtComponent boundary_lib FUNCTION_CALL parts#() @L49 -> vue#makeStore(ComponentOptions) boundary_lib FUNCTION_CALL reg#() @L18 -> vue#defineComponent(ComponentOptions) callback_registered FUNCTION_CALL parts#() @L40 -> parts#(?) +callback_registered JSX_COMPONENT_CALL app#() @L? -> app#onPing() known_edge FUNCTION_CALL app#onPing() @L12 -> app#helper() known_edge FUNCTION_CALL button#() @L10 -> button#leafButton() known_edge FUNCTION_CALL card#render() @L12 -> card#leafCard() diff --git a/graph/typescript/engine/call-edge-generation/call_chain.dl b/graph/typescript/engine/call-edge-generation/call_chain.dl index 56518799..890a08ad 100644 --- a/graph/typescript/engine/call-edge-generation/call_chain.dl +++ b/graph/typescript/engine/call-edge-generation/call_chain.dl @@ -134,6 +134,14 @@ call_chain_edge(ce, caller, "-", m, "client", "callback_registered", kind) :- caller != m, invocation_site(ce, kind). +// …and a function written inside a JSX `{…}` (resolution/value-flow.dl, jsx_handed_value). The `{…}` is +// the site: an intrinsic tag has no call site of its own, and the expression positions the edge. +call_chain_edge(r, caller, "-", m, "client", "callback_registered", "JSX_COMPONENT_CALL") :- + jsx_handed_value(r, m), + method_prov(m, "client"), + jsx_root_caller(r, caller), + caller != m. + // ── call_runs_edge(FromMethod, ToMethod, Prov) ────────────────────────────── // The chain a consumer should WALK: overload signatures re-pointed to the body that // actually executes (resolution/overload-sets.dl). Kept separate from diff --git a/graph/typescript/engine/call-edge-generation/calls.dl b/graph/typescript/engine/call-edge-generation/calls.dl index f90275c3..b4b87c8f 100644 --- a/graph/typescript/engine/call-edge-generation/calls.dl +++ b/graph/typescript/engine/call-edge-generation/calls.dl @@ -151,5 +151,42 @@ library_value(e, spec) :- expr_kind("client", k, _, e), library_value(e, spec) :- expr_referenced("client", "VARIABLE", v, e), var_initializer("client", _, i, v), !var_reassigned(v), library_value(i, spec). call_passes_project_instance(e) :- expr_child("client", e, "ARGUMENT", _, a), expr_type(a, "client", _). +// …and the runtime's own objects: `JSON.parse(s)`, `Promise.resolve(v).then(f)`, `Object.keys(o).forEach(g)`, +// `document.createElement('a')`, `console.error(e)`. With no standard library staged the name resolves to nothing +// in the run (AMBIENT_GLOBAL), so every such call was an untyped `x.parse()` to the consumers, and a project method +// that happens to be named parse, resolve, all, keys, error or getMetadata was given every one of them as a by-name +// caller — and every test around them. The receiver is the platform's, never the project's: a program that declares +// the name itself binds it as its own variable, not as an ambient global, and stays out. Only the objects whose +// members are called through the name; `window`, `globalThis` and `self` are left out, since a program may hang its +// own functions on them. +library_value(e, n) :- expr_referenced("client", "AMBIENT_GLOBAL", _, e), expr_name("client", n, e), + builtin_global_object(n). +builtin_global_object("JSON"). +builtin_global_object("Math"). +builtin_global_object("Object"). +builtin_global_object("Reflect"). +builtin_global_object("Promise"). +builtin_global_object("Array"). +builtin_global_object("Number"). +builtin_global_object("String"). +builtin_global_object("Boolean"). +builtin_global_object("BigInt"). +builtin_global_object("Symbol"). +builtin_global_object("Date"). +builtin_global_object("RegExp"). +builtin_global_object("Error"). +builtin_global_object("Intl"). +builtin_global_object("Atomics"). +builtin_global_object("ArrayBuffer"). +builtin_global_object("console"). +builtin_global_object("process"). +builtin_global_object("Buffer"). +builtin_global_object("document"). +builtin_global_object("navigator"). +builtin_global_object("localStorage"). +builtin_global_object("sessionStorage"). +builtin_global_object("performance"). +builtin_global_object("crypto"). +builtin_global_object("URL"). library_receiver(ce, spec) :- call_site("client", _, _, rk, recv, ce, _), receiver_kind_is_value(rk), !call_resolved(ce), library_value(recv, spec). diff --git a/graph/typescript/engine/resolution/contextual-params.dl b/graph/typescript/engine/resolution/contextual-params.dl index c9f675e1..99b05b5a 100644 --- a/graph/typescript/engine/resolution/contextual-params.dl +++ b/graph/typescript/engine/resolution/contextual-params.dl @@ -278,6 +278,20 @@ builtin_elem_ref(parent, x) :- ref_is_readonly_operator(parent), builtin_elem_ref(child, x). builtin_elem_ref(ref, x) :- ref_via_alias(ref, rhs), builtin_elem_ref(rhs, x). +// A tuple's elements are its members: `[Module, ...Module[]]`. +builtin_elem_ref(ref, child) :- type_ref(rp, "TUPLE", _, _, _, _, _, ref), + type_ref_parent(rp, ref, _, child). +// A TYPE VARIABLE constrained to a list holds that list's elements: `(ms: Ms)`, +// `(...modules: Ms)`, then `modules.map((m) => m.init())`. The +// constraint is what every element is at least, as for a `` receiver. Both routes +// ref_type_target takes: the exact link to the declaration, and the name in scope where it is absent. +builtin_elem_ref(ref, x) :- type_ref_type_param(rp, tp, ref), + type_param_constraint(rp, cref, tp), + builtin_elem_ref(cref, x). +builtin_elem_ref(ref, x) :- type_ref(_, "TYPE_VARIABLE", _, tn, _, owner, _, ref), + !type_ref_type_param(_, _, ref), + type_var_constraint_in_scope(owner, tn, cref), + builtin_elem_ref(cref, x). builtin_elem_ref(ref, a) :- ref_via_alias(ref, rhs), builtin_elem_ref(rhs, x), type_ref(_, "TYPE_VARIABLE", _, n, _, _, _, x), @@ -339,6 +353,13 @@ lambda_param_ref(lp, c) :- call_site("client", _, "then", _, recv, ce, _), !param_type_ref(_, _, lp). lambda_param_type(lp, prov, t) :- lambda_param_ref(lp, c), ref_type_target(c, prov, t). +// …and its SHAPE, where the element is an object type written as a type alias or a literal rather +// than a class or an interface: `type Plugin = { init(): void }`, then `plugins.map((p) => p.init())`. +// The element has no type declaration to dispatch on, only members, so the type clause above gave +// the parameter nothing and every call on it was unresolved; an annotated parameter of the same type +// resolved, through param_shape_target. The element reference is the same one, read as a shape. +lambda_param_shape(lp, sh) :- lambda_param_ref(lp, c), + ref_shape_target(c, sh). // The parameter's reference is its declared reference, as an annotation's would be. expr_decl_ref(e, c) :- expr_referenced("client", "PARAMETER", p, e), lambda_param_ref(p, c). diff --git a/graph/typescript/engine/resolution/type-resolution.dl b/graph/typescript/engine/resolution/type-resolution.dl index 2c8e5ef2..0d3cff9f 100644 --- a/graph/typescript/engine/resolution/type-resolution.dl +++ b/graph/typescript/engine/resolution/type-resolution.dl @@ -587,6 +587,19 @@ ref_element_target(ref, prov, t) :- type_ref(rp, k, _, _, _, _, _, ref), ref_kind_is_array(k), type_ref_parent(rp, ref, _, child), ref_type_target(child, prov, t). +// …and of a TYPE VARIABLE constrained to an array or a tuple: `(ms: Ms)`, +// `(...modules: Ms)`, then `modules.map((m) => m.init())`. The +// constraint is what every element is at least, exactly as it is for a `` receiver +// (ref_type_target above), so the callback's parameter is typed by the constraint's element. Without it +// the element of a generic list had no type, and every call on it was unresolved. Same two routes as +// ref_type_target: the exact link to the declaration, and the name in scope where the link is absent. +ref_element_target(ref, prov, t) :- type_ref_type_param(rp, tp, ref), + type_param_constraint(rp, cref, tp), + ref_element_target(cref, prov, t). +ref_element_target(ref, prov, t) :- type_ref(_, "TYPE_VARIABLE", _, tn, _, owner, _, ref), + !type_ref_type_param(_, _, ref), + type_var_constraint_in_scope(owner, tn, cref), + ref_element_target(cref, prov, t). // ── ref_element_shape(Ref, Shape) — an element that is ITSELF a function type ── // `fns: readonly ((data: T) => boolean)[]` then `fns.every((fn) => fn(1))`. The element diff --git a/graph/typescript/engine/resolution/value-flow.dl b/graph/typescript/engine/resolution/value-flow.dl index 75e9d129..284c6e88 100644 --- a/graph/typescript/engine/resolution/value-flow.dl +++ b/graph/typescript/engine/resolution/value-flow.dl @@ -294,6 +294,69 @@ proxy_trap_name("preventExtensions"). proxy_trap_name("apply"). proxy_trap_name("construct"). +// …AND A FUNCTION WRITTEN AS A JSX ATTRIBUTE OR CHILD. ``, +// ` + + ) + } +} + +export function mountBoard(): unknown { + return +} + +// a function component handing a module function and a render callback to elements +export function Toolbar(): unknown { + return ( + + ) +} + +export function List(props: { renderItem: (i: number) => string }): unknown { + return
    {props.renderItem.length}
+} + +// CONTROL: a class with a render method that is never written as a tag is not a component: building it runs no render +export class Report { + render(): number { return measure(1) } +} +export function makeReport(): Report { + return new Report() +} + +// CONTROL: an element whose attributes are data hands nothing over +export function Badge(): unknown { + return +} diff --git a/tests/cases/typescript/jsx-handed-functions/src/jsx.d.ts b/tests/cases/typescript/jsx-handed-functions/src/jsx.d.ts new file mode 100644 index 00000000..e5c8fd2d --- /dev/null +++ b/tests/cases/typescript/jsx-handed-functions/src/jsx.d.ts @@ -0,0 +1,13 @@ +// the project supplies its own JSX namespace and component base, so the case needs no framework installed +declare global { + namespace JSX { + type Element = unknown + interface ElementClass { render(): unknown } + interface ElementAttributesProperty { props: unknown } + interface IntrinsicElements { [name: string]: unknown } + } +} +export class Component

{ + props: P + constructor(props: P) { this.props = props } +} diff --git a/tests/cases/typescript/jsx-handed-functions/src/work.ts b/tests/cases/typescript/jsx-handed-functions/src/work.ts new file mode 100644 index 00000000..46ad4ace --- /dev/null +++ b/tests/cases/typescript/jsx-handed-functions/src/work.ts @@ -0,0 +1,7 @@ +export function hitTest(x: number): number { return x * 2 } +export function persist(id: string): string { return id.trim() } +export function startClock(): number { return Date.now() } +export function stopClock(): number { return 0 } +export function onSave(): string { return "saved" } +export function drawRow(i: number): string { return String(i) } +export function measure(n: number): number { return n + 1 } diff --git a/tests/cases/typescript/jsx-handed-functions/tsconfig.json b/tests/cases/typescript/jsx-handed-functions/tsconfig.json new file mode 100644 index 00000000..f75ffe5b --- /dev/null +++ b/tests/cases/typescript/jsx-handed-functions/tsconfig.json @@ -0,0 +1 @@ +{ "compilerOptions": { "jsx": "preserve", "strict": true } }