From 849a35d47ae0154a7a8cdc48b7a3d6e002394b7d Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Sun, 25 Oct 2026 23:17:30 +0100 Subject: [PATCH] Read RETURNS TABLE as the set of rows of its columns sql_signature reads RETURNS TABLE (...) as PostgreSQL defines it, a set of rows whose columns are its OUT arguments: a set of record whose columns are the table's, or a set of the type of its one column. meta/sql-columns.json states the field of the returned structure feeding each column where a column's name or a field no column reads leaves it to the statement: the rows of pixelAsPoints read the geom, the value, the x and the y of each Pixel, and those of geoPoseFrames each field of a GeoPoseFrame by the column of its name. A row whose columns nothing feeds still stops the catalog, but for a signature passing a SQL array where the function takes one element of it (_loops_over_an_array): that signature is the wrapper's loop of calls, not a projection of the function, and attach_row_sources gives it up as attach_claims_fit gives up an unfit shared signature. dumpValues(raster, integer[], boolean) calls raster_dump_values once per band of its array, each row's nband a band of the argument, and the function keeps its one-band signature dumpValues(raster, integer, boolean). Measured over MobilityDB master 1a7a1a0e11: the catalog differs from master's in geopose_frames and raster_pixel_as_points, whose signatures state their rows, and in raster_dump_values, which keeps its one-band signature; _loops_over_an_array holds for that one signature of the catalog. Over that catalog the JVM generators of JMEOS main answer pixelAsPoints on the typed Spark and Flink surfaces, 1305 SQL functions and 9373 overloads where master's catalog gives 1304 and 9370, refuse geoPoseFrames as setret:column/return:text, and the Spark gaps ledger is unchanged; MobilityDuck main's generator writes the same files. tests/test_sqlfn_rows.py reads a RETURNS TABLE of several columns and of one, gives up the unfed row of a signature looping over an array, and stops on the unfed row of a signature that does not; the suite passes, 593 tests and none skipped. --- meta/sql-columns.json | 8 +++++ parser/sqlfn.py | 64 +++++++++++++++++++++++++++++++++++----- tests/test_sqlfn_rows.py | 63 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 8 deletions(-) diff --git a/meta/sql-columns.json b/meta/sql-columns.json index 4fc5c15..b3ae0fd 100644 --- a/meta/sql-columns.json +++ b/meta/sql-columns.json @@ -79,6 +79,14 @@ "valuePercent": { "note": "A ValueCount holds the count and the percent of a value; the rows of valuePercent read its value and its percent, as PostGIS's ST_ValuePercent does.", "columns": {"value": {"field": "value"}, "percent": {"field": "percent"}} + }, + "pixelAsPoints": { + "note": "A Pixel holds its column, its row, whether it is nodata, its value and its point; the rows of pixelAsPoints read its point, its value, its column and its row, as PostGIS's ST_PixelAsPoints does.", + "columns": {"geom": {"field": "geom"}, "val": {"field": "value"}, "x": {"field": "x"}, "y": {"field": "y"}} + }, + "geoPoseFrames": { + "note": "A GeoPoseFrame holds the fields of one row of the geopose_frames table, each read by the column of its name.", + "columns": {"frame_id": {"field": "frame_id"}, "authority": {"field": "authority"}, "code": {"field": "code"}, "name": {"field": "name"}, "srid": {"field": "srid"}, "is_geographic": {"field": "is_geographic"}, "description": {"field": "description"}} } } } diff --git a/parser/sqlfn.py b/parser/sqlfn.py index 6193d94..849eb16 100644 --- a/parser/sqlfn.py +++ b/parser/sqlfn.py @@ -333,7 +333,10 @@ def sql_statements(sql_src): for stmt in _create_fn_stmts(text, bodies=True): stmts.append(stmt) argdecls, ret = stmt[1], stmt[2] - if ret: + table = _table_columns(ret) + if table: + vocab.update(t for _, t in table) # each column of RETURNS TABLE is a type + elif ret: vocab.add(ret) # a RETURNS clause is always a type for a in argdecls: bt = _bare_type(a) @@ -342,13 +345,25 @@ def sql_statements(sql_src): return stmts, vocab, composites +def _table_columns(ret): + """The `(name, type)` columns of a `RETURNS TABLE (...)` clause, or None for any other + return type.""" + m = re.match(r"TABLE\s*\((.*)\)$", ret or "", re.I | re.S) + return [_column(c) for c in _split_top_commas(m.group(1)) if c.strip()] if m else None + + def sql_signature(sqlname, argdecls, ret, retset, vocab, composites): """The SQL signature {sqlName, args, required, argDefaults, ret, retSet, columns} a - CREATE FUNCTION states, as #_wrapper_sql_sigs lists it.""" + CREATE FUNCTION states, as #_wrapper_sql_sigs lists it. `RETURNS TABLE (...)` is + PostgreSQL's set of rows whose columns are its OUT arguments: a set of `record`, or of + the type of its one column, as `pixelAsPoints` returns its rows `(geom, val, x, y)`.""" + table = _table_columns(ret) + if table: + ret, retset = ("record" if len(table) > 1 else table[0][1]), True indecls = [a for a in argdecls if _is_in_arg(a)] outcols = [_column(_OUTMODE.sub("", a.strip())) for a in argdecls if _OUTMODE.match(a.strip())] - columns = outcols or composites.get(ret) + columns = table or outcols or composites.get(ret) return {"sqlName": sqlname, "args": [_arg_type(a, vocab) for a in indecls], "required": sum(1 for a in indecls if not re.search(r"\bDEFAULT\b", a, re.I)), "argDefaults": [_arg_default(a) for a in indecls], @@ -705,7 +720,12 @@ def attach_row_sources(idl, declared=None): Runs once the object model and the type relations are attached, since a column is matched to a C value by type. A row whose columns no C value and no declaration feeds stops the catalog: guessing a column's source would hand a - binding a row PostgreSQL does not return.""" + binding a row PostgreSQL does not return. Such a row of a signature passing a SQL + array where the function takes one element of it (#_loops_over_an_array) is the + wrapper's loop of calls, not a projection of the function, and the function gives the + signature up, as #attach_claims_fit gives up an unfit shared signature: + `dumpValues(raster, integer[], boolean)` calls `raster_dump_values` once per band of + its array, and each row's `nband` is a band of the argument.""" declared = declared_columns() if declared is None else declared sqlc = _sql_ctypes(idl) structs = {s["name"]: s for s in idl.get("structs") or ()} @@ -713,16 +733,21 @@ def attach_row_sources(idl, declared=None): for f in idl["functions"]: ar = (f.get("shape") or {}).get("arrayReturn") struct = structs.get(_c_base(ar["element"]["c"])[0]) if ar else None + unfit = [] for s in f.get("sqlSignatures") or (): if not s.get("columns"): continue columns = _column_sources(f, s, sqlc, declared, struct) - if columns is None: + if columns is None and _loops_over_an_array(f, s, sqlc): + unfit.append(s) + elif columns is None: unfed.append(f"{f['name']}: {s.get('sqlName', f.get('sqlfn'))}" f"({', '.join(s['args'])}) RETURNS {s['ret']}") else: s["columns"] = columns n += 1 + if unfit: + f["sqlSignatures"] = [s for s in f["sqlSignatures"] if s not in unfit] if unfed: raise ValueError( "SQL rows with a column MEOS states no source for; state it in " @@ -737,6 +762,19 @@ def _signature_fits(func, sig, sqlc): two counts differ (an argument left to its default). The parameters `sig` binds, the out-parameters and the length of each input array take no argument: a SQL array is the C array together with its count (`shape.inputArrays`).""" + got = _signature_inputs(func, sig) + if got is None: + return None + inputs, types = got + + def fit(order): + return all(_fits(t, *_c_base(p.get("cType")), sqlc) for t, p in zip(types, order)) + return fit(inputs) or any(fit(o) for o in permutations(inputs)) + + +def _signature_inputs(func, sig): + """(the C input parameters of `func`, the SQL types of the arguments of `sig`) as + #_signature_fits matches them, or None when the two counts differ.""" shape = func.get("shape") or {} bound = sig.get("boundArgs") or shape.get("boundArgs") or {} out = set(shape.get("outParams") or ()) @@ -750,10 +788,20 @@ def _signature_fits(func, sig, sqlc): for a in sig["args"]: t = re.sub(r"\s*\(.*\)$", "", a.strip().lower()) types.append(SQL_ALIASES.get(t, t)) + return inputs, types - def fit(order): - return all(_fits(t, *_c_base(p.get("cType")), sqlc) for t, p in zip(types, order)) - return fit(inputs) or any(fit(o) for o in permutations(inputs)) + +def _loops_over_an_array(func, sig, sqlc): + """Whether `sig` passes a SQL array where `func` takes one element of it, an argument + `X[]` at a C parameter of the type of an `X` (#_signature_inputs): the wrapper calls + the function once per element, as `dumpValues(raster, integer[], boolean)` calls + `raster_dump_values` once per band.""" + got = _signature_inputs(func, sig) + if got is None: + return False + inputs, types = got + return any(t.endswith("[]") and _fits(t[:-2], *_c_base(p.get("cType")), sqlc) + and not _fits(t, *_c_base(p.get("cType")), sqlc) for t, p in zip(types, inputs)) def attach_claims_fit(idl): diff --git a/tests/test_sqlfn_rows.py b/tests/test_sqlfn_rows.py index 55a9265..a8c4a9f 100644 --- a/tests/test_sqlfn_rows.py +++ b/tests/test_sqlfn_rows.py @@ -75,6 +75,15 @@ RETURNS SETOF integer AS 'MODULE_PATHNAME', 'Set_unnest' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; +CREATE FUNCTION pixelAsPoints(rast raster, band integer DEFAULT 1, + exclude_nodata_value boolean DEFAULT true) + RETURNS TABLE(geom geometry, val double precision, x integer, y integer) + AS 'MODULE_PATHNAME', 'Raster_pixel_as_points' + LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; +CREATE FUNCTION bandNumbers(raster) + RETURNS TABLE (nband integer) + AS 'MODULE_PATHNAME', 'Raster_band_numbers' + LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION dumpAsPolygons(rast raster, band integer DEFAULT 1, exclude_nodata_value boolean DEFAULT true) RETURNS SETOF geomval @@ -117,6 +126,20 @@ def test_a_composite_of_another_extension_gives_its_declared_members(self): self.assertEqual((s["ret"], s["retSet"]), ("geomval", True)) self.assertEqual(s["columns"], [("geom", "geometry"), ("val", "double precision")]) + def test_returns_table_is_a_set_of_records_of_its_columns(self): + """As #test_out_arguments_are_columns_not_args: RETURNS TABLE is PostgreSQL's + set of rows whose columns are OUT arguments.""" + s = _sigs()["Raster_pixel_as_points"][0] + self.assertEqual((s["ret"], s["retSet"]), ("record", True)) + self.assertEqual(s["args"], ["raster", "integer", "boolean"]) + self.assertEqual(s["columns"], [("geom", "geometry"), ("val", "double precision"), + ("x", "integer"), ("y", "integer")]) + + def test_returns_table_of_one_column_is_a_set_of_its_type(self): + """As #test_one_value_has_no_columns, for the one column of a RETURNS TABLE.""" + s = _sigs()["Raster_band_numbers"][0] + self.assertEqual((s["ret"], s["retSet"], s["columns"]), ("integer", True, None)) + def test_one_value_has_no_columns(self): self.assertIsNone(_sigs()["Set_unnest"][0]["columns"]) @@ -331,8 +354,48 @@ def test_a_row_without_columns_is_left_alone(self): self.assertEqual(n, 0) +def _dump_values(args): + """raster_dump_values over the SQL signature `args` returning the rows (nband, + valarray) a wrapper loop over bands answers, and its one-band signature.""" + return { + "name": "raster_dump_values", + "sqlfn": "dumpValues", + "params": [{"name": "rast", "cType": "const Raster *"}, + {"name": "band", "cType": "int"}, + {"name": "exclude_nodata", "cType": "bool"}, + {"name": "nodata", "cType": "bool **"}, _count()], + "returnType": {"c": "double *"}, + "shape": {"arrayReturn": {"lengthFrom": {"kind": "param", "name": "count"}, + "element": {"c": "double"}}, + "outputArrays": [{"param": "nodata"}], "outParams": ["nodata", "count"]}, + "sqlSignatures": [ + {"args": args, "ret": "record", "retSet": True, + "columns": _cols(("nband", "integer"), ("valarray", "double precision[]"))}, + {"args": ["raster", "integer", "boolean"], "ret": "double precision[]"}]} + + +def _attach_raster(func): + idl = copy.deepcopy(CATALOG) + idl["objectModel"]["classes"]["Raster"] = {"cType": "Raster"} + idl["functions"] = [func] + idl, _ = attach_row_sources(idl, {}) + return idl["functions"][0]["sqlSignatures"] + + class UnfedRowTests(unittest.TestCase): + def test_an_unfed_row_of_an_unfit_signature_is_given_up(self): + """As #attach_claims_fit gives up an unfit shared signature: dumpValues(raster, + integer[], boolean) loops over the bands of its array, so no C value feeds nband + and its arguments do not fit the one band the C function takes.""" + sigs = _attach_raster(_dump_values(["raster", "integer[]", "boolean"])) + self.assertEqual([s["args"] for s in sigs], [["raster", "integer", "boolean"]]) + + def test_an_unfed_row_of_a_fitting_signature_still_stops_the_catalog(self): + """As #test_a_column_no_c_value_fits_stops_the_catalog, for a signature that fits.""" + with self.assertRaisesRegex(ValueError, "dumpValues"): + _attach_raster(_dump_values(["raster", "integer", "boolean"])) + def test_a_column_no_c_value_fits_stops_the_catalog(self): func = copy.deepcopy(VALUE_TIME_SPLIT) func["sqlSignatures"][0]["columns"][0]["type"] = "text"