diff --git a/meta/sql-columns.json b/meta/sql-columns.json index b3ae0fd..1a9634e 100644 --- a/meta/sql-columns.json +++ b/meta/sql-columns.json @@ -7,6 +7,13 @@ "columns": [["geom", "geometry"], ["val", "double precision"]] } }, + "arrays": { + "dumpValues": { + "note": "raster_dump_values answers the values of a band row by row and, in nodata, true for a nodata pixel; its wrapper answers them as a height by width array, NULL for a nodata pixel, as PostGIS's ST_DumpValues does, each row raster_width of the raster long.", + "nulls": "nodata", + "rowLength": {"call": "raster_width", "args": ["rast"]} + } + }, "rows": { "geomval": { "note": "A GeomVal holds a polygon and the value of the pixels it covers; the rows of geomval read its geom and its val, as PostGIS's ST_DumpAsPolygons does.", diff --git a/meta/sql-columns.schema.json b/meta/sql-columns.schema.json index 800afff..f870b29 100644 --- a/meta/sql-columns.schema.json +++ b/meta/sql-columns.schema.json @@ -36,6 +36,42 @@ } } }, + "arrays": { + "type": "object", + "description": "The layout a wrapper gives the SQL array a function returns beyond the C array, keyed by SQL name.", + "additionalProperties": { + "type": "object", + "required": ["note"], + "minProperties": 2, + "additionalProperties": false, + "properties": { + "note": { + "type": "string", + "description": "What the wrapper lays out, and why MEOS does not state it.", + "minLength": 1 + }, + "nulls": { + "description": "The output array whose true elements are NULL elements of the SQL array.", + "type": "string", + "pattern": "^[a-z_][a-z0-9_]*$" + }, + "rowLength": { + "description": "The public function, and the parameters of the function it takes, answering the length of each row of a two-dimensional SQL array whose rows the C array lays out one after the other.", + "type": "object", + "required": ["call", "args"], + "additionalProperties": false, + "properties": { + "call": {"type": "string", "pattern": "^[a-z_][a-z0-9_]*$"}, + "args": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "pattern": "^[a-z_][a-z0-9_]*$"} + } + } + } + } + } + }, "rows": { "type": "object", "description": "Keyed by the composite type a function returns, or by its SQL name when it returns record.", diff --git a/parser/sqlfn.py b/parser/sqlfn.py index 849eb16..aca51c0 100644 --- a/parser/sqlfn.py +++ b/parser/sqlfn.py @@ -541,6 +541,55 @@ def declared_composites(path=_COLUMNS_META): for key, entry in (doc.get("composites") or {}).items()} +def declared_arrays(path=_COLUMNS_META): + """The layout a wrapper gives the SQL array a function returns beyond the C array, + stated in `meta/sql-columns.json` beside the rows #declared_columns reads, keyed by + SQL name: {sqlName: {"nulls": ..., "rowLength": {"call": ..., "args": [...]}}}.""" + doc = json.loads(Path(path).read_text()) + return {key: {k: v for k, v in entry.items() if k != "note"} + for key, entry in (doc.get("arrays") or {}).items()} + + +def attach_array_shapes(idl, declared=None): + """(idl, number of signatures stating their array's layout) once each SQL signature + returning an array whose layout only its wrapper gives states it as `arrayShape`: + `nulls`, the output array whose true elements are NULL elements of the SQL array, and + `rowLength`, the public function and the parameters of the function answering the + length of each row of a two-dimensional SQL array, its rows laid out one after the + other in the C array. `dumpValues(raster, integer, boolean)` answers the values of a + band as a height by width array, NULL where `nodata` is true, each row `raster_width` + of the raster long, as #declared_columns states what a wrapper computes of a row. + + A statement naming no output array of its function, a call the catalog does not + hold or parameters the function does not take, or a statement no signature takes, + stops the catalog: a binding reading it would lay out an array PostgreSQL does not + return.""" + declared = declared_arrays() if declared is None else declared + names = {f["name"] for f in idl["functions"]} + bad, used, n = [], set(), 0 + for f in idl["functions"]: + outs = {a["param"] for a in (f.get("shape") or {}).get("outputArrays") or ()} + params = {p["name"] for p in f.get("params") or ()} + for s in f.get("sqlSignatures") or (): + key = s.get("sqlName") or f.get("sqlfn") + st = declared.get(key) + if not st or not str(s.get("ret") or "").endswith("[]"): + continue + used.add(key) + rl = st.get("rowLength") + if ("nulls" in st and st["nulls"] not in outs) or \ + (rl and (rl["call"] not in names or not set(rl["args"]) <= params)): + bad.append(f"{f['name']}: {key}({', '.join(s['args'])})") + continue + s["arrayShape"] = dict(st) + n += 1 + bad += [f"{key}: no SQL signature returning an array" for key in declared if key not in used] + if bad: + raise ValueError("SQL array layouts meta/sql-columns.json states that no function " + "takes:\n " + "\n ".join(sorted(set(bad)))) + return idl, n + + def _c_base(ctype): """A C type without `const`, `struct` and its pointer levels, with the `` spellings read as MEOS's own (`int64_t` is `int64`), and the number of diff --git a/run.py b/run.py index 9134240..7fcaf74 100644 --- a/run.py +++ b/run.py @@ -27,7 +27,7 @@ from parser.enrich import enrich_idl, restate_wire from parser.codecs import state_type_encodings from parser.sqlfn import (attach_sqlfn_map, attach_aggfn_map, attach_row_sources, - attach_claims_fit, + attach_array_shapes, attach_claims_fit, attach_sqlaggfn_map, lint_container_family_csqlfn, lint_ea_sqlfn, lint_positional_sqlfn, lint_sqlfn_case_collisions, state_deployed_sqlfn) @@ -423,6 +423,12 @@ def main(): # type; a row with a column nothing feeds stops the catalog. idl, nrows = attach_row_sources(idl) print(f" SQL rows with every column's C source: {nrows}", file=sys.stderr) + # State the layout a wrapper gives a returned SQL array beyond the C array, its NULL + # elements and the length of its rows, read against the SQL signatures; a statement no + # function takes stops the catalog. + if SQL_SRC.exists(): + idl, narrays = attach_array_shapes(idl) + print(f" SQL arrays stating their layout: {narrays}", file=sys.stderr) # State each SQL function composed over the functions of another type, its casts, # call and restore named by MEOS function, read from the deployed SQL bodies and diff --git a/tests/test_sqlfn_rows.py b/tests/test_sqlfn_rows.py index a8c4a9f..475f78c 100644 --- a/tests/test_sqlfn_rows.py +++ b/tests/test_sqlfn_rows.py @@ -23,7 +23,8 @@ from pathlib import Path from parser.sqlfn import (_wrapper_sql_sigs, attach_row_sources, attach_sqlfn_map, - declared_columns, declared_composites) + attach_array_shapes, declared_arrays, declared_columns, + declared_composites) ROOT = Path(__file__).resolve().parents[1] COLUMNS = ROOT / "meta" / "sql-columns.json" @@ -497,6 +498,66 @@ def test_a_composite_member_without_a_type_is_refused(self): with self.assertRaises(jsonschema.ValidationError): jsonschema.validate(doc, json.loads(SCHEMA.read_text())) + def test_the_array_layouts_are_read_by_sql_name(self): + """As #test_the_declarations_are_read_by_row reads the rows.""" + self.assertEqual(declared_arrays(COLUMNS)["dumpValues"], + {"nulls": "nodata", + "rowLength": {"call": "raster_width", "args": ["rast"]}}) + + def test_a_row_length_without_its_call_is_refused(self): + """As #test_an_unknown_source_is_refused refuses a row's column.""" + import jsonschema + doc = json.loads(COLUMNS.read_text()) + del doc["arrays"]["dumpValues"]["rowLength"]["call"] + with self.assertRaises(jsonschema.ValidationError): + jsonschema.validate(doc, json.loads(SCHEMA.read_text())) + + +DUMP_VALUES_LAYOUT = {"dumpValues": {"nulls": "nodata", + "rowLength": {"call": "raster_width", "args": ["rast"]}}} + + +def _array_idl(layout=None): + """raster_dump_values over its one-band signature returning double precision[], and + raster_width, through #attach_array_shapes with the statement `layout`.""" + func = _dump_values(["raster", "integer[]", "boolean"]) + func["sqlSignatures"] = func["sqlSignatures"][1:] + width = {"name": "raster_width", "sqlfn": "width", + "params": [{"name": "rast", "cType": "const Raster *"}], + "returnType": {"c": "int"}} + idl, n = attach_array_shapes({"functions": [func, width]}, + DUMP_VALUES_LAYOUT if layout is None else layout) + return idl["functions"][0]["sqlSignatures"][0], n + + +class ArrayShapeTests(unittest.TestCase): + + def test_an_array_states_its_null_mask_and_the_length_of_its_rows(self): + """As #test_a_stated_source_feeds_its_column feeds a row's column from its statement.""" + sig, n = _array_idl() + self.assertEqual(n, 1) + self.assertEqual(sig["arrayShape"], DUMP_VALUES_LAYOUT["dumpValues"]) + + def test_a_null_mask_naming_no_output_array_stops_the_catalog(self): + """As #test_a_stated_source_naming_no_c_value_stops_the_catalog.""" + layout = {"dumpValues": dict(DUMP_VALUES_LAYOUT["dumpValues"], nulls="values")} + with self.assertRaisesRegex(ValueError, "dumpValues"): + _array_idl(layout) + + def test_a_row_length_calling_no_catalog_function_stops_the_catalog(self): + """As #test_a_stated_source_naming_no_c_value_stops_the_catalog.""" + layout = {"dumpValues": dict(DUMP_VALUES_LAYOUT["dumpValues"], + rowLength={"call": "raster_cols", "args": ["rast"]})} + with self.assertRaisesRegex(ValueError, "dumpValues"): + _array_idl(layout) + + def test_a_layout_no_signature_takes_stops_the_catalog(self): + """As #test_a_stated_source_naming_no_c_value_stops_the_catalog, for a statement + whose SQL name returns no array.""" + layout = dict(DUMP_VALUES_LAYOUT, pixelValues={"nulls": "nodata"}) + with self.assertRaisesRegex(ValueError, "pixelValues"): + _array_idl(layout) + if __name__ == "__main__": unittest.main()