diff --git a/age--1.8.0--y.y.y.sql b/age--1.8.0--y.y.y.sql index 4c85b28ca..097d066ee 100644 --- a/age--1.8.0--y.y.y.sql +++ b/age--1.8.0--y.y.y.sql @@ -22,7 +22,7 @@ --* This is a TEMPLATE for upgrading from the previous version of Apache AGE --* Please adjust the below ALTER EXTENSION to reflect the -- correct version it ---* is upgrading to. +-- is upgrading to. -- This will only work within a major version of PostgreSQL, not across -- major versions. @@ -33,3 +33,35 @@ --* Please add all additions, deletions, and modifications to the end of this --* file. We need to keep the order of these changes. --* REMOVE ALL LINES ABOVE, and this one, that start with --* + +-- +-- Add delimiter parameter to load_labels_from_file and load_edges_from_file +-- +-- Issue #2449: Both load_labels_from_file and load_edges_from_file now accept +-- an optional delimiter parameter (default ',') to support non-CSV delimiters +-- such as pipe-delimited files. + +-- Drop and recreate load_labels_from_file with new delimiter parameter +DROP FUNCTION IF EXISTS ag_catalog.load_labels_from_file(name, name, text, bool, bool); + +CREATE FUNCTION ag_catalog.load_labels_from_file(graph_name name, + label_name name, + file_path text, + id_field_exists bool default true, + load_as_agtype bool default false, + delimiter text default ',') + RETURNS void + LANGUAGE c + AS 'MODULE_PATHNAME'; + +-- Drop and recreate load_edges_from_file with new delimiter parameter +DROP FUNCTION IF EXISTS ag_catalog.load_edges_from_file(name, name, text, bool); + +CREATE FUNCTION ag_catalog.load_edges_from_file(graph_name name, + label_name name, + file_path text, + load_as_agtype bool default false, + delimiter text default ',') + RETURNS void + LANGUAGE c + AS 'MODULE_PATHNAME'; diff --git a/regress/age_load/data/pipe_edges.csv b/regress/age_load/data/pipe_edges.csv new file mode 100644 index 000000000..776fdf728 --- /dev/null +++ b/regress/age_load/data/pipe_edges.csv @@ -0,0 +1,3 @@ +start_id|start_vertex_type|end_id|end_vertex_type|distance +1|City|2|City|2500 +2|City|3|City|1050 diff --git a/regress/age_load/data/pipe_vertices.csv b/regress/age_load/data/pipe_vertices.csv new file mode 100644 index 000000000..48e6d1e26 --- /dev/null +++ b/regress/age_load/data/pipe_vertices.csv @@ -0,0 +1,4 @@ +id|name|country +1|Moscow|Russia +2|Berlin|Germany +3|Paris|France diff --git a/regress/expected/age_load.out b/regress/expected/age_load.out index 17c5ecc27..01afcb640 100644 --- a/regress/expected/age_load.out +++ b/regress/expected/age_load.out @@ -511,6 +511,72 @@ NOTICE: graph "agload_delim" has been dropped (1 row) +-- +-- Test delimiter parameter for pipe-delimited files +-- +SELECT create_graph('agload_pipe'); +NOTICE: graph "agload_pipe" has been created + create_graph +-------------- + +(1 row) + +SELECT create_vlabel('agload_pipe', 'City'); +NOTICE: VLabel "City" has been created + create_vlabel +--------------- + +(1 row) + +SELECT create_elabel('agload_pipe', 'Connected'); +NOTICE: ELabel "Connected" has been created + create_elabel +--------------- + +(1 row) + +-- pipe-delimited vertex file with delimiter parameter +SELECT load_labels_from_file('agload_pipe', 'City', 'age_load/pipe_vertices.csv', true, false, '|'); + load_labels_from_file +----------------------- + +(1 row) + +-- pipe-delimited edge file with delimiter parameter +SELECT load_edges_from_file('agload_pipe', 'Connected', 'age_load/pipe_edges.csv', false, '|'); + load_edges_from_file +---------------------- + +(1 row) + +-- verify data loaded correctly +SELECT * FROM cypher('agload_pipe', $$ MATCH (n:City) RETURN n.name, n.country ORDER BY n.name $$) AS (name agtype, country agtype); + name | country +----------+----------- + "Berlin" | "Germany" + "Moscow" | "Russia" + "Paris" | "France" +(3 rows) + +SELECT * FROM cypher('agload_pipe', $$ MATCH (a:City)-[e:Connected]->(b:City) RETURN a.name, b.name, e.distance ORDER BY a.name $$) AS (a_name agtype, b_name agtype, distance agtype); + a_name | b_name | distance +----------+----------+---------- + "Berlin" | "Paris" | "1050" + "Moscow" | "Berlin" | "2500" +(2 rows) + +SELECT drop_graph('agload_pipe', true); +NOTICE: drop cascades to 4 other objects +DETAIL: drop cascades to table agload_pipe._ag_label_vertex +drop cascades to table agload_pipe._ag_label_edge +drop cascades to table agload_pipe."City" +drop cascades to table agload_pipe."Connected" +NOTICE: graph "agload_pipe" has been dropped + drop_graph +------------ + +(1 row) + -- -- Test security and permissions -- diff --git a/regress/sql/age_load.sql b/regress/sql/age_load.sql index 196b09806..3492b57c9 100644 --- a/regress/sql/age_load.sql +++ b/regress/sql/age_load.sql @@ -223,6 +223,25 @@ SELECT load_labels_from_file('agload_delim', 'V', 'age_load/labels_long_row.csv' SELECT drop_graph('agload_delim', true); +-- +-- Test delimiter parameter for pipe-delimited files +-- +SELECT create_graph('agload_pipe'); +SELECT create_vlabel('agload_pipe', 'City'); +SELECT create_elabel('agload_pipe', 'Connected'); + +-- pipe-delimited vertex file with delimiter parameter +SELECT load_labels_from_file('agload_pipe', 'City', 'age_load/pipe_vertices.csv', true, false, '|'); + +-- pipe-delimited edge file with delimiter parameter +SELECT load_edges_from_file('agload_pipe', 'Connected', 'age_load/pipe_edges.csv', false, '|'); + +-- verify data loaded correctly +SELECT * FROM cypher('agload_pipe', $$ MATCH (n:City) RETURN n.name, n.country ORDER BY n.name $$) AS (name agtype, country agtype); +SELECT * FROM cypher('agload_pipe', $$ MATCH (a:City)-[e:Connected]->(b:City) RETURN a.name, b.name, e.distance ORDER BY a.name $$) AS (a_name agtype, b_name agtype, distance agtype); + +SELECT drop_graph('agload_pipe', true); + -- -- Test security and permissions -- diff --git a/sql/age_main.sql b/sql/age_main.sql index 233d0d23f..5b1044d5a 100644 --- a/sql/age_main.sql +++ b/sql/age_main.sql @@ -167,7 +167,8 @@ CREATE FUNCTION ag_catalog.load_labels_from_file(graph_name name, label_name name, file_path text, id_field_exists bool default true, - load_as_agtype bool default false) + load_as_agtype bool default false, + delimiter text default ',') RETURNS void LANGUAGE c AS 'MODULE_PATHNAME'; @@ -175,7 +176,8 @@ CREATE FUNCTION ag_catalog.load_labels_from_file(graph_name name, CREATE FUNCTION ag_catalog.load_edges_from_file(graph_name name, label_name name, file_path text, - load_as_agtype bool default false) + load_as_agtype bool default false, + delimiter text default ',') RETURNS void LANGUAGE c AS 'MODULE_PATHNAME'; diff --git a/src/backend/utils/load/ag_load_edges.c b/src/backend/utils/load/ag_load_edges.c index 01585bab0..912547c01 100644 --- a/src/backend/utils/load/ag_load_edges.c +++ b/src/backend/utils/load/ag_load_edges.c @@ -134,7 +134,7 @@ static void process_edge_row(char **fields, int nfields, * Create COPY options for CSV parsing. * Returns a List of DefElem nodes. */ -static List *create_copy_options(void) +static List *create_copy_options(char delimiter) { List *options = NIL; @@ -150,6 +150,17 @@ static List *create_copy_options(void) (Node *) makeBoolean(false), -1)); + /* DELIMITER */ + { + char delimiter_str[2]; + delimiter_str[0] = delimiter; + delimiter_str[1] = '\0'; + options = lappend(options, + makeDefElem("delimiter", + (Node *) makeString(pstrdup(delimiter_str)), + -1)); + } + return options; } @@ -161,7 +172,8 @@ int create_edges_from_csv_file(char *file_path, Oid graph_oid, char *label_name, int label_id, - bool load_as_agtype) + bool load_as_agtype, + char delimiter) { Relation label_rel; Oid label_relid; @@ -196,7 +208,7 @@ int create_edges_from_csv_file(char *file_path, init_batch_insert(&batch_state, label_name, graph_oid); /* Create COPY options for CSV parsing */ - copy_options = create_copy_options(); + copy_options = create_copy_options(delimiter); /* Create a minimal ParseState for BeginCopyFrom */ pstate = make_parsestate(NULL); diff --git a/src/backend/utils/load/ag_load_labels.c b/src/backend/utils/load/ag_load_labels.c index 236d47a1d..927c9008e 100644 --- a/src/backend/utils/load/ag_load_labels.c +++ b/src/backend/utils/load/ag_load_labels.c @@ -120,7 +120,7 @@ static void process_vertex_row(char **fields, int nfields, * Create COPY options for csv parsing. * Returns a List of DefElem nodes. */ -static List *create_copy_options(void) +static List *create_copy_options(char delimiter) { List *options = NIL; @@ -136,6 +136,17 @@ static List *create_copy_options(void) (Node *) makeBoolean(false), -1)); + /* DELIMITER */ + { + char delimiter_str[2]; + delimiter_str[0] = delimiter; + delimiter_str[1] = '\0'; + options = lappend(options, + makeDefElem("delimiter", + (Node *) makeString(pstrdup(delimiter_str)), + -1)); + } + return options; } @@ -148,7 +159,8 @@ int create_labels_from_csv_file(char *file_path, char *label_name, int label_id, bool id_field_exists, - bool load_as_agtype) + bool load_as_agtype, + char delimiter) { Relation label_rel; Oid label_relid; @@ -193,7 +205,7 @@ int create_labels_from_csv_file(char *file_path, init_batch_insert(&batch_state, label_name, graph_oid); /* Create COPY options for CSV parsing */ - copy_options = create_copy_options(); + copy_options = create_copy_options(delimiter); /* Create a minimal ParseState for BeginCopyFrom */ pstate = make_parsestate(NULL); diff --git a/src/backend/utils/load/age_load.c b/src/backend/utils/load/age_load.c index e4f10d7e4..d655ef2eb 100644 --- a/src/backend/utils/load/age_load.c +++ b/src/backend/utils/load/age_load.c @@ -576,6 +576,7 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) int32 label_id; bool id_field_exists; bool load_as_agtype; + char delimiter; if (PG_ARGISNULL(0)) { @@ -604,6 +605,23 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) id_field_exists = PG_GETARG_BOOL(3); load_as_agtype = PG_GETARG_BOOL(4); + if (PG_NARGS() > 5 && !PG_ARGISNULL(5)) + { + text *delim_text = PG_GETARG_TEXT_P(5); + char *delim_str = text_to_cstring(delim_text); + if (strlen(delim_str) != 1) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("delimiter must be a single character"))); + } + delimiter = delim_str[0]; + } + else + { + delimiter = ','; + } + graph_name_str = NameStr(*graph_name); label_name_str = NameStr(*label_name); @@ -625,7 +643,7 @@ Datum load_labels_from_file(PG_FUNCTION_ARGS) create_labels_from_csv_file(file_path_str, graph_name_str, graph_oid, label_name_str, label_id, id_field_exists, - load_as_agtype); + load_as_agtype, delimiter); free(file_path_str); @@ -645,6 +663,7 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) Oid label_relid; int32 label_id; bool load_as_agtype; + char delimiter; if (PG_ARGISNULL(0)) { @@ -672,6 +691,23 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) file_name = PG_GETARG_TEXT_P(2); load_as_agtype = PG_GETARG_BOOL(3); + if (PG_NARGS() > 4 && !PG_ARGISNULL(4)) + { + text *delim_text = PG_GETARG_TEXT_P(4); + char *delim_str = text_to_cstring(delim_text); + if (strlen(delim_str) != 1) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("delimiter must be a single character"))); + } + delimiter = delim_str[0]; + } + else + { + delimiter = ','; + } + graph_name_str = NameStr(*graph_name); label_name_str = NameStr(*label_name); @@ -692,7 +728,8 @@ Datum load_edges_from_file(PG_FUNCTION_ARGS) check_rls_for_load(label_relid); create_edges_from_csv_file(file_path_str, graph_name_str, graph_oid, - label_name_str, label_id, load_as_agtype); + label_name_str, label_id, load_as_agtype, + delimiter); free(file_path_str); diff --git a/src/include/utils/load/ag_load_edges.h b/src/include/utils/load/ag_load_edges.h index 4db00d93a..b2f336489 100644 --- a/src/include/utils/load/ag_load_edges.h +++ b/src/include/utils/load/ag_load_edges.h @@ -39,6 +39,6 @@ */ int create_edges_from_csv_file(char *file_path, char *graph_name, Oid graph_oid, char *label_name, int label_id, - bool load_as_agtype); + bool load_as_agtype, char delimiter); #endif /* AG_LOAD_EDGES_H */ diff --git a/src/include/utils/load/ag_load_labels.h b/src/include/utils/load/ag_load_labels.h index c3d517f30..5a9c0246b 100644 --- a/src/include/utils/load/ag_load_labels.h +++ b/src/include/utils/load/ag_load_labels.h @@ -39,6 +39,7 @@ */ int create_labels_from_csv_file(char *file_path, char *graph_name, Oid graph_oid, char *label_name, int label_id, - bool id_field_exists, bool load_as_agtype); + bool id_field_exists, bool load_as_agtype, + char delimiter); #endif /* AG_LOAD_LABELS_H */