diff --git a/changelog.yml b/changelog.yml index be0b6232..c9242810 100644 --- a/changelog.yml +++ b/changelog.yml @@ -1,6 +1,8 @@ - version: 3.8.0 summary: date: unreleased + added: + - "`Relation#on_conflict`, `#do_update` and `#do_nothing` for building INSERT ... ON CONFLICT statements, with a DSL for the DO UPDATE part. Create commands and changesets built on such a relation become upserts (@flash-gordon)" fixed: - "Use dataset to fetch record after insert so that read types are properly applied in Create command (@katafrakt)" changed: diff --git a/lib/rom/sql/commands/create.rb b/lib/rom/sql/commands/create.rb index e548d233..b5cc9d2f 100644 --- a/lib/rom/sql/commands/create.rb +++ b/lib/rom/sql/commands/create.rb @@ -11,6 +11,9 @@ module Commands class Create < ROM::Commands::Create adapter :sql + # Lets a relation carrying ON CONFLICT be used via `relation.command(:create)` + restrictable true + include ErrorWrapper use :associates diff --git a/lib/rom/sql/errors.rb b/lib/rom/sql/errors.rb index ee18c25a..f717582b 100644 --- a/lib/rom/sql/errors.rb +++ b/lib/rom/sql/errors.rb @@ -19,6 +19,7 @@ module SQL MissingPrimaryKeyError = Class.new(StandardError) MigrationError = Class.new(StandardError) UnsupportedConversion = Class.new(MigrationError) + UnsupportedFeatureError = Class.new(StandardError) ERROR_MAP = { Sequel::DatabaseError => DatabaseError, diff --git a/lib/rom/sql/relation/writing.rb b/lib/rom/sql/relation/writing.rb index a8101725..2524d2a7 100644 --- a/lib/rom/sql/relation/writing.rb +++ b/lib/rom/sql/relation/writing.rb @@ -1,9 +1,106 @@ # frozen_string_literal: true +require 'rom/sql/upsert_dsl' + module ROM module SQL class Relation < ROM::Relation module Writing + # Handle conflicts with unique constraints on insert + # + # Returns a relation carrying an ON CONFLICT clause, so any insert + # through it becomes an upsert: `insert`, `multi_insert`, `import`, + # `command(:create)` and changesets. Conflicts are ignored unless + # `do_update` is chained. + # + # @example ignore conflicting rows + # users.on_conflict(:email).insert(name: 'Jane', email: 'jane@doe.org') + # + # @example infer a partial unique index + # users.on_conflict(:email) { active.is(true) }.insert(...) + # + # @example use a named constraint + # users.on_conflict(constraint: :users_email_key).insert(...) + # + # @param [Array] target Columns of the unique index + # @param [Symbol] constraint Name of the constraint, takes precedence over target + # + # @yield Index predicate built with the restriction DSL + # + # @return [Relation] + # + # @api public + def on_conflict(*target, constraint: nil, &block) + opts = conflict_options.except(:target, :constraint, :conflict_where) + opts[:constraint] = constraint if constraint + opts[:target] = target.map { |t| t.is_a?(Attribute) ? t.name : t } unless target.empty? + opts[:conflict_where] = schema.canonical.restriction(&block) if block + + new(insert_conflict(opts)) + end + + # Update the existing row on conflict + # + # Without arguments every column except the primary key and the + # conflict target is set from the excluded row, i.e. the one proposed + # for insertion. Listed columns restrict that, pairs set arbitrary + # values, and the block gives access to the upsert DSL. + # + # @example set columns from the excluded row + # users.on_conflict(:email).do_update(:name, :updated_at).insert(...) + # + # @example set values and expressions + # users.on_conflict(:email).do_update(removed_at: nil).insert(...) + # + # @example use the DSL + # users.on_conflict(:email).do_update { + # set(last_seen_at: greatest(last_seen_at, excluded[:last_seen_at])) + # .where(updated_at < excluded[:updated_at]) + # }.multi_insert(tuples) + # + # @param [Array] columns Columns to set from the excluded row + # @param [Hash] pairs Column-value pairs + # + # @yield Block evaluated with the upsert DSL, returns `set(...)` or a hash + # + # @return [Relation] + # + # @see UpsertDSL + # + # @api public + def do_update(*columns, **pairs, &) + assignments, condition = conflict_update(columns, pairs, &) + + opts = conflict_options.except(:update_where).merge(update: assignments) + opts[:update_where] = condition if condition + + new(insert_conflict(opts)) + end + + # Ignore the conflicting row + # + # This is the default of `on_conflict`, the method exists to revert + # a previous `do_update` + # + # @return [Relation] + # + # @api public + def do_nothing + new(insert_conflict(conflict_options.except(:update, :update_where))) + end + + # Row proposed for insertion in ON CONFLICT DO UPDATE + # + # @example + # users.on_conflict(:email).do_update(name: users.excluded[:name]) + # + # @return [SQL::Schema] Schema with attributes qualified with `excluded` + # + # @api public + def excluded + schema.qualified(:excluded) + end + # Add upsert option (only PostgreSQL >= 9.5) # Uses internal Sequel implementation # Default - ON CONFLICT DO NOTHING @@ -128,6 +225,53 @@ def import(other, options = EMPTY_HASH) ) end end + + private + + # @api private + def insert_conflict(opts) + unless dataset.respond_to?(:insert_conflict) + raise UnsupportedFeatureError, + "ON CONFLICT is not supported by #{dataset.db.database_type}" + end + + dataset.insert_conflict(opts) + end + + # Options of the ON CONFLICT clause set on the dataset + # + # @api private + def conflict_options + opts = dataset.opts[:insert_conflict] || dataset.opts[:insert_on_conflict] + opts.is_a?(Hash) ? opts : EMPTY_HASH + end + + # Assignments and condition of the DO UPDATE clause + # + # @return [Array(Hash, Object)] + # + # @api private + def conflict_update(columns, pairs, &block) + assignments = columns.to_h { |column| [column, excluded[column]] }.merge(pairs) + condition = nil + + if block + dsl_assignments, condition = UpsertDSL.new(schema).call(&block) + assignments = assignments.merge(dsl_assignments) + end + + assignments = default_conflict_assignments if assignments.empty? + + [assignments.to_h { |k, v| [k.is_a?(Attribute) ? k.name : k, v] }, condition] + end + + # Every column except the primary key and the conflict target + # + # @api private + def default_conflict_assignments + keys = schema.primary_key.map(&:name) + Array(conflict_options[:target]) + (schema.map(&:name) - keys).to_h { |name| [name, excluded[name]] } + end end end end diff --git a/lib/rom/sql/upsert_dsl.rb b/lib/rom/sql/upsert_dsl.rb new file mode 100644 index 00000000..d757d95c --- /dev/null +++ b/lib/rom/sql/upsert_dsl.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +require 'rom/sql/restriction_dsl' + +module ROM + module SQL + # DSL for the DO UPDATE part of INSERT ... ON CONFLICT statements + # + # Attributes refer to the existing row, `excluded` gives access to + # the row proposed for insertion, like PostgreSQL's EXCLUDED pseudo-table. + # Functions are resolved through the virtual row, as in the restriction DSL. + # + # @example + # users.on_conflict(:email).do_update { + # set(name: excluded[:name], updated_at: excluded[:updated_at]) + # .where(updated_at < excluded[:updated_at]) + # } + # + # @api public + class UpsertDSL < RestrictionDSL + # Immutable description of the update: assignments and an optional condition + # + # Every method returns a new instance, so the parts are chained + # + # @api public + class Update + # @!attribute [r] schema + # @return [SQL::Schema] + attr_reader :schema + + # @!attribute [r] assignments + # @return [Hash] Column-value pairs for the SET clause + attr_reader :assignments + + # @!attribute [r] condition + # @return [Object, nil] Condition for the WHERE clause + attr_reader :condition + + # @api private + def initialize(schema, assignments = EMPTY_HASH, condition = nil) + @schema = schema + @assignments = assignments + @condition = condition + freeze + end + + # Add assignments, later ones take precedence + # + # @param [Hash] pairs Column-value pairs + # + # @return [Update] + # + # @api public + def set(**pairs) + self.class.new(schema, assignments.merge(pairs), condition) + end + + # Restrict the update to rows matching the condition + # + # @param [Hash, Object] condition + # + # @return [Update] + # + # @api public + def where(condition) + self.class.new(schema, assignments, qualify(condition)) + end + + private + + # Bare column names in DO UPDATE ... WHERE are ambiguous + # between the table and EXCLUDED + # + # @api private + def qualify(condition) + return condition unless condition.is_a?(Hash) + + condition.to_h { |key, value| + [key.is_a?(Symbol) && schema.key?(key) ? schema[key].qualified : key, value] + } + end + end + + # Row proposed for insertion + # + # @return [SQL::Schema] Schema with attributes qualified with `excluded` + # + # @api public + def excluded + @excluded ||= schema.qualified(:excluded) + end + + # Start the update with assignments + # + # @see Update#set + # + # @api public + def set(**pairs) + Update.new(schema).set(**pairs) + end + + # Start the update with a condition + # + # @see Update#where + # + # @api public + def where(condition) + Update.new(schema).where(condition) + end + + # @return [Array(Hash, Object)] Assignments and condition + # + # @api private + def call(&) + result = super + + case result + when Update then [result.assignments, result.condition] + when ::Hash then [result, nil] + else + ::Kernel.raise ::ArgumentError, "expected set(...) or a hash, got #{result.inspect}" + end + end + end + end +end diff --git a/spec/integration/commands/create_spec.rb b/spec/integration/commands/create_spec.rb index e8bc45f5..aacc9e2f 100644 --- a/spec/integration/commands/create_spec.rb +++ b/spec/integration/commands/create_spec.rb @@ -350,4 +350,36 @@ def self.[](input) ]) end end + + describe 'on a relation with conflict handling' do + let(:task) { { title: 'task 1' } } + + seed do + create_user.call(name: 'Jane') + create_task.call(task) + end + + it 'returns nothing when the conflict is ignored' do + expect(tasks.on_conflict(:title).command(:create).call(task)).to eql([]) + end + + it 'returns the updated tuple' do + command = tasks.on_conflict(:title).do_update(:user_id).command(:create) + + expect(command.call(task.merge(user_id: 1))).to eql([{ id: 1, user_id: 1, title: 'task 1' }]) + end + + it 'returns inserted and updated tuples' do + command = tasks.on_conflict(:title).do_update(:user_id).command(:create) + + expect(command.call([task.merge(user_id: 1), title: 'task 2', user_id: 1])).to match([ + { id: 1, user_id: 1, title: 'task 1' }, + hash_including(user_id: 1, title: 'task 2') + ]) + end + + it 'leaves the plain command failing on conflict' do + expect { tasks.command(:create).call(task) }.to raise_error(ROM::SQL::UniqueConstraintError) + end + end end diff --git a/spec/unit/relation/do_update_spec.rb b/spec/unit/relation/do_update_spec.rb new file mode 100644 index 00000000..a020a1bc --- /dev/null +++ b/spec/unit/relation/do_update_spec.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +RSpec.describe ROM::Relation, '#do_update' do + subject(:relation) { relations[:tasks] } + + include_context 'users and tasks' + + # tasks are seeded with explicit ids + seed do |example| + conn[:users].insert name: 'Jack' + conn.run("SELECT setval('tasks_id_seq', (SELECT max(id) FROM tasks))") if postgres?(example) + end + + with_adapters(:postgres, :sqlite) do + setup_tables do + conn.add_index :tasks, :title, unique: true + end + + let(:conflicting) { relation.on_conflict(:title) } + + let(:joes_task) { relation.by_pk(1).one } + + it 'updates every column but the key and the target from the excluded row' do + conflicting.do_update.insert(user_id: 1, title: "Joe's task") + + expect(joes_task).to eql(id: 1, user_id: 1, title: "Joe's task") + end + + it 'updates listed columns from the excluded row' do + conflicting.do_update(:user_id).insert(user_id: 1, title: "Joe's task") + + expect(joes_task[:user_id]).to be(1) + end + + it 'sets literal values' do + conflicting.do_update(user_id: nil).insert(user_id: 1, title: "Joe's task") + + expect(joes_task[:user_id]).to be_nil + end + + it 'sets expressions built outside the DSL' do + conflicting.do_update(user_id: relation.excluded[:user_id]).insert(user_id: 1, title: "Joe's task") + + expect(joes_task[:user_id]).to be(1) + end + + it 'accepts a hash returned from the block' do + conflicting.do_update { { user_id: excluded[:user_id] } }.insert(user_id: 1, title: "Joe's task") + + expect(joes_task[:user_id]).to be(1) + end + + it 'sets values built with the DSL' do + conflicting.do_update { set(user_id: user_id + excluded[:user_id]) }.insert(user_id: 1, title: "Joe's task") + + expect(joes_task[:user_id]).to be(3) + end + + it 'updates rows matching the where condition' do + conflicting + .do_update { set(user_id: excluded[:user_id]).where(user_id > excluded[:user_id]) } + .insert(user_id: 1, title: "Joe's task") + + expect(joes_task[:user_id]).to be(1) + end + + it 'skips rows not matching the where condition' do + conflicting + .do_update { set(user_id: excluded[:user_id]).where(user_id < excluded[:user_id]) } + .insert(user_id: 1, title: "Joe's task") + + expect(joes_task[:user_id]).to be(2) + end + + it 'accepts hash conditions in where' do + conflicting + .do_update { set(user_id: excluded[:user_id]).where(user_id: 2) } + .insert(user_id: 1, title: "Joe's task") + + expect(joes_task[:user_id]).to be(1) + end + + it 'merges listed columns with the block' do + conflicting.do_update(:user_id) { where(user_id: 2) }.insert(user_id: 1, title: "Joe's task") + + expect(joes_task[:user_id]).to be(1) + end + + it 'lets a later set win' do + conflicting + .do_update { set(user_id: nil).set(user_id: excluded[:user_id]) } + .insert(user_id: 1, title: "Joe's task") + + expect(joes_task[:user_id]).to be(1) + end + + it 'handles conflicts in multi_insert' do + conflicting.do_update(:user_id).multi_insert( + [{ user_id: 1, title: "Joe's task" }, { user_id: 2, title: "Jane's task" }, { user_id: 1, title: 'New task' }] + ) + + expect(relation.by_pk(1).one[:user_id]).to be(1) + expect(relation.by_pk(2).one[:user_id]).to be(2) + expect(relation.where(title: 'New task').one).to include(user_id: 1) + end + + it 'raises when the block returns neither set nor a hash' do + expect { + conflicting.do_update { user_id } + }.to raise_error(ArgumentError, /set/) + end + end +end diff --git a/spec/unit/relation/on_conflict_spec.rb b/spec/unit/relation/on_conflict_spec.rb new file mode 100644 index 00000000..f7898524 --- /dev/null +++ b/spec/unit/relation/on_conflict_spec.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +RSpec.describe ROM::Relation, '#on_conflict' do + subject(:relation) { relations[:tasks] } + + include_context 'users and tasks' + + let(:joes_task) { { id: 1, user_id: 2, title: "Joe's task" } } + + # tasks are seeded with explicit ids + seed do |example| + conn.run("SELECT setval('tasks_id_seq', (SELECT max(id) FROM tasks))") if postgres?(example) + end + + with_adapters(:postgres, :sqlite) do + context 'with a unique constraint' do + setup_tables do |example| + if sqlite?(example) + conn.add_index :tasks, :title, unique: true + else + conn.run 'ALTER TABLE tasks ADD CONSTRAINT tasks_title_key UNIQUE (title)' + end + end + + it 'does nothing on conflict with the target' do + relation.on_conflict(:title).insert(user_id: 1, title: "Joe's task") + + expect(relation.by_pk(1).one).to eql(joes_task) + expect(relation.count).to be(2) + end + + it 'does nothing on conflict with any constraint' do + relation.on_conflict.insert(user_id: 1, title: "Joe's task") + + expect(relation.by_pk(1).one).to eql(joes_task) + expect(relation.count).to be(2) + end + + it 'accepts attributes as the target' do + relation.on_conflict(relation[:title]).insert(user_id: 1, title: "Joe's task") + + expect(relation.by_pk(1).one).to eql(joes_task) + end + + it 'inserts rows that do not conflict' do + relation.on_conflict(:title).insert(user_id: 1, title: 'Another task') + + expect(relation.count).to be(3) + end + + it 'handles conflicts in multi_insert' do + relation.on_conflict(:title).multi_insert( + [{ user_id: 1, title: "Joe's task" }, { user_id: 1, title: 'Another task' }] + ) + + expect(relation.by_pk(1).one).to eql(joes_task) + expect(relation.count).to be(3) + end + + it 'resets a previous do_update with do_nothing' do + relation.on_conflict(:title).do_update(:user_id).do_nothing.insert(user_id: 1, title: "Joe's task") + + expect(relation.by_pk(1).one).to eql(joes_task) + end + + it 'keeps the insert failing without on_conflict' do + expect { + relation.insert(user_id: 1, title: "Joe's task") + }.to raise_error(Sequel::UniqueConstraintViolation) + end + end + + context 'with a partial unique index' do + setup_tables do + conn.run 'CREATE UNIQUE INDEX tasks_title_partial_index ON tasks (title) WHERE user_id = 1' + end + + it 'infers the index from the predicate in the block' do + relation.on_conflict(:title) { user_id.is(1) }.insert(user_id: 1, title: "Jane's task") + + expect(relation.where(title: "Jane's task").count).to be(1) + end + + it 'inserts rows outside the predicate' do + relation.on_conflict(:title) { user_id.is(1) }.insert(user_id: 2, title: "Jane's task") + + expect(relation.where(title: "Jane's task").count).to be(2) + end + end + end + + with_adapters(:postgres) do + setup_tables do + conn.run 'ALTER TABLE tasks ADD CONSTRAINT tasks_title_key UNIQUE (title)' + end + + it 'does nothing on conflict with a named constraint' do + relation.on_conflict(constraint: :tasks_title_key).insert(user_id: 1, title: "Joe's task") + + expect(relation.by_pk(1).one).to eql(joes_task) + end + end + + with_adapters(:mysql) do + it 'raises an error when the database does not support ON CONFLICT' do + expect { + relation.on_conflict(:title) + }.to raise_error(ROM::SQL::UnsupportedFeatureError, /ON CONFLICT is not supported by mysql/) + end + end +end diff --git a/spec/unit/upsert_dsl_spec.rb b/spec/unit/upsert_dsl_spec.rb new file mode 100644 index 00000000..fe127f75 --- /dev/null +++ b/spec/unit/upsert_dsl_spec.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ROM::SQL::UpsertDSL, :sqlite, helpers: true do + include_context 'database setup' + + subject(:dsl) do + ROM::SQL::UpsertDSL.new(schema) + end + + let(:schema) do + define_schema( + :users, + id: ROM::SQL::Types::Serial, + name: ROM::SQL::Types::String, + updated_at: ROM::SQL::Types::Time + ) + end + + let(:ds) do + conn[:users] + end + + describe '#excluded' do + it 'qualifies attributes with the excluded table' do + expect(ds.literal(dsl.excluded[:name])).to eql('`excluded`.`name`') + end + end + + describe '#call' do + it 'returns assignments from a hash' do + set, where = dsl.call { { name: excluded[:name] } } + + expect(ds.literal(set[:name])).to eql('`excluded`.`name`') + expect(where).to be_nil + end + + it 'returns assignments from set' do + set, where = dsl.call { set(name: excluded[:name], updated_at: excluded[:updated_at]) } + + expect(set.keys).to eql(%i[name updated_at]) + expect(ds.literal(set[:name])).to eql('`excluded`.`name`') + expect(where).to be_nil + end + + it 'returns the condition from where' do + _, where = dsl.call { set(name: excluded[:name]).where(updated_at < excluded[:updated_at]) } + + expect(ds.literal(where)).to eql('(`updated_at` < `excluded`.`updated_at`)') + end + + it 'allows where before set' do + set, where = dsl.call { where(updated_at < excluded[:updated_at]).set(name: excluded[:name]) } + + expect(set.keys).to eql(%i[name]) + expect(ds.literal(where)).to eql('(`updated_at` < `excluded`.`updated_at`)') + end + + it 'qualifies hash conditions with the table' do + _, where = dsl.call { set(name: excluded[:name]).where(name: 'Jane') } + + expect(ds.literal(where)).to eql("(`users`.`name` = 'Jane')") + end + + it 'lets a later set win' do + set, = dsl.call { set(name: 'Jane').set(name: excluded[:name]) } + + expect(ds.literal(set[:name])).to eql('`excluded`.`name`') + end + + it 'resolves functions with the virtual row' do + set, = dsl.call { set(name: coalesce(name, excluded[:name])) } + + expect(ds.literal(set[:name])).to eql('coalesce(`name`, `excluded`.`name`)') + end + + it 'raises when the block returns neither set nor a hash' do + expect { dsl.call { name } }.to raise_error(ArgumentError, /set/) + end + end +end