From 6ffedc558e735bccd285fe6b0814b5c9b998d87b Mon Sep 17 00:00:00 2001 From: Radu Berinde Date: Thu, 27 Feb 2020 14:33:25 -0800 Subject: [PATCH] opt: add inbound FK checks for upsert This change adds inbound FK checks for upsert and switches execution over to the new style FK checks for upsert. Similar to UPDATE, the inbound FK checks run on the set difference between "old" values for the FK columns and "new" values. Release note (performance improvement): improved execution plans of foreign key checks for UPSERT and INSERT ON CONFLICT in some cases (in particular multi-region). --- pkg/sql/logictest/testdata/logic_test/fk_opt | 151 ++ pkg/sql/opt/bench/stub_factory.go | 1 + pkg/sql/opt/exec/execbuilder/mutation.go | 6 + pkg/sql/opt/exec/factory.go | 10 +- pkg/sql/opt/norm/custom_funcs.go | 4 +- pkg/sql/opt/optbuilder/insert.go | 3 + pkg/sql/opt/optbuilder/mutation_builder.go | 204 ++- .../opt/optbuilder/testdata/fk-checks-upsert | 1318 +++++++++++++++++ pkg/sql/opt_exec_factory.go | 29 +- pkg/sql/tablewriter_upsert_opt.go | 15 +- 10 files changed, 1708 insertions(+), 33 deletions(-) diff --git a/pkg/sql/logictest/testdata/logic_test/fk_opt b/pkg/sql/logictest/testdata/logic_test/fk_opt index e96aff5a25bc..1529d69a80a0 100644 --- a/pkg/sql/logictest/testdata/logic_test/fk_opt +++ b/pkg/sql/logictest/testdata/logic_test/fk_opt @@ -133,6 +133,157 @@ DROP TABLE child2 statement ok DROP TABLE parent2 +# Upsert +# ------ + +statement ok +CREATE TABLE parent (p INT PRIMARY KEY, other INT) + +statement ok +CREATE TABLE child (c INT PRIMARY KEY, p INT NOT NULL REFERENCES parent(p)) + +statement ok +INSERT INTO parent VALUES (1), (2) + +# Insert case. +statement ok +INSERT INTO child VALUES (1, 1) ON CONFLICT (c) DO UPDATE SET p = 2 + +statement error foreign key +INSERT INTO child VALUES (2, 10) ON CONFLICT (c) DO UPDATE SET p = 2 + +# Update case. +statement ok +INSERT INTO child VALUES (1, 1) ON CONFLICT (c) DO UPDATE SET p = 1 + +statement ok +INSERT INTO child VALUES (1, 10) ON CONFLICT (c) DO UPDATE SET p = 1 + +statement error foreign key +INSERT INTO child VALUES (1, 10) ON CONFLICT (c) DO UPDATE SET p = 10 + +statement ok +TRUNCATE child + +statement ok +INSERT INTO child VALUES (1, 1) + +# Both insert and update case. + +# Both insert and update are invalid. + +statement error foreign key +INSERT INTO child VALUES (1, 1), (2, 3) ON CONFLICT (c) DO UPDATE SET p = 3 + +# Insert is invalid, update is valid. + +statement error foreign key +INSERT INTO child VALUES (1, 2), (2, 3) ON CONFLICT (c) DO UPDATE SET p = 1 + +# Insert is valid, update is invalid. + +statement error foreign key +INSERT INTO child VALUES (1, 2), (2, 1) ON CONFLICT (c) DO UPDATE SET p = 3 + +# Both insert and update are valid. + +statement ok +INSERT INTO child VALUES (1, 2), (2, 1) ON CONFLICT (c) DO UPDATE SET p = 2 + +statement ok +DROP TABLE child + +statement ok +DROP TABLE parent + +# Pseudo-deletions + +statement ok +CREATE TABLE parent (a INT PRIMARY KEY, b INT, UNIQUE (b)) + +statement ok +CREATE TABLE child (a INT PRIMARY KEY, b INT REFERENCES parent (b)) + +statement ok +INSERT INTO parent VALUES (1, 2) + +statement ok +INSERT INTO child VALUES (10, 2) + +statement error pq: upsert on table "parent" violates foreign key constraint "fk_b_ref_parent" on table "child"\nDETAIL: Key \(b\)=\(2\) is still referenced from table "child"\. +UPSERT INTO parent VALUES (1, 3) + +statement ok +INSERT INTO parent VALUES (1, 3), (2, 2) ON CONFLICT (a) DO UPDATE SET b = 3 + +query II +SELECT * FROM child +---- +10 2 + +query II rowsort +SELECT * FROM parent +---- +1 3 +2 2 + +# child references the second '2' column in parent. This mutation removes that +# row via an update, and is disallowed. +statement error pq: insert on table "parent" violates foreign key constraint "fk_b_ref_parent" on table "child"\nDETAIL: Key \(b\)=\(2\) is still referenced from table "child"\. +INSERT INTO parent VALUES (2, 2) ON CONFLICT (a) DO UPDATE SET b = parent.b - 1 + +# This mutation *also* removes that row, but also via an update, introduces a +# new one, making it acceptable. +statement ok +INSERT INTO parent VALUES (2, 2), (1, 3) ON CONFLICT (a) DO UPDATE SET b = parent.b - 1 + +query II rowsort +SELECT * FROM parent +---- +1 2 +2 1 + +statement ok +DROP TABLE child + +statement ok +DROP TABLE parent + +# Self-reference. + +statement ok +CREATE TABLE self (k int primary key, a int unique, b int references self(a)) + +statement error pq: upsert on table "self" violates foreign key constraint "fk_b_ref_self"\nDETAIL: Key \(b\)=\(2\) is not present in table "self" +UPSERT INTO self VALUES (1, 1, 2) + +statement ok +UPSERT INTO self VALUES (1, 1, 1) + +statement ok +UPSERT INTO self VALUES (1, 1, 1) + +statement error pq: upsert on table "self" violates foreign key constraint "fk_b_ref_self"\nDETAIL: Key \(b\)=\(2\) is not present in table "self" +UPSERT INTO self VALUES (1, 1, 2) + +statement ok +UPSERT INTO self VALUES (1, 2, 2) + +statement error pq: upsert on table "self" violates foreign key constraint "fk_b_ref_self"\nDETAIL: Key \(b\)=\(2\) is not present in table "self" +UPSERT INTO self(k,a) VALUES (1, 1) + +statement ok +UPSERT INTO self VALUES (1, 1, 2), (2, 2, 1) + +statement ok +INSERT INTO self VALUES (2, 2, 2) ON CONFLICT (k) DO UPDATE SET b = self.b + 1 + +statement error pq: insert on table "self" violates foreign key constraint "fk_b_ref_self"\nDETAIL: Key \(b\)=\(3\) is not present in table "self" +INSERT INTO self VALUES (2, 2, 2) ON CONFLICT (k) DO UPDATE SET b = self.b + 1 + +statement ok +DROP TABLE self + # --- Tests that follow are copied from the fk tests and adjusted as needed. statement ok diff --git a/pkg/sql/opt/bench/stub_factory.go b/pkg/sql/opt/bench/stub_factory.go index ece28e51d43b..6f00a6d34d1f 100644 --- a/pkg/sql/opt/bench/stub_factory.go +++ b/pkg/sql/opt/bench/stub_factory.go @@ -279,6 +279,7 @@ func (f *stubFactory) ConstructUpsert( returnCols exec.ColumnOrdinalSet, checks exec.CheckOrdinalSet, allowAutoCommit bool, + skipFKChecks bool, ) (exec.Node, error) { return struct{}{}, nil } diff --git a/pkg/sql/opt/exec/execbuilder/mutation.go b/pkg/sql/opt/exec/execbuilder/mutation.go index 9b57a0292f0c..739c40da6ff4 100644 --- a/pkg/sql/opt/exec/execbuilder/mutation.go +++ b/pkg/sql/opt/exec/execbuilder/mutation.go @@ -383,6 +383,7 @@ func (b *Builder) buildUpsert(ups *memo.UpsertExpr) (execPlan, error) { updateColOrds := ordinalSetFromColList(ups.UpdateCols) returnColOrds := ordinalSetFromColList(ups.ReturnCols) checkOrds := ordinalSetFromColList(ups.CheckCols) + disableExecFKs := !ups.FKFallback node, err := b.factory.ConstructUpsert( input.root, tab, @@ -393,11 +394,16 @@ func (b *Builder) buildUpsert(ups *memo.UpsertExpr) (execPlan, error) { returnColOrds, checkOrds, b.allowAutoCommit && len(ups.Checks) == 0, + disableExecFKs, ) if err != nil { return execPlan{}, err } + if err := b.buildFKChecks(ups.Checks); err != nil { + return execPlan{}, err + } + // If UPSERT returns rows, they contain all non-mutation columns from the // table, in the same order they're defined in the table. Each output column // value is taken from an insert, fetch, or update column, depending on the diff --git a/pkg/sql/opt/exec/factory.go b/pkg/sql/opt/exec/factory.go index e648780f8542..08d98100c692 100644 --- a/pkg/sql/opt/exec/factory.go +++ b/pkg/sql/opt/exec/factory.go @@ -425,6 +425,11 @@ type Factory interface { // transaction (if appropriate, i.e. if it is in an implicit transaction). // This is false if there are multiple mutations in a statement, or the output // of the mutation is processed through side-effecting expressions. + // + // If skipFKChecks is set, foreign keys are not checked as part of the + // execution of the upsert for the insert half. This is used when the FK + // checks are planned by the optimizer and are run separately as plan + // postqueries. ConstructUpsert( input Node, table cat.Table, @@ -435,6 +440,7 @@ type Factory interface { returnCols ColumnOrdinalSet, checks CheckOrdinalSet, allowAutoCommit bool, + skipFKChecks bool, ) (Node, error) // ConstructDelete creates a node that implements a DELETE statement. The @@ -451,8 +457,8 @@ type Factory interface { // of the mutation is processed through side-effecting expressions. // // If skipFKChecks is set, foreign keys are not checked as part of the - // execution of the insertion. This is used when the FK checks are planned by - // the optimizer and are run separately as plan postqueries. + // execution of the delete. This is used when the FK checks are planned + // by the optimizer and are run separately as plan postqueries. ConstructDelete( input Node, table cat.Table, diff --git a/pkg/sql/opt/norm/custom_funcs.go b/pkg/sql/opt/norm/custom_funcs.go index f422e93dea10..3c682720783d 100644 --- a/pkg/sql/opt/norm/custom_funcs.go +++ b/pkg/sql/opt/norm/custom_funcs.go @@ -2084,9 +2084,11 @@ func (c *CustomFuncs) WithUses(r opt.Expr) props.WithUsesMap { switch e := r.(type) { case memo.RelExpr: relProps := e.Relational() + + // Lazily calculate and store the WithUses value. if !relProps.IsAvailable(props.WithUses) { - relProps.SetAvailable(props.WithUses) relProps.Shared.Rule.WithUses = c.deriveWithUses(r) + relProps.SetAvailable(props.WithUses) } return relProps.Shared.Rule.WithUses diff --git a/pkg/sql/opt/optbuilder/insert.go b/pkg/sql/opt/optbuilder/insert.go index 0d94b48ae9b6..a0911279b3e9 100644 --- a/pkg/sql/opt/optbuilder/insert.go +++ b/pkg/sql/opt/optbuilder/insert.go @@ -353,6 +353,9 @@ func (b *Builder) buildInsert(ins *tree.Insert, inScope *scope) (outScope *scope // values specified for them. // 3. Each update value is the same as the corresponding insert value. // +// TODO(radu): once FKs no longer require indexes, this function will have to +// take FKs into account explicitly. +// // TODO(andyk): The fast path is currently only enabled when the UPSERT alias // is explicitly selected by the user. It's possible to fast path some queries // of the form INSERT ... ON CONFLICT, but the utility is low and there are lots diff --git a/pkg/sql/opt/optbuilder/mutation_builder.go b/pkg/sql/opt/optbuilder/mutation_builder.go index a5c25d9cc4ac..165920c83296 100644 --- a/pkg/sql/opt/optbuilder/mutation_builder.go +++ b/pkg/sql/opt/optbuilder/mutation_builder.go @@ -876,6 +876,33 @@ func (mb *mutationBuilder) parseDefaultOrComputedExpr(colID opt.ColumnID) tree.E // buildFKChecks* methods populate mb.checks with queries that check the // integrity of foreign key relations that involve modified rows. +// +// The foreign key checks are queries that run after the statement (including +// the relevant mutation) completes; any row that is returned by these +// FK check queries indicates a foreign key violation. +// +// In the case of insert, each FK check query is an anti-join with the left side +// being a WithScan of the mutation input and the right side being the +// referenced table. A simple example of an insert with a FK check: +// +// insert child +// ├── ... +// ├── input binding: &1 +// └── f-k-checks +// └── f-k-checks-item: child(p) -> parent(p) +// └── anti-join (hash) +// ├── columns: column2:5!null +// ├── with-scan &1 +// │ ├── columns: column2:5!null +// │ └── mapping: +// │ └── column2:4 => column2:5 +// ├── scan parent +// │ └── columns: parent.p:6!null +// └── filters +// └── column2:5 = parent.p:6 +// +// See testdata/fk-checks-insert for more examples. +// func (mb *mutationBuilder) buildFKChecksForInsert() { if mb.tab.OutboundForeignKeyCount() == 0 { // No relevant FKs. @@ -899,6 +926,32 @@ func (mb *mutationBuilder) buildFKChecksForInsert() { // buildFKChecks* methods populate mb.checks with queries that check the // integrity of foreign key relations that involve modified rows. +// +// The foreign key checks are queries that run after the statement (including +// the relevant mutation) completes; any row that is returned by these +// FK check queries indicates a foreign key violation. +// +// In the case of delete, each FK check query is a semi-join with the left side +// being a WithScan of the mutation input and the right side being the +// referencing table. For example: +// delete parent +// ├── ... +// ├── input binding: &1 +// └── f-k-checks +// └── f-k-checks-item: child(p) -> parent(p) +// └── semi-join (hash) +// ├── columns: p:7!null +// ├── with-scan &1 +// │ ├── columns: p:7!null +// │ └── mapping: +// │ └── parent.p:5 => p:7 +// ├── scan child +// │ └── columns: child.p:9!null +// └── filters +// └── p:7 = child.p:9 +// +// See testdata/fk-checks-delete for more examples. +// func (mb *mutationBuilder) buildFKChecksForDelete() { if mb.tab.InboundForeignKeyCount() == 0 { // No relevant FKs. @@ -931,6 +984,63 @@ func (mb *mutationBuilder) buildFKChecksForDelete() { // buildFKChecks* methods populate mb.checks with queries that check the // integrity of foreign key relations that involve modified rows. +// +// The foreign key checks are queries that run after the statement (including +// the relevant mutation) completes; any row that is returned by these +// FK check queries indicates a foreign key violation. +// +// In the case of update, there are two types of FK check queries: +// +// - insertion-side checks are very similar to the checks we issue for insert; +// they are an anti-join with the left side being a WithScan of the "new" +// values for each row. For example: +// update child +// ├── ... +// ├── input binding: &1 +// └── f-k-checks +// └── f-k-checks-item: child(p) -> parent(p) +// └── anti-join (hash) +// ├── columns: column5:6!null +// ├── with-scan &1 +// │ ├── columns: column5:6!null +// │ └── mapping: +// │ └── column5:5 => column5:6 +// ├── scan parent +// │ └── columns: parent.p:8!null +// └── filters +// └── column5:6 = parent.p:8 +// +// - deletion-side checks are similar to the checks we issue for delete; they +// are a semi-join but the left side input is more complicated: it is an +// Except between a WithScan of the "old" values and a WithScan of the "new" +// values for each row (this is the set of values that are effectively +// removed from the table). For example: +// update parent +// ├── ... +// ├── input binding: &1 +// └── f-k-checks +// └── f-k-checks-item: child(p) -> parent(p) +// └── semi-join (hash) +// ├── columns: p:8!null +// ├── except +// │ ├── columns: p:8!null +// │ ├── left columns: p:8!null +// │ ├── right columns: column7:9 +// │ ├── with-scan &1 +// │ │ ├── columns: p:8!null +// │ │ └── mapping: +// │ │ └── parent.p:5 => p:8 +// │ └── with-scan &1 +// │ ├── columns: column7:9!null +// │ └── mapping: +// │ └── column7:7 => column7:9 +// ├── scan child +// │ └── columns: child.p:11!null +// └── filters +// └── p:8 = child.p:11 +// +// Only FK relations that involve updated columns result in FK checks. +// func (mb *mutationBuilder) buildFKChecksForUpdate() { if mb.tab.OutboundForeignKeyCount() == 0 && mb.tab.InboundForeignKeyCount() == 0 { return @@ -992,12 +1102,36 @@ func (mb *mutationBuilder) buildFKChecksForUpdate() { return } + // Construct an Except expression for the set difference between "old" + // FK values and "new" FK values. + // + // The simplest example to see why this is necessary is when we are + // "updating" a value to the same value, e.g: + // UPDATE child SET c = c + // Here we are not removing any values from the column, so we must not + // check for orphaned rows or we will be generating bogus FK violation + // errors. + // + // There are more complicated cases where one row replaces the value from + // another row, e.g. + // UPDATE child SET c = c+1 + // when we have existing consecutive values. These cases are sketchy because + // depending on the order in which the mutations are applied, they may or + // may not result in unique index violations (but if they go through, the FK + // checks should be accurate). + // + // Note that the same reasoning could be applied to the insertion checks, + // but in that case, it is not a correctness issue: it's always ok to + // recheck that an existing row is not orphan. It's not really desirable for + // performance either: we would be incurring extra cost (more complicated + // expressions, scanning the input buffer twice) for a rare case. + oldRows, colsForOldRow, _ := h.makeFKInputScan(fkInputScanFetchedVals) newRows, colsForNewRow, _ := h.makeFKInputScan(fkInputScanNewVals) // The rows that no longer exist are the ones that were "deleted" by virtue // of being updated _from_, minus the ones that were "added" by virtue of - // being updated _to_. Note that this could equivalently be ExceptAll. + // being updated _to_. deletedRows := mb.b.factory.ConstructExcept( oldRows, newRows, @@ -1012,6 +1146,25 @@ func (mb *mutationBuilder) buildFKChecksForUpdate() { } } +// buildFKChecks* methods populate mb.checks with queries that check the +// integrity of foreign key relations that involve modified rows. +// +// The foreign key checks are queries that run after the statement (including +// the relevant mutation) completes; any row that is returned by these +// FK check queries indicates a foreign key violation. +// +// The case of upsert is very similar to update; see buildFKChecksForUpdate. +// The main difference is that for update, the "new" values were readily +// available, whereas for upsert, the "new" values can be the result of an +// expression of the form: +// CASE WHEN canary IS NULL THEN inserter-value ELSE updated-value END +// These expressions are already projected as part of the mutation input and are +// directly accessible through WithScan. +// +// Only FK relations that involve updated columns result in deletion-side FK +// checks. The insertion-side FK checks are always needed (similar to insert) +// because any of the rows might result in an insert rather than an update. +// func (mb *mutationBuilder) buildFKChecksForUpsert() { numOutbound := mb.tab.OutboundForeignKeyCount() numInbound := mb.tab.InboundForeignKeyCount() @@ -1019,6 +1172,7 @@ func (mb *mutationBuilder) buildFKChecksForUpsert() { if numOutbound == 0 && numInbound == 0 { return } + if !mb.b.evalCtx.SessionData.OptimizerFKs { mb.fkFallback = true return @@ -1029,6 +1183,52 @@ func (mb *mutationBuilder) buildFKChecksForUpsert() { for i := 0; i < numOutbound; i++ { mb.addInsertionCheck(i) } + + for i := 0; i < numInbound; i++ { + // Verify that at least one FK column is updated by the Upsert; columns that + // are not updated can get new values (through the insert path) but existing + // values are never removed. + if !mb.inboundFKColsUpdated(i) { + continue + } + + h := &mb.fkCheckHelper + if !h.initWithInboundFK(mb, i) { + continue + } + + if a := h.fk.UpdateReferenceAction(); a != tree.Restrict && a != tree.NoAction { + // Bail, so that exec FK checks pick up on FK checks and perform them. + mb.checks = nil + mb.fkFallback = true + return + } + + // Construct an Except expression for the set difference between "old" FK + // values and "new" FK values. See buildFKChecksForUpdate for more details. + // + // Note that technically, to get "old" values for the updated rows we should + // be selecting only the rows that correspond to updates, as opposed to + // insertions (using a "canaryCol IS NOT NULL" condition). But the rows we + // would filter out have all-null fetched values anyway and will never match + // in the semi join. + oldRows, colsForOldRow, _ := h.makeFKInputScan(fkInputScanFetchedVals) + newRows, colsForNewRow, _ := h.makeFKInputScan(fkInputScanNewVals) + + // The rows that no longer exist are the ones that were "deleted" by virtue + // of being updated _from_, minus the ones that were "added" by virtue of + // being updated _to_. + deletedRows := mb.b.factory.ConstructExcept( + oldRows, + newRows, + &memo.SetPrivate{ + LeftCols: colsForOldRow, + RightCols: colsForNewRow, + OutCols: colsForOldRow, + }, + ) + mb.addDeletionCheck(h, deletedRows, colsForOldRow) + } } // addInsertionCheck adds a FK check for rows which are added to a table. @@ -1333,7 +1533,7 @@ func (h *fkCheckHelper) initWithInboundFK(mb *mutationBuilder, fkOrdinal int) (o } panic(err) } - // We need SELECT privileges on the referenced table. + // We need SELECT privileges on the origin table. mb.b.checkPrivilege(opt.DepByID(originID), ref, privilege.SELECT) h.otherTab = ref.(cat.Table) diff --git a/pkg/sql/opt/optbuilder/testdata/fk-checks-upsert b/pkg/sql/opt/optbuilder/testdata/fk-checks-upsert index 519e6a1ec368..9447987df86d 100644 --- a/pkg/sql/opt/optbuilder/testdata/fk-checks-upsert +++ b/pkg/sql/opt/optbuilder/testdata/fk-checks-upsert @@ -1154,3 +1154,1321 @@ upsert cmulti └── filters ├── upsert_b:21 = pq.p:24 └── z:22 = q:25 + +# --------------------------------------- +# Inbound FK tests with single FK column +# --------------------------------------- + +# No need to check inbound FKs since PK values never get removed by an upsert. +build +UPSERT INTO p VALUES (1, 1), (2, 2) +---- +upsert p + ├── columns: + ├── upsert-mapping: + │ ├── column1:3 => p:1 + │ └── column2:4 => other:2 + └── values + ├── columns: column1:3!null column2:4!null + ├── (1, 1) + └── (2, 2) + +exec-ddl +CREATE TABLE p1 (p INT PRIMARY KEY, other INT, INDEX(other)) +---- + +exec-ddl +CREATE TABLE p1c (c INT PRIMARY KEY, p INT NOT NULL DEFAULT 5 REFERENCES p1(p)) +---- + +# No need to check inbound FKs since PK values never get removed by an upsert. +build +UPSERT INTO p1 VALUES (1, 1), (2, 2) +---- +upsert p1 + ├── columns: + ├── canary column: 5 + ├── fetch columns: p:5 other:6 + ├── insert-mapping: + │ ├── column1:3 => p:1 + │ └── column2:4 => other:2 + ├── update-mapping: + │ └── column2:4 => other:2 + └── project + ├── columns: upsert_p:7 column1:3!null column2:4!null p:5 other:6 + ├── left-join (hash) + │ ├── columns: column1:3!null column2:4!null p:5 other:6 + │ ├── upsert-distinct-on + │ │ ├── columns: column1:3!null column2:4!null + │ │ ├── grouping columns: column1:3!null + │ │ ├── values + │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ ├── (1, 1) + │ │ │ └── (2, 2) + │ │ └── aggregations + │ │ └── first-agg [as=column2:4] + │ │ └── column2:4 + │ ├── scan p1 + │ │ └── columns: p:5!null other:6 + │ └── filters + │ └── column1:3 = p:5 + └── projections + └── CASE WHEN p:5 IS NULL THEN column1:3 ELSE p:5 END [as=upsert_p:7] + +# This statement can modify existing values of p so we need to perform the FK +# check. +build +INSERT INTO p1 VALUES (100, 1), (200, 1) ON CONFLICT (p) DO UPDATE SET p = excluded.p + 1 +---- +upsert p1 + ├── columns: + ├── canary column: 5 + ├── fetch columns: p1.p:5 other:6 + ├── insert-mapping: + │ ├── column1:3 => p1.p:1 + │ └── column2:4 => other:2 + ├── update-mapping: + │ └── upsert_p:8 => p1.p:1 + ├── input binding: &1 + ├── project + │ ├── columns: upsert_p:8!null upsert_other:9 column1:3!null column2:4!null p1.p:5 other:6 column7:7!null + │ ├── project + │ │ ├── columns: column7:7!null column1:3!null column2:4!null p1.p:5 other:6 + │ │ ├── left-join (hash) + │ │ │ ├── columns: column1:3!null column2:4!null p1.p:5 other:6 + │ │ │ ├── upsert-distinct-on + │ │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ │ ├── grouping columns: column1:3!null + │ │ │ │ ├── values + │ │ │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ │ │ ├── (100, 1) + │ │ │ │ │ └── (200, 1) + │ │ │ │ └── aggregations + │ │ │ │ └── first-agg [as=column2:4] + │ │ │ │ └── column2:4 + │ │ │ ├── scan p1 + │ │ │ │ └── columns: p1.p:5!null other:6 + │ │ │ └── filters + │ │ │ └── column1:3 = p1.p:5 + │ │ └── projections + │ │ └── column1:3 + 1 [as=column7:7] + │ └── projections + │ ├── CASE WHEN p1.p:5 IS NULL THEN column1:3 ELSE column7:7 END [as=upsert_p:8] + │ └── CASE WHEN p1.p:5 IS NULL THEN column2:4 ELSE other:6 END [as=upsert_other:9] + └── f-k-checks + └── f-k-checks-item: p1c(p) -> p1(p) + └── semi-join (hash) + ├── columns: p:10 + ├── except + │ ├── columns: p:10 + │ ├── left columns: p:10 + │ ├── right columns: upsert_p:11 + │ ├── with-scan &1 + │ │ ├── columns: p:10 + │ │ └── mapping: + │ │ └── p1.p:5 => p:10 + │ └── with-scan &1 + │ ├── columns: upsert_p:11!null + │ └── mapping: + │ └── upsert_p:8 => upsert_p:11 + ├── scan p1c + │ └── columns: p1c.p:13!null + └── filters + └── p:10 = p1c.p:13 + +# No need to check the inbound FK: we never modify existing values of p. +build +INSERT INTO p1 VALUES (100, 1), (200, 1) ON CONFLICT (p) DO UPDATE SET other = p1.other + 1 +---- +upsert p1 + ├── columns: + ├── canary column: 5 + ├── fetch columns: p:5 other:6 + ├── insert-mapping: + │ ├── column1:3 => p:1 + │ └── column2:4 => other:2 + ├── update-mapping: + │ └── upsert_other:9 => other:2 + └── project + ├── columns: upsert_p:8 upsert_other:9 column1:3!null column2:4!null p:5 other:6 column7:7 + ├── project + │ ├── columns: column7:7 column1:3!null column2:4!null p:5 other:6 + │ ├── left-join (hash) + │ │ ├── columns: column1:3!null column2:4!null p:5 other:6 + │ │ ├── upsert-distinct-on + │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ ├── grouping columns: column1:3!null + │ │ │ ├── values + │ │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ │ ├── (100, 1) + │ │ │ │ └── (200, 1) + │ │ │ └── aggregations + │ │ │ └── first-agg [as=column2:4] + │ │ │ └── column2:4 + │ │ ├── scan p1 + │ │ │ └── columns: p:5!null other:6 + │ │ └── filters + │ │ └── column1:3 = p:5 + │ └── projections + │ └── other:6 + 1 [as=column7:7] + └── projections + ├── CASE WHEN p:5 IS NULL THEN column1:3 ELSE p:5 END [as=upsert_p:8] + └── CASE WHEN p:5 IS NULL THEN column2:4 ELSE column7:7 END [as=upsert_other:9] + +# Similar tests when the FK column is not the PK. +exec-ddl +CREATE TABLE p2 (p INT PRIMARY KEY, fk INT UNIQUE) +---- + +exec-ddl +CREATE TABLE p2c (c INT PRIMARY KEY, fk INT REFERENCES p2(fk)) +---- + +build +UPSERT INTO p2 VALUES (1, 1), (2, 2) +---- +upsert p2 + ├── columns: + ├── canary column: 5 + ├── fetch columns: p:5 p2.fk:6 + ├── insert-mapping: + │ ├── column1:3 => p:1 + │ └── column2:4 => p2.fk:2 + ├── update-mapping: + │ └── column2:4 => p2.fk:2 + ├── input binding: &1 + ├── project + │ ├── columns: upsert_p:7 column1:3!null column2:4!null p:5 p2.fk:6 + │ ├── left-join (hash) + │ │ ├── columns: column1:3!null column2:4!null p:5 p2.fk:6 + │ │ ├── upsert-distinct-on + │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ ├── grouping columns: column1:3!null + │ │ │ ├── values + │ │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ │ ├── (1, 1) + │ │ │ │ └── (2, 2) + │ │ │ └── aggregations + │ │ │ └── first-agg [as=column2:4] + │ │ │ └── column2:4 + │ │ ├── scan p2 + │ │ │ └── columns: p:5!null p2.fk:6 + │ │ └── filters + │ │ └── column1:3 = p:5 + │ └── projections + │ └── CASE WHEN p:5 IS NULL THEN column1:3 ELSE p:5 END [as=upsert_p:7] + └── f-k-checks + └── f-k-checks-item: p2c(fk) -> p2(fk) + └── semi-join (hash) + ├── columns: fk:8 + ├── except + │ ├── columns: fk:8 + │ ├── left columns: fk:8 + │ ├── right columns: column2:9 + │ ├── with-scan &1 + │ │ ├── columns: fk:8 + │ │ └── mapping: + │ │ └── p2.fk:6 => fk:8 + │ └── with-scan &1 + │ ├── columns: column2:9!null + │ └── mapping: + │ └── column2:4 => column2:9 + ├── scan p2c + │ └── columns: p2c.fk:11 + └── filters + └── fk:8 = p2c.fk:11 + +# This statement never removes existing values of the fk column; FK check is +# not needed. +build +INSERT INTO p2 VALUES (1, 1), (2, 2) ON CONFLICT (p) DO UPDATE SET p = excluded.p + 1 +---- +upsert p2 + ├── columns: + ├── canary column: 5 + ├── fetch columns: p:5 fk:6 + ├── insert-mapping: + │ ├── column1:3 => p:1 + │ └── column2:4 => fk:2 + ├── update-mapping: + │ └── upsert_p:8 => p:1 + └── project + ├── columns: upsert_p:8!null upsert_fk:9 column1:3!null column2:4!null p:5 fk:6 column7:7!null + ├── project + │ ├── columns: column7:7!null column1:3!null column2:4!null p:5 fk:6 + │ ├── left-join (hash) + │ │ ├── columns: column1:3!null column2:4!null p:5 fk:6 + │ │ ├── upsert-distinct-on + │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ ├── grouping columns: column1:3!null + │ │ │ ├── values + │ │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ │ ├── (1, 1) + │ │ │ │ └── (2, 2) + │ │ │ └── aggregations + │ │ │ └── first-agg [as=column2:4] + │ │ │ └── column2:4 + │ │ ├── scan p2 + │ │ │ └── columns: p:5!null fk:6 + │ │ └── filters + │ │ └── column1:3 = p:5 + │ └── projections + │ └── column1:3 + 1 [as=column7:7] + └── projections + ├── CASE WHEN p:5 IS NULL THEN column1:3 ELSE column7:7 END [as=upsert_p:8] + └── CASE WHEN p:5 IS NULL THEN column2:4 ELSE fk:6 END [as=upsert_fk:9] + +# This statement can change existing values of the fk column, so the FK check +# is needed. +build +INSERT INTO p2 VALUES (1, 1), (2, 2) ON CONFLICT (p) DO UPDATE SET fk = excluded.fk + 1 +---- +upsert p2 + ├── columns: + ├── canary column: 5 + ├── fetch columns: p:5 p2.fk:6 + ├── insert-mapping: + │ ├── column1:3 => p:1 + │ └── column2:4 => p2.fk:2 + ├── update-mapping: + │ └── upsert_fk:9 => p2.fk:2 + ├── input binding: &1 + ├── project + │ ├── columns: upsert_p:8 upsert_fk:9!null column1:3!null column2:4!null p:5 p2.fk:6 column7:7!null + │ ├── project + │ │ ├── columns: column7:7!null column1:3!null column2:4!null p:5 p2.fk:6 + │ │ ├── left-join (hash) + │ │ │ ├── columns: column1:3!null column2:4!null p:5 p2.fk:6 + │ │ │ ├── upsert-distinct-on + │ │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ │ ├── grouping columns: column1:3!null + │ │ │ │ ├── values + │ │ │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ │ │ ├── (1, 1) + │ │ │ │ │ └── (2, 2) + │ │ │ │ └── aggregations + │ │ │ │ └── first-agg [as=column2:4] + │ │ │ │ └── column2:4 + │ │ │ ├── scan p2 + │ │ │ │ └── columns: p:5!null p2.fk:6 + │ │ │ └── filters + │ │ │ └── column1:3 = p:5 + │ │ └── projections + │ │ └── column2:4 + 1 [as=column7:7] + │ └── projections + │ ├── CASE WHEN p:5 IS NULL THEN column1:3 ELSE p:5 END [as=upsert_p:8] + │ └── CASE WHEN p:5 IS NULL THEN column2:4 ELSE column7:7 END [as=upsert_fk:9] + └── f-k-checks + └── f-k-checks-item: p2c(fk) -> p2(fk) + └── semi-join (hash) + ├── columns: fk:10 + ├── except + │ ├── columns: fk:10 + │ ├── left columns: fk:10 + │ ├── right columns: upsert_fk:11 + │ ├── with-scan &1 + │ │ ├── columns: fk:10 + │ │ └── mapping: + │ │ └── p2.fk:6 => fk:10 + │ └── with-scan &1 + │ ├── columns: upsert_fk:11!null + │ └── mapping: + │ └── upsert_fk:9 => upsert_fk:11 + ├── scan p2c + │ └── columns: p2c.fk:13 + └── filters + └── fk:10 = p2c.fk:13 + +# This statement never removes existing values of the fk column; the FK check is +# not needed. +build +INSERT INTO p2 VALUES (1, 1), (2, 2) ON CONFLICT (fk) DO UPDATE SET p = excluded.p + 1 +---- +upsert p2 + ├── columns: + ├── canary column: 5 + ├── fetch columns: p:5 fk:6 + ├── insert-mapping: + │ ├── column1:3 => p:1 + │ └── column2:4 => fk:2 + ├── update-mapping: + │ └── upsert_p:8 => p:1 + └── project + ├── columns: upsert_p:8!null upsert_fk:9 column1:3!null column2:4!null p:5 fk:6 column7:7!null + ├── project + │ ├── columns: column7:7!null column1:3!null column2:4!null p:5 fk:6 + │ ├── left-join (hash) + │ │ ├── columns: column1:3!null column2:4!null p:5 fk:6 + │ │ ├── upsert-distinct-on + │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ ├── grouping columns: column2:4!null + │ │ │ ├── values + │ │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ │ ├── (1, 1) + │ │ │ │ └── (2, 2) + │ │ │ └── aggregations + │ │ │ └── first-agg [as=column1:3] + │ │ │ └── column1:3 + │ │ ├── scan p2 + │ │ │ └── columns: p:5!null fk:6 + │ │ └── filters + │ │ └── column2:4 = fk:6 + │ └── projections + │ └── column1:3 + 1 [as=column7:7] + └── projections + ├── CASE WHEN p:5 IS NULL THEN column1:3 ELSE column7:7 END [as=upsert_p:8] + └── CASE WHEN p:5 IS NULL THEN column2:4 ELSE fk:6 END [as=upsert_fk:9] + +build +INSERT INTO p2 VALUES (1, 1), (2, 2) ON CONFLICT (fk) DO UPDATE SET fk = excluded.fk + 1 +---- +upsert p2 + ├── columns: + ├── canary column: 5 + ├── fetch columns: p:5 p2.fk:6 + ├── insert-mapping: + │ ├── column1:3 => p:1 + │ └── column2:4 => p2.fk:2 + ├── update-mapping: + │ └── upsert_fk:9 => p2.fk:2 + ├── input binding: &1 + ├── project + │ ├── columns: upsert_p:8 upsert_fk:9!null column1:3!null column2:4!null p:5 p2.fk:6 column7:7!null + │ ├── project + │ │ ├── columns: column7:7!null column1:3!null column2:4!null p:5 p2.fk:6 + │ │ ├── left-join (hash) + │ │ │ ├── columns: column1:3!null column2:4!null p:5 p2.fk:6 + │ │ │ ├── upsert-distinct-on + │ │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ │ ├── grouping columns: column2:4!null + │ │ │ │ ├── values + │ │ │ │ │ ├── columns: column1:3!null column2:4!null + │ │ │ │ │ ├── (1, 1) + │ │ │ │ │ └── (2, 2) + │ │ │ │ └── aggregations + │ │ │ │ └── first-agg [as=column1:3] + │ │ │ │ └── column1:3 + │ │ │ ├── scan p2 + │ │ │ │ └── columns: p:5!null p2.fk:6 + │ │ │ └── filters + │ │ │ └── column2:4 = p2.fk:6 + │ │ └── projections + │ │ └── column2:4 + 1 [as=column7:7] + │ └── projections + │ ├── CASE WHEN p:5 IS NULL THEN column1:3 ELSE p:5 END [as=upsert_p:8] + │ └── CASE WHEN p:5 IS NULL THEN column2:4 ELSE column7:7 END [as=upsert_fk:9] + └── f-k-checks + └── f-k-checks-item: p2c(fk) -> p2(fk) + └── semi-join (hash) + ├── columns: fk:10 + ├── except + │ ├── columns: fk:10 + │ ├── left columns: fk:10 + │ ├── right columns: upsert_fk:11 + │ ├── with-scan &1 + │ │ ├── columns: fk:10 + │ │ └── mapping: + │ │ └── p2.fk:6 => fk:10 + │ └── with-scan &1 + │ ├── columns: upsert_fk:11!null + │ └── mapping: + │ └── upsert_fk:9 => upsert_fk:11 + ├── scan p2c + │ └── columns: p2c.fk:13 + └── filters + └── fk:10 = p2c.fk:13 + +# This partial upsert never removes existing values of the fk column; the FK +# check is not needed. +build +UPSERT INTO p2(p) VALUES (1), (2) +---- +upsert p2 + ├── columns: + ├── canary column: 5 + ├── fetch columns: p:5 fk:6 + ├── insert-mapping: + │ ├── column1:3 => p:1 + │ └── column4:4 => fk:2 + └── project + ├── columns: upsert_p:7 upsert_fk:8 column1:3!null column4:4 p:5 fk:6 + ├── left-join (hash) + │ ├── columns: column1:3!null column4:4 p:5 fk:6 + │ ├── upsert-distinct-on + │ │ ├── columns: column1:3!null column4:4 + │ │ ├── grouping columns: column1:3!null + │ │ ├── project + │ │ │ ├── columns: column4:4 column1:3!null + │ │ │ ├── values + │ │ │ │ ├── columns: column1:3!null + │ │ │ │ ├── (1,) + │ │ │ │ └── (2,) + │ │ │ └── projections + │ │ │ └── NULL::INT8 [as=column4:4] + │ │ └── aggregations + │ │ └── first-agg [as=column4:4] + │ │ └── column4:4 + │ ├── scan p2 + │ │ └── columns: p:5!null fk:6 + │ └── filters + │ └── column1:3 = p:5 + └── projections + ├── CASE WHEN p:5 IS NULL THEN column1:3 ELSE p:5 END [as=upsert_p:7] + └── CASE WHEN p:5 IS NULL THEN column4:4 ELSE fk:6 END [as=upsert_fk:8] + +# ------------------------------------------ +# Inbound FK tests with multiple FK columns +# ------------------------------------------ + +build +UPSERT INTO pq VALUES (1, 1, 1, 1), (2, 2, 2, 2) +---- +upsert pq + ├── columns: + ├── canary column: 9 + ├── fetch columns: k:9 pq.p:10 pq.q:11 pq.other:12 + ├── insert-mapping: + │ ├── column1:5 => k:1 + │ ├── column2:6 => pq.p:2 + │ ├── column3:7 => pq.q:3 + │ └── column4:8 => pq.other:4 + ├── update-mapping: + │ ├── column2:6 => pq.p:2 + │ ├── column3:7 => pq.q:3 + │ └── column4:8 => pq.other:4 + ├── input binding: &1 + ├── project + │ ├── columns: upsert_k:13 column1:5!null column2:6!null column3:7!null column4:8!null k:9 pq.p:10 pq.q:11 pq.other:12 + │ ├── left-join (hash) + │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null k:9 pq.p:10 pq.q:11 pq.other:12 + │ │ ├── upsert-distinct-on + │ │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null + │ │ │ ├── grouping columns: column1:5!null + │ │ │ ├── values + │ │ │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null + │ │ │ │ ├── (1, 1, 1, 1) + │ │ │ │ └── (2, 2, 2, 2) + │ │ │ └── aggregations + │ │ │ ├── first-agg [as=column2:6] + │ │ │ │ └── column2:6 + │ │ │ ├── first-agg [as=column3:7] + │ │ │ │ └── column3:7 + │ │ │ └── first-agg [as=column4:8] + │ │ │ └── column4:8 + │ │ ├── scan pq + │ │ │ └── columns: k:9!null pq.p:10 pq.q:11 pq.other:12 + │ │ └── filters + │ │ └── column1:5 = k:9 + │ └── projections + │ └── CASE WHEN k:9 IS NULL THEN column1:5 ELSE k:9 END [as=upsert_k:13] + └── f-k-checks + ├── f-k-checks-item: cpq(p,q) -> pq(p,q) + │ └── semi-join (hash) + │ ├── columns: p:14 q:15 + │ ├── except + │ │ ├── columns: p:14 q:15 + │ │ ├── left columns: p:14 q:15 + │ │ ├── right columns: column2:16 column3:17 + │ │ ├── with-scan &1 + │ │ │ ├── columns: p:14 q:15 + │ │ │ └── mapping: + │ │ │ ├── pq.p:10 => p:14 + │ │ │ └── pq.q:11 => q:15 + │ │ └── with-scan &1 + │ │ ├── columns: column2:16!null column3:17!null + │ │ └── mapping: + │ │ ├── column2:6 => column2:16 + │ │ └── column3:7 => column3:17 + │ ├── scan cpq + │ │ └── columns: cpq.p:19 cpq.q:20 + │ └── filters + │ ├── p:14 = cpq.p:19 + │ └── q:15 = cpq.q:20 + └── f-k-checks-item: cmulti(b,c) -> pq(p,q) + └── semi-join (hash) + ├── columns: p:22 q:23 + ├── except + │ ├── columns: p:22 q:23 + │ ├── left columns: p:22 q:23 + │ ├── right columns: column2:24 column3:25 + │ ├── with-scan &1 + │ │ ├── columns: p:22 q:23 + │ │ └── mapping: + │ │ ├── pq.p:10 => p:22 + │ │ └── pq.q:11 => q:23 + │ └── with-scan &1 + │ ├── columns: column2:24!null column3:25!null + │ └── mapping: + │ ├── column2:6 => column2:24 + │ └── column3:7 => column3:25 + ├── scan cmulti + │ └── columns: b:27!null cmulti.c:28 + └── filters + ├── p:22 = b:27 + └── q:23 = cmulti.c:28 + +# Partial UPSERT doesn't remove (p,q) values; FK check not needed. +build +UPSERT INTO pq (k) VALUES (1), (2) +---- +upsert pq + ├── columns: + ├── canary column: 7 + ├── fetch columns: k:7 p:8 q:9 other:10 + ├── insert-mapping: + │ ├── column1:5 => k:1 + │ ├── column6:6 => p:2 + │ ├── column6:6 => q:3 + │ └── column6:6 => other:4 + └── project + ├── columns: upsert_k:11 upsert_p:12 upsert_q:13 upsert_other:14 column1:5!null column6:6 k:7 p:8 q:9 other:10 + ├── left-join (hash) + │ ├── columns: column1:5!null column6:6 k:7 p:8 q:9 other:10 + │ ├── upsert-distinct-on + │ │ ├── columns: column1:5!null column6:6 + │ │ ├── grouping columns: column1:5!null + │ │ ├── project + │ │ │ ├── columns: column6:6 column1:5!null + │ │ │ ├── values + │ │ │ │ ├── columns: column1:5!null + │ │ │ │ ├── (1,) + │ │ │ │ └── (2,) + │ │ │ └── projections + │ │ │ └── NULL::INT8 [as=column6:6] + │ │ └── aggregations + │ │ └── first-agg [as=column6:6] + │ │ └── column6:6 + │ ├── scan pq + │ │ └── columns: k:7!null p:8 q:9 other:10 + │ └── filters + │ └── column1:5 = k:7 + └── projections + ├── CASE WHEN k:7 IS NULL THEN column1:5 ELSE k:7 END [as=upsert_k:11] + ├── CASE WHEN k:7 IS NULL THEN column6:6 ELSE p:8 END [as=upsert_p:12] + ├── CASE WHEN k:7 IS NULL THEN column6:6 ELSE q:9 END [as=upsert_q:13] + └── CASE WHEN k:7 IS NULL THEN column6:6 ELSE other:10 END [as=upsert_other:14] + +build +UPSERT INTO pq (k,q) VALUES (1, 1), (2, 2) +---- +upsert pq + ├── columns: + ├── canary column: 8 + ├── fetch columns: k:8 pq.p:9 pq.q:10 pq.other:11 + ├── insert-mapping: + │ ├── column1:5 => k:1 + │ ├── column7:7 => pq.p:2 + │ ├── column2:6 => pq.q:3 + │ └── column7:7 => pq.other:4 + ├── update-mapping: + │ └── column2:6 => pq.q:3 + ├── input binding: &1 + ├── project + │ ├── columns: upsert_k:12 upsert_p:13 upsert_other:14 column1:5!null column2:6!null column7:7 k:8 pq.p:9 pq.q:10 pq.other:11 + │ ├── left-join (hash) + │ │ ├── columns: column1:5!null column2:6!null column7:7 k:8 pq.p:9 pq.q:10 pq.other:11 + │ │ ├── upsert-distinct-on + │ │ │ ├── columns: column1:5!null column2:6!null column7:7 + │ │ │ ├── grouping columns: column1:5!null + │ │ │ ├── project + │ │ │ │ ├── columns: column7:7 column1:5!null column2:6!null + │ │ │ │ ├── values + │ │ │ │ │ ├── columns: column1:5!null column2:6!null + │ │ │ │ │ ├── (1, 1) + │ │ │ │ │ └── (2, 2) + │ │ │ │ └── projections + │ │ │ │ └── NULL::INT8 [as=column7:7] + │ │ │ └── aggregations + │ │ │ ├── first-agg [as=column2:6] + │ │ │ │ └── column2:6 + │ │ │ └── first-agg [as=column7:7] + │ │ │ └── column7:7 + │ │ ├── scan pq + │ │ │ └── columns: k:8!null pq.p:9 pq.q:10 pq.other:11 + │ │ └── filters + │ │ └── column1:5 = k:8 + │ └── projections + │ ├── CASE WHEN k:8 IS NULL THEN column1:5 ELSE k:8 END [as=upsert_k:12] + │ ├── CASE WHEN k:8 IS NULL THEN column7:7 ELSE pq.p:9 END [as=upsert_p:13] + │ └── CASE WHEN k:8 IS NULL THEN column7:7 ELSE pq.other:11 END [as=upsert_other:14] + └── f-k-checks + ├── f-k-checks-item: cpq(p,q) -> pq(p,q) + │ └── semi-join (hash) + │ ├── columns: p:15 q:16 + │ ├── except + │ │ ├── columns: p:15 q:16 + │ │ ├── left columns: p:15 q:16 + │ │ ├── right columns: upsert_p:17 column2:18 + │ │ ├── with-scan &1 + │ │ │ ├── columns: p:15 q:16 + │ │ │ └── mapping: + │ │ │ ├── pq.p:9 => p:15 + │ │ │ └── pq.q:10 => q:16 + │ │ └── with-scan &1 + │ │ ├── columns: upsert_p:17 column2:18!null + │ │ └── mapping: + │ │ ├── upsert_p:13 => upsert_p:17 + │ │ └── column2:6 => column2:18 + │ ├── scan cpq + │ │ └── columns: cpq.p:20 cpq.q:21 + │ └── filters + │ ├── p:15 = cpq.p:20 + │ └── q:16 = cpq.q:21 + └── f-k-checks-item: cmulti(b,c) -> pq(p,q) + └── semi-join (hash) + ├── columns: p:23 q:24 + ├── except + │ ├── columns: p:23 q:24 + │ ├── left columns: p:23 q:24 + │ ├── right columns: upsert_p:25 column2:26 + │ ├── with-scan &1 + │ │ ├── columns: p:23 q:24 + │ │ └── mapping: + │ │ ├── pq.p:9 => p:23 + │ │ └── pq.q:10 => q:24 + │ └── with-scan &1 + │ ├── columns: upsert_p:25 column2:26!null + │ └── mapping: + │ ├── upsert_p:13 => upsert_p:25 + │ └── column2:6 => column2:26 + ├── scan cmulti + │ └── columns: b:28!null cmulti.c:29 + └── filters + ├── p:23 = b:28 + └── q:24 = cmulti.c:29 + +# Statement doesn't remove (p,q) values; FK check not needed. +build +INSERT INTO pq VALUES (1, 1, 1, 1), (2, 2, 2, 2) ON CONFLICT (p,q) DO UPDATE SET k = pq.k + 1 +---- +upsert pq + ├── columns: + ├── canary column: 9 + ├── fetch columns: k:9 p:10 q:11 other:12 + ├── insert-mapping: + │ ├── column1:5 => k:1 + │ ├── column2:6 => p:2 + │ ├── column3:7 => q:3 + │ └── column4:8 => other:4 + ├── update-mapping: + │ └── upsert_k:14 => k:1 + └── project + ├── columns: upsert_k:14 upsert_p:15 upsert_q:16 upsert_other:17 column1:5!null column2:6!null column3:7!null column4:8!null k:9 p:10 q:11 other:12 column13:13 + ├── project + │ ├── columns: column13:13 column1:5!null column2:6!null column3:7!null column4:8!null k:9 p:10 q:11 other:12 + │ ├── left-join (hash) + │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null k:9 p:10 q:11 other:12 + │ │ ├── upsert-distinct-on + │ │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null + │ │ │ ├── grouping columns: column2:6!null column3:7!null + │ │ │ ├── values + │ │ │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null + │ │ │ │ ├── (1, 1, 1, 1) + │ │ │ │ └── (2, 2, 2, 2) + │ │ │ └── aggregations + │ │ │ ├── first-agg [as=column1:5] + │ │ │ │ └── column1:5 + │ │ │ └── first-agg [as=column4:8] + │ │ │ └── column4:8 + │ │ ├── scan pq + │ │ │ └── columns: k:9!null p:10 q:11 other:12 + │ │ └── filters + │ │ ├── column2:6 = p:10 + │ │ └── column3:7 = q:11 + │ └── projections + │ └── k:9 + 1 [as=column13:13] + └── projections + ├── CASE WHEN k:9 IS NULL THEN column1:5 ELSE column13:13 END [as=upsert_k:14] + ├── CASE WHEN k:9 IS NULL THEN column2:6 ELSE p:10 END [as=upsert_p:15] + ├── CASE WHEN k:9 IS NULL THEN column3:7 ELSE q:11 END [as=upsert_q:16] + └── CASE WHEN k:9 IS NULL THEN column4:8 ELSE other:12 END [as=upsert_other:17] + +build +INSERT INTO pq VALUES (1, 1, 1, 1), (2, 2, 2, 2) ON CONFLICT (p,q) DO UPDATE SET p = pq.p + 1 +---- +upsert pq + ├── columns: + ├── canary column: 9 + ├── fetch columns: k:9 pq.p:10 pq.q:11 pq.other:12 + ├── insert-mapping: + │ ├── column1:5 => k:1 + │ ├── column2:6 => pq.p:2 + │ ├── column3:7 => pq.q:3 + │ └── column4:8 => pq.other:4 + ├── update-mapping: + │ └── upsert_p:15 => pq.p:2 + ├── input binding: &1 + ├── project + │ ├── columns: upsert_k:14 upsert_p:15 upsert_q:16 upsert_other:17 column1:5!null column2:6!null column3:7!null column4:8!null k:9 pq.p:10 pq.q:11 pq.other:12 column13:13 + │ ├── project + │ │ ├── columns: column13:13 column1:5!null column2:6!null column3:7!null column4:8!null k:9 pq.p:10 pq.q:11 pq.other:12 + │ │ ├── left-join (hash) + │ │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null k:9 pq.p:10 pq.q:11 pq.other:12 + │ │ │ ├── upsert-distinct-on + │ │ │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null + │ │ │ │ ├── grouping columns: column2:6!null column3:7!null + │ │ │ │ ├── values + │ │ │ │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null + │ │ │ │ │ ├── (1, 1, 1, 1) + │ │ │ │ │ └── (2, 2, 2, 2) + │ │ │ │ └── aggregations + │ │ │ │ ├── first-agg [as=column1:5] + │ │ │ │ │ └── column1:5 + │ │ │ │ └── first-agg [as=column4:8] + │ │ │ │ └── column4:8 + │ │ │ ├── scan pq + │ │ │ │ └── columns: k:9!null pq.p:10 pq.q:11 pq.other:12 + │ │ │ └── filters + │ │ │ ├── column2:6 = pq.p:10 + │ │ │ └── column3:7 = pq.q:11 + │ │ └── projections + │ │ └── pq.p:10 + 1 [as=column13:13] + │ └── projections + │ ├── CASE WHEN k:9 IS NULL THEN column1:5 ELSE k:9 END [as=upsert_k:14] + │ ├── CASE WHEN k:9 IS NULL THEN column2:6 ELSE column13:13 END [as=upsert_p:15] + │ ├── CASE WHEN k:9 IS NULL THEN column3:7 ELSE pq.q:11 END [as=upsert_q:16] + │ └── CASE WHEN k:9 IS NULL THEN column4:8 ELSE pq.other:12 END [as=upsert_other:17] + └── f-k-checks + ├── f-k-checks-item: cpq(p,q) -> pq(p,q) + │ └── semi-join (hash) + │ ├── columns: p:18 q:19 + │ ├── except + │ │ ├── columns: p:18 q:19 + │ │ ├── left columns: p:18 q:19 + │ │ ├── right columns: upsert_p:20 upsert_q:21 + │ │ ├── with-scan &1 + │ │ │ ├── columns: p:18 q:19 + │ │ │ └── mapping: + │ │ │ ├── pq.p:10 => p:18 + │ │ │ └── pq.q:11 => q:19 + │ │ └── with-scan &1 + │ │ ├── columns: upsert_p:20 upsert_q:21 + │ │ └── mapping: + │ │ ├── upsert_p:15 => upsert_p:20 + │ │ └── upsert_q:16 => upsert_q:21 + │ ├── scan cpq + │ │ └── columns: cpq.p:23 cpq.q:24 + │ └── filters + │ ├── p:18 = cpq.p:23 + │ └── q:19 = cpq.q:24 + └── f-k-checks-item: cmulti(b,c) -> pq(p,q) + └── semi-join (hash) + ├── columns: p:26 q:27 + ├── except + │ ├── columns: p:26 q:27 + │ ├── left columns: p:26 q:27 + │ ├── right columns: upsert_p:28 upsert_q:29 + │ ├── with-scan &1 + │ │ ├── columns: p:26 q:27 + │ │ └── mapping: + │ │ ├── pq.p:10 => p:26 + │ │ └── pq.q:11 => q:27 + │ └── with-scan &1 + │ ├── columns: upsert_p:28 upsert_q:29 + │ └── mapping: + │ ├── upsert_p:15 => upsert_p:28 + │ └── upsert_q:16 => upsert_q:29 + ├── scan cmulti + │ └── columns: b:31!null cmulti.c:32 + └── filters + ├── p:26 = b:31 + └── q:27 = cmulti.c:32 + +# Statement never removes (p,q) values; FK check not needed. +build +INSERT INTO pq VALUES (1, 1, 1, 1), (2, 2, 2, 2) ON CONFLICT (k) DO UPDATE SET other = 5 +---- +upsert pq + ├── columns: + ├── canary column: 9 + ├── fetch columns: k:9 p:10 q:11 other:12 + ├── insert-mapping: + │ ├── column1:5 => k:1 + │ ├── column2:6 => p:2 + │ ├── column3:7 => q:3 + │ └── column4:8 => other:4 + ├── update-mapping: + │ └── upsert_other:17 => other:4 + └── project + ├── columns: upsert_k:14 upsert_p:15 upsert_q:16 upsert_other:17!null column1:5!null column2:6!null column3:7!null column4:8!null k:9 p:10 q:11 other:12 column13:13!null + ├── project + │ ├── columns: column13:13!null column1:5!null column2:6!null column3:7!null column4:8!null k:9 p:10 q:11 other:12 + │ ├── left-join (hash) + │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null k:9 p:10 q:11 other:12 + │ │ ├── upsert-distinct-on + │ │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null + │ │ │ ├── grouping columns: column1:5!null + │ │ │ ├── values + │ │ │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null + │ │ │ │ ├── (1, 1, 1, 1) + │ │ │ │ └── (2, 2, 2, 2) + │ │ │ └── aggregations + │ │ │ ├── first-agg [as=column2:6] + │ │ │ │ └── column2:6 + │ │ │ ├── first-agg [as=column3:7] + │ │ │ │ └── column3:7 + │ │ │ └── first-agg [as=column4:8] + │ │ │ └── column4:8 + │ │ ├── scan pq + │ │ │ └── columns: k:9!null p:10 q:11 other:12 + │ │ └── filters + │ │ └── column1:5 = k:9 + │ └── projections + │ └── 5 [as=column13:13] + └── projections + ├── CASE WHEN k:9 IS NULL THEN column1:5 ELSE k:9 END [as=upsert_k:14] + ├── CASE WHEN k:9 IS NULL THEN column2:6 ELSE p:10 END [as=upsert_p:15] + ├── CASE WHEN k:9 IS NULL THEN column3:7 ELSE q:11 END [as=upsert_q:16] + └── CASE WHEN k:9 IS NULL THEN column4:8 ELSE column13:13 END [as=upsert_other:17] + +build +INSERT INTO pq VALUES (1, 1, 1, 1), (2, 2, 2, 2) ON CONFLICT (k) DO UPDATE SET q = 5 +---- +upsert pq + ├── columns: + ├── canary column: 9 + ├── fetch columns: k:9 pq.p:10 pq.q:11 pq.other:12 + ├── insert-mapping: + │ ├── column1:5 => k:1 + │ ├── column2:6 => pq.p:2 + │ ├── column3:7 => pq.q:3 + │ └── column4:8 => pq.other:4 + ├── update-mapping: + │ └── upsert_q:16 => pq.q:3 + ├── input binding: &1 + ├── project + │ ├── columns: upsert_k:14 upsert_p:15 upsert_q:16!null upsert_other:17 column1:5!null column2:6!null column3:7!null column4:8!null k:9 pq.p:10 pq.q:11 pq.other:12 column13:13!null + │ ├── project + │ │ ├── columns: column13:13!null column1:5!null column2:6!null column3:7!null column4:8!null k:9 pq.p:10 pq.q:11 pq.other:12 + │ │ ├── left-join (hash) + │ │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null k:9 pq.p:10 pq.q:11 pq.other:12 + │ │ │ ├── upsert-distinct-on + │ │ │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null + │ │ │ │ ├── grouping columns: column1:5!null + │ │ │ │ ├── values + │ │ │ │ │ ├── columns: column1:5!null column2:6!null column3:7!null column4:8!null + │ │ │ │ │ ├── (1, 1, 1, 1) + │ │ │ │ │ └── (2, 2, 2, 2) + │ │ │ │ └── aggregations + │ │ │ │ ├── first-agg [as=column2:6] + │ │ │ │ │ └── column2:6 + │ │ │ │ ├── first-agg [as=column3:7] + │ │ │ │ │ └── column3:7 + │ │ │ │ └── first-agg [as=column4:8] + │ │ │ │ └── column4:8 + │ │ │ ├── scan pq + │ │ │ │ └── columns: k:9!null pq.p:10 pq.q:11 pq.other:12 + │ │ │ └── filters + │ │ │ └── column1:5 = k:9 + │ │ └── projections + │ │ └── 5 [as=column13:13] + │ └── projections + │ ├── CASE WHEN k:9 IS NULL THEN column1:5 ELSE k:9 END [as=upsert_k:14] + │ ├── CASE WHEN k:9 IS NULL THEN column2:6 ELSE pq.p:10 END [as=upsert_p:15] + │ ├── CASE WHEN k:9 IS NULL THEN column3:7 ELSE column13:13 END [as=upsert_q:16] + │ └── CASE WHEN k:9 IS NULL THEN column4:8 ELSE pq.other:12 END [as=upsert_other:17] + └── f-k-checks + ├── f-k-checks-item: cpq(p,q) -> pq(p,q) + │ └── semi-join (hash) + │ ├── columns: p:18 q:19 + │ ├── except + │ │ ├── columns: p:18 q:19 + │ │ ├── left columns: p:18 q:19 + │ │ ├── right columns: upsert_p:20 upsert_q:21 + │ │ ├── with-scan &1 + │ │ │ ├── columns: p:18 q:19 + │ │ │ └── mapping: + │ │ │ ├── pq.p:10 => p:18 + │ │ │ └── pq.q:11 => q:19 + │ │ └── with-scan &1 + │ │ ├── columns: upsert_p:20 upsert_q:21!null + │ │ └── mapping: + │ │ ├── upsert_p:15 => upsert_p:20 + │ │ └── upsert_q:16 => upsert_q:21 + │ ├── scan cpq + │ │ └── columns: cpq.p:23 cpq.q:24 + │ └── filters + │ ├── p:18 = cpq.p:23 + │ └── q:19 = cpq.q:24 + └── f-k-checks-item: cmulti(b,c) -> pq(p,q) + └── semi-join (hash) + ├── columns: p:26 q:27 + ├── except + │ ├── columns: p:26 q:27 + │ ├── left columns: p:26 q:27 + │ ├── right columns: upsert_p:28 upsert_q:29 + │ ├── with-scan &1 + │ │ ├── columns: p:26 q:27 + │ │ └── mapping: + │ │ ├── pq.p:10 => p:26 + │ │ └── pq.q:11 => q:27 + │ └── with-scan &1 + │ ├── columns: upsert_p:28 upsert_q:29!null + │ └── mapping: + │ ├── upsert_p:15 => upsert_p:28 + │ └── upsert_q:16 => upsert_q:29 + ├── scan cmulti + │ └── columns: b:31!null cmulti.c:32 + └── filters + ├── p:26 = b:31 + └── q:27 = cmulti.c:32 + +# ------------------------------------- +# Inbound + outbound combination tests +# ------------------------------------- + +exec-ddl +CREATE TABLE tab1 ( + a INT PRIMARY KEY, + b INT UNIQUE +) +---- + +exec-ddl +CREATE TABLE tab2 ( + c INT PRIMARY KEY, + d INT REFERENCES tab1(b), + e INT UNIQUE +) +---- + +exec-ddl +CREATE TABLE tab3 ( + f INT PRIMARY KEY, + g INT REFERENCES tab2(e) +) +---- + +build +UPSERT INTO tab2 VALUES (1,NULL,NULL), (2,2,2) +---- +upsert tab2 + ├── columns: + ├── canary column: 7 + ├── fetch columns: c:7 d:8 tab2.e:9 + ├── insert-mapping: + │ ├── column1:4 => c:1 + │ ├── column2:5 => d:2 + │ └── column3:6 => tab2.e:3 + ├── update-mapping: + │ ├── column2:5 => d:2 + │ └── column3:6 => tab2.e:3 + ├── input binding: &1 + ├── project + │ ├── columns: upsert_c:10 column1:4!null column2:5 column3:6 c:7 d:8 tab2.e:9 + │ ├── left-join (hash) + │ │ ├── columns: column1:4!null column2:5 column3:6 c:7 d:8 tab2.e:9 + │ │ ├── upsert-distinct-on + │ │ │ ├── columns: column1:4!null column2:5 column3:6 + │ │ │ ├── grouping columns: column1:4!null + │ │ │ ├── values + │ │ │ │ ├── columns: column1:4!null column2:5 column3:6 + │ │ │ │ ├── (1, NULL::INT8, NULL::INT8) + │ │ │ │ └── (2, 2, 2) + │ │ │ └── aggregations + │ │ │ ├── first-agg [as=column2:5] + │ │ │ │ └── column2:5 + │ │ │ └── first-agg [as=column3:6] + │ │ │ └── column3:6 + │ │ ├── scan tab2 + │ │ │ └── columns: c:7!null d:8 tab2.e:9 + │ │ └── filters + │ │ └── column1:4 = c:7 + │ └── projections + │ └── CASE WHEN c:7 IS NULL THEN column1:4 ELSE c:7 END [as=upsert_c:10] + └── f-k-checks + ├── f-k-checks-item: tab2(d) -> tab1(b) + │ └── anti-join (hash) + │ ├── columns: column2:11!null + │ ├── select + │ │ ├── columns: column2:11!null + │ │ ├── with-scan &1 + │ │ │ ├── columns: column2:11 + │ │ │ └── mapping: + │ │ │ └── column2:5 => column2:11 + │ │ └── filters + │ │ └── column2:11 IS NOT NULL + │ ├── scan tab1 + │ │ └── columns: b:13 + │ └── filters + │ └── column2:11 = b:13 + └── f-k-checks-item: tab3(g) -> tab2(e) + └── semi-join (hash) + ├── columns: e:14 + ├── except + │ ├── columns: e:14 + │ ├── left columns: e:14 + │ ├── right columns: column3:15 + │ ├── with-scan &1 + │ │ ├── columns: e:14 + │ │ └── mapping: + │ │ └── tab2.e:9 => e:14 + │ └── with-scan &1 + │ ├── columns: column3:15 + │ └── mapping: + │ └── column3:6 => column3:15 + ├── scan tab3 + │ └── columns: g:17 + └── filters + └── e:14 = g:17 + +build +INSERT INTO tab2 VALUES (1,1,1) ON CONFLICT (c) DO UPDATE SET e = tab2.e + 1 +---- +upsert tab2 + ├── columns: + ├── canary column: 7 + ├── fetch columns: c:7 d:8 tab2.e:9 + ├── insert-mapping: + │ ├── column1:4 => c:1 + │ ├── column2:5 => d:2 + │ └── column3:6 => tab2.e:3 + ├── update-mapping: + │ └── upsert_e:13 => tab2.e:3 + ├── input binding: &1 + ├── project + │ ├── columns: upsert_c:11 upsert_d:12 upsert_e:13 column1:4!null column2:5!null column3:6!null c:7 d:8 tab2.e:9 column10:10 + │ ├── project + │ │ ├── columns: column10:10 column1:4!null column2:5!null column3:6!null c:7 d:8 tab2.e:9 + │ │ ├── left-join (hash) + │ │ │ ├── columns: column1:4!null column2:5!null column3:6!null c:7 d:8 tab2.e:9 + │ │ │ ├── upsert-distinct-on + │ │ │ │ ├── columns: column1:4!null column2:5!null column3:6!null + │ │ │ │ ├── grouping columns: column1:4!null + │ │ │ │ ├── values + │ │ │ │ │ ├── columns: column1:4!null column2:5!null column3:6!null + │ │ │ │ │ └── (1, 1, 1) + │ │ │ │ └── aggregations + │ │ │ │ ├── first-agg [as=column2:5] + │ │ │ │ │ └── column2:5 + │ │ │ │ └── first-agg [as=column3:6] + │ │ │ │ └── column3:6 + │ │ │ ├── scan tab2 + │ │ │ │ └── columns: c:7!null d:8 tab2.e:9 + │ │ │ └── filters + │ │ │ └── column1:4 = c:7 + │ │ └── projections + │ │ └── tab2.e:9 + 1 [as=column10:10] + │ └── projections + │ ├── CASE WHEN c:7 IS NULL THEN column1:4 ELSE c:7 END [as=upsert_c:11] + │ ├── CASE WHEN c:7 IS NULL THEN column2:5 ELSE d:8 END [as=upsert_d:12] + │ └── CASE WHEN c:7 IS NULL THEN column3:6 ELSE column10:10 END [as=upsert_e:13] + └── f-k-checks + ├── f-k-checks-item: tab2(d) -> tab1(b) + │ └── anti-join (hash) + │ ├── columns: upsert_d:14!null + │ ├── select + │ │ ├── columns: upsert_d:14!null + │ │ ├── with-scan &1 + │ │ │ ├── columns: upsert_d:14 + │ │ │ └── mapping: + │ │ │ └── upsert_d:12 => upsert_d:14 + │ │ └── filters + │ │ └── upsert_d:14 IS NOT NULL + │ ├── scan tab1 + │ │ └── columns: b:16 + │ └── filters + │ └── upsert_d:14 = b:16 + └── f-k-checks-item: tab3(g) -> tab2(e) + └── semi-join (hash) + ├── columns: e:17 + ├── except + │ ├── columns: e:17 + │ ├── left columns: e:17 + │ ├── right columns: upsert_e:18 + │ ├── with-scan &1 + │ │ ├── columns: e:17 + │ │ └── mapping: + │ │ └── tab2.e:9 => e:17 + │ └── with-scan &1 + │ ├── columns: upsert_e:18 + │ └── mapping: + │ └── upsert_e:13 => upsert_e:18 + ├── scan tab3 + │ └── columns: g:20 + └── filters + └── e:17 = g:20 + +# Statement never removes values from e column; the inbound check is not necessary. +build +INSERT INTO tab2 VALUES (1,1,1) ON CONFLICT (e) DO UPDATE SET d = tab2.d + 1 +---- +upsert tab2 + ├── columns: + ├── canary column: 7 + ├── fetch columns: c:7 d:8 e:9 + ├── insert-mapping: + │ ├── column1:4 => c:1 + │ ├── column2:5 => d:2 + │ └── column3:6 => e:3 + ├── update-mapping: + │ └── upsert_d:12 => d:2 + ├── input binding: &1 + ├── project + │ ├── columns: upsert_c:11 upsert_d:12 upsert_e:13 column1:4!null column2:5!null column3:6!null c:7 d:8 e:9 column10:10 + │ ├── project + │ │ ├── columns: column10:10 column1:4!null column2:5!null column3:6!null c:7 d:8 e:9 + │ │ ├── left-join (hash) + │ │ │ ├── columns: column1:4!null column2:5!null column3:6!null c:7 d:8 e:9 + │ │ │ ├── upsert-distinct-on + │ │ │ │ ├── columns: column1:4!null column2:5!null column3:6!null + │ │ │ │ ├── grouping columns: column3:6!null + │ │ │ │ ├── values + │ │ │ │ │ ├── columns: column1:4!null column2:5!null column3:6!null + │ │ │ │ │ └── (1, 1, 1) + │ │ │ │ └── aggregations + │ │ │ │ ├── first-agg [as=column1:4] + │ │ │ │ │ └── column1:4 + │ │ │ │ └── first-agg [as=column2:5] + │ │ │ │ └── column2:5 + │ │ │ ├── scan tab2 + │ │ │ │ └── columns: c:7!null d:8 e:9 + │ │ │ └── filters + │ │ │ └── column3:6 = e:9 + │ │ └── projections + │ │ └── d:8 + 1 [as=column10:10] + │ └── projections + │ ├── CASE WHEN c:7 IS NULL THEN column1:4 ELSE c:7 END [as=upsert_c:11] + │ ├── CASE WHEN c:7 IS NULL THEN column2:5 ELSE column10:10 END [as=upsert_d:12] + │ └── CASE WHEN c:7 IS NULL THEN column3:6 ELSE e:9 END [as=upsert_e:13] + └── f-k-checks + └── f-k-checks-item: tab2(d) -> tab1(b) + └── anti-join (hash) + ├── columns: upsert_d:14!null + ├── select + │ ├── columns: upsert_d:14!null + │ ├── with-scan &1 + │ │ ├── columns: upsert_d:14 + │ │ └── mapping: + │ │ └── upsert_d:12 => upsert_d:14 + │ └── filters + │ └── upsert_d:14 IS NOT NULL + ├── scan tab1 + │ └── columns: b:16 + └── filters + └── upsert_d:14 = b:16 + +exec-ddl +CREATE TABLE self ( + a INT, + b INT, + c INT, + d INT, + PRIMARY KEY (a,b), + UNIQUE (b,d), + UNIQUE (c), + FOREIGN KEY (a,b) REFERENCES self(b,d), + FOREIGN KEY (d) REFERENCES self(c) +) +---- + +build +UPSERT INTO self SELECT x, y, z, w FROM xyzw +---- +upsert self + ├── columns: + ├── canary column: 10 + ├── fetch columns: a:10 self.b:11 self.c:12 self.d:13 + ├── insert-mapping: + │ ├── x:5 => a:1 + │ ├── y:6 => self.b:2 + │ ├── xyzw.z:7 => self.c:3 + │ └── xyzw.w:8 => self.d:4 + ├── update-mapping: + │ ├── xyzw.z:7 => self.c:3 + │ └── xyzw.w:8 => self.d:4 + ├── input binding: &1 + ├── project + │ ├── columns: upsert_a:14 upsert_b:15 x:5 y:6 xyzw.z:7 xyzw.w:8 a:10 self.b:11 self.c:12 self.d:13 + │ ├── left-join (hash) + │ │ ├── columns: x:5 y:6 xyzw.z:7 xyzw.w:8 a:10 self.b:11 self.c:12 self.d:13 + │ │ ├── upsert-distinct-on + │ │ │ ├── columns: x:5 y:6 xyzw.z:7 xyzw.w:8 + │ │ │ ├── grouping columns: x:5 y:6 + │ │ │ ├── project + │ │ │ │ ├── columns: x:5 y:6 xyzw.z:7 xyzw.w:8 + │ │ │ │ └── scan xyzw + │ │ │ │ └── columns: x:5 y:6 xyzw.z:7 xyzw.w:8 rowid:9!null + │ │ │ └── aggregations + │ │ │ ├── first-agg [as=xyzw.z:7] + │ │ │ │ └── xyzw.z:7 + │ │ │ └── first-agg [as=xyzw.w:8] + │ │ │ └── xyzw.w:8 + │ │ ├── scan self + │ │ │ └── columns: a:10!null self.b:11!null self.c:12 self.d:13 + │ │ └── filters + │ │ ├── x:5 = a:10 + │ │ └── y:6 = self.b:11 + │ └── projections + │ ├── CASE WHEN a:10 IS NULL THEN x:5 ELSE a:10 END [as=upsert_a:14] + │ └── CASE WHEN a:10 IS NULL THEN y:6 ELSE self.b:11 END [as=upsert_b:15] + └── f-k-checks + ├── f-k-checks-item: self(a,b) -> self(b,d) + │ └── anti-join (hash) + │ ├── columns: upsert_a:16 upsert_b:17 + │ ├── with-scan &1 + │ │ ├── columns: upsert_a:16 upsert_b:17 + │ │ └── mapping: + │ │ ├── upsert_a:14 => upsert_a:16 + │ │ └── upsert_b:15 => upsert_b:17 + │ ├── scan self + │ │ └── columns: self.b:19!null self.d:21 + │ └── filters + │ ├── upsert_a:16 = self.b:19 + │ └── upsert_b:17 = self.d:21 + ├── f-k-checks-item: self(d) -> self(c) + │ └── anti-join (hash) + │ ├── columns: w:22!null + │ ├── select + │ │ ├── columns: w:22!null + │ │ ├── with-scan &1 + │ │ │ ├── columns: w:22 + │ │ │ └── mapping: + │ │ │ └── xyzw.w:8 => w:22 + │ │ └── filters + │ │ └── w:22 IS NOT NULL + │ ├── scan self + │ │ └── columns: self.c:25 + │ └── filters + │ └── w:22 = self.c:25 + ├── f-k-checks-item: self(a,b) -> self(b,d) + │ └── semi-join (hash) + │ ├── columns: b:27 d:28 + │ ├── except + │ │ ├── columns: b:27 d:28 + │ │ ├── left columns: b:27 d:28 + │ │ ├── right columns: upsert_b:29 w:30 + │ │ ├── with-scan &1 + │ │ │ ├── columns: b:27 d:28 + │ │ │ └── mapping: + │ │ │ ├── self.b:11 => b:27 + │ │ │ └── self.d:13 => d:28 + │ │ └── with-scan &1 + │ │ ├── columns: upsert_b:29 w:30 + │ │ └── mapping: + │ │ ├── upsert_b:15 => upsert_b:29 + │ │ └── xyzw.w:8 => w:30 + │ ├── scan self + │ │ └── columns: a:31!null self.b:32!null + │ └── filters + │ ├── b:27 = a:31 + │ └── d:28 = self.b:32 + └── f-k-checks-item: self(d) -> self(c) + └── semi-join (hash) + ├── columns: c:35 + ├── except + │ ├── columns: c:35 + │ ├── left columns: c:35 + │ ├── right columns: z:36 + │ ├── with-scan &1 + │ │ ├── columns: c:35 + │ │ └── mapping: + │ │ └── self.c:12 => c:35 + │ └── with-scan &1 + │ ├── columns: z:36 + │ └── mapping: + │ └── xyzw.z:7 => z:36 + ├── scan self + │ └── columns: self.d:40 + └── filters + └── c:35 = self.d:40 diff --git a/pkg/sql/opt_exec_factory.go b/pkg/sql/opt_exec_factory.go index 9f3cded81ef3..a63351b69e72 100644 --- a/pkg/sql/opt_exec_factory.go +++ b/pkg/sql/opt_exec_factory.go @@ -1383,10 +1383,7 @@ func (ef *execFactory) ConstructUpdate( } } - // Create the table updater, which does the bulk of the work. In the HP, - // the updater derives the columns that need to be fetched. By contrast, the - // CBO will have already determined the set of fetch and update columns, and - // passes those sets into the updater (which will basically be a no-op). + // Create the table updater, which does the bulk of the work. ru, err := row.MakeUpdater( ctx, ef.planner.txn, @@ -1506,6 +1503,7 @@ func (ef *execFactory) ConstructUpsert( returnColOrdSet exec.ColumnOrdinalSet, checks exec.CheckOrdinalSet, allowAutoCommit bool, + skipFKChecks bool, ) (exec.Node, error) { ctx := ef.planner.extendedEvalCtx.Context @@ -1516,25 +1514,28 @@ func (ef *execFactory) ConstructUpsert( fetchColDescs := makeColDescList(table, fetchColOrdSet) updateColDescs := makeColDescList(table, updateColOrdSet) - // Determine the foreign key tables involved in the upsert. - fkTables, err := ef.makeFkMetadata(tabDesc, row.CheckUpdates) - if err != nil { - return nil, err + var fkTables row.FkTableMetadata + checkFKs := row.SkipFKs + if !skipFKChecks { + checkFKs = row.CheckFKs + + // Determine the foreign key tables involved in the upsert. + var err error + fkTables, err = ef.makeFkMetadata(tabDesc, row.CheckUpdates) + if err != nil { + return nil, err + } } // Create the table inserter, which does the bulk of the insert-related work. ri, err := row.MakeInserter( - ctx, ef.planner.txn, tabDesc, insertColDescs, row.CheckFKs, fkTables, &ef.planner.alloc, + ctx, ef.planner.txn, tabDesc, insertColDescs, checkFKs, fkTables, &ef.planner.alloc, ) if err != nil { return nil, err } // Create the table updater, which does the bulk of the update-related work. - // In the HP, the updater derives the columns that need to be fetched. By - // contrast, the CBO will have already determined the set of fetch and update - // columns, and passes those sets into the updater (which will basically be a - // no-op). ru, err := row.MakeUpdater( ctx, ef.planner.txn, @@ -1543,7 +1544,7 @@ func (ef *execFactory) ConstructUpsert( updateColDescs, fetchColDescs, row.UpdaterDefault, - row.CheckFKs, + checkFKs, ef.planner.EvalContext(), &ef.planner.alloc, ) diff --git a/pkg/sql/tablewriter_upsert_opt.go b/pkg/sql/tablewriter_upsert_opt.go index 41f87cfd9486..6be0020f6e9e 100644 --- a/pkg/sql/tablewriter_upsert_opt.go +++ b/pkg/sql/tablewriter_upsert_opt.go @@ -171,20 +171,7 @@ func (tu *optTableUpserter) init( ) } - var err error - tu.ru, err = row.MakeUpdater( - ctx, - txn, - tu.tableDesc(), - tu.fkTables, - tu.updateCols, - tu.fetchCols, - row.UpdaterDefault, - row.CheckFKs, - evalCtx, - tu.alloc, - ) - return err + return nil } // flushAndStartNewBatch is part of the tableWriter interface.