diff --git a/pkg/dbutil/table.go b/pkg/dbutil/table.go index 0286bf1e..45a05807 100644 --- a/pkg/dbutil/table.go +++ b/pkg/dbutil/table.go @@ -30,6 +30,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" _ "github.com/pingcap/tidb/pkg/planner/core" // to setup expression.EvalAstExpr. See: https://github.com/pingcap/tidb/blob/a94cff903cd1e7f3b050db782da84273ef5592f4/planner/core/optimizer.go#L202 "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/types" _ "github.com/pingcap/tidb/pkg/types/parser_driver" // for parser driver "github.com/pingcap/tidb/pkg/util/collate" @@ -43,6 +44,9 @@ const ( func init() { collate.SetNewCollationEnabledForTest(false) + + // Enable constraint check + variable.EnableCheckConstraint.Store(true) } // addClusteredAnnotation add the `/*T![clustered_index] NONCLUSTERED */` for primary key of create table info diff --git a/sync_diff_inspector/config/config.go b/sync_diff_inspector/config/config.go index 7fd99cca..93bb8a9e 100644 --- a/sync_diff_inspector/config/config.go +++ b/sync_diff_inspector/config/config.go @@ -370,6 +370,8 @@ type Config struct { // set true if want to compare rows // set false won't compare rows. ExportFixSQL bool `toml:"export-fix-sql" json:"export-fix-sql"` + // whether check table constraints. + CheckConstraints bool `toml:"check-constraints" json:"check-constraints"` // only check table struct without table data. CheckStructOnly bool `toml:"check-struct-only" json:"check-struct-only"` // experimental feature: only check table data without table struct @@ -412,6 +414,7 @@ func NewConfig() *Config { fs.StringVar(&cfg.DMTask, "dm-task", "", "identifier of dm task") fs.IntVar(&cfg.CheckThreadCount, "check-thread-count", 4, "how many goroutines are created to check data") fs.BoolVar(&cfg.ExportFixSQL, "export-fix-sql", true, "set true if want to compare rows or set to false will only compare checksum") + fs.BoolVar(&cfg.CheckConstraints, "check-constraints", false, "check table constraints") fs.BoolVar(&cfg.CheckStructOnly, "check-struct-only", false, "ignore check table's data") fs.BoolVar(&cfg.SkipNonExistingTable, "skip-non-existing-table", false, "skip validation for tables that don't exist upstream or downstream") fs.BoolVar(&cfg.CheckDataOnly, "check-data-only", false, "ignore check table's struct") diff --git a/sync_diff_inspector/diff.go b/sync_diff_inspector/diff.go index 16106c32..42fde9aa 100644 --- a/sync_diff_inspector/diff.go +++ b/sync_diff_inspector/diff.go @@ -290,7 +290,7 @@ func (df *Diff) Equal(ctx context.Context) error { return nil } -func (df *Diff) StructEqual(ctx context.Context) error { +func (df *Diff) StructEqual(ctx context.Context, checkConstraint bool) error { tables := df.downstream.GetTables() tableIndex := 0 if df.startRange != nil { @@ -300,7 +300,7 @@ func (df *Diff) StructEqual(ctx context.Context) error { isEqual, isSkip, isAllTableExist := false, true, tables[tableIndex].TableLack if common.AllTableExist(isAllTableExist) { var err error - isEqual, isSkip, err = df.compareStruct(ctx, tableIndex) + isEqual, isSkip, err = df.compareStruct(ctx, tableIndex, checkConstraint) if err != nil { return errors.Trace(err) } @@ -311,13 +311,13 @@ func (df *Diff) StructEqual(ctx context.Context) error { return nil } -func (df *Diff) compareStruct(ctx context.Context, tableIndex int) (isEqual bool, isSkip bool, err error) { +func (df *Diff) compareStruct(ctx context.Context, tableIndex int, checkConstraint bool) (isEqual bool, isSkip bool, err error) { sourceTableInfos, err := df.upstream.GetSourceStructInfo(ctx, tableIndex) if err != nil { return false, true, errors.Trace(err) } table := df.downstream.GetTables()[tableIndex] - isEqual, isSkip = utils.CompareStruct(sourceTableInfos, table.Info) + isEqual, isSkip = utils.CompareStruct(sourceTableInfos, table.Info, checkConstraint) table.IgnoreDataCheck = isSkip return isEqual, isSkip, nil } @@ -506,7 +506,14 @@ func (df *Diff) BinGenerate(ctx context.Context, targetSource source.Source, tab return df.binSearch(ctx, targetSource, tableRange, count, tableDiff, indexColumns) } -func (df *Diff) binSearch(ctx context.Context, targetSource source.Source, tableRange *splitter.RangeInfo, count int64, tableDiff *common.TableDiff, indexColumns []*model.ColumnInfo) (*splitter.RangeInfo, error) { +func (df *Diff) binSearch( + ctx context.Context, + targetSource source.Source, + tableRange *splitter.RangeInfo, + count int64, + tableDiff *common.TableDiff, + indexColumns []*model.ColumnInfo, +) (*splitter.RangeInfo, error) { if count <= splitter.SplitThreshold { return tableRange, nil } diff --git a/sync_diff_inspector/main.go b/sync_diff_inspector/main.go index 702f24c6..7a60a1ea 100644 --- a/sync_diff_inspector/main.go +++ b/sync_diff_inspector/main.go @@ -124,7 +124,7 @@ func checkSyncState(ctx context.Context, cfg *config.Config) bool { defer d.Close() if !cfg.CheckDataOnly { - err = d.StructEqual(ctx) + err = d.StructEqual(ctx, cfg.CheckConstraints) if err != nil { fmt.Printf("An error occured while comparing table structure: %s, please check log info in %s for full details\n", err, filepath.Join(cfg.Task.OutputDir, config.LogFileName)) diff --git a/sync_diff_inspector/utils/utils.go b/sync_diff_inspector/utils/utils.go index 382bfe02..f2570a74 100644 --- a/sync_diff_inspector/utils/utils.go +++ b/sync_diff_inspector/utils/utils.go @@ -365,7 +365,7 @@ func sameProperties(c1, c2 *model.ColumnInfo) bool { // // isEqual : result of comparing tables' columns and indices // isPanic : the differences of tables' struct can not be ignored. Need to skip data comparing. -func CompareStruct(upstreamTableInfos []*model.TableInfo, downstreamTableInfo *model.TableInfo) (isEqual bool, isPanic bool) { +func CompareStruct(upstreamTableInfos []*model.TableInfo, downstreamTableInfo *model.TableInfo, checkConstraint bool) (isEqual bool, isPanic bool) { // compare columns for _, upstreamTableInfo := range upstreamTableInfos { if len(upstreamTableInfo.Columns) != len(downstreamTableInfo.Columns) { @@ -419,6 +419,28 @@ func CompareStruct(upstreamTableInfos []*model.TableInfo, downstreamTableInfo *m } } + // compare constraints + if checkConstraint { + downstreamConstraints := downstreamTableInfo.Constraints + sort.Slice(downstreamConstraints, func(i, j int) bool { + return downstreamConstraints[i].Name.L < downstreamConstraints[j].Name.L + }) + for _, upstreamTableInfo := range upstreamTableInfos { + upstreamConstraints := upstreamTableInfo.Constraints + if len(upstreamConstraints) != len(downstreamConstraints) { + return false, true + } + sort.Slice(upstreamConstraints, func(i, j int) bool { + return upstreamConstraints[i].Name.L < upstreamConstraints[j].Name.L + }) + for i, upConstrain := range upstreamConstraints { + if upConstrain.ExprString != downstreamConstraints[i].ExprString { + return false, true + } + } + } + } + // compare indices deleteIndicesSet := make(map[string]struct{}) unilateralIndicesSet := make(map[string]struct{}) diff --git a/sync_diff_inspector/utils/utils_test.go b/sync_diff_inspector/utils/utils_test.go index 2131fa22..b91f684e 100644 --- a/sync_diff_inspector/utils/utils_test.go +++ b/sync_diff_inspector/utils/utils_test.go @@ -528,7 +528,7 @@ func TestCompareStruct(t *testing.T) { var isEqual bool var isPanic bool - isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo}, tableInfo) + isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo}, tableInfo, false) require.True(t, isEqual) require.False(t, isPanic) @@ -537,7 +537,7 @@ func TestCompareStruct(t *testing.T) { tableInfo2, err := dbutil.GetTableInfoBySQL(createTableSQL2, parser.New()) require.NoError(t, err) - isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo) + isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false) require.False(t, isEqual) require.True(t, isPanic) @@ -546,7 +546,7 @@ func TestCompareStruct(t *testing.T) { tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New()) require.NoError(t, err) - isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo) + isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false) require.False(t, isEqual) require.True(t, isPanic) @@ -555,7 +555,7 @@ func TestCompareStruct(t *testing.T) { tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New()) require.NoError(t, err) - isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo) + isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false) require.True(t, isEqual) require.False(t, isPanic) @@ -563,7 +563,7 @@ func TestCompareStruct(t *testing.T) { tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New()) require.NoError(t, err) - isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo) + isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false) require.True(t, isEqual) require.False(t, isPanic) @@ -572,7 +572,7 @@ func TestCompareStruct(t *testing.T) { tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New()) require.NoError(t, err) - isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo) + isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false) require.False(t, isEqual) require.True(t, isPanic) @@ -581,7 +581,7 @@ func TestCompareStruct(t *testing.T) { tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New()) require.NoError(t, err) - isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo) + isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false) require.False(t, isEqual) require.True(t, isPanic) @@ -592,7 +592,7 @@ func TestCompareStruct(t *testing.T) { tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New()) require.NoError(t, err) - isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo) + isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false) require.False(t, isEqual) require.False(t, isPanic) require.Equal(t, len(tableInfo.Indices), 1) @@ -607,7 +607,7 @@ func TestCompareStruct(t *testing.T) { tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New()) require.NoError(t, err) - isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo) + isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false) require.False(t, isEqual) require.False(t, isPanic) require.Equal(t, len(tableInfo.Indices), 1)