Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sync-diff-inspector: add check constraints and partition comparation #832

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+52 −16
Open

sync-diff-inspector: add check constraints and partition comparation #832

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pkg/dbutil/table.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/mysql"
_ "github.com/pingcap/tidb/pkg/planner/core" // to setup expression.EvalAstExpr. See: https://github.com/pingcap/tidb/blob/a94cff903cd1e7f3b050db782da84273ef5592f4/planner/core/optimizer.go#L202
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/types"
_ "github.com/pingcap/tidb/pkg/types/parser_driver" // for parser driver
"github.com/pingcap/tidb/pkg/util/collate"
Expand All @@ -43,6 +44,9 @@ const (

func init() {
collate.SetNewCollationEnabledForTest(false)

// Enable constraint check
variable.EnableCheckConstraint.Store(true)
}

// addClusteredAnnotation add the `/*T![clustered_index] NONCLUSTERED */` for primary key of create table info
Expand Down
3 changes: 3 additions & 0 deletions sync_diff_inspector/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,8 @@ type Config struct {
// set true if want to compare rows
// set false won't compare rows.
ExportFixSQL bool `toml:"export-fix-sql" json:"export-fix-sql"`
// whether check table constraints.
CheckConstraints bool `toml:"check-constraints" json:"check-constraints"`
// only check table struct without table data.
CheckStructOnly bool `toml:"check-struct-only" json:"check-struct-only"`
// experimental feature: only check table data without table struct
Expand Down Expand Up @@ -412,6 +414,7 @@ func NewConfig() *Config {
fs.StringVar(&cfg.DMTask, "dm-task", "", "identifier of dm task")
fs.IntVar(&cfg.CheckThreadCount, "check-thread-count", 4, "how many goroutines are created to check data")
fs.BoolVar(&cfg.ExportFixSQL, "export-fix-sql", true, "set true if want to compare rows or set to false will only compare checksum")
fs.BoolVar(&cfg.CheckConstraints, "check-constraints", false, "check table constraints")
fs.BoolVar(&cfg.CheckStructOnly, "check-struct-only", false, "ignore check table's data")
fs.BoolVar(&cfg.SkipNonExistingTable, "skip-non-existing-table", false, "skip validation for tables that don't exist upstream or downstream")
fs.BoolVar(&cfg.CheckDataOnly, "check-data-only", false, "ignore check table's struct")
Expand Down
17 changes: 12 additions & 5 deletions sync_diff_inspector/diff.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ func (df *Diff) Equal(ctx context.Context) error {
return nil
}

func (df *Diff) StructEqual(ctx context.Context) error {
func (df *Diff) StructEqual(ctx context.Context, checkConstraint bool) error {
tables := df.downstream.GetTables()
tableIndex := 0
if df.startRange != nil {
Expand All @@ -300,7 +300,7 @@ func (df *Diff) StructEqual(ctx context.Context) error {
isEqual, isSkip, isAllTableExist := false, true, tables[tableIndex].TableLack
if common.AllTableExist(isAllTableExist) {
var err error
isEqual, isSkip, err = df.compareStruct(ctx, tableIndex)
isEqual, isSkip, err = df.compareStruct(ctx, tableIndex, checkConstraint)
if err != nil {
return errors.Trace(err)
}
Expand All @@ -311,13 +311,13 @@ func (df *Diff) StructEqual(ctx context.Context) error {
return nil
}

func (df *Diff) compareStruct(ctx context.Context, tableIndex int) (isEqual bool, isSkip bool, err error) {
func (df *Diff) compareStruct(ctx context.Context, tableIndex int, checkConstraint bool) (isEqual bool, isSkip bool, err error) {
sourceTableInfos, err := df.upstream.GetSourceStructInfo(ctx, tableIndex)
if err != nil {
return false, true, errors.Trace(err)
}
table := df.downstream.GetTables()[tableIndex]
isEqual, isSkip = utils.CompareStruct(sourceTableInfos, table.Info)
isEqual, isSkip = utils.CompareStruct(sourceTableInfos, table.Info, checkConstraint)
table.IgnoreDataCheck = isSkip
return isEqual, isSkip, nil
}
Expand Down Expand Up @@ -506,7 +506,14 @@ func (df *Diff) BinGenerate(ctx context.Context, targetSource source.Source, tab
return df.binSearch(ctx, targetSource, tableRange, count, tableDiff, indexColumns)
}

func (df *Diff) binSearch(ctx context.Context, targetSource source.Source, tableRange *splitter.RangeInfo, count int64, tableDiff *common.TableDiff, indexColumns []*model.ColumnInfo) (*splitter.RangeInfo, error) {
func (df *Diff) binSearch(
ctx context.Context,
targetSource source.Source,
tableRange *splitter.RangeInfo,
count int64,
tableDiff *common.TableDiff,
indexColumns []*model.ColumnInfo,
) (*splitter.RangeInfo, error) {
if count <= splitter.SplitThreshold {
return tableRange, nil
}
Expand Down
2 changes: 1 addition & 1 deletion sync_diff_inspector/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func checkSyncState(ctx context.Context, cfg *config.Config) bool {
defer d.Close()

if !cfg.CheckDataOnly {
err = d.StructEqual(ctx)
err = d.StructEqual(ctx, cfg.CheckConstraints)
if err != nil {
fmt.Printf("An error occured while comparing table structure: %s, please check log info in %s for full details\n",
err, filepath.Join(cfg.Task.OutputDir, config.LogFileName))
Expand Down
24 changes: 23 additions & 1 deletion sync_diff_inspector/utils/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ func sameProperties(c1, c2 *model.ColumnInfo) bool {
//
// isEqual : result of comparing tables' columns and indices
// isPanic : the differences of tables' struct can not be ignored. Need to skip data comparing.
func CompareStruct(upstreamTableInfos []*model.TableInfo, downstreamTableInfo *model.TableInfo) (isEqual bool, isPanic bool) {
func CompareStruct(upstreamTableInfos []*model.TableInfo, downstreamTableInfo *model.TableInfo, checkConstraint bool) (isEqual bool, isPanic bool) {
// compare columns
for _, upstreamTableInfo := range upstreamTableInfos {
if len(upstreamTableInfo.Columns) != len(downstreamTableInfo.Columns) {
Expand Down Expand Up @@ -419,6 +419,28 @@ func CompareStruct(upstreamTableInfos []*model.TableInfo, downstreamTableInfo *m
}
}

// compare constraints
if checkConstraint {
downstreamConstraints := downstreamTableInfo.Constraints
sort.Slice(downstreamConstraints, func(i, j int) bool {
return downstreamConstraints[i].Name.L < downstreamConstraints[j].Name.L
})
for _, upstreamTableInfo := range upstreamTableInfos {
upstreamConstraints := upstreamTableInfo.Constraints
if len(upstreamConstraints) != len(downstreamConstraints) {
return false, true
}
sort.Slice(upstreamConstraints, func(i, j int) bool {
return upstreamConstraints[i].Name.L < upstreamConstraints[j].Name.L
})
for i, upConstrain := range upstreamConstraints {
if upConstrain.ExprString != downstreamConstraints[i].ExprString {
return false, true
}
}
}
}

// compare indices
deleteIndicesSet := make(map[string]struct{})
unilateralIndicesSet := make(map[string]struct{})
Expand Down
18 changes: 9 additions & 9 deletions sync_diff_inspector/utils/utils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -528,7 +528,7 @@ func TestCompareStruct(t *testing.T) {

var isEqual bool
var isPanic bool
isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo}, tableInfo)
isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo}, tableInfo, false)
require.True(t, isEqual)
require.False(t, isPanic)

Expand All @@ -537,7 +537,7 @@ func TestCompareStruct(t *testing.T) {
tableInfo2, err := dbutil.GetTableInfoBySQL(createTableSQL2, parser.New())
require.NoError(t, err)

isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo)
isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false)
require.False(t, isEqual)
require.True(t, isPanic)

Expand All @@ -546,7 +546,7 @@ func TestCompareStruct(t *testing.T) {
tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New())
require.NoError(t, err)

isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo)
isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false)
require.False(t, isEqual)
require.True(t, isPanic)

Expand All @@ -555,15 +555,15 @@ func TestCompareStruct(t *testing.T) {
tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New())
require.NoError(t, err)

isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo)
isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false)
require.True(t, isEqual)
require.False(t, isPanic)

createTableSQL2 = "create table `test`(`a` int(11), `b` varchar(10), `c` float, `d` datetime, primary key(`a`, `b`), index(`c`))"
tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New())
require.NoError(t, err)

isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo)
isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false)
require.True(t, isEqual)
require.False(t, isPanic)

Expand All @@ -572,7 +572,7 @@ func TestCompareStruct(t *testing.T) {
tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New())
require.NoError(t, err)

isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo)
isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false)
require.False(t, isEqual)
require.True(t, isPanic)

Expand All @@ -581,7 +581,7 @@ func TestCompareStruct(t *testing.T) {
tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New())
require.NoError(t, err)

isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo)
isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false)
require.False(t, isEqual)
require.True(t, isPanic)

Expand All @@ -592,7 +592,7 @@ func TestCompareStruct(t *testing.T) {
tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New())
require.NoError(t, err)

isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo)
isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false)
require.False(t, isEqual)
require.False(t, isPanic)
require.Equal(t, len(tableInfo.Indices), 1)
Expand All @@ -607,7 +607,7 @@ func TestCompareStruct(t *testing.T) {
tableInfo2, err = dbutil.GetTableInfoBySQL(createTableSQL2, parser.New())
require.NoError(t, err)

isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo)
isEqual, isPanic = CompareStruct([]*model.TableInfo{tableInfo, tableInfo2}, tableInfo, false)
require.False(t, isEqual)
require.False(t, isPanic)
require.Equal(t, len(tableInfo.Indices), 1)
Expand Down