@@ -48,6 +48,13 @@ type ClientMgr interface {
4848 Close ()
4949}
5050
51+ // Checksum is the checksum of some backup files calculated by CollectChecksums.
52+ type Checksum struct {
53+ Crc64Xor uint64
54+ TotalKvs uint64
55+ TotalBytes uint64
56+ }
57+
5158// Maximum total sleep time(in ms) for kv/cop commands.
5259const (
5360 backupFineGrainedMaxBackoff = 80000
@@ -748,8 +755,59 @@ func SendBackup(
748755 return nil
749756}
750757
751- // FastChecksum check data integrity by xor all(sst_checksum) per table
752- func (bc * Client ) FastChecksum () (bool , error ) {
758+ // ChecksumMatches tests whether the "local" checksum matches the checksum from TiKV.
759+ func (bc * Client ) ChecksumMatches (local []Checksum ) (bool , error ) {
760+ if len (local ) != len (bc .backupMeta .Schemas ) {
761+ return false , nil
762+ }
763+
764+ for i , schema := range bc .backupMeta .Schemas {
765+ localChecksum := local [i ]
766+ dbInfo := & model.DBInfo {}
767+ err := json .Unmarshal (schema .Db , dbInfo )
768+ if err != nil {
769+ log .Error ("failed in fast checksum, and cannot parse db info." )
770+ return false , err
771+ }
772+ tblInfo := & model.TableInfo {}
773+ err = json .Unmarshal (schema .Table , tblInfo )
774+ if err != nil {
775+ log .Error ("failed in fast checksum, and cannot parse table info." )
776+ return false , err
777+ }
778+ if localChecksum .Crc64Xor != schema .Crc64Xor ||
779+ localChecksum .TotalBytes != schema .TotalBytes ||
780+ localChecksum .TotalKvs != schema .TotalKvs {
781+ log .Error ("failed in fast checksum" ,
782+ zap .Stringer ("db" , dbInfo .Name ),
783+ zap .Stringer ("table" , tblInfo .Name ),
784+ zap .Uint64 ("origin tidb crc64" , schema .Crc64Xor ),
785+ zap .Uint64 ("calculated crc64" , localChecksum .Crc64Xor ),
786+ zap .Uint64 ("origin tidb total kvs" , schema .TotalKvs ),
787+ zap .Uint64 ("calculated total kvs" , localChecksum .TotalKvs ),
788+ zap .Uint64 ("origin tidb total bytes" , schema .TotalBytes ),
789+ zap .Uint64 ("calculated total bytes" , localChecksum .TotalBytes ),
790+ )
791+ return false , nil
792+ }
793+ log .Info ("fast checksum success" ,
794+ zap .String ("database" , dbInfo .Name .L ),
795+ zap .String ("table" , tblInfo .Name .L ))
796+ }
797+ return true , nil
798+ }
799+
800+ // CollectFileInfo collects ungrouped file summary information, like kv count and size.
801+ func (bc * Client ) CollectFileInfo () {
802+ for _ , file := range bc .backupMeta .Files {
803+ summary .CollectSuccessUnit (summary .TotalKV , 1 , file .TotalKvs )
804+ summary .CollectSuccessUnit (summary .TotalBytes , 1 , file .TotalBytes )
805+ }
806+ }
807+
808+ // CollectChecksums check data integrity by xor all(sst_checksum) per table
809+ // it returns the checksum of all local files.
810+ func (bc * Client ) CollectChecksums () ([]Checksum , error ) {
753811 start := time .Now ()
754812 defer func () {
755813 elapsed := time .Since (start )
@@ -758,19 +816,20 @@ func (bc *Client) FastChecksum() (bool, error) {
758816
759817 dbs , err := utils .LoadBackupTables (& bc .backupMeta )
760818 if err != nil {
761- return false , err
819+ return nil , err
762820 }
763821
822+ checksums := make ([]Checksum , 0 , len (bc .backupMeta .Schemas ))
764823 for _ , schema := range bc .backupMeta .Schemas {
765824 dbInfo := & model.DBInfo {}
766825 err = json .Unmarshal (schema .Db , dbInfo )
767826 if err != nil {
768- return false , err
827+ return nil , err
769828 }
770829 tblInfo := & model.TableInfo {}
771830 err = json .Unmarshal (schema .Table , tblInfo )
772831 if err != nil {
773- return false , err
832+ return nil , err
774833 }
775834 tbl := dbs [dbInfo .Name .String ()].GetTable (tblInfo .Name .String ())
776835
@@ -785,25 +844,16 @@ func (bc *Client) FastChecksum() (bool, error) {
785844
786845 summary .CollectSuccessUnit (summary .TotalKV , 1 , totalKvs )
787846 summary .CollectSuccessUnit (summary .TotalBytes , 1 , totalBytes )
788-
789- if schema .Crc64Xor == checksum && schema .TotalKvs == totalKvs && schema .TotalBytes == totalBytes {
790- log .Info ("fast checksum success" , zap .Stringer ("db" , dbInfo .Name ), zap .Stringer ("table" , tblInfo .Name ))
791- } else {
792- log .Error ("failed in fast checksum" ,
793- zap .String ("database" , dbInfo .Name .String ()),
794- zap .String ("table" , tblInfo .Name .String ()),
795- zap .Uint64 ("origin tidb crc64" , schema .Crc64Xor ),
796- zap .Uint64 ("calculated crc64" , checksum ),
797- zap .Uint64 ("origin tidb total kvs" , schema .TotalKvs ),
798- zap .Uint64 ("calculated total kvs" , totalKvs ),
799- zap .Uint64 ("origin tidb total bytes" , schema .TotalBytes ),
800- zap .Uint64 ("calculated total bytes" , totalBytes ),
801- )
802- return false , nil
847+ log .Info ("fast checksum calculated" , zap .Stringer ("db" , dbInfo .Name ), zap .Stringer ("table" , tblInfo .Name ))
848+ localChecksum := Checksum {
849+ Crc64Xor : checksum ,
850+ TotalKvs : totalKvs ,
851+ TotalBytes : totalBytes ,
803852 }
853+ checksums = append (checksums , localChecksum )
804854 }
805855
806- return true , nil
856+ return checksums , nil
807857}
808858
809859// CompleteMeta wait response of admin checksum from TiDB to complete backup meta
@@ -815,3 +865,14 @@ func (bc *Client) CompleteMeta(backupSchemas *Schemas) error {
815865 bc .backupMeta .Schemas = schemas
816866 return nil
817867}
868+
869+ // CopyMetaFrom copies schema metadata directly from pending backupSchemas, without calculating checksum.
870+ // use this when user skip the checksum generating.
871+ func (bc * Client ) CopyMetaFrom (backupSchemas * Schemas ) {
872+ schemas := make ([]* kvproto.Schema , 0 , len (backupSchemas .schemas ))
873+ for _ , v := range backupSchemas .schemas {
874+ schema := v
875+ schemas = append (schemas , & schema )
876+ }
877+ bc .backupMeta .Schemas = schemas
878+ }
0 commit comments