add integration test for reparo (#416)

Author: WangXiangUSTC

Makefile

@@ -36,7 +36,7 @@ all: dev install
 
 dev: check test build
 
-build: pump drainer
+build: pump drainer reparo
 
 pump:
 	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/pump cmd/pump/main.go
@@ -63,6 +63,7 @@ integration_test: build
 	@which bin/pd-server
 	@which bin/drainer
 	@which bin/pump
+	@which bin/reparo
 	tests/run.sh
 
 fmt:

tests/_utils/run_reparo

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -ue
+
+OUT_DIR=/tmp/tidb_binlog_test
+
+killall reparo || true
+
+
+config=${TEST_DIR-.}/reparo.toml
+log=$OUT_DIR/reparo.log
+
+echo "[$(date)] <<<<<< START IN TEST ${TEST_NAME-} FOR: $config >>>>>>" >> "$log"
+
+if [ -f "$config" ]
+then
+	reparo -config $config -log-file $log >> $log 2>&1
+else
+	reapro -log-file $log >> $log 2>&1
+fi

tests/reparo/drainer.toml

@@ -0,0 +1,41 @@
+# drainer Configuration.
+
+# addr (i.e. 'host:port') to listen on for drainer connections
+# will register this addr into etcd
+# addr = "127.0.0.1:8249"
+
+# the interval time (in seconds) of detect pumps' status
+detect-interval = 10
+
+# drainer meta data directory path
+data-dir = "data.drainer"
+
+# a comma separated list of PD endpoints
+pd-urls = "http://127.0.0.1:2379"
+
+# syncer Configuration.
+[syncer]
+
+# disable sync these schema
+ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql"
+
+# number of binlog events in a transaction batch
+txn-batch = 1
+
+# work count to execute binlogs
+worker-count = 1
+
+disable-dispatch = false
+
+# safe mode will split update to delete and insert
+safe-mode = false
+
+# downstream storage, equal to --dest-db-type
+# valid values are "mysql", "pb", "tidb", "flash", "kafka"
+db-type = "pb"
+
+#[syncer.to]
+#dir = "/data/data.drainer"
+#compression = "gzip"
+
+

tests/reparo/reparo.toml

@@ -0,0 +1,28 @@
+# drainer 输出的 protobuf 格式 binlog 文件的存储路径
+data-dir = "data.drainer"
+
+# log-file = ""
+# log-rotate = "hour"
+log-level = "info"
+
+# start-datetime and stop-datetime enable you to pick a range of binlog to reparo.
+# The datetime format is like '2018-02-28 12:12:12'.
+#start-datetime = "2018-10-24 00:00:00"
+#stop-datetime = "2018-11-26 00:00:00"
+
+# Start-tso is similar to start-datetime, but in pd-server tso format. e.g.  395181938313123110
+# Stop-tso is is similar to stop-datetime, but in pd-server tso format. e.g. 395181938313123110
+# start-tso = 0
+# stop-tso = 0
+
+# dest-type choose a destination, which value can be "mysql", "print".
+# for print, it just prints decoded value.
+dest-type = "mysql"
+#compression = "gzip"
+
+# 如果指定的 dest-type 为 mysql 或 tidb，需要配置 dest-db。
+[dest-db]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = ""

tests/reparo/run.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+
+run_drainer &
+
+run_sql "DROP DATABASE IF EXISTS reparo_test;"
+run_sql "CREATE DATABASE reparo_test"
+run_sql "CREATE TABLE reparo_test.test(id int, name varchar(10), PRIMARY KEY(id))"
+
+run_sql "INSERT INTO reparo_test.test VALUES(1, 'a'), (2, 'b')"
+run_sql "INSERT INTO reparo_test.test VALUES(3, 'c'), (4, 'd')"
+run_sql "UPDATE reparo_test.test SET name = 'bb' where id = 2"
+run_sql "DELETE FROM reparo_test.test WHERE name = 'bb'"
+run_sql "INSERT INTO reparo_test.test VALUES(5, 'e')"
+
+sleep 5
+
+run_reparo &
+
+sleep 5
+
+down_run_sql "SELECT count(*) FROM reparo_test.test"
+check_contains "count(*): 4"
+
+# clean up
+run_sql "DROP DATABASE IF EXISTS reparo_test"
+
+killall drainer