diff --git a/Makefile b/Makefile index 317449d2e..5582e5981 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ all: dev install dev: check test build -build: pump drainer +build: pump drainer reparo pump: $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/pump cmd/pump/main.go @@ -63,6 +63,7 @@ integration_test: build @which bin/pd-server @which bin/drainer @which bin/pump + @which bin/reparo tests/run.sh fmt: diff --git a/tests/_utils/run_reparo b/tests/_utils/run_reparo new file mode 100755 index 000000000..9c320b0b4 --- /dev/null +++ b/tests/_utils/run_reparo @@ -0,0 +1,20 @@ +#!/bin/sh + +set -ue + +OUT_DIR=/tmp/tidb_binlog_test + +killall reparo || true + + +config=${TEST_DIR-.}/reparo.toml +log=$OUT_DIR/reparo.log + +echo "[$(date)] <<<<<< START IN TEST ${TEST_NAME-} FOR: $config >>>>>>" >> "$log" + +if [ -f "$config" ] +then + reparo -config $config -log-file $log >> $log 2>&1 +else + reapro -log-file $log >> $log 2>&1 +fi diff --git a/tests/reparo/drainer.toml b/tests/reparo/drainer.toml new file mode 100644 index 000000000..d3f8a7379 --- /dev/null +++ b/tests/reparo/drainer.toml @@ -0,0 +1,41 @@ +# drainer Configuration. + +# addr (i.e. 'host:port') to listen on for drainer connections +# will register this addr into etcd +# addr = "127.0.0.1:8249" + +# the interval time (in seconds) of detect pumps' status +detect-interval = 10 + +# drainer meta data directory path +data-dir = "data.drainer" + +# a comma separated list of PD endpoints +pd-urls = "http://127.0.0.1:2379" + +# syncer Configuration. +[syncer] + +# disable sync these schema +ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" + +# number of binlog events in a transaction batch +txn-batch = 1 + +# work count to execute binlogs +worker-count = 1 + +disable-dispatch = false + +# safe mode will split update to delete and insert +safe-mode = false + +# downstream storage, equal to --dest-db-type +# valid values are "mysql", "pb", "tidb", "flash", "kafka" +db-type = "pb" + +#[syncer.to] +#dir = "/data/data.drainer" +#compression = "gzip" + + diff --git a/tests/reparo/reparo.toml b/tests/reparo/reparo.toml new file mode 100644 index 000000000..8eda6d904 --- /dev/null +++ b/tests/reparo/reparo.toml @@ -0,0 +1,28 @@ +# drainer 输出的 protobuf 格式 binlog 文件的存储路径 +data-dir = "data.drainer" + +# log-file = "" +# log-rotate = "hour" +log-level = "info" + +# start-datetime and stop-datetime enable you to pick a range of binlog to reparo. +# The datetime format is like '2018-02-28 12:12:12'. +#start-datetime = "2018-10-24 00:00:00" +#stop-datetime = "2018-11-26 00:00:00" + +# Start-tso is similar to start-datetime, but in pd-server tso format. e.g. 395181938313123110 +# Stop-tso is is similar to stop-datetime, but in pd-server tso format. e.g. 395181938313123110 +# start-tso = 0 +# stop-tso = 0 + +# dest-type choose a destination, which value can be "mysql", "print". +# for print, it just prints decoded value. +dest-type = "mysql" +#compression = "gzip" + +# 如果指定的 dest-type 为 mysql 或 tidb,需要配置 dest-db。 +[dest-db] +host = "127.0.0.1" +port = 3306 +user = "root" +password = "" \ No newline at end of file diff --git a/tests/reparo/run.sh b/tests/reparo/run.sh new file mode 100755 index 000000000..fc4cb3b3c --- /dev/null +++ b/tests/reparo/run.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +set -e + +cd "$(dirname "$0")" + +run_drainer & + +run_sql "DROP DATABASE IF EXISTS reparo_test;" +run_sql "CREATE DATABASE reparo_test" +run_sql "CREATE TABLE reparo_test.test(id int, name varchar(10), PRIMARY KEY(id))" + +run_sql "INSERT INTO reparo_test.test VALUES(1, 'a'), (2, 'b')" +run_sql "INSERT INTO reparo_test.test VALUES(3, 'c'), (4, 'd')" +run_sql "UPDATE reparo_test.test SET name = 'bb' where id = 2" +run_sql "DELETE FROM reparo_test.test WHERE name = 'bb'" +run_sql "INSERT INTO reparo_test.test VALUES(5, 'e')" + +sleep 5 + +run_reparo & + +sleep 5 + +down_run_sql "SELECT count(*) FROM reparo_test.test" +check_contains "count(*): 4" + +# clean up +run_sql "DROP DATABASE IF EXISTS reparo_test" + +killall drainer