From c5e83e863734c0898004e5a3367c3ecdcca0786f Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Tue, 25 Dec 2018 13:49:35 +0800 Subject: [PATCH 1/3] add integration test for reparo (#416) --- Makefile | 3 ++- tests/_utils/run_reparo | 20 +++++++++++++++++++ tests/reparo/drainer.toml | 41 +++++++++++++++++++++++++++++++++++++++ tests/reparo/reparo.toml | 28 ++++++++++++++++++++++++++ tests/reparo/run.sh | 31 +++++++++++++++++++++++++++++ 5 files changed, 122 insertions(+), 1 deletion(-) create mode 100755 tests/_utils/run_reparo create mode 100644 tests/reparo/drainer.toml create mode 100644 tests/reparo/reparo.toml create mode 100755 tests/reparo/run.sh diff --git a/Makefile b/Makefile index 317449d2e..5582e5981 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ all: dev install dev: check test build -build: pump drainer +build: pump drainer reparo pump: $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/pump cmd/pump/main.go @@ -63,6 +63,7 @@ integration_test: build @which bin/pd-server @which bin/drainer @which bin/pump + @which bin/reparo tests/run.sh fmt: diff --git a/tests/_utils/run_reparo b/tests/_utils/run_reparo new file mode 100755 index 000000000..9c320b0b4 --- /dev/null +++ b/tests/_utils/run_reparo @@ -0,0 +1,20 @@ +#!/bin/sh + +set -ue + +OUT_DIR=/tmp/tidb_binlog_test + +killall reparo || true + + +config=${TEST_DIR-.}/reparo.toml +log=$OUT_DIR/reparo.log + +echo "[$(date)] <<<<<< START IN TEST ${TEST_NAME-} FOR: $config >>>>>>" >> "$log" + +if [ -f "$config" ] +then + reparo -config $config -log-file $log >> $log 2>&1 +else + reapro -log-file $log >> $log 2>&1 +fi diff --git a/tests/reparo/drainer.toml b/tests/reparo/drainer.toml new file mode 100644 index 000000000..d3f8a7379 --- /dev/null +++ b/tests/reparo/drainer.toml @@ -0,0 +1,41 @@ +# drainer Configuration. + +# addr (i.e. 'host:port') to listen on for drainer connections +# will register this addr into etcd +# addr = "127.0.0.1:8249" + +# the interval time (in seconds) of detect pumps' status +detect-interval = 10 + +# drainer meta data directory path +data-dir = "data.drainer" + +# a comma separated list of PD endpoints +pd-urls = "http://127.0.0.1:2379" + +# syncer Configuration. +[syncer] + +# disable sync these schema +ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" + +# number of binlog events in a transaction batch +txn-batch = 1 + +# work count to execute binlogs +worker-count = 1 + +disable-dispatch = false + +# safe mode will split update to delete and insert +safe-mode = false + +# downstream storage, equal to --dest-db-type +# valid values are "mysql", "pb", "tidb", "flash", "kafka" +db-type = "pb" + +#[syncer.to] +#dir = "/data/data.drainer" +#compression = "gzip" + + diff --git a/tests/reparo/reparo.toml b/tests/reparo/reparo.toml new file mode 100644 index 000000000..8eda6d904 --- /dev/null +++ b/tests/reparo/reparo.toml @@ -0,0 +1,28 @@ +# drainer 输出的 protobuf 格式 binlog 文件的存储路径 +data-dir = "data.drainer" + +# log-file = "" +# log-rotate = "hour" +log-level = "info" + +# start-datetime and stop-datetime enable you to pick a range of binlog to reparo. +# The datetime format is like '2018-02-28 12:12:12'. +#start-datetime = "2018-10-24 00:00:00" +#stop-datetime = "2018-11-26 00:00:00" + +# Start-tso is similar to start-datetime, but in pd-server tso format. e.g. 395181938313123110 +# Stop-tso is is similar to stop-datetime, but in pd-server tso format. e.g. 395181938313123110 +# start-tso = 0 +# stop-tso = 0 + +# dest-type choose a destination, which value can be "mysql", "print". +# for print, it just prints decoded value. +dest-type = "mysql" +#compression = "gzip" + +# 如果指定的 dest-type 为 mysql 或 tidb，需要配置 dest-db。 +[dest-db] +host = "127.0.0.1" +port = 3306 +user = "root" +password = "" \ No newline at end of file diff --git a/tests/reparo/run.sh b/tests/reparo/run.sh new file mode 100755 index 000000000..fc4cb3b3c --- /dev/null +++ b/tests/reparo/run.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +set -e + +cd "$(dirname "$0")" + +run_drainer & + +run_sql "DROP DATABASE IF EXISTS reparo_test;" +run_sql "CREATE DATABASE reparo_test" +run_sql "CREATE TABLE reparo_test.test(id int, name varchar(10), PRIMARY KEY(id))" + +run_sql "INSERT INTO reparo_test.test VALUES(1, 'a'), (2, 'b')" +run_sql "INSERT INTO reparo_test.test VALUES(3, 'c'), (4, 'd')" +run_sql "UPDATE reparo_test.test SET name = 'bb' where id = 2" +run_sql "DELETE FROM reparo_test.test WHERE name = 'bb'" +run_sql "INSERT INTO reparo_test.test VALUES(5, 'e')" + +sleep 5 + +run_reparo & + +sleep 5 + +down_run_sql "SELECT count(*) FROM reparo_test.test" +check_contains "count(*): 4" + +# clean up +run_sql "DROP DATABASE IF EXISTS reparo_test" + +killall drainer From 7145da369c745975d27cdedf3fe07bc6c310f581 Mon Sep 17 00:00:00 2001 From: WangXiangUSTC <347249478@qq.com> Date: Mon, 21 Jan 2019 17:18:28 +0800 Subject: [PATCH 2/3] add sleep time --- tests/reparo/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/reparo/run.sh b/tests/reparo/run.sh index fc4cb3b3c..06063d0c0 100755 --- a/tests/reparo/run.sh +++ b/tests/reparo/run.sh @@ -20,7 +20,7 @@ sleep 5 run_reparo & -sleep 5 +sleep 15 down_run_sql "SELECT count(*) FROM reparo_test.test" check_contains "count(*): 4" From d40505b7a734c3bd4ab7b1dc319e7f52bd7f0809 Mon Sep 17 00:00:00 2001 From: WangXiangUSTC <347249478@qq.com> Date: Tue, 22 Jan 2019 10:28:12 +0800 Subject: [PATCH 3/3] change some chinese to english --- tests/reparo/reparo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/reparo/reparo.toml b/tests/reparo/reparo.toml index 8eda6d904..64dc7761a 100644 --- a/tests/reparo/reparo.toml +++ b/tests/reparo/reparo.toml @@ -1,4 +1,4 @@ -# drainer 输出的 protobuf 格式 binlog 文件的存储路径 +# where the protobuf format binlog files saved by drainer data-dir = "data.drainer" # log-file = "" @@ -20,7 +20,7 @@ log-level = "info" dest-type = "mysql" #compression = "gzip" -# 如果指定的 dest-type 为 mysql 或 tidb，需要配置 dest-db。 +# if dest-type is "mysql" or "tidb", need set dest-db. [dest-db] host = "127.0.0.1" port = 3306