Skip to content

Commit a61225c

Browse files
lonngwinkyao
lonng
authored and
winkyao
committed
cherry-pick: replace gofail with the new failpoint implementation (pingcap#10327)
1 parent c089bb9 commit a61225c

26 files changed

+287
-247
lines changed

‎Makefile‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ PACKAGES := $$($(PACKAGE_LIST))
2525
PACKAGE_DIRECTORIES := $(PACKAGE_LIST) | sed 's|github.com/pingcap/$(PROJECT)/||'
2626
FILES := $$(find $$($(PACKAGE_DIRECTORIES)) -name "*.go" | grep -vE "vendor")
2727

28-
GOFAIL_ENABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|_tools)" | xargs tools/bin/gofail enable)
29-
GOFAIL_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|_tools)" | xargs tools/bin/gofail disable)
28+
FAILPOINT_ENABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|tools)" | xargs tools/bin/failpoint-ctl enable)
29+
FAILPOINT_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|tools)" | xargs tools/bin/failpoint-ctl disable)
3030

3131
LDFLAGS += -X "github.com/pingcap/parser/mysql.TiDBReleaseVersion=$(shell git describe --tags --dirty)"
3232
LDFLAGS += -X "github.com/pingcap/tidb/util/printer.TiDBBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')"
@@ -112,34 +112,34 @@ test: checklist gotest explaintest
112112
explaintest: server
113113
@cd cmd/explaintest && ./run-tests.sh -s ../../bin/tidb-server
114114

115-
gotest: gofail-enable
115+
gotest: failpoint-enable
116116
ifeq ("$(TRAVIS_COVERAGE)", "1")
117117
@echo "Running in TRAVIS_COVERAGE mode."
118118
@export log_level=error; \
119119
go get github.com/go-playground/overalls
120120
go get github.com/mattn/goveralls
121-
$(OVERALLS) -project=github.com/pingcap/tidb -covermode=count -ignore='.git,vendor,cmd,docs,LICENSES' || { $(GOFAIL_DISABLE); exit 1; }
122-
$(GOVERALLS) -service=travis-ci -coverprofile=overalls.coverprofile || { $(GOFAIL_DISABLE); exit 1; }
121+
$(OVERALLS) -project=github.com/pingcap/tidb -covermode=count -ignore='.git,vendor,cmd,docs,LICENSES' || { $(FAILPOINT_DISABLE); exit 1; }
122+
$(GOVERALLS) -service=travis-ci -coverprofile=overalls.coverprofile || { $(FAILPOINT_DISABLE); exit 1; }
123123
else
124124
@echo "Running in native mode."
125125
@export log_level=error; \
126-
$(GOTEST) -ldflags '$(TEST_LDFLAGS)' -cover $(PACKAGES) || { $(GOFAIL_DISABLE); exit 1; }
126+
$(GOTEST) -ldflags '$(TEST_LDFLAGS)' -cover $(PACKAGES) || { $(FAILPOINT_DISABLE); exit 1; }
127127
endif
128-
@$(GOFAIL_DISABLE)
128+
@$(FAILPOINT_DISABLE)
129129

130-
race: gofail-enable
130+
race: failpoint-enable
131131
@export log_level=debug; \
132-
$(GOTEST) -timeout 20m -race $(PACKAGES) || { $(GOFAIL_DISABLE); exit 1; }
133-
@$(GOFAIL_DISABLE)
132+
$(GOTEST) -timeout 20m -race $(PACKAGES) || { $(FAILPOINT_DISABLE); exit 1; }
133+
@$(FAILPOINT_DISABLE)
134134

135-
leak: gofail-enable
135+
leak: failpoint-enable
136136
@export log_level=debug; \
137-
$(GOTEST) -tags leak $(PACKAGES) || { $(GOFAIL_DISABLE); exit 1; }
138-
@$(GOFAIL_DISABLE)
137+
$(GOTEST) -tags leak $(PACKAGES) || { $(FAILPOINT_DISABLE); exit 1; }
138+
@$(FAILPOINT_DISABLE)
139139

140-
tikv_integration_test: gofail-enable
141-
$(GOTEST) ./store/tikv/. -with-tikv=true || { $(GOFAIL_DISABLE); exit 1; }
142-
@$(GOFAIL_DISABLE)
140+
tikv_integration_test: failpoint-enable
141+
$(GOTEST) ./store/tikv/. -with-tikv=true || { $(FAILPOINT_DISABLE); exit 1; }
142+
@$(FAILPOINT_DISABLE)
143143

144144
RACE_FLAG =
145145
ifeq ("$(WITH_RACE)", "1")
@@ -181,13 +181,13 @@ importer:
181181
checklist:
182182
cat checklist.md
183183

184-
gofail-enable: tools/bin/gofail
184+
failpoint-enable: tools/bin/failpoint-ctl
185185
# Converting gofail failpoints...
186-
@$(GOFAIL_ENABLE)
186+
@$(FAILPOINT_ENABLE)
187187

188-
gofail-disable: tools/bin/gofail
188+
failpoint-disable: tools/bin/failpoint-ctl
189189
# Restoring gofail failpoints...
190-
@$(GOFAIL_DISABLE)
190+
@$(FAILPOINT_DISABLE)
191191

192192
tools/bin/megacheck: tools/check/go.mod
193193
cd tools/check; \
@@ -213,5 +213,5 @@ tools/bin/errcheck: tools/check/go.mod
213213
cd tools/check; \
214214
$(GO) build -o ../bin/errcheck github.com/kisielk/errcheck
215215

216-
tools/bin/gofail: go.mod
217-
$(GO) build -o $@ github.com/pingcap/gofail
216+
tools/bin/failpoint-ctl: go.mod
217+
$(GO) build -o $@ github.com/pingcap/failpoint/failpoint-ctl

‎ddl/column.go‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"time"
1919

2020
"github.com/pingcap/errors"
21+
"github.com/pingcap/failpoint"
2122
"github.com/pingcap/parser/ast"
2223
"github.com/pingcap/parser/model"
2324
"github.com/pingcap/parser/mysql"
@@ -137,10 +138,11 @@ func onAddColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error)
137138
if err != nil {
138139
return ver, errors.Trace(err)
139140
}
140-
// gofail: var errorBeforeDecodeArgs bool
141-
// if errorBeforeDecodeArgs {
142-
// return ver, errors.New("occur an error before decode args")
143-
// }
141+
failpoint.Inject("errorBeforeDecodeArgs", func(val failpoint.Value) {
142+
if val.(bool) {
143+
failpoint.Return(ver, errors.New("occur an error before decode args"))
144+
}
145+
})
144146
col := &model.ColumnInfo{}
145147
pos := &ast.ColumnPosition{}
146148
offset := 0
@@ -336,12 +338,13 @@ func doModifyColumn(t *meta.Meta, job *model.Job, newCol *model.ColumnInfo, oldN
336338
}
337339
}
338340

339-
// gofail: var uninitializedOffsetAndState bool
340-
// if uninitializedOffsetAndState {
341-
// if newCol.State != model.StatePublic {
342-
// return ver, errors.New("the column state is wrong")
343-
// }
344-
// }
341+
failpoint.Inject("uninitializedOffsetAndState", func(val failpoint.Value) {
342+
if val.(bool) {
343+
if newCol.State != model.StatePublic {
344+
failpoint.Return(ver, errors.New("the column state is wrong"))
345+
}
346+
}
347+
})
345348

346349
// We need the latest column's offset and state. This information can be obtained from the store.
347350
newCol.Offset = oldCol.Offset

‎ddl/fail_db_test.go‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import (
2121

2222
. "github.com/pingcap/check"
2323
"github.com/pingcap/errors"
24-
gofail "github.com/pingcap/gofail/runtime"
24+
"github.com/pingcap/failpoint"
2525
"github.com/pingcap/parser"
2626
"github.com/pingcap/parser/model"
2727
"github.com/pingcap/tidb/ddl"
@@ -71,9 +71,10 @@ func (s *testFailDBSuite) TearDownSuite(c *C) {
7171

7272
// TestHalfwayCancelOperations tests the case that the schema is correct after the execution of operations are cancelled halfway.
7373
func (s *testFailDBSuite) TestHalfwayCancelOperations(c *C) {
74-
gofail.Enable("github.com/pingcap/tidb/ddl/truncateTableErr", `return(true)`)
75-
defer gofail.Disable("github.com/pingcap/tidb/ddl/truncateTableErr")
76-
74+
c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/truncateTableErr", `return(true)`), IsNil)
75+
defer func() {
76+
c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/truncateTableErr"), IsNil)
77+
}()
7778
// test for truncating table
7879
_, err := s.se.Execute(context.Background(), "create database cancel_job_db")
7980
c.Assert(err, IsNil)
@@ -110,8 +111,11 @@ func (s *testFailDBSuite) TestHalfwayCancelOperations(c *C) {
110111
c.Assert(err, IsNil)
111112

112113
// test for renaming table
113-
gofail.Enable("github.com/pingcap/tidb/ddl/renameTableErr", `return(true)`)
114-
defer gofail.Disable("github.com/pingcap/tidb/ddl/renameTableErr")
114+
c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/renameTableErr", `return(true)`), IsNil)
115+
defer func() {
116+
c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/renameTableErr"), IsNil)
117+
}()
118+
115119
_, err = s.se.Execute(context.Background(), "create table tx(a int)")
116120
c.Assert(err, IsNil)
117121
_, err = s.se.Execute(context.Background(), "insert into tx values(1)")
@@ -155,15 +159,17 @@ func (s *testStateChangeSuite) TestInitializeOffsetAndState(c *C) {
155159
c.Assert(err, IsNil)
156160
defer s.se.Execute(context.Background(), "drop table t")
157161

158-
gofail.Enable("github.com/pingcap/tidb/ddl/uninitializedOffsetAndState", `return(true)`)
162+
c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/uninitializedOffsetAndState", `return(true)`), IsNil)
159163
_, err = s.se.Execute(context.Background(), "ALTER TABLE t MODIFY COLUMN b int FIRST;")
160164
c.Assert(err, IsNil)
161-
gofail.Disable("github.com/pingcap/tidb/ddl/uninitializedOffsetAndState")
165+
c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/uninitializedOffsetAndState"), IsNil)
162166
}
163167

164168
func (s *testDBSuite) TestUpdateHandleFailed(c *C) {
165-
gofail.Enable("github.com/pingcap/tidb/ddl/errorUpdateReorgHandle", `return(true)`)
166-
defer gofail.Disable("github.com/pingcap/tidb/ddl/errorUpdateReorgHandle")
169+
c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/errorUpdateReorgHandle", `1*return`), IsNil)
170+
defer func() {
171+
c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/errorUpdateReorgHandle"), IsNil)
172+
}()
167173
tk := testkit.NewTestKit(c, s.store)
168174
tk.MustExec("create database if not exists test_handle_failed")
169175
defer tk.MustExec("drop database test_handle_failed")
@@ -177,8 +183,10 @@ func (s *testDBSuite) TestUpdateHandleFailed(c *C) {
177183
}
178184

179185
func (s *testDBSuite) TestAddIndexFailed(c *C) {
180-
gofail.Enable("github.com/pingcap/tidb/ddl/mockAddIndexErr", `return(true)`)
181-
defer gofail.Disable("github.com/pingcap/tidb/ddl/mockAddIndexErr")
186+
c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/mockAddIndexErr", `1*return`), IsNil)
187+
defer func() {
188+
c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/mockAddIndexErr"), IsNil)
189+
}()
182190
tk := testkit.NewTestKit(c, s.store)
183191
tk.MustExec("create database if not exists test_add_index_failed")
184192
defer tk.MustExec("drop database test_add_index_failed")
@@ -278,8 +286,10 @@ func (s *testDBSuite) TestAddIndexWorkerNum(c *C) {
278286
ddl.TestCheckWorkerNumber = lastSetWorkerCnt
279287
defer tk.MustExec(fmt.Sprintf("set @@global.tidb_ddl_reorg_worker_cnt=%d", originDDLAddIndexWorkerCnt))
280288

281-
gofail.Enable("github.com/pingcap/tidb/ddl/checkIndexWorkerNum", `return(true)`)
282-
defer gofail.Disable("github.com/pingcap/tidb/ddl/checkIndexWorkerNum")
289+
c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/checkIndexWorkerNum", `return(true)`), IsNil)
290+
defer func() {
291+
c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/checkIndexWorkerNum"), IsNil)
292+
}()
283293

284294
sessionExecInGoroutine(c, s.store, "create index c3_index on test_add_index (c3)", done)
285295
checkNum := 0

‎ddl/fail_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ package ddl
1515

1616
import (
1717
. "github.com/pingcap/check"
18-
gofail "github.com/pingcap/gofail/runtime"
18+
"github.com/pingcap/failpoint"
1919
"github.com/pingcap/parser/ast"
2020
"github.com/pingcap/parser/model"
2121
"github.com/pingcap/tidb/types"
@@ -51,10 +51,10 @@ func (s *testColumnChangeSuite) TestFailBeforeDecodeArgs(c *C) {
5151
stateCnt++
5252
} else if job.SchemaState == model.StateWriteReorganization {
5353
if first {
54-
gofail.Enable("github.com/pingcap/tidb/ddl/errorBeforeDecodeArgs", `return(true)`)
54+
c.Assert(failpoint.Enable("github.com/pingcap/tidb/ddl/errorBeforeDecodeArgs", `return(true)`), IsNil)
5555
first = false
5656
} else {
57-
gofail.Disable("github.com/pingcap/tidb/ddl/errorBeforeDecodeArgs")
57+
c.Assert(failpoint.Disable("github.com/pingcap/tidb/ddl/errorBeforeDecodeArgs"), IsNil)
5858
}
5959
}
6060
}

‎ddl/index.go‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"time"
2222

2323
"github.com/pingcap/errors"
24+
"github.com/pingcap/failpoint"
2425
"github.com/pingcap/parser/ast"
2526
"github.com/pingcap/parser/model"
2627
"github.com/pingcap/parser/mysql"
@@ -837,8 +838,6 @@ func (w *addIndexWorker) handleBackfillTask(d *ddlCtx, task *reorgIndexTask) *ad
837838
return result
838839
}
839840

840-
var gofailMockAddindexErrOnceGuard bool
841-
842841
func (w *addIndexWorker) run(d *ddlCtx) {
843842
logutil.Logger(ddlLogCtx).Info("[ddl] add index worker start", zap.Int("workerID", w.id))
844843
defer func() {
@@ -857,13 +856,13 @@ func (w *addIndexWorker) run(d *ddlCtx) {
857856
}
858857

859858
logutil.Logger(ddlLogCtx).Debug("[ddl] add index worker got task", zap.Int("workerID", w.id), zap.String("task", task.String()))
860-
// gofail: var mockAddIndexErr bool
861-
//if w.id == 0 && mockAddIndexErr && !gofailMockAddindexErrOnceGuard {
862-
// gofailMockAddindexErrOnceGuard = true
863-
// result := &addIndexResult{addedCount: 0, nextHandle: 0, err: errors.Errorf("mock add index error")}
864-
// w.resultCh <- result
865-
// continue
866-
//}
859+
failpoint.Inject("mockAddIndexErr", func() {
860+
if w.id == 0 {
861+
result := &addIndexResult{addedCount: 0, nextHandle: 0, err: errors.Errorf("mock add index error")}
862+
w.resultCh <- result
863+
failpoint.Continue()
864+
}
865+
})
867866

868867
// Dynamic change batch size.
869868
w.batchCnt = int(variable.GetDDLReorgBatchSize())
@@ -1137,20 +1136,21 @@ func (w *worker) addPhysicalTableIndex(t table.PhysicalTable, indexInfo *model.I
11371136
closeAddIndexWorkers(workers)
11381137
}
11391138

1140-
// gofail: var checkIndexWorkerNum bool
1141-
// if checkIndexWorkerNum {
1142-
// num := int(atomic.LoadInt32(&TestCheckWorkerNumber))
1143-
// if num != 0 {
1144-
// if num > len(kvRanges) {
1145-
// if len(idxWorkers) != len(kvRanges) {
1146-
// return errors.Errorf("check index worker num error, len kv ranges is: %v, check index worker num is: %v, actual index num is: %v", len(kvRanges), num, len(idxWorkers))
1147-
// }
1148-
// } else if num != len(idxWorkers) {
1149-
// return errors.Errorf("check index worker num error, len kv ranges is: %v, check index worker num is: %v, actual index num is: %v", len(kvRanges), num, len(idxWorkers))
1150-
// }
1151-
// TestCheckWorkerNumCh <- struct{}{}
1152-
// }
1153-
//}
1139+
failpoint.Inject("checkIndexWorkerNum", func(val failpoint.Value) {
1140+
if val.(bool) {
1141+
num := int(atomic.LoadInt32(&TestCheckWorkerNumber))
1142+
if num != 0 {
1143+
if num > len(kvRanges) {
1144+
if len(idxWorkers) != len(kvRanges) {
1145+
failpoint.Return(errors.Errorf("check index worker num error, len kv ranges is: %v, check index worker num is: %v, actual index num is: %v", len(kvRanges), num, len(idxWorkers)))
1146+
}
1147+
} else if num != len(idxWorkers) {
1148+
failpoint.Return(errors.Errorf("check index worker num error, len kv ranges is: %v, check index worker num is: %v, actual index num is: %v", len(kvRanges), num, len(idxWorkers)))
1149+
}
1150+
TestCheckWorkerNumCh <- struct{}{}
1151+
}
1152+
}
1153+
})
11541154

11551155
logutil.Logger(ddlLogCtx).Info("[ddl] start add index workers to reorg index", zap.Int("workerCnt", len(idxWorkers)), zap.Int("regionCnt", len(kvRanges)), zap.Int64("startHandle", startHandle), zap.Int64("endHandle", endHandle))
11561156
remains, err := w.sendRangeTaskToWorkers(t, idxWorkers, reorgInfo, &totalAddedCount, kvRanges)

‎ddl/reorg.go‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"time"
2222

2323
"github.com/pingcap/errors"
24+
"github.com/pingcap/failpoint"
2425
"github.com/pingcap/parser/model"
2526
"github.com/pingcap/parser/mysql"
2627
"github.com/pingcap/parser/terror"
@@ -313,8 +314,6 @@ func (d *ddlCtx) GetTableMaxRowID(startTS uint64, tbl table.PhysicalTable) (maxR
313314
return maxRowID, false, nil
314315
}
315316

316-
var gofailOnceGuard bool
317-
318317
// getTableRange gets the start and end handle of a table (or partition).
319318
func getTableRange(d *ddlCtx, tbl table.PhysicalTable, snapshotVer uint64, priority int) (startHandle, endHandle int64, err error) {
320319
startHandle = math.MinInt64
@@ -376,12 +375,9 @@ func getReorgInfo(d *ddlCtx, t *meta.Meta, job *model.Job, tbl table.Table) (*re
376375
}
377376
logutil.Logger(ddlLogCtx).Info("[ddl] job get table range", zap.Int64("jobID", job.ID), zap.Int64("physicalTableID", pid), zap.Int64("startHandle", start), zap.Int64("endHandle", end))
378377

379-
// gofail: var errorUpdateReorgHandle bool
380-
// if errorUpdateReorgHandle && !gofailOnceGuard {
381-
// // only return error once.
382-
// gofailOnceGuard = true
383-
// return &info, errors.New("occur an error when update reorg handle.")
384-
// }
378+
failpoint.Inject("errorUpdateReorgHandle", func() (*reorgInfo, error) {
379+
return &info, errors.New("occur an error when update reorg handle")
380+
})
385381
err = t.UpdateDDLReorgHandle(job, start, end, pid)
386382
if err != nil {
387383
return &info, errors.Trace(err)

‎ddl/table.go‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"strings"
2020

2121
"github.com/pingcap/errors"
22+
"github.com/pingcap/failpoint"
2223
"github.com/pingcap/parser/model"
2324
"github.com/pingcap/tidb/ddl/util"
2425
"github.com/pingcap/tidb/infoschema"
@@ -257,11 +258,12 @@ func onTruncateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ erro
257258
job.State = model.JobStateCancelled
258259
return ver, errors.Trace(err)
259260
}
260-
// gofail: var truncateTableErr bool
261-
// if truncateTableErr {
262-
// job.State = model.JobStateCancelled
263-
// return ver, errors.New("occur an error after dropping table.")
264-
// }
261+
failpoint.Inject("truncateTableErr", func(val failpoint.Value) {
262+
if val.(bool) {
263+
job.State = model.JobStateCancelled
264+
failpoint.Return(ver, errors.New("occur an error after dropping table"))
265+
}
266+
})
265267

266268
var oldPartitionIDs []int64
267269
if tblInfo.GetPartitionInfo() != nil {
@@ -388,11 +390,14 @@ func onRenameTable(t *meta.Meta, job *model.Job) (ver int64, _ error) {
388390
job.State = model.JobStateCancelled
389391
return ver, errors.Trace(err)
390392
}
391-
// gofail: var renameTableErr bool
392-
// if renameTableErr {
393-
// job.State = model.JobStateCancelled
394-
// return ver, errors.New("occur an error after renaming table.")
395-
// }
393+
394+
failpoint.Inject("renameTableErr", func(val failpoint.Value) {
395+
if val.(bool) {
396+
job.State = model.JobStateCancelled
397+
failpoint.Return(ver, errors.New("occur an error after renaming table"))
398+
}
399+
})
400+
396401
tblInfo.Name = tableName
397402
err = t.CreateTable(newSchemaID, tblInfo)
398403
if err != nil {

‎executor/aggregate.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -673,6 +673,7 @@ func (e *HashAggExec) execute(ctx context.Context) (err error) {
673673
if err != nil {
674674
return errors.Trace(err)
675675
}
676+
676677
// no more data.
677678
if e.childResult.NumRows() == 0 {
678679
return nil

0 commit comments

Comments
 (0)
⚡