Skip to content

Data race when running the unit tests #3601

Closed
#3603
Closed
Data race when running the unit tests#3601
#3603
Labels
severity/majortype/bugThe issue is confirmed as a bug.
@rleungx

Description

@rleungx
rleungx
opened on Apr 16, 2021

Bug Report

[2021-04-15T08:00:15.125Z] ==================
[2021-04-15T08:00:15.125Z] WARNING: DATA RACE
[2021-04-15T08:00:15.125Z] Write at 0x00c007dd7410 by goroutine 363:
[2021-04-15T08:00:15.125Z]   runtime.mapassign()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/runtime/map.go:574 +0x0
[2021-04-15T08:00:15.125Z]   reflect.mapassign()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/runtime/map.go:1329 +0x3e
[2021-04-15T08:00:15.125Z]   encoding/json.(*decodeState).object()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/encoding/json/decode.go:865 +0x115e
[2021-04-15T08:00:15.125Z]   encoding/json.(*decodeState).value()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/encoding/json/decode.go:429 +0xaf
[2021-04-15T08:00:15.125Z]   encoding/json.(*decodeState).object()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/encoding/json/decode.go:823 +0x21db
[2021-04-15T08:00:15.125Z]   encoding/json.(*decodeState).value()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/encoding/json/decode.go:429 +0xaf
[2021-04-15T08:00:15.125Z]   encoding/json.(*decodeState).unmarshal()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/encoding/json/decode.go:179 +0x254
[2021-04-15T08:00:15.125Z]   encoding/json.Unmarshal()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/encoding/json/decode.go:106 +0x1e4
[2021-04-15T08:00:15.125Z]   github.com/tikv/pd/pkg/apiutil.ReadJSON()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/src/github.com/pingcap/pd/pkg/apiutil/apiutil.go:65 +0x287
[2021-04-15T08:00:15.125Z]   github.com/tikv/pd/pkg/apiutil.ReadJSONRespondError()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/src/github.com/pingcap/pd/pkg/apiutil/apiutil.go:97 +0x77
[2021-04-15T08:00:15.125Z]   github.com/tikv/pd/server/api.(*confHandler).SetSchedule()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/src/github.com/pingcap/pd/server/api/config.go:359 +0x280
[2021-04-15T08:00:15.125Z]   github.com/tikv/pd/server/api.(*confHandler).SetSchedule-fm()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/src/github.com/pingcap/pd/server/api/config.go:357 +0x5f
[2021-04-15T08:00:15.125Z]   net/http.HandlerFunc.ServeHTTP()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/net/http/server.go:2007 +0x51
[2021-04-15T08:00:15.125Z]   github.com/gorilla/mux.(*Router).ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/gorilla/mux@v1.7.4/mux.go:210 +0x13e
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.Wrap.func1()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:46 +0x68
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.HandlerFunc.ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:29 +0x5b
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.middleware.ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:38 +0x116
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.middleware.ServeHTTP-fm()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:37 +0x71
[2021-04-15T08:00:15.125Z]   github.com/tikv/pd/pkg/apiutil/serverapi.(*redirector).ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/src/github.com/pingcap/pd/pkg/apiutil/serverapi/middleware.go:98 +0xc92
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.middleware.ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:38 +0x116
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.middleware.ServeHTTP-fm()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:37 +0x71
[2021-04-15T08:00:15.125Z]   github.com/tikv/pd/pkg/apiutil/serverapi.(*runtimeServiceValidator).ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/src/github.com/pingcap/pd/pkg/apiutil/serverapi/middleware.go:54 +0x114
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.middleware.ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:38 +0x116
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.(*Negroni).ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:96 +0x17a
[2021-04-15T08:00:15.125Z]   github.com/gorilla/mux.(*Router).ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/gorilla/mux@v1.7.4/mux.go:210 +0x13e
[2021-04-15T08:00:15.125Z]   github.com/gorilla/mux.(*Router).ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/gorilla/mux@v1.7.4/mux.go:210 +0x13e
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.Wrap.func1()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:46 +0x68
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.HandlerFunc.ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:29 +0x5b
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.middleware.ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:38 +0x116
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.middleware.ServeHTTP-fm()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:37 +0x71
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.(*Recovery).ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/recovery.go:193 +0xbb
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.middleware.ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:38 +0x116
[2021-04-15T08:00:15.125Z]   github.com/urfave/negroni.(*Negroni).ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/urfave/negroni@v0.3.0/negroni.go:96 +0x17a
[2021-04-15T08:00:15.125Z]   net/http.(*ServeMux).ServeHTTP()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/net/http/server.go:2387 +0x288
[2021-04-15T08:00:15.125Z]   go.etcd.io/etcd/embed.(*accessController).ServeHTTP()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/embed/serve.go:359 +0x323
[2021-04-15T08:00:15.125Z]   net/http.serverHandler.ServeHTTP()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/net/http/server.go:2802 +0xce
[2021-04-15T08:00:15.125Z]   net/http.(*conn).serve()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/net/http/server.go:1890 +0x837
[2021-04-15T08:00:15.125Z] 
[2021-04-15T08:00:15.125Z] Previous read at 0x00c007dd7410 by goroutine 487:
[2021-04-15T08:00:15.125Z]   runtime.mapiternext()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/runtime/map.go:851 +0x0
[2021-04-15T08:00:15.125Z]   github.com/tikv/pd/server/statistics.(*storeStatistics).Collect()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/src/github.com/pingcap/pd/server/statistics/store_collection.go:190 +0x1b17
[2021-04-15T08:00:15.125Z]   github.com/tikv/pd/server/cluster.(*RaftCluster).collectMetrics()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/src/github.com/pingcap/pd/server/statistics/store_collection.go:236 +0x201
[2021-04-15T08:00:15.125Z]   github.com/tikv/pd/server/cluster.(*RaftCluster).runBackgroundJobs()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/src/github.com/pingcap/pd/server/cluster/cluster.go:325 +0x12c
[2021-04-15T08:00:15.125Z] 
[2021-04-15T08:00:15.125Z] Goroutine 363 (running) created at:
[2021-04-15T08:00:15.125Z]   net/http.(*Server).Serve()
[2021-04-15T08:00:15.125Z]       /usr/local/go/src/net/http/server.go:2927 +0x5be
[2021-04-15T08:00:15.125Z]   go.etcd.io/etcd/embed.(*serveCtx).serve.func3()
[2021-04-15T08:00:15.125Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/embed/serve.go:135 +0x4c
[2021-04-15T08:00:15.126Z] 
[2021-04-15T08:00:15.126Z] Goroutine 487 (running) created at:
[2021-04-15T08:00:15.126Z]   github.com/tikv/pd/server/cluster.(*RaftCluster).Start()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/src/github.com/pingcap/pd/server/cluster/cluster.go:265 +0xb42
[2021-04-15T08:00:15.126Z]   github.com/tikv/pd/server.(*Server).bootstrapCluster()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/src/github.com/pingcap/pd/server/server.go:633 +0x14bc
[2021-04-15T08:00:15.126Z]   github.com/tikv/pd/server.(*Server).Bootstrap()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/src/github.com/pingcap/pd/server/grpc_service.go:211 +0x3e1
[2021-04-15T08:00:15.126Z]   github.com/pingcap/kvproto/pkg/pdpb._PD_Bootstrap_Handler.func1()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/pingcap/kvproto@v0.0.0-20210219064844-c1844a4775d6/pkg/pdpb/pdpb.pb.go:5338 +0xa1
[2021-04-15T08:00:15.126Z]   github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.0.1-0.20190118093823-f849b5445de4/chain.go:31 +0x2d3
[2021-04-15T08:00:15.126Z]   github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:107 +0xff
[2021-04-15T08:00:15.126Z]   github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.0.1-0.20190118093823-f849b5445de4/chain.go:34 +0x1d9
[2021-04-15T08:00:15.126Z]   go.etcd.io/etcd/etcdserver/api/v3rpc.newUnaryInterceptor.func1()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/etcdserver/api/v3rpc/interceptor.go:64 +0x18a
[2021-04-15T08:00:15.126Z]   github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.0.1-0.20190118093823-f849b5445de4/chain.go:34 +0x1d9
[2021-04-15T08:00:15.126Z]   go.etcd.io/etcd/etcdserver/api/v3rpc.newLogUnaryInterceptor.func1()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/etcdserver/api/v3rpc/interceptor.go:71 +0xda
[2021-04-15T08:00:15.126Z]   github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.0.1-0.20190118093823-f849b5445de4/chain.go:39 +0x33f
[2021-04-15T08:00:15.126Z]   github.com/pingcap/kvproto/pkg/pdpb._PD_Bootstrap_Handler()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/github.com/pingcap/kvproto@v0.0.0-20210219064844-c1844a4775d6/pkg/pdpb/pdpb.pb.go:5340 +0x1d8
[2021-04-15T08:00:15.126Z]   google.golang.org/grpc.(*Server).processUnaryRPC()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/google.golang.org/grpc@v1.26.0/server.go:1024 +0x983
[2021-04-15T08:00:15.126Z]   google.golang.org/grpc.(*Server).handleStream()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/google.golang.org/grpc@v1.26.0/server.go:1313 +0x1343
[2021-04-15T08:00:15.126Z]   google.golang.org/grpc.(*Server).serveStreams.func1.1()
[2021-04-15T08:00:15.126Z]       /home/jenkins/agent/workspace/pd_test/go/pkg/mod/google.golang.org/grpc@v1.26.0/server.go:722 +0xc8
[2021-04-15T08:00:15.126Z] ==================

What did you do?

run tests

What did you expect to see?

succeed

What did you see instead?

failed due to the data race

What version of PD are you using (pd-server -V)?

master

Metadata

Metadata

Assignees

No one assigned

    Labels

    severity/majortype/bugThe issue is confirmed as a bug.

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions
      ⚡