oss: detect ECS region and use internal endpoint automaticaly

br/pkg/pdutil/BUILD.bazel

@@ -10,11 +10,11 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//br/pkg/errors",
-        "//br/pkg/httputil",
         "//pkg/kv",
         "//pkg/store/pdtypes",
         "//pkg/tablecodec",
         "//pkg/util/codec",
+        "//pkg/util/httputil",
         "@com_github_coreos_go_semver//semver",
         "@com_github_docker_go_units//:go-units",
         "@com_github_google_uuid//:uuid",

br/pkg/pdutil/utils.go

@@ -13,10 +13,10 @@ import (
 
 	"github.com/pingcap/errors"
 	berrors "github.com/pingcap/tidb/br/pkg/errors"
-	"github.com/pingcap/tidb/br/pkg/httputil"
 	"github.com/pingcap/tidb/pkg/store/pdtypes"
 	"github.com/pingcap/tidb/pkg/tablecodec"
 	"github.com/pingcap/tidb/pkg/util/codec"
+	"github.com/pingcap/tidb/pkg/util/httputil"
 	pd "github.com/tikv/pd/client/http"
 )
 

br/pkg/stream/BUILD.bazel

@@ -21,7 +21,6 @@ go_library(
         "//br/pkg/encryption",
         "//br/pkg/errors",
         "//br/pkg/glue",
-        "//br/pkg/httputil",
         "//br/pkg/logutil",
         "//br/pkg/metautil",
         "//br/pkg/restore/ingestrec",
@@ -43,6 +42,7 @@ go_library(
         "//pkg/tablecodec",
         "//pkg/util",
         "//pkg/util/codec",
+        "//pkg/util/httputil",
         "//pkg/util/mathutil",
         "//pkg/util/table-filter",
         "//pkg/util/versioninfo",

br/pkg/stream/stream_status.go

@@ -21,11 +21,11 @@ import (
 	"github.com/pingcap/log"
 	"github.com/pingcap/tidb/br/pkg/conn"
 	"github.com/pingcap/tidb/br/pkg/glue"
-	"github.com/pingcap/tidb/br/pkg/httputil"
 	"github.com/pingcap/tidb/br/pkg/logutil"
 	. "github.com/pingcap/tidb/br/pkg/streamhelper"
 	"github.com/pingcap/tidb/br/pkg/utils"
 	"github.com/pingcap/tidb/pkg/objstore"
+	"github.com/pingcap/tidb/pkg/util/httputil"
 	"github.com/tikv/client-go/v2/oracle"
 	pd "github.com/tikv/pd/client"
 	"go.uber.org/zap"

br/pkg/task/BUILD.bazel

@@ -30,7 +30,6 @@ go_library(
         "//br/pkg/encryption",
         "//br/pkg/errors",
         "//br/pkg/glue",
-        "//br/pkg/httputil",
         "//br/pkg/logutil",
         "//br/pkg/metautil",
         "//br/pkg/pdutil",
@@ -74,6 +73,7 @@ go_library(
         "//pkg/util/codec",
         "//pkg/util/collate",
         "//pkg/util/engine",
+        "//pkg/util/httputil",
         "//pkg/util/table-filter",
         "@com_github_docker_go_units//:go-units",
         "@com_github_fatih_color//:color",

br/pkg/task/restore.go

@@ -29,7 +29,6 @@ import (
 	connutil "github.com/pingcap/tidb/br/pkg/conn/util"
 	berrors "github.com/pingcap/tidb/br/pkg/errors"
 	"github.com/pingcap/tidb/br/pkg/glue"
-	"github.com/pingcap/tidb/br/pkg/httputil"
 	"github.com/pingcap/tidb/br/pkg/logutil"
 	"github.com/pingcap/tidb/br/pkg/metautil"
 	"github.com/pingcap/tidb/br/pkg/pdutil"
@@ -59,6 +58,7 @@ import (
 	"github.com/pingcap/tidb/pkg/util/codec"
 	"github.com/pingcap/tidb/pkg/util/collate"
 	"github.com/pingcap/tidb/pkg/util/engine"
+	"github.com/pingcap/tidb/pkg/util/httputil"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/tikv/client-go/v2/tikv"

br/pkg/task/restore_raw.go

@@ -12,14 +12,14 @@ import (
 	"github.com/pingcap/tidb/br/pkg/conn"
 	berrors "github.com/pingcap/tidb/br/pkg/errors"
 	"github.com/pingcap/tidb/br/pkg/glue"
-	"github.com/pingcap/tidb/br/pkg/httputil"
 	"github.com/pingcap/tidb/br/pkg/logutil"
 	"github.com/pingcap/tidb/br/pkg/metautil"
 	"github.com/pingcap/tidb/br/pkg/restore"
 	snapclient "github.com/pingcap/tidb/br/pkg/restore/snap_client"
 	restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
 	"github.com/pingcap/tidb/br/pkg/rtree"
 	"github.com/pingcap/tidb/br/pkg/summary"
+	"github.com/pingcap/tidb/pkg/util/httputil"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"go.uber.org/zap"

br/pkg/task/stream.go

@@ -40,7 +40,6 @@ import (
 	"github.com/pingcap/tidb/br/pkg/encryption"
 	berrors "github.com/pingcap/tidb/br/pkg/errors"
 	"github.com/pingcap/tidb/br/pkg/glue"
-	"github.com/pingcap/tidb/br/pkg/httputil"
 	"github.com/pingcap/tidb/br/pkg/logutil"
 	"github.com/pingcap/tidb/br/pkg/metautil"
 	"github.com/pingcap/tidb/br/pkg/pdutil"
@@ -67,6 +66,7 @@ import (
 	"github.com/pingcap/tidb/pkg/tablecodec"
 	"github.com/pingcap/tidb/pkg/util/cdcutil"
 	"github.com/pingcap/tidb/pkg/util/codec"
+	"github.com/pingcap/tidb/pkg/util/httputil"
 	"github.com/spf13/pflag"
 	"github.com/tikv/client-go/v2/oracle"
 	pd "github.com/tikv/pd/client"

br/tests/br_key_locked/BUILD.bazel

@@ -9,14 +9,14 @@ go_library(
     importpath = "github.com/pingcap/tidb/br/tests/br_key_locked",
     visibility = ["//visibility:private"],
     deps = [
-        "//br/pkg/httputil",
         "//br/pkg/task",
         "//pkg/config",
         "//pkg/kv",
         "//pkg/meta/model",
         "//pkg/store/driver",
         "//pkg/tablecodec",
         "//pkg/util/codec",
+        "//pkg/util/httputil",
         "@com_github_pingcap_errors//:errors",
         "@com_github_pingcap_kvproto//pkg/kvrpcpb",
         "@com_github_pingcap_kvproto//pkg/metapb",

br/tests/br_key_locked/locker.go

@@ -34,13 +34,13 @@ import (
 	"github.com/pingcap/errors"
 	"github.com/pingcap/kvproto/pkg/kvrpcpb"
 	"github.com/pingcap/log"
-	"github.com/pingcap/tidb/br/pkg/httputil"
 	"github.com/pingcap/tidb/br/pkg/task"
 	"github.com/pingcap/tidb/pkg/config"
 	"github.com/pingcap/tidb/pkg/kv"
 	"github.com/pingcap/tidb/pkg/meta/model"
 	"github.com/pingcap/tidb/pkg/store/driver"
 	"github.com/pingcap/tidb/pkg/tablecodec"
+	"github.com/pingcap/tidb/pkg/util/httputil"
 	"github.com/tikv/client-go/v2/oracle"
 	"github.com/tikv/client-go/v2/tikv"
 	"github.com/tikv/client-go/v2/tikvrpc"

pkg/lightning/common/BUILD.bazel

@@ -21,7 +21,6 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//br/pkg/errors",
-        "//br/pkg/httputil",
         "//br/pkg/logutil",
         "//pkg/errno",
         "//pkg/ingestor/errdef",
@@ -38,6 +37,7 @@ go_library(
         "//pkg/util/codec",
         "//pkg/util/dbutil",
         "//pkg/util/format",
+        "//pkg/util/httputil",
         "//pkg/util/logutil",
         "@com_github_cockroachdb_pebble//:pebble",
         "@com_github_go_sql_driver_mysql//:mysql",
@@ -118,7 +118,7 @@ go_test(
     ],
     embed = [":common"],
     flaky = True,
-    shard_count = 30,
+    shard_count = 29,
     deps = [
         "//br/pkg/errors",
         "//pkg/autoid_service",

pkg/lightning/common/security.go

@@ -23,8 +23,8 @@ import (
 	"strings"
 
 	"github.com/pingcap/errors"
-	"github.com/pingcap/tidb/br/pkg/httputil"
 	"github.com/pingcap/tidb/pkg/util"
+	"github.com/pingcap/tidb/pkg/util/httputil"
 	"github.com/tikv/client-go/v2/config"
 	pd "github.com/tikv/pd/client"
 	"google.golang.org/grpc"
@@ -132,7 +132,7 @@ func (tc *TLS) WrapListener(l net.Listener) net.Listener {
 
 // GetJSON performs a GET request to the given path and unmarshals the response
 func (tc *TLS) GetJSON(ctx context.Context, path string, v any) error {
-	return GetJSON(ctx, tc.client, tc.url+path, v)
+	return httputil.GetJSON(ctx, tc.client, tc.url+path, v)
 }
 
 // ToPDSecurityOption converts the TLS configuration to a PD security option.

pkg/lightning/common/util.go

@@ -20,11 +20,9 @@ import (
 	"crypto/tls"
 	"database/sql"
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net"
-	"net/http"
 	"os"
 	"runtime"
 	"strconv"
@@ -414,42 +412,6 @@ func SchemaExists(ctx context.Context, db dbutil.QueryExecutor, schema string) (
 	}
 }
 
-// GetJSON fetches a page and parses it as JSON. The parsed result will be
-// stored into the `v`. The variable `v` must be a pointer to a type that can be
-// unmarshalled from JSON.
-//
-// Example:
-//
-//	client := &http.Client{}
-//	var resp struct { IP string }
-//	if err := util.GetJSON(client, "http://api.ipify.org/?format=json", &resp); err != nil {
-//		return errors.Trace(err)
-//	}
-//	fmt.Println(resp.IP)
-func GetJSON(ctx context.Context, client *http.Client, url string, v any) error {
-	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
-	if err != nil {
-		return errors.Trace(err)
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return errors.Trace(err)
-	}
-
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return errors.Trace(err)
-		}
-		return errors.Errorf("get %s http status code != 200, message %s", url, string(body))
-	}
-
-	return errors.Trace(json.NewDecoder(resp.Body).Decode(v))
-}
-
 // KillMySelf sends sigint to current process, used in integration test only
 //
 // Only works on Unix. Signaling on Windows is not supported.

pkg/lightning/common/util_test.go

@@ -17,13 +17,9 @@ package common_test
 import (
 	"context"
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
 	"io"
-	"net/http"
-	"net/http/httptest"
 	"testing"
-	"time"
 
 	"github.com/DATA-DOG/go-sqlmock"
 	"github.com/go-sql-driver/mysql"
@@ -44,46 +40,6 @@ func TestDirNotExist(t *testing.T) {
 	require.False(t, common.IsDirExists("not-exists"))
 }
 
-func TestGetJSON(t *testing.T) {
-	type TestPayload struct {
-		Username string `json:"username"`
-		Password string `json:"password"`
-	}
-	request := TestPayload{
-		Username: "lightning",
-		Password: "lightning-ctl",
-	}
-
-	ctx := context.Background()
-	// Mock success response
-	handle := func(res http.ResponseWriter, _ *http.Request) {
-		res.WriteHeader(http.StatusOK)
-		err := json.NewEncoder(res).Encode(request)
-		require.NoError(t, err)
-	}
-	testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
-		handle(res, req)
-	}))
-	defer testServer.Close()
-
-	client := &http.Client{Timeout: time.Second}
-
-	response := TestPayload{}
-	err := common.GetJSON(ctx, client, "http://localhost:1", &response)
-	require.Error(t, err)
-	err = common.GetJSON(ctx, client, testServer.URL, &response)
-	require.NoError(t, err)
-	require.Equal(t, request, response)
-
-	// Mock `StatusNoContent` response
-	handle = func(res http.ResponseWriter, _ *http.Request) {
-		res.WriteHeader(http.StatusNoContent)
-	}
-	err = common.GetJSON(ctx, client, testServer.URL, &response)
-	require.Error(t, err)
-	require.Regexp(t, ".*http status code != 200.*", err.Error())
-}
-
 func TestConnect(t *testing.T) {
 	plainPsw := "dQAUoDiyb1ucWZk7"
 

pkg/objstore/ossstore/BUILD.bazel

@@ -19,6 +19,7 @@ go_library(
         "//pkg/objstore/s3like",
         "//pkg/objstore/storeapi",
         "//pkg/util",
+        "//pkg/util/httputil",
         "//pkg/util/logutil",
         "@com_github_alibabacloud_go_tea//tea",
         "@com_github_aliyun_alibabacloud_oss_go_sdk_v2//oss",
@@ -42,7 +43,7 @@ go_test(
     ],
     embed = [":ossstore"],
     flaky = True,
-    shard_count = 8,
+    shard_count = 10,
     deps = [
         "//dumpling/context",
         "//pkg/objstore/ossstore/mock",

pkg/objstore/ossstore/logger.go

@@ -16,7 +16,6 @@ package ossstore
 
 import (
 	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
-	"github.com/pingcap/log"
 	tidblogutil "github.com/pingcap/tidb/pkg/util/logutil"
 	"go.uber.org/zap"
 )
@@ -25,9 +24,14 @@ type logPrinter struct {
 	logger *zap.Logger
 }
 
-func newLogPrinter(extraFields ...zap.Field) *logPrinter {
+func newLogPrinter(logger *zap.Logger) *logPrinter {
 	return &logPrinter{
-		logger: log.L().WithOptions(zap.AddCallerSkip(1)).With(extraFields...),
+		// skip 3 levels to reach the original caller
+		// top: this logger
+		// top-1: standardLogger.printf of OSS
+		// top-2: standardLogger.Info/Error/Debug/Warn of OSS
+		// top-3: the actual caller in OSS SDK
+		logger: logger.WithOptions(zap.AddCallerSkip(3)),
 	}
 }