*: add the new SEMv2 config and implement most of the functions

cmd/tidb-server/BUILD.bazel

@@ -52,6 +52,7 @@ go_library(
         "//pkg/util/printer",
         "//pkg/util/redact",
         "//pkg/util/sem",
+        "//pkg/util/sem/v2:sem",
         "//pkg/util/signal",
         "//pkg/util/stmtsummary/v2:stmtsummary",
         "//pkg/util/sys/linux",

cmd/tidb-server/main.go

@@ -77,6 +77,7 @@ import (
 	"github.com/pingcap/tidb/pkg/util/printer"
 	"github.com/pingcap/tidb/pkg/util/redact"
 	"github.com/pingcap/tidb/pkg/util/sem"
+	semv2 "github.com/pingcap/tidb/pkg/util/sem/v2"
 	"github.com/pingcap/tidb/pkg/util/signal"
 	stmtsummaryv2 "github.com/pingcap/tidb/pkg/util/stmtsummary/v2"
 	"github.com/pingcap/tidb/pkg/util/sys/linux"
@@ -368,6 +369,7 @@ func main() {
 		logutil.BgLogger().Warn("internal check is enabled, this should NOT happen in the production environment")
 	}
 	setGlobalVars()
+	setupSEM()
 	err = setCPUAffinity()
 	terror.MustNil(err)
 	cgmon.StartCgroupMonitor()
@@ -879,10 +881,6 @@ func setGlobalVars() {
 	}
 	vardef.GlobalLogMaxDays.Store(int32(config.GetGlobalConfig().Log.File.MaxDays))
 
-	if cfg.Security.EnableSEM {
-		sem.Enable()
-	}
-
 	// For CI environment we default enable prepare-plan-cache.
 	if config.CheckTableBeforeDrop { // only for test
 		variable.SetSysVar(vardef.TiDBEnablePrepPlanCache, variable.BoolToOnOff(true))
@@ -1098,3 +1096,18 @@ func enablePyroscope() {
 		}
 	}
 }
+
+func setupSEM() {
+	cfg := config.GetGlobalConfig()
+
+	if cfg.Security.EnableSEM {
+		if cfg.Security.SEMConfig != "" {
+			err := semv2.Enable(cfg.Security.SEMConfig)
+			if err != nil {
+				logutil.BgLogger().Fatal("failed to enable SEM", zap.Error(err))
+			}
+		} else {
+			sem.Enable()
+		}
+	}
+}

pkg/config/config.go

@@ -601,6 +601,8 @@ type Security struct {
 	SpilledFileEncryptionMethod string `toml:"spilled-file-encryption-method" json:"spilled-file-encryption-method"`
 	// EnableSEM prevents SUPER users from having full access.
 	EnableSEM bool `toml:"enable-sem" json:"enable-sem"`
+	// SEMConfig represents the path to the SEM configuration file.
+	SEMConfig string `toml:"sem-config" json:"sem-config"`
 	// Allow automatic TLS certificate generation
 	AutoTLS         bool   `toml:"auto-tls" json:"auto-tls"`
 	MinTLSVersion   string `toml:"tls-version" json:"tls-version"`
@@ -1069,6 +1071,7 @@ var defaultConf = Config{
 	Security: Security{
 		SpilledFileEncryptionMethod: SpilledFileEncryptionMethodPlaintext,
 		EnableSEM:                   false,
+		SEMConfig:                   "",
 		AutoTLS:                     false,
 		RSAKeySize:                  4096,
 		AuthTokenJWKS:               "",

pkg/ddl/BUILD.bazel

@@ -370,7 +370,7 @@ go_test(
         "//pkg/util/generic",
         "//pkg/util/mathutil",
         "//pkg/util/mock",
-        "//pkg/util/sem",
+        "//pkg/util/sem/compat",
         "//pkg/util/sqlexec",
         "//pkg/util/timeutil",
         "@com_github_fsouza_fake_gcs_server//fakestorage",

pkg/ddl/db_cache_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/pingcap/tidb/pkg/testkit"
 	"github.com/pingcap/tidb/pkg/testkit/external"
 	"github.com/pingcap/tidb/pkg/util/dbterror"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	sem "github.com/pingcap/tidb/pkg/util/sem/compat"
 	"github.com/stretchr/testify/require"
 )
 
@@ -164,9 +164,13 @@ func TestCacheTableSizeLimit(t *testing.T) {
 }
 
 func TestIssue34069(t *testing.T) {
+	testIssue34069(t, sem.V1)
+	testIssue34069(t, sem.V2)
+}
+
+func testIssue34069(t *testing.T, semVer string) {
 	store := testkit.CreateMockStore(t)
-	sem.Enable()
-	defer sem.Disable()
+	defer sem.SwitchToSEMForTest(t, semVer)()
 
 	tk := testkit.NewTestKit(t, store)
 	require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil, nil))

pkg/executor/BUILD.bazel

@@ -249,6 +249,8 @@ go_library(
         "//pkg/util/rowDecoder",
         "//pkg/util/rowcodec",
         "//pkg/util/sem",
+        "//pkg/util/sem/compat",
+        "//pkg/util/sem/v2:sem",
         "//pkg/util/servermemorylimit",
         "//pkg/util/set",
         "//pkg/util/size",

pkg/executor/brie.go

@@ -50,7 +50,7 @@ import (
 	"github.com/pingcap/tidb/pkg/util/dbterror/exeerrors"
 	"github.com/pingcap/tidb/pkg/util/dbterror/plannererrors"
 	"github.com/pingcap/tidb/pkg/util/printer"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	semv1 "github.com/pingcap/tidb/pkg/util/sem"
 	"github.com/pingcap/tidb/pkg/util/syncutil"
 	filter "github.com/pingcap/tidb/pkg/util/table-filter"
 	"github.com/tikv/client-go/v2/oracle"
@@ -299,14 +299,17 @@ func (b *executorBuilder) buildBRIE(s *ast.BRIEStmt, schema *expression.Schema)
 		storage.ExtractQueryParameters(storageURL, &cfg.S3)
 	case "gs", "gcs":
 		storage.ExtractQueryParameters(storageURL, &cfg.GCS)
+
+	// Only check `semv1.IsEnabled()` because in SEM v2, the statement will be limited by `RESTRICTED_SQL` configuration in
+	// `(b *PlanBuilder).Build`. `sql_rule.go` is used to define the highly customized SQL rules to filter these statements.
 	case "hdfs":
-		if sem.IsEnabled() {
+		if semv1.IsEnabled() {
 			// Storage is not permitted to be hdfs when SEM is enabled.
 			b.err = plannererrors.ErrNotSupportedWithSem.GenWithStackByArgs("hdfs storage")
 			return nil
 		}
 	case "local", "file", "":
-		if sem.IsEnabled() {
+		if semv1.IsEnabled() {
 			// Storage is not permitted to be local when SEM is enabled.
 			b.err = plannererrors.ErrNotSupportedWithSem.GenWithStackByArgs("local storage")
 			return nil

pkg/executor/importer/import.go

@@ -67,7 +67,7 @@ import (
 	"github.com/pingcap/tidb/pkg/util/filter"
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	"github.com/pingcap/tidb/pkg/util/naming"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	semv1 "github.com/pingcap/tidb/pkg/util/sem"
 	"github.com/pingcap/tidb/pkg/util/stringutil"
 	pd "github.com/tikv/pd/client"
 	"go.uber.org/zap"
@@ -658,7 +658,9 @@ func (p *Plan) initOptions(ctx context.Context, seCtx sessionctx.Context, option
 	}
 	p.specifiedOptions = specifiedOptions
 
-	if kerneltype.IsNextGen() && sem.IsEnabled() {
+	// Only check `semv1.IsEnabled()` because in SEM v2, the statement will be limited by `RESTRICTED_SQL` configuration in
+	// `(b *PlanBuilder).Build`. `sql_rule.go` is used to define the highly customized SQL rules to filter these statements.
+	if kerneltype.IsNextGen() && semv1.IsEnabled() {
 		if p.DataSourceType == DataSourceTypeQuery {
 			return plannererrors.ErrNotSupportedWithSem.GenWithStackByArgs("IMPORT INTO from select")
 		}

pkg/executor/infoschema_reader.go

@@ -78,7 +78,7 @@ import (
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	"github.com/pingcap/tidb/pkg/util/memory"
 	"github.com/pingcap/tidb/pkg/util/resourcegrouptag"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	sem "github.com/pingcap/tidb/pkg/util/sem/compat"
 	"github.com/pingcap/tidb/pkg/util/servermemorylimit"
 	"github.com/pingcap/tidb/pkg/util/set"
 	"github.com/pingcap/tidb/pkg/util/stringutil"

pkg/executor/set.go

@@ -40,7 +40,8 @@ import (
 	disttaskutil "github.com/pingcap/tidb/pkg/util/disttask"
 	"github.com/pingcap/tidb/pkg/util/gcutil"
 	"github.com/pingcap/tidb/pkg/util/logutil"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	sem "github.com/pingcap/tidb/pkg/util/sem/compat"
+	semv2 "github.com/pingcap/tidb/pkg/util/sem/v2"
 	"github.com/tikv/client-go/v2/oracle/oracles"
 	"go.uber.org/zap"
 )
@@ -136,6 +137,14 @@ func (e *SetExecutor) setSysVariable(ctx context.Context, name string, v *expres
 		}
 	}
 
+	// Check read-only system variables in SEM mode.
+	if semv2.IsEnabled() && semv2.IsReadOnlyVariable(v.Name) {
+		pm := privilege.GetPrivilegeManager(e.Ctx())
+		if !pm.RequestDynamicVerification(sessionVars.ActiveRoles, "RESTRICTED_VARIABLES_ADMIN", false) {
+			return plannererrors.ErrSpecificAccessDenied.GenWithStackByArgs("RESTRICTED_VARIABLES_ADMIN")
+		}
+	}
+
 	if sysVar.IsNoop && !vardef.EnableNoopVariables.Load() {
 		// The variable is a noop. For compatibility we allow it to still
 		// be changed, but we append a warning since users might be expecting

pkg/executor/show.go

@@ -80,7 +80,7 @@ import (
 	"github.com/pingcap/tidb/pkg/util/hint"
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	"github.com/pingcap/tidb/pkg/util/memory"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	sem "github.com/pingcap/tidb/pkg/util/sem/compat"
 	"github.com/pingcap/tidb/pkg/util/set"
 	"github.com/pingcap/tidb/pkg/util/sqlexec"
 	"github.com/pingcap/tidb/pkg/util/stringutil"

pkg/executor/simple.go

@@ -66,7 +66,7 @@ import (
 	"github.com/pingcap/tidb/pkg/util/hack"
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	pwdValidator "github.com/pingcap/tidb/pkg/util/password-validation"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	sem "github.com/pingcap/tidb/pkg/util/sem/compat"
 	"github.com/pingcap/tidb/pkg/util/sqlescape"
 	"github.com/pingcap/tidb/pkg/util/sqlexec"
 	"github.com/pingcap/tidb/pkg/util/timeutil"

pkg/expression/BUILD.bazel

@@ -124,7 +124,7 @@ go_library(
         "//pkg/util/plancodec",
         "//pkg/util/printer",
         "//pkg/util/redact",
-        "//pkg/util/sem",
+        "//pkg/util/sem/compat",
         "//pkg/util/set",
         "//pkg/util/size",
         "//pkg/util/sqlexec",

pkg/expression/aggregation/main_test.go

@@ -28,6 +28,7 @@ func TestMain(m *testing.M) {
 		goleak.IgnoreTopFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"),
 		goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
 		goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
+		goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
 	}
 	goleak.VerifyTestMain(m, opts...)
 }

pkg/expression/extension.go

@@ -27,7 +27,7 @@ import (
 	"github.com/pingcap/tidb/pkg/sessionctx/variable"
 	"github.com/pingcap/tidb/pkg/types"
 	"github.com/pingcap/tidb/pkg/util/chunk"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	sem "github.com/pingcap/tidb/pkg/util/sem/compat"
 )
 
 var extensionFuncs sync.Map

pkg/extension/BUILD.bazel

@@ -61,7 +61,7 @@ go_test(
         "//pkg/types",
         "//pkg/util/chunk",
         "//pkg/util/mock",
-        "//pkg/util/sem",
+        "//pkg/util/sem/compat",
         "@com_github_pingcap_errors//:errors",
         "@com_github_stretchr_testify//mock",
         "@com_github_stretchr_testify//require",

pkg/extension/function_test.go

@@ -31,7 +31,7 @@ import (
 	"github.com/pingcap/tidb/pkg/types"
 	"github.com/pingcap/tidb/pkg/util/chunk"
 	"github.com/pingcap/tidb/pkg/util/mock"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	sem "github.com/pingcap/tidb/pkg/util/sem/compat"
 	"github.com/stretchr/testify/require"
 )
 
@@ -270,9 +270,13 @@ func checkFuncList(t *testing.T, orgList []string, customFuncs ...string) {
 }
 
 func TestExtensionFuncPrivilege(t *testing.T) {
+	testExtensionFuncPrivilege(t, sem.V1)
+	testExtensionFuncPrivilege(t, sem.V2)
+}
+
+func testExtensionFuncPrivilege(t *testing.T, semVer string) {
 	defer func() {
 		extension.Reset()
-		sem.Disable()
 	}()
 
 	extension.Reset()
@@ -403,7 +407,7 @@ func TestExtensionFuncPrivilege(t *testing.T) {
 	tk1.MustQuery("select custom_both_dyn_priv_func()").Check(testkit.Rows("ghi"))
 	tk1.MustQuery("select custom_eval_int_func()").Check(testkit.Rows("1"))
 
-	sem.Enable()
+	defer sem.SwitchToSEMForTest(t, semVer)()
 
 	// root in sem
 	require.NoError(t, tk1.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "localhost"}, nil, nil, nil))
@@ -461,7 +465,6 @@ func TestExtensionFuncPrivilege(t *testing.T) {
 func TestShouldNotOptimizeExtensionFunc(t *testing.T) {
 	defer func() {
 		extension.Reset()
-		sem.Disable()
 	}()
 
 	extension.Reset()

pkg/extension/registry_test.go

@@ -26,7 +26,7 @@ import (
 	"github.com/pingcap/tidb/pkg/sessionctx/vardef"
 	"github.com/pingcap/tidb/pkg/sessionctx/variable"
 	"github.com/pingcap/tidb/pkg/testkit"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	sem "github.com/pingcap/tidb/pkg/util/sem/compat"
 	"github.com/stretchr/testify/require"
 )
 
@@ -184,6 +184,11 @@ func TestRegisterExtensionWithSysVars(t *testing.T) {
 }
 
 func TestSetVariablePrivilege(t *testing.T) {
+	testSetVariablePrivilege(t, sem.V1)
+	testSetVariablePrivilege(t, sem.V2)
+}
+
+func testSetVariablePrivilege(t *testing.T, semVer string) {
 	defer extension.Reset()
 
 	sysVar1 := &variable.SysVar{
@@ -227,7 +232,6 @@ func TestSetVariablePrivilege(t *testing.T) {
 	tk2 := testkit.NewTestKit(t, store)
 	require.NoError(t, tk2.Session().Auth(&auth.UserIdentity{Username: "u2", Hostname: "localhost"}, nil, nil, nil))
 
-	sem.Disable()
 	tk1.MustExec("set @@var1=7")
 	tk1.MustQuery("select @@var1").Check(testkit.Rows("7"))
 
@@ -249,8 +253,7 @@ func TestSetVariablePrivilege(t *testing.T) {
 	tk2.MustExec("set @@global.var1=18")
 	tk2.MustQuery("select @@global.var1").Check(testkit.Rows("18"))
 
-	sem.Enable()
-	defer sem.Disable()
+	defer sem.SwitchToSEMForTest(t, semVer)()
 
 	require.EqualError(t, tk1.ExecToErr("set @@global.var1=27"), "[planner:1227]Access denied; you need (at least one of) the restricted_priv3 privilege(s) for this operation")
 	tk1.MustQuery("select @@global.var1").Check(testkit.Rows("18"))
@@ -264,6 +267,11 @@ func TestSetVariablePrivilege(t *testing.T) {
 }
 
 func TestCustomAccessCheck(t *testing.T) {
+	testCustomAccessCheck(t, sem.V1)
+	testCustomAccessCheck(t, sem.V2)
+}
+
+func testCustomAccessCheck(t *testing.T, semVer string) {
 	defer extension.Reset()
 	extension.Reset()
 
@@ -331,8 +339,7 @@ func TestCustomAccessCheck(t *testing.T) {
 	tk2.MustExec("update t1 set v=12 where id<2")
 	tk2.MustQuery("select * from t1 where id=1").Check(testkit.Rows("1 12"))
 
-	sem.Enable()
-	defer sem.Disable()
+	defer sem.SwitchToSEMForTest(t, semVer)()
 
 	require.EqualError(t, tk1.ExecToErr("update t1 set v=21 where id=1"), "[planner:8121]privilege check for 'Update' fail")
 	require.EqualError(t, tk1.ExecToErr("update t1 set v=21 where id<2"), "[planner:8121]privilege check for 'Update' fail")

pkg/infoschema/BUILD.bazel

@@ -56,7 +56,7 @@ go_library(
         "//pkg/util/intest",
         "//pkg/util/logutil",
         "//pkg/util/mock",
-        "//pkg/util/sem",
+        "//pkg/util/sem/compat",
         "//pkg/util/set",
         "//pkg/util/size",
         "//pkg/util/stmtsummary",

pkg/infoschema/cluster.go

@@ -26,7 +26,7 @@ import (
 	"github.com/pingcap/tidb/pkg/sessionctx"
 	"github.com/pingcap/tidb/pkg/types"
 	"github.com/pingcap/tidb/pkg/util/intest"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	sem "github.com/pingcap/tidb/pkg/util/sem/compat"
 )
 
 // Cluster table indicates that these tables need to get data from other tidb nodes, which may get from all other nodes, or may get from the ddl owner.

pkg/infoschema/tables.go

@@ -58,7 +58,7 @@ import (
 	"github.com/pingcap/tidb/pkg/util/deadlockhistory"
 	"github.com/pingcap/tidb/pkg/util/execdetails"
 	"github.com/pingcap/tidb/pkg/util/logutil"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	sem "github.com/pingcap/tidb/pkg/util/sem/compat"
 	"github.com/pingcap/tidb/pkg/util/set"
 	"github.com/pingcap/tidb/pkg/util/stmtsummary"
 	"github.com/tikv/client-go/v2/tikv"

pkg/planner/core/BUILD.bazel

@@ -188,6 +188,8 @@ go_library(
         "//pkg/util/redact",
         "//pkg/util/rowcodec",
         "//pkg/util/sem",
+        "//pkg/util/sem/compat",
+        "//pkg/util/sem/v2:sem",
         "//pkg/util/set",
         "//pkg/util/size",
         "//pkg/util/sqlexec",

pkg/planner/core/expression_rewriter.go

@@ -46,7 +46,7 @@ import (
 	"github.com/pingcap/tidb/pkg/util/dbterror/plannererrors"
 	"github.com/pingcap/tidb/pkg/util/hint"
 	"github.com/pingcap/tidb/pkg/util/intest"
-	"github.com/pingcap/tidb/pkg/util/sem"
+	sem "github.com/pingcap/tidb/pkg/util/sem/compat"
 	"github.com/pingcap/tidb/pkg/util/stringutil"
 )