Merge pull request #5385 from dolthub/aaron/nbs-table-file-finalizer-panic-always-on

go/store/nbs: Make the panic in the finalizer for an unclosed table file index always on.
This commit is contained in:
Aaron Son
2023-03-29 14:53:14 -07:00
committed by GitHub
45 changed files with 744 additions and 223 deletions
+2
View File
@@ -68,6 +68,7 @@ func TestInit(t *testing.T) {
gCfg.SetStrings(test.GlobalConfig)
result := InitCmd{}.Exec(context.Background(), "dolt init", test.Args, dEnv)
defer dEnv.DoltDB.Close()
require.Equalf(t, test.ExpectSuccess, result == 0, "- Expected success: %t; result: %t;", test.ExpectSuccess, result == 0)
@@ -88,6 +89,7 @@ func TestInitTwice(t *testing.T) {
result := InitCmd{}.Exec(context.Background(), "dolt init",
[]string{"-name", "Bill Billerson", "-email", "bigbillieb@fake.horse"}, dEnv)
require.True(t, result == 0, "First init should succeed")
defer dEnv.DoltDB.Close()
result = InitCmd{}.Exec(context.Background(), "dolt init",
[]string{"-name", "Bill Billerson", "-email", "bigbillieb@fake.horse"}, dEnv)
+2
View File
@@ -34,6 +34,7 @@ import (
func TestLog(t *testing.T) {
dEnv := createUninitializedEnv()
err := dEnv.InitRepo(context.Background(), types.Format_Default, "Bill Billerson", "bigbillieb@fake.horse", env.DefaultInitBranch)
defer dEnv.DoltDB.Close()
if err != nil {
t.Error("Failed to init repo")
@@ -52,6 +53,7 @@ func TestLogSigterm(t *testing.T) {
dEnv := createUninitializedEnv()
err := dEnv.InitRepo(context.Background(), types.Format_Default, "Bill Billerson", "bigbillieb@fake.horse", env.DefaultInitBranch)
defer dEnv.DoltDB.Close()
if err != nil {
t.Error("Failed to init repo")
+12
View File
@@ -48,6 +48,7 @@ func TestSqlConsole(t *testing.T) {
t.Run("SQL console opens and exits", func(t *testing.T) {
dEnv, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
args := []string{}
commandStr := "dolt sql"
@@ -75,6 +76,7 @@ func TestSqlBatchMode(t *testing.T) {
t.Run(test.query, func(t *testing.T) {
dEnv, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
args := []string{"-b", "-q", test.query}
@@ -113,6 +115,7 @@ func TestSqlSelect(t *testing.T) {
t.Run(test.query, func(t *testing.T) {
dEnv, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
args := []string{"-q", test.query}
@@ -138,6 +141,7 @@ func TestSqlShow(t *testing.T) {
t.Run(test.query, func(t *testing.T) {
dEnv, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
args := []string{"-q", test.query}
@@ -166,6 +170,7 @@ func TestCreateTable(t *testing.T) {
for _, test := range tests {
t.Run(test.query, func(t *testing.T) {
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
working, err := dEnv.WorkingRoot(context.Background())
assert.Nil(t, err, "Unexpected error")
has, err := working.HasTable(context.Background(), tableName)
@@ -210,6 +215,7 @@ func TestShowTables(t *testing.T) {
t.Run(test.query, func(t *testing.T) {
dEnv, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
args := []string{"-q", test.query}
commandStr := "dolt sql"
@@ -240,6 +246,7 @@ func TestAlterTable(t *testing.T) {
t.Run(test.query, func(t *testing.T) {
dEnv, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
args := []string{"-q", test.query}
commandStr := "dolt sql"
@@ -266,6 +273,7 @@ func TestDropTable(t *testing.T) {
t.Run(test.query, func(t *testing.T) {
dEnv, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
args := []string{"-q", test.query}
commandStr := "dolt sql"
@@ -383,6 +391,7 @@ func TestInsert(t *testing.T) {
ctx := context.Background()
dEnv, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
args := []string{"-q", test.query}
@@ -463,6 +472,7 @@ func TestUpdate(t *testing.T) {
ctx := context.Background()
dEnv, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
args := []string{"-q", test.query}
@@ -536,6 +546,7 @@ func TestDelete(t *testing.T) {
t.Run(test.query, func(t *testing.T) {
dEnv, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
args := []string{"-q", test.query}
@@ -563,6 +574,7 @@ func TestDelete(t *testing.T) {
func TestCommitHooksNoErrors(t *testing.T) {
dEnv, err := sqle.CreateEnvWithSeedData()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
sqle.AddDoltSystemVariables()
sql.SystemVariables.SetGlobal(dsess.SkipReplicationErrors, true)
+5
View File
@@ -53,6 +53,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/events"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/nbs"
"github.com/dolthub/dolt/go/store/util/tempfiles"
)
@@ -145,6 +146,10 @@ func main() {
func runMain() int {
args := os.Args[1:]
if os.Getenv("DOLT_VERBOSE_ASSERT_TABLE_FILES_CLOSED") == "" {
nbs.TableIndexGCFinalizerWithStackTrace = false
}
csMetrics := false
ignoreLockFile := false
if len(args) > 0 {
@@ -184,6 +184,7 @@ func TestEmptyInMemoryRepoCreation(t *testing.T) {
if err != nil {
t.Fatal("Failed to load db")
}
defer ddb.Close()
err = ddb.WriteEmptyRepo(context.Background(), "master", "Bill Billerson", "bigbillieb@fake.horse")
@@ -151,6 +151,7 @@ func TestFeatureVersion(t *testing.T) {
doltdb.DoltFeatureVersion = oldVersion
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
doltdb.DoltFeatureVersion = DoltFeatureVersionCopy
for _, cmd := range test.setup {
+1
View File
@@ -110,6 +110,7 @@ var gcSetupCommon = []testCommand{
func testGarbageCollection(t *testing.T, test gcTest) {
ctx := context.Background()
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
for _, c := range gcSetupCommon {
exitCode := c.cmd.Exec(ctx, c.cmd.Name(), c.args, dEnv)
@@ -451,6 +451,7 @@ func TestInferSchema(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
wrCl, err := dEnv.FS.OpenForWrite(importFilePath, os.ModePerm)
require.NoError(t, err)
+2
View File
@@ -144,6 +144,7 @@ func TestInitRepo(t *testing.T) {
dEnv, _ := createTestEnv(false, false)
err := dEnv.InitRepo(context.Background(), types.Format_Default, "aoeu aoeu", "aoeu@aoeu.org", DefaultInitBranch)
require.NoError(t, err)
defer dEnv.DoltDB.Close()
_, err = dEnv.WorkingRoot(context.Background())
require.NoError(t, err)
@@ -168,6 +169,7 @@ func TestMigrateWorkingSet(t *testing.T) {
err = dEnv.InitRepo(context.Background(), types.Format_Default, "aoeu aoeu", "aoeu@aoeu.org", DefaultInitBranch)
require.NoError(t, err)
defer dEnv.DoltDB.Close()
ws, err := dEnv.WorkingSet(context.Background())
require.NoError(t, err)
@@ -122,6 +122,7 @@ func TestMerge(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
ctx := context.Background()
dEnv := dtu.CreateTestEnv()
defer dEnv.DoltDB.Close()
for _, tc := range setupCommon {
tc.exec(t, ctx, dEnv)
@@ -241,6 +242,7 @@ func TestMergeConflicts(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
ctx := context.Background()
dEnv := dtu.CreateTestEnv()
defer dEnv.DoltDB.Close()
for _, tc := range setupCommon {
tc.exec(t, ctx, dEnv)
@@ -281,6 +283,7 @@ const (
func TestMergeConcurrency(t *testing.T) {
ctx := context.Background()
dEnv := setupConcurrencyTest(t, ctx)
defer dEnv.DoltDB.Close()
_, eng := engineFromEnvironment(ctx, dEnv)
eg, ctx := errgroup.WithContext(ctx)
@@ -106,6 +106,7 @@ func TestKeylessMerge(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
ctx := context.Background()
dEnv := dtu.CreateTestEnv()
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
@@ -257,6 +258,7 @@ func TestKeylessMergeConflicts(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
dEnv := dtu.CreateTestEnv()
defer dEnv.DoltDB.Close()
setupTest(t, ctx, dEnv, test.setup)
root, err := dEnv.WorkingRoot(ctx)
@@ -270,6 +272,7 @@ func TestKeylessMergeConflicts(t *testing.T) {
t.Run(test.name+"_resolved_ours", func(t *testing.T) {
dEnv := dtu.CreateTestEnv()
defer dEnv.DoltDB.Close()
setupTest(t, ctx, dEnv, test.setup)
@@ -287,6 +290,7 @@ func TestKeylessMergeConflicts(t *testing.T) {
})
t.Run(test.name+"_resolved_theirs", func(t *testing.T) {
dEnv := dtu.CreateTestEnv()
defer dEnv.DoltDB.Close()
setupTest(t, ctx, dEnv, test.setup)
+4 -3
View File
@@ -305,7 +305,8 @@ func TestMergeCommits(t *testing.T) {
t.Skip()
}
vrw, ns, rightCommitHash, ancCommitHash, root, mergeRoot, ancRoot, expectedRows, expectedArtifacts := setupMergeTest(t)
ddb, vrw, ns, rightCommitHash, ancCommitHash, root, mergeRoot, ancRoot, expectedRows, expectedArtifacts := setupMergeTest(t)
defer ddb.Close()
merger, err := NewMerger(root, mergeRoot, ancRoot, rightCommitHash, ancCommitHash, vrw, ns)
if err != nil {
t.Fatal(err)
@@ -422,7 +423,7 @@ func sortTests(t []testRow) {
})
}
func setupMergeTest(t *testing.T) (types.ValueReadWriter, tree.NodeStore, doltdb.Rootish, doltdb.Rootish, *doltdb.RootValue, *doltdb.RootValue, *doltdb.RootValue, durable.Index, prolly.ArtifactMap) {
func setupMergeTest(t *testing.T) (*doltdb.DoltDB, types.ValueReadWriter, tree.NodeStore, doltdb.Rootish, doltdb.Rootish, *doltdb.RootValue, *doltdb.RootValue, *doltdb.RootValue, durable.Index, prolly.ArtifactMap) {
ddb := mustMakeEmptyRepo(t)
vrw := ddb.ValueReadWriter()
ns := ddb.NodeStore()
@@ -520,7 +521,7 @@ func setupMergeTest(t *testing.T) (types.ValueReadWriter, tree.NodeStore, doltdb
expectedArtifacts, err := artEditor.Flush(context.Background())
require.NoError(t, err)
return vrw, ns, rightCm, baseCm, root, mergeRoot, ancRoot, durable.IndexFromProllyMap(expectedRows), expectedArtifacts
return ddb, vrw, ns, rightCm, baseCm, root, mergeRoot, ancRoot, durable.IndexFromProllyMap(expectedRows), expectedArtifacts
}
func setupNomsMergeTest(t *testing.T) (types.ValueReadWriter, tree.NodeStore, doltdb.Rootish, doltdb.Rootish, *doltdb.RootValue, *doltdb.RootValue, *doltdb.RootValue, types.Map, types.Map, *MergeStats) {
@@ -558,6 +558,7 @@ func testMergeSchemas(t *testing.T, test mergeSchemaTest) {
}
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
ctx := context.Background()
for _, c := range setupCommon {
@@ -601,6 +602,7 @@ func testMergeSchemasWithConflicts(t *testing.T, test mergeSchemaConflictTest) {
}
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
ctx := context.Background()
for _, c := range setupCommon {
c.exec(t, ctx, dEnv)
@@ -655,6 +657,7 @@ func testMergeSchemasWithConflicts(t *testing.T, test mergeSchemaConflictTest) {
func testMergeForeignKeys(t *testing.T, test mergeForeignKeyTest) {
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
ctx := context.Background()
for _, c := range setupForeignKeyTests {
c.exec(t, ctx, dEnv)
@@ -134,7 +134,8 @@ func TestExists(t *testing.T) {
//NewDataLocation("file.nbf", ""),
}
_, root, fs := createRootAndFS()
ddb, root, fs := createRootAndFS()
defer ddb.Close()
for _, loc := range testLocations {
t.Run(loc.String(), func(t *testing.T) {
@@ -193,6 +194,7 @@ func TestCreateRdWr(t *testing.T) {
}
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(context.Background())
require.NoError(t, err)
dEnv.FS.WriteFile(testSchemaFileName, []byte(testSchema))
@@ -213,6 +213,7 @@ func setupFilterBranchTests(t *testing.T) *env.DoltEnv {
func testFilterBranch(t *testing.T, test filterBranchTest) {
ctx := context.Background()
dEnv := setupFilterBranchTests(t)
defer dEnv.DoltDB.Close()
for _, c := range test.setup {
exitCode := c.cmd.Exec(ctx, c.cmd.Name(), c.args, dEnv)
require.Equal(t, 0, exitCode)
@@ -53,7 +53,9 @@ func testSchemaSerialization(t *testing.T, sch schema.Schema) {
func parseSchemaString(t *testing.T, s string) schema.Schema {
ctx := context.Background()
root, err := dtestutils.CreateTestEnv().WorkingRoot(ctx)
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
_, sch, err := sqlutil.ParseCreateTableStatement(ctx, root, s)
require.NoError(t, err)
@@ -50,7 +50,8 @@ func TestSqlIntegration(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx := context.Background()
root := runTestSql(t, ctx, test.setup)
ddb, root := runTestSql(t, ctx, test.setup)
defer ddb.Close()
tbl, ok, err := root.GetTable(ctx, tblName)
require.NoError(t, err)
@@ -86,7 +87,8 @@ func TestSchemaOrdering(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx := context.Background()
root := runTestSql(t, ctx, []string{test.query})
ddb, root := runTestSql(t, ctx, []string{test.query})
defer ddb.Close()
tbl, ok, err := root.GetTable(ctx, "t")
require.NoError(t, err)
@@ -147,7 +149,8 @@ func TestGetKeyTags(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx := context.Background()
root := runTestSql(t, ctx, test.setup)
ddb, root := runTestSql(t, ctx, test.setup)
defer ddb.Close()
tbl, ok, err := root.GetTable(ctx, tblName)
require.NoError(t, err)
@@ -170,7 +173,7 @@ func TestGetKeyTags(t *testing.T) {
}
}
func runTestSql(t *testing.T, ctx context.Context, setup []string) *doltdb.RootValue {
func runTestSql(t *testing.T, ctx context.Context, setup []string) (*doltdb.DoltDB, *doltdb.RootValue) {
dEnv := dtestutils.CreateTestEnv()
cmd := commands.SqlCmd{}
for _, query := range setup {
@@ -179,5 +182,5 @@ func runTestSql(t *testing.T, ctx context.Context, setup []string) *doltdb.RootV
}
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
return root
return dEnv.DoltDB, root
}
@@ -84,6 +84,7 @@ func TestRenameTable(t *testing.T) {
t.Run(tt.description, func(t *testing.T) {
ctx := context.Background()
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
@@ -228,6 +229,7 @@ func TestAddColumnToTable(t *testing.T) {
ctx := context.Background()
dEnv, err := makePeopleTable(ctx, dtestutils.CreateTestEnv())
require.NoError(t, err)
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
@@ -434,6 +436,7 @@ func TestDropPks(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
ctx := context.Background()
tmpDir, err := dEnv.TempTableFilesDir()
require.NoError(t, err)
@@ -746,6 +749,7 @@ func TestModifyColumn(t *testing.T) {
ctx := context.Background()
dEnv, err := makePeopleTable(ctx, dtestutils.CreateTestEnv())
require.NoError(t, err)
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
assert.NoError(t, err)
@@ -30,6 +30,7 @@ import (
func TestInsertIntoQueryCatalogTable(t *testing.T) {
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
ctx := context.Background()
root, _ := dEnv.WorkingRoot(ctx)
@@ -1135,6 +1135,7 @@ var BranchControlTests = []BranchControlTest{
func TestBranchControl(t *testing.T) {
for _, test := range BranchControlTests {
harness := newDoltHarness(t)
defer harness.Close()
t.Run(test.Name, func(t *testing.T) {
engine, err := harness.NewEngine(t)
require.NoError(t, err)
@@ -1187,6 +1188,7 @@ func TestBranchControl(t *testing.T) {
func TestBranchControlBlocks(t *testing.T) {
for _, test := range BranchControlBlockTests {
harness := newDoltHarness(t)
defer harness.Close()
t.Run(test.Name, func(t *testing.T) {
engine, err := harness.NewEngine(t)
require.NoError(t, err)
File diff suppressed because it is too large Load Diff
@@ -80,6 +80,7 @@ var defaultSkippedQueries = []string{
// Setup sets the setup scripts for this DoltHarness's engine
func (d *DoltHarness) Setup(setupData ...[]setup.SetupScript) {
d.closeProvider()
d.engine = nil
d.provider = nil
d.setupData = nil
@@ -284,6 +285,7 @@ func (d *DoltHarness) SupportsKeylessTables() bool {
}
func (d *DoltHarness) NewDatabases(names ...string) []sql.Database {
d.closeProvider()
d.engine = nil
d.provider = nil
@@ -359,8 +361,31 @@ func (d *DoltHarness) NewDatabaseProvider() sql.MutableDatabaseProvider {
return d.provider
}
func (d *DoltHarness) Close() {
d.closeProvider()
}
func (d *DoltHarness) closeProvider() {
if d.provider != nil {
dbs := sqle.AllDbs(sql.NewEmptyContext(), d.provider)
for _, db := range dbs {
d.t.Logf("closing %v", db)
require.NoError(d.t, db.DbData().Ddb.Close())
}
}
if d.session != nil {
dbs := sqle.AllDbs(sql.NewEmptyContext(), d.session.Provider())
for _, db := range dbs {
d.t.Logf("session had database %v", db)
}
}
}
func (d *DoltHarness) newProvider() sql.MutableDatabaseProvider {
d.closeProvider()
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
store := dEnv.DoltDB.ValueReadWriter().(*types.ValueStore)
store.SetValidateContentAddresses(true)
@@ -31,6 +31,7 @@ import (
"github.com/dolthub/dolt/go/cmd/dolt/commands/sqlserver"
"github.com/dolthub/dolt/go/libraries/doltcore/dtestutils"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
)
// DoltBranchMultiSessionScriptTests contain tests that need to be run in a multi-session server environment
@@ -431,9 +432,10 @@ func TestDropDatabaseMultiSessionBehavior(t *testing.T) {
func testMultiSessionScriptTests(t *testing.T, tests []queries.ScriptTest) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
sc, serverConfig := startServer(t, true, "", "")
dEnv, sc, serverConfig := startServer(t, true, "", "")
err := sc.WaitForStart()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
conn1, sess1 := newConnection(t, serverConfig)
conn2, sess2 := newConnection(t, serverConfig)
@@ -539,7 +541,7 @@ func assertResultsEqual(t *testing.T, expected []sql.Row, rows *gosql.Rows) {
}
// startServer will start sql-server with given host, unix socket file path and whether to use specific port, which is defined randomly.
func startServer(t *testing.T, withPort bool, host string, unixSocketPath string) (*sqlserver.ServerController, sqlserver.ServerConfig) {
func startServer(t *testing.T, withPort bool, host string, unixSocketPath string) (*env.DoltEnv, *sqlserver.ServerController, sqlserver.ServerConfig) {
dEnv := dtestutils.CreateTestEnv()
serverConfig := sqlserver.DefaultServerConfig()
@@ -562,7 +564,7 @@ func startServer(t *testing.T, withPort bool, host string, unixSocketPath string
err := sc.WaitForStart()
require.NoError(t, err)
return sc, serverConfig
return dEnv, sc, serverConfig
}
// newConnection takes sqlserver.serverConfig and opens a connection, and will return that connection with a new session
@@ -581,8 +583,9 @@ func TestDoltServerRunningUnixSocket(t *testing.T) {
const defaultUnixSocketPath = "/tmp/mysql.sock"
// Running unix socket server
sc, serverConfig := startServer(t, false, "", defaultUnixSocketPath)
dEnv, sc, serverConfig := startServer(t, false, "", defaultUnixSocketPath)
sc.WaitForStart()
defer dEnv.DoltDB.Close()
require.True(t, strings.Contains(sqlserver.ConnectionString(serverConfig, "dolt"), "unix"))
// default unix socket connection works
@@ -628,8 +631,9 @@ func TestDoltServerRunningUnixSocket(t *testing.T) {
require.NoFileExists(t, defaultUnixSocketPath)
// Running TCP socket server
tcpSc, tcpServerConfig := startServer(t, true, "0.0.0.0", "")
dEnv, tcpSc, tcpServerConfig := startServer(t, true, "0.0.0.0", "")
tcpSc.WaitForStart()
defer dEnv.DoltDB.Close()
require.False(t, strings.Contains(sqlserver.ConnectionString(tcpServerConfig, "dolt"), "unix"))
t.Run("host and port specified, there should not be unix socket created", func(t *testing.T) {
@@ -35,6 +35,7 @@ func TestDoltTransactionCommitOneClient(t *testing.T) {
// In this test, we're setting only one client to match transaction commits to dolt commits.
// Autocommit is disabled for the enabled client, as it's the recommended way to use this feature.
harness := newDoltHarness(t)
defer harness.Close()
harness.Setup(setup.MydbData)
enginetest.TestTransactionScript(t, harness, queries.TransactionTest{
Name: "dolt commit on transaction commit one client",
@@ -176,6 +177,7 @@ func TestDoltTransactionCommitTwoClients(t *testing.T) {
// In this test, we're setting both clients to match transaction commits to dolt commits.
// Autocommit is disabled, as it's the recommended way to use this feature.
harness := newDoltHarness(t)
defer harness.Close()
enginetest.TestTransactionScript(t, harness, queries.TransactionTest{
Name: "dolt commit on transaction commit two clients",
SetUpScript: []string{
@@ -336,6 +338,7 @@ func TestDoltTransactionCommitAutocommit(t *testing.T) {
// In this test, each insertion from both clients cause a commit as autocommit is enabled.
// Not the recommended way to use the feature, but it's permitted.
harness := newDoltHarness(t)
defer harness.Close()
enginetest.TestTransactionScript(t, harness, queries.TransactionTest{
Name: "dolt commit with autocommit",
SetUpScript: []string{
@@ -414,6 +417,7 @@ func TestDoltTransactionCommitAutocommit(t *testing.T) {
func TestDoltTransactionCommitLateFkResolution(t *testing.T) {
harness := newDoltHarness(t)
defer harness.Close()
enginetest.TestTransactionScript(t, harness, queries.TransactionTest{
Name: "delayed foreign key resolution with transaction commits",
SetUpScript: []string{
@@ -26,6 +26,7 @@ import (
func TestSysbenchTransactionCV(t *testing.T) {
harness := newDoltHarness(t)
defer harness.Close()
enginetest.TestTransactionScript(t, harness, queries.TransactionTest{
Name: "Sysbench Transactions Shouldn't Cause Constraint Violations",
SetUpScript: []string{
@@ -183,7 +183,8 @@ var (
)
func TestDoltIndexEqual(t *testing.T) {
ctx, root, indexMap := doltIndexSetup(t)
ddb, ctx, root, indexMap := doltIndexSetup(t)
defer ddb.Close()
tests := []doltIndexTestCase{
{
@@ -305,7 +306,8 @@ func TestDoltIndexEqual(t *testing.T) {
}
func TestDoltIndexGreaterThan(t *testing.T) {
ctx, root, indexMap := doltIndexSetup(t)
ddb, ctx, root, indexMap := doltIndexSetup(t)
defer ddb.Close()
tests := []struct {
indexName string
@@ -446,7 +448,8 @@ func TestDoltIndexGreaterThan(t *testing.T) {
}
func TestDoltIndexGreaterThanOrEqual(t *testing.T) {
ctx, root, indexMap := doltIndexSetup(t)
ddb, ctx, root, indexMap := doltIndexSetup(t)
defer ddb.Close()
tests := []struct {
indexName string
@@ -583,7 +586,8 @@ func TestDoltIndexGreaterThanOrEqual(t *testing.T) {
}
func TestDoltIndexLessThan(t *testing.T) {
ctx, root, indexMap := doltIndexSetup(t)
ddb, ctx, root, indexMap := doltIndexSetup(t)
defer ddb.Close()
tests := []struct {
indexName string
@@ -729,7 +733,8 @@ func TestDoltIndexLessThan(t *testing.T) {
}
func TestDoltIndexLessThanOrEqual(t *testing.T) {
ctx, root, indexMap := doltIndexSetup(t)
ddb, ctx, root, indexMap := doltIndexSetup(t)
defer ddb.Close()
tests := []struct {
indexName string
@@ -876,7 +881,8 @@ func TestDoltIndexLessThanOrEqual(t *testing.T) {
}
func TestDoltIndexBetween(t *testing.T) {
ctx, root, indexMap := doltIndexSetup(t)
ddb, ctx, root, indexMap := doltIndexSetup(t)
defer ddb.Close()
tests := []doltIndexBetweenTestCase{
{
@@ -1329,7 +1335,7 @@ func testDoltIndex(t *testing.T, ctx *sql.Context, root *doltdb.RootValue, keys
requireUnorderedRowsEqual(t, pkSch.Schema, convertSqlRowToInt64(expectedRows), readRows)
}
func doltIndexSetup(t *testing.T) (*sql.Context, *doltdb.RootValue, map[string]index.DoltIndex) {
func doltIndexSetup(t *testing.T) (*doltdb.DoltDB, *sql.Context, *doltdb.RootValue, map[string]index.DoltIndex) {
ctx := NewTestSQLCtx(context.Background())
dEnv := dtestutils.CreateTestEnv()
root, err := dEnv.WorkingRoot(ctx)
@@ -1402,7 +1408,7 @@ INSERT INTO types VALUES (1, 4, '2020-05-14 12:00:03', 1.1, 'd', 1.1, 'a,c', '00
}
}
return ctx, root, indexMap
return dEnv.DoltDB, ctx, root, indexMap
}
func NewTestSQLCtx(ctx context.Context) *sql.Context {
@@ -151,6 +151,7 @@ func TestDbRevision(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
ctx := context.Background()
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
setup := append(setupCommon, test.setup...)
for _, c := range setup {
@@ -33,6 +33,7 @@ import (
func TestHistoryTable(t *testing.T) {
SkipByDefaultInCI(t)
dEnv := setupHistoryTests(t)
defer dEnv.DoltDB.Close()
for _, test := range historyTableTests() {
t.Run(test.name, func(t *testing.T) {
testHistoryTable(t, test, dEnv)
@@ -20128,6 +20128,7 @@ INSERT INTO join_result VALUES ('stock','ZYNE','us','2017-11-01',9.7,9.93,9.41,9
func TestCreateTables(t *testing.T) {
SkipByDefaultInCI(t)
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
ctx := context.Background()
root, _ := dEnv.WorkingRoot(ctx)
@@ -20150,6 +20151,7 @@ func TestInserts(t *testing.T) {
}
SkipByDefaultInCI(t)
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
ctx := context.Background()
root, _ := dEnv.WorkingRoot(ctx)
@@ -20179,6 +20181,7 @@ func TestInsertsWithIndexes(t *testing.T) {
}
SkipByDefaultInCI(t)
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
ctx := context.Background()
root, _ := dEnv.WorkingRoot(ctx)
@@ -20214,6 +20217,7 @@ func TestInsertsWithIndexes(t *testing.T) {
func TestJoin(t *testing.T) {
SkipByDefaultInCI(t)
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
ctx := context.Background()
root, _ := dEnv.WorkingRoot(ctx)
@@ -20262,6 +20266,7 @@ func assertResultRowsEqual(t *testing.T, expected, actual []sql.Row) {
func TestExplain(t *testing.T) {
SkipByDefaultInCI(t)
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
ctx := context.Background()
root, _ := dEnv.WorkingRoot(ctx)
@@ -51,6 +51,13 @@ type DoltHarness struct {
sess *dsess.DoltSession
}
func (h *DoltHarness) Close() {
dbs := dsql.AllDbs(sql.NewEmptyContext(), h.sess.Provider())
for _, db := range dbs {
db.DbData().Ddb.Close()
}
}
func (h *DoltHarness) EngineStr() string {
return "mysql"
}
@@ -100,6 +100,7 @@ func TestDoltHarness(t *testing.T) {
fs := filesys.NewInMemFS([]string{}, nil, tmp)
dEnv := createTestEnvWithFS(fs, wd)
defer dEnv.DoltDB.Close()
// We run this several times in a row to make sure that the same dolt env can be used in multiple setup / teardown
// cycles
@@ -108,6 +109,7 @@ func TestDoltHarness(t *testing.T) {
h := &DoltHarness{}
err := innerInit(h, dEnv)
assert.Equal(t, nil, err)
defer h.Close()
// setup repo with statements
for _, test := range statementTests {
@@ -31,6 +31,7 @@ import (
func TestCommitHooksNoErrors(t *testing.T) {
dEnv, err := CreateEnvWithSeedData()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
AddDoltSystemVariables()
sql.SystemVariables.SetGlobal(dsess.SkipReplicationErrors, true)
@@ -62,6 +62,7 @@ func TestSqlBatchInserts(t *testing.T) {
ctx := context.Background()
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
@@ -159,6 +160,7 @@ func TestSqlBatchInsertIgnoreReplace(t *testing.T) {
ctx := context.Background()
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
@@ -204,6 +206,7 @@ func TestSqlBatchInsertErrors(t *testing.T) {
ctx := context.Background()
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
tmpDir, err := dEnv.TempTableFilesDir()
require.NoError(t, err)
+19 -2
View File
@@ -259,6 +259,7 @@ func TestCreateTable(t *testing.T) {
ctx := context.Background()
dEnv, err := CreateEmptyTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
@@ -334,6 +335,7 @@ func TestDropTable(t *testing.T) {
ctx := context.Background()
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
@@ -508,6 +510,7 @@ func TestAddColumn(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
ctx := context.Background()
root, err := dEnv.WorkingRoot(ctx)
@@ -629,6 +632,7 @@ func TestRenameColumn(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
ctx := context.Background()
root, _ := dEnv.WorkingRoot(ctx)
@@ -744,6 +748,7 @@ func TestRenameTableStatements(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
ctx := context.Background()
root, err := dEnv.WorkingRoot(ctx)
@@ -820,6 +825,7 @@ func TestAlterSystemTables(t *testing.T) {
t.Run("Create", func(t *testing.T) {
setup()
defer dEnv.DoltDB.Close()
for _, tableName := range append(systemTableNames, reservedTableNames...) {
assertFails(t, dEnv, fmt.Sprintf("create table %s (a int primary key not null)", tableName), "reserved")
}
@@ -827,8 +833,12 @@ func TestAlterSystemTables(t *testing.T) {
t.Run("Drop", func(t *testing.T) {
setup()
for _, tableName := range append(systemTableNames) {
expectedErr := "system tables cannot be dropped or altered"
defer dEnv.DoltDB.Close()
for _, tableName := range systemTableNames {
expectedErr := "system table"
if strings.HasPrefix(tableName, "dolt_diff") || strings.HasPrefix(tableName, "dolt_history") {
expectedErr = "system tables cannot be dropped or altered"
}
assertFails(t, dEnv, fmt.Sprintf("drop table %s", tableName), expectedErr)
}
for _, tableName := range reservedTableNames {
@@ -838,6 +848,7 @@ func TestAlterSystemTables(t *testing.T) {
t.Run("Rename", func(t *testing.T) {
setup()
defer dEnv.DoltDB.Close()
for _, tableName := range systemTableNames {
expectedErr := "system table"
if strings.HasPrefix(tableName, "dolt_diff") || strings.HasPrefix(tableName, "dolt_history") {
@@ -852,6 +863,7 @@ func TestAlterSystemTables(t *testing.T) {
t.Run("Alter", func(t *testing.T) {
setup()
defer dEnv.DoltDB.Close()
for _, tableName := range append(systemTableNames, reservedTableNames...) {
expectedErr := "cannot be altered"
if strings.HasPrefix(tableName, "dolt_diff") || strings.HasPrefix(tableName, "dolt_history") {
@@ -1061,6 +1073,7 @@ func TestParseCreateTableStatement(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
ctx := context.Background()
root, _ := dEnv.WorkingRoot(ctx)
@@ -1081,6 +1094,7 @@ func TestParseCreateTableStatement(t *testing.T) {
func TestIndexOverwrite(t *testing.T) {
ctx := context.Background()
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
if err != nil {
panic(err)
@@ -1185,6 +1199,7 @@ INSERT INTO child_non_unq VALUES ('1', 1), ('2', NULL), ('3', 3), ('4', 3), ('5'
func TestDropPrimaryKey(t *testing.T) {
ctx := context.Background()
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
if err != nil {
panic(err)
@@ -1242,6 +1257,7 @@ func TestDropPrimaryKey(t *testing.T) {
func TestDropIndex(t *testing.T) {
ctx := context.Background()
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
if err != nil {
panic(err)
@@ -1295,6 +1311,7 @@ func TestDropIndex(t *testing.T) {
func TestCreateIndexUnique(t *testing.T) {
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(context.Background())
if err != nil {
panic(err)
@@ -229,6 +229,7 @@ func testDeleteQuery(t *testing.T, test DeleteTest) {
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
if test.AdditionalSetup != nil {
test.AdditionalSetup(t, dEnv)
@@ -432,6 +432,7 @@ func testInsertQuery(t *testing.T, test InsertTest) {
dEnv, err := CreateEmptyTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
if test.AdditionalSetup != nil {
test.AdditionalSetup(t, dEnv)
@@ -97,6 +97,7 @@ func TestExecutePersist(t *testing.T) {
func testPersistQuery(t *testing.T, test PersistTest) {
dEnv, err := CreateEmptyTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
if test.AdditionalSetup != nil {
test.AdditionalSetup(t, dEnv)
@@ -302,6 +302,7 @@ func testReplaceQuery(t *testing.T, test ReplaceTest) {
dEnv, err := CreateEmptyTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
if test.AdditionalSetup != nil {
test.AdditionalSetup(t, dEnv)
@@ -1344,6 +1344,7 @@ func testSelectQuery(t *testing.T, test SelectTest) {
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
if test.AdditionalSetup != nil {
test.AdditionalSetup(t, dEnv)
@@ -399,6 +399,7 @@ func testUpdateQuery(t *testing.T, test UpdateTest) {
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB.Close()
if test.AdditionalSetup != nil {
test.AdditionalSetup(t, dEnv)
@@ -153,6 +153,7 @@ func TestTableEditorForeignKeyCascade(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
dEnv, initialRoot := setupEditorFkTest(t)
defer dEnv.DoltDB.Close()
testRoot, err := ExecuteSql(dEnv, initialRoot, `
ALTER TABLE two ADD FOREIGN KEY (v1) REFERENCES one(v1) ON DELETE CASCADE ON UPDATE CASCADE;
@@ -202,6 +203,7 @@ func TestTableEditorForeignKeySetNull(t *testing.T) {
for _, test := range tests {
t.Run(test.sqlStatement, func(t *testing.T) {
dEnv, initialRoot := setupEditorFkTest(t)
defer dEnv.DoltDB.Close()
testRoot, err := ExecuteSql(dEnv, initialRoot, `
ALTER TABLE two ADD FOREIGN KEY (v1) REFERENCES one(v1) ON DELETE SET NULL ON UPDATE SET NULL;`)
@@ -284,6 +286,7 @@ func TestTableEditorForeignKeyRestrict(t *testing.T) {
for _, test := range tests {
t.Run(test.setup+test.trigger, func(t *testing.T) {
dEnv, initialRoot := setupEditorFkTest(t)
defer dEnv.DoltDB.Close()
testRoot, err := ExecuteSql(dEnv, initialRoot, fmt.Sprintf(`
ALTER TABLE two ADD FOREIGN KEY (v1) REFERENCES one(v1) %s;
@@ -355,6 +358,7 @@ func TestTableEditorForeignKeyViolations(t *testing.T) {
for _, test := range tests {
t.Run(test.setup+test.trigger, func(t *testing.T) {
dEnv, initialRoot := setupEditorFkTest(t)
defer dEnv.DoltDB.Close()
testRoot, err := ExecuteSql(dEnv, initialRoot, `
ALTER TABLE two ADD FOREIGN KEY (v1) REFERENCES one(v1) ON DELETE CASCADE ON UPDATE CASCADE;
@@ -376,6 +380,7 @@ ALTER TABLE three ADD FOREIGN KEY (v1, v2) REFERENCES two(v1, v2) ON DELETE CASC
func TestTableEditorSelfReferentialForeignKeyRestrict(t *testing.T) {
dEnv, initialRoot := setupEditorFkTest(t)
defer dEnv.DoltDB.Close()
ctx := context.Background()
root := initialRoot
@@ -446,6 +451,7 @@ func TestTableEditorSelfReferentialForeignKeyRestrict(t *testing.T) {
func TestTableEditorSelfReferentialForeignKeyCascade(t *testing.T) {
dEnv, initialRoot := setupEditorFkTest(t)
defer dEnv.DoltDB.Close()
ctx := context.Background()
root := initialRoot
@@ -546,6 +552,7 @@ func TestTableEditorSelfReferentialForeignKeyCascade(t *testing.T) {
func TestTableEditorSelfReferentialForeignKeySetNull(t *testing.T) {
dEnv, initialRoot := setupEditorFkTest(t)
defer dEnv.DoltDB.Close()
ctx := context.Background()
root := initialRoot
@@ -857,6 +864,7 @@ func TestTableEditorKeylessFKCascade(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
dEnv, initialRoot := setupEditorKeylessFkTest(t)
defer dEnv.DoltDB.Close()
testRoot, err := ExecuteSql(dEnv, initialRoot, `
ALTER TABLE two ADD FOREIGN KEY (v1) REFERENCES one(v1) ON DELETE CASCADE ON UPDATE CASCADE;
@@ -120,6 +120,7 @@ UPDATE onepk SET pk1 = v1 + pk1 ORDER BY pk1 DESC;
for _, test := range tests {
t.Run(test.sqlStatement, func(t *testing.T) {
dEnv, initialRoot := setupEditorIndexTest(t)
defer dEnv.DoltDB.Close()
root := initialRoot
for _, sqlStatement := range strings.Split(test.sqlStatement, ";") {
@@ -282,6 +283,7 @@ UPDATE oneuni SET v1 = v1 + pk1;
for _, test := range tests {
t.Run(test.sqlStatement, func(t *testing.T) {
dEnv, initialRoot := setupEditorIndexTest(t)
defer dEnv.DoltDB.Close()
root := initialRoot
var err error
+1
View File
@@ -28,6 +28,7 @@ import (
// Not an exhaustive test of views -- we rely on bats tests for end-to-end verification.
func TestViews(t *testing.T) {
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
ctx := context.Background()
root, _ := dEnv.WorkingRoot(ctx)
@@ -88,6 +88,7 @@ func TestEndToEnd(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
ctx := context.Background()
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB.Close()
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
+12 -23
View File
@@ -21,7 +21,6 @@ import (
"errors"
"fmt"
"io"
"os"
"runtime"
"runtime/debug"
"sync/atomic"
@@ -35,24 +34,11 @@ var (
ErrWrongCopySize = errors.New("could not copy enough bytes")
)
// By setting the environment variable DOLT_ASSERT_TABLE_FILES_CLOSED to any
// non-empty string, dolt will run some sanity checks on table file lifecycle
// management. In particular, dolt will install a GC finalizer on the table
// file index buffer to assert that it has been properly closed at the time
// that it gets garbage collected.
//
// This is mostly intended for developers. It isa recommended mode in tests and
// can make sense in other contexts as well. At the time of this writing---
// (2023/02, aaron@)---lifecycle management in tests in particular is not good
// enough to globally enable this.
// By setting this to false, you can make tablefile index creation cheaper. In
// exchange, the panics which leaked table files create do not come with as
// much information.
var TableIndexAssertClosedWithGCFinalizer bool
func init() {
if os.Getenv("DOLT_ASSERT_TABLE_FILES_CLOSED") != "" {
TableIndexAssertClosedWithGCFinalizer = true
}
}
var TableIndexGCFinalizerWithStackTrace = true
type tableIndex interface {
// entrySuffixMatches returns true if the entry at index |idx| matches
@@ -278,10 +264,15 @@ func newOnHeapTableIndex(indexBuff []byte, offsetsBuff1 []byte, count uint32, to
refCnt := new(int32)
*refCnt = 1
if TableIndexAssertClosedWithGCFinalizer {
if TableIndexGCFinalizerWithStackTrace {
stack := string(debug.Stack())
runtime.SetFinalizer(refCnt, func(i *int32) {
panic(fmt.Sprintf("OnHeapTableIndex not closed:\n%s", stack))
panic(fmt.Sprintf("OnHeapTableIndex %x not closed:\n%s", refCnt, stack))
})
} else {
runtime.SetFinalizer(refCnt, func(i *int32) {
panic(fmt.Sprintf("OnHeapTableIndex %x was not closed", refCnt))
})
}
@@ -539,9 +530,7 @@ func (ti onHeapTableIndex) Close() error {
return nil
}
if TableIndexAssertClosedWithGCFinalizer {
runtime.SetFinalizer(ti.refCnt, nil)
}
runtime.SetFinalizer(ti.refCnt, nil)
ti.q.ReleaseQuotaBytes(len(ti.prefixTuples) + len(ti.offsets1) + len(ti.offsets2) + len(ti.suffixes) + len(ti.footer))
return nil
}