Add support for dolt_global_tables "immediate" mode.

This commit is contained in:
Nick Tobey
2025-10-06 13:12:04 -07:00
parent f206e7392c
commit a750a4de1c
3 changed files with 506 additions and 7 deletions
+125 -6
View File
@@ -18,6 +18,7 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"io"
"strings"
"time"
@@ -59,6 +60,7 @@ var ErrInvalidTableName = errors.NewKind("Invalid table name %s.")
var ErrReservedTableName = errors.NewKind("Invalid table name %s. Table names beginning with `dolt_` are reserved for internal use")
var ErrReservedDiffTableName = errors.NewKind("Invalid table name %s. Table names beginning with `__DATABASE__` are reserved for internal use")
var ErrSystemTableAlter = errors.NewKind("Cannot alter table %s: system tables cannot be dropped or altered")
var ErrInvalidGlobalsTableOptions = errors.NewKind("Invalid global table options %s: only valid value is 'immediate'.")
// Database implements sql.Database for a dolt DB.
type Database struct {
@@ -268,6 +270,10 @@ func (db Database) GetGlobalState() globalstate.GlobalState {
// GetTableInsensitive is used when resolving tables in queries. It returns a best-effort case-insensitive match for
// the table name given.
func (db Database) GetTableInsensitive(ctx *sql.Context, tblName string) (sql.Table, bool, error) {
return db.getTableInsensitive(ctx, tblName, true)
}
func (db Database) getTableInsensitive(ctx *sql.Context, tblName string, readGlobalTables bool) (sql.Table, bool, error) {
// We start by first checking whether the input table is a temporary table. Temporary tables with name `x` take
// priority over persisted tables of name `x`.
ds := dsess.DSessFromSess(ctx.Session)
@@ -280,11 +286,15 @@ func (db Database) GetTableInsensitive(ctx *sql.Context, tblName string) (sql.Ta
return nil, false, err
}
return db.getTableInsensitive(ctx, nil, ds, root, tblName, "")
return db.getTableInsensitiveWithRoot(ctx, nil, ds, root, tblName, "", readGlobalTables)
}
func (db Database) GetTableInsensitiveAsOf(ctx *sql.Context, tableName string, asOf interface{}) (sql.Table, bool, error) {
return db.getTableInsensitiveAsOf(ctx, tableName, asOf, true)
}
// GetTableInsensitiveAsOf implements sql.VersionedDatabase
func (db Database) GetTableInsensitiveAsOf(ctx *sql.Context, tableName string, asOf interface{}) (sql.Table, bool, error) {
func (db Database) getTableInsensitiveAsOf(ctx *sql.Context, tableName string, asOf interface{}, readGlobalTables bool) (sql.Table, bool, error) {
if asOf == nil {
return db.GetTableInsensitive(ctx, tableName)
}
@@ -297,7 +307,7 @@ func (db Database) GetTableInsensitiveAsOf(ctx *sql.Context, tableName string, a
sess := dsess.DSessFromSess(ctx.Session)
table, ok, err := db.getTableInsensitive(ctx, head, sess, root, tableName, asOf)
table, ok, err := db.getTableInsensitiveWithRoot(ctx, head, sess, root, tableName, asOf, readGlobalTables)
if err != nil {
return nil, false, err
}
@@ -307,7 +317,7 @@ func (db Database) GetTableInsensitiveAsOf(ctx *sql.Context, tableName string, a
if doltdb.IsReadOnlySystemTable(doltdb.TableName{Name: tableName, Schema: db.schemaName}) {
// currently, system tables do not need to be "locked to root"
// see comment below in getTableInsensitive
// see comment below in getTableInsensitiveWithRoot
return table, ok, nil
}
@@ -330,9 +340,19 @@ func (db Database) GetTableInsensitiveAsOf(ctx *sql.Context, tableName string, a
}
}
func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds *dsess.DoltSession, root doltdb.RootValue, tblName string, asOf interface{}) (sql.Table, bool, error) {
func (db Database) getTableInsensitiveWithRoot(ctx *sql.Context, head *doltdb.Commit, ds *dsess.DoltSession, root doltdb.RootValue, tblName string, asOf interface{}, readGlobalTables bool) (sql.Table, bool, error) {
lwrName := strings.ToLower(tblName)
if readGlobalTables {
globalTable, exists, err := db.getGlobalTable(ctx, root, lwrName)
if err != nil {
return nil, false, err
}
if exists {
return globalTable, true, nil
}
}
// TODO: these tables that cache a root value at construction time should not, they need to get it from the session
// at runtime
switch {
@@ -476,7 +496,7 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds
}
}
srcTable, ok, err := db.getTableInsensitive(ctx, head, ds, root, tname.Name, asOf)
srcTable, ok, err := db.getTableInsensitiveWithRoot(ctx, head, ds, root, tname.Name, asOf, true)
if err != nil {
return nil, false, err
} else if !ok {
@@ -868,6 +888,17 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds
versionableTable := backingTable.(dtables.VersionableTable)
dt, found = dtables.NewQueryCatalogTable(ctx, versionableTable), true
}
case doltdb.GlobalTablesTableName, doltdb.GetGlobalTablesTableName():
backingTable, _, err := db.getTable(ctx, root, doltdb.GlobalTablesTableName)
if err != nil {
return nil, false, err
}
if backingTable == nil {
dt, found = dtables.NewEmptyGlobalTablesTable(ctx), true
} else {
versionableTable := backingTable.(dtables.VersionableTable)
dt, found = dtables.NewGlobalTablesTable(ctx, versionableTable), true
}
case doltdb.GetTestsTableName():
backingTable, _, err := db.getTable(ctx, root, doltdb.GetTestsTableName())
if err != nil {
@@ -905,6 +936,94 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds
return resolveOverriddenNonexistentTable(ctx, tblName, db)
}
// getGlobalTable checks whether the table name maps onto a table in another root via the dolt_global_tables system table
func (db Database) getGlobalTable(ctx *sql.Context, root doltdb.RootValue, lwrName string) (sql.Table, bool, error) {
globalTablesTableName := doltdb.TableName{
Name: doltdb.GetGlobalTablesTableName(),
Schema: db.schemaName,
}
globalsTable, globalsTableExists, err := root.GetTable(ctx, globalTablesTableName)
if err != nil {
return nil, false, err
}
if !globalsTableExists {
return nil, false, nil
}
index, err := globalsTable.GetRowData(ctx)
if err != nil {
return nil, false, err
}
globalTablesSchema, err := globalsTable.GetSchema(ctx)
if err != nil {
return nil, false, err
}
m := durable.MapFromIndex(index)
keyDesc, valueDesc := globalTablesSchema.GetMapDescriptors(m.NodeStore())
globalTablesMap, err := m.IterAll(ctx)
if err != nil {
return nil, false, err
}
var globalTablesEntry doltdb.GlobalTablesEntry
// check if there's an entry for this table. If so, resolve that reference.
for {
keyTuple, valueTuple, err := globalTablesMap.Next(ctx)
if err == io.EOF {
break
}
if err != nil {
return nil, false, err
}
globalsEntryTableName, err := doltdb.GetGlobalTablesNameColumn(ctx, keyDesc, keyTuple)
if err != nil {
return nil, false, err
}
globalsEntryTableName = strings.ToLower(globalsEntryTableName)
matchesPattern, err := doltdb.MatchTablePattern(globalsEntryTableName, lwrName)
if err != nil {
return nil, false, err
}
if !matchesPattern {
continue
}
globalTablesEntry = doltdb.GetGlobalTablesRef(ctx, valueDesc, valueTuple)
if globalTablesEntry.NewTableName == "" {
globalTablesEntry.NewTableName = lwrName
}
if globalTablesEntry.Ref == "" {
globalTablesEntry.Ref = db.revision
}
if globalTablesEntry.Options != "immediate" {
return nil, false, ErrInvalidGlobalsTableOptions.New(globalTablesEntry.Options)
}
// If the ref is a branch, we get the working set, not the head.
_, exists, err := isBranch(ctx, db, globalTablesEntry.Ref)
if exists {
referencedBranch, err := RevisionDbForBranch(ctx, db, globalTablesEntry.Ref, db.requestedName)
if err != nil {
return nil, false, err
}
return referencedBranch.(Database).getTableInsensitive(ctx, globalTablesEntry.NewTableName, false)
} else {
// If we couldn't resolve it as a database revision, treat it as a noms ref.
// This lets us resolve branch heads like 'heads/$branchName' or remotes refs like '$remote/$branchName'
return db.getTableInsensitiveAsOf(ctx, globalTablesEntry.NewTableName, globalTablesEntry.Ref, false)
}
}
return nil, false, nil
}
// getRootsForBranch uses the specified |ddb| to look up a branch named |branch|, and return the
// Roots for it. If there is no branch named |branch|, then an error is returned.
func getRootsForBranch(ctx *sql.Context, ddb *doltdb.DoltDB, branch string) (doltdb.Roots, error) {
+1 -1
View File
@@ -658,7 +658,7 @@ type WritableDoltTable struct {
var _ doltTableInterface = (*WritableDoltTable)(nil)
// WritableDoltTableWrapper is an interface that allows a table to be returned as an sql.Table, but actually be a wrapped
// fake table. Specifically, databases.getTableInsensitive will returns an sql.Table, and there are cases where we
// fake table. Specifically, databases.getTableInsensitiveWithRoot will returns an sql.Table, and there are cases where we
// want to return a table that hasn't been materialized yet.
type WritableDoltTableWrapper interface {
// Unwrap returns the underlying WritableDoltTable, nil returns are expected when the wrapped table hasn't been materialized
+380
View File
@@ -0,0 +1,380 @@
#!/usr/bin/env bats
load $BATS_TEST_DIRNAME/helper/common.bash
setup() {
setup_common
}
teardown() {
assert_feature_version
teardown_common
}
@test "global: basic case" {
dolt checkout -b other
dolt sql <<SQL
CALL dolt_checkout('main');
CREATE TABLE aliased_table (pk char(8) PRIMARY KEY);
INSERT INTO aliased_table VALUES ("amzmapqt");
CALL dolt_checkout('other');
INSERT INTO dolt_global_tables(table_name, target_ref, ref_table, options) VALUES
("table_alias_branch", "main", "aliased_table", "immediate");
SQL
run dolt sql -q "select * from table_alias_branch;"
[ "$status" -eq 0 ]
[[ "$output" =~ "amzmapqt" ]] || false
# Global tables appear in "show create", but the output matches the aliased table.
run dolt sql -q "show create table table_alias_branch"
[ "$status" -eq 0 ]
[[ "$output" =~ "aliased_table" ]] || false
}
@test "global: branch name reflects the working set of the referenced branch" {
dolt checkout -b other
dolt sql <<SQL
CALL dolt_checkout('main');
CREATE TABLE aliased_table (pk char(8) PRIMARY KEY);
INSERT INTO aliased_table VALUES ("amzmapqt");
CALL dolt_checkout('other');
INSERT INTO dolt_global_tables(table_name, target_ref, ref_table, options) VALUES
("table_alias_branch", "main", "aliased_table", "immediate");
SQL
run dolt sql -q "select * from table_alias_branch;"
[ "$status" -eq 0 ]
[[ "$output" =~ "amzmapqt" ]] || false
}
@test "global: branch ref reflects the committed version of the parent" {
dolt checkout -b other
dolt sql <<SQL
CALL DOLT_CHECKOUT('main');
CREATE TABLE aliased_table (pk char(8) PRIMARY KEY);
INSERT INTO aliased_table VALUES ('amzmapqt');
CALL DOLT_COMMIT('-Am', 'create table');
INSERT INTO aliased_table VALUES ('eesekkgo');
CALL DOLT_CHECKOUT('other');
INSERT INTO dolt_global_tables(table_name, target_ref, ref_table, options) VALUES
("table_alias_branch_ref", "heads/main", "aliased_table", "immediate");
SQL
run dolt sql -q "select * from table_alias_branch_ref;"
[ "$status" -eq 0 ]
[[ "$output" =~ "amzmapqt" ]] || false
! [[ "$output" =~ "eesekkgo" ]] || false
}
@test "global: tag and hash" {
dolt checkout -b other
dolt sql <<SQL
CALL DOLT_CHECKOUT('main');
CREATE TABLE aliased_table (pk char(8) PRIMARY KEY);
INSERT INTO aliased_table VALUES ("amzmapqt");
CALL DOLT_COMMIT('-Am', 'commit');
CALL DOLT_TAG('v1.0');
CALL DOLT_CHECKOUT('other');
INSERT INTO dolt_global_tables(table_name, target_ref, ref_table, options) VALUES
("table_alias_tag", "v1.0", "aliased_table", "immediate"),
("table_alias_tag_ref", "tags/v1.0", "aliased_table", "immediate"),
("table_alias_hash", DOLT_HASHOF('v1.0'), "aliased_table", "immediate");
SQL
run dolt sql -q "select * from table_alias_tag_ref;"
[ "$status" -eq 0 ]
[[ "$output" =~ "amzmapqt" ]] || false
run dolt sql -q "select * from table_alias_tag;"
[ "$status" -eq 0 ]
[[ "$output" =~ "amzmapqt" ]] || false
run dolt sql -q "select * from table_alias_hash;"
[ "$status" -eq 0 ]
[[ "$output" =~ "amzmapqt" ]] || false
# These global tables are read-only because they reference a read-only ref
run dolt sql <<SQL
INSERT INTO table_alias_tag_ref VALUES ("eesekkgo");
SQL
[ "$status" -eq 1 ]
[[ "$output" =~ "table doesn't support INSERT INTO" ]] || false
run dolt sql <<SQL
INSERT INTO table_alias_tag VALUES ("eesekkgo");
SQL
[ "$status" -eq 1 ]
[[ "$output" =~ "table doesn't support INSERT INTO" ]] || false
run dolt sql <<SQL
INSERT INTO table_alias_hash VALUES ("eesekkgo");
SQL
[ "$status" -eq 1 ]
[[ "$output" =~ "table doesn't support INSERT INTO" ]] || false
}
@test "global: remote ref" {
mkdir child
dolt checkout -b other
dolt sql <<SQL
CALL DOLT_CHECKOUT('main');
CREATE TABLE aliased_table (pk char(8) PRIMARY KEY);
INSERT INTO aliased_table VALUES ("amzmapqt");
CALL DOLT_COMMIT('-Am', 'create table');
CALL DOLT_REMOTE('add', 'remote_db', 'file://./remote');
CALL DOLT_PUSH('remote_db', 'main');
-- drop table so it is only accessible from the remote ref
DROP TABLE aliased_table;
CALL DOLT_COMMIT('-am', 'drop table');
CALL DOLT_CHECKOUT('other');
INSERT INTO dolt_global_tables(table_name, target_ref, ref_table, options) VALUES
("table_alias_remote_branch", "remote_db/main", "aliased_table", "immediate"),
("table_alias_remote_ref", "remotes/remote_db/main", "aliased_table", "immediate");
SQL
run dolt sql -q "select * from table_alias_remote_branch;"
[ "$status" -eq 0 ]
[[ "$output" =~ "amzmapqt" ]] || false
run dolt sql -q "select * from table_alias_remote_ref;"
[ "$status" -eq 0 ]
[[ "$output" =~ "amzmapqt" ]] || false
}
@test "global: default ref" {
# If unspecified, the ref defaults to the current HEAD.
# This allows one table to alias another table on the same branch.
dolt sql <<SQL
CREATE TABLE aliased_table (pk char(8) PRIMARY KEY);
INSERT INTO aliased_table VALUES ("amzmapqt");
INSERT INTO dolt_global_tables(table_name, ref_table, options) VALUES
("table_alias", "aliased_table", "immediate");
SQL
run dolt sql -q "select * from table_alias;"
[ "$status" -eq 0 ]
[[ "$output" =~ "amzmapqt" ]] || false
}
@test "global: default table_name" {
# If unspecified, the parent table name defaults to the same table name as the child
dolt checkout -b other
dolt sql <<SQL
CALL DOLT_CHECKOUT('main');
CREATE TABLE table_alias (pk char(8) PRIMARY KEY);
INSERT INTO table_alias VALUES ("amzmapqt");
CALL DOLT_CHECKOUT('other');
INSERT INTO dolt_global_tables(table_name, target_ref, options) VALUES
("table_alias", "main", "immediate");
SQL
run dolt sql -q "select * from table_alias;"
[ "$status" -eq 0 ]
[[ "$output" =~ "amzmapqt" ]] || false
}
@test "global: wildcard table_name" {
# The wildcard syntax matches the wildcard syntax used in dolt_ignore
dolt checkout -b other
dolt sql <<SQL
INSERT INTO dolt_global_tables(table_name, target_ref, options) VALUES
("global_*", "main", "immediate");
CALL DOLT_CHECKOUT('main');
CREATE TABLE global_table1 (pk char(8) PRIMARY KEY);
CREATE TABLE global_table2 (pk char(8) PRIMARY KEY);
CREATE TABLE not_global_table (pk char(8) PRIMARY KEY);
INSERT INTO global_table1 VALUES ("amzmapqt");
INSERT INTO global_table2 VALUES ("eesekkgo");
INSERT INTO not_global_table VALUES ("pzdxwmbd");
SQL
run dolt sql -q "select * from global_table1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "amzmapqt" ]] || false
run dolt sql -q "select * from global_table2;"
[ "$status" -eq 0 ]
[[ "$output" =~ "eesekkgo" ]] || false
run dolt sql -q "select * from not_global_table;"
[ "$status" -eq 1 ]
[[ "$output" =~ "table not found" ]] || false
}
@test "global: creating a global table creates it on the appropriate branch" {
dolt checkout -b other
dolt sql <<SQL
INSERT INTO dolt_global_tables(table_name, target_ref, options) VALUES
("global_table", "main", "immediate");
CREATE TABLE global_table (pk char(8) PRIMARY KEY);
INSERT INTO global_table VALUES ("amzmapqt");
SQL
run dolt ls main
[ "$status" -eq 0 ]
[[ "$output" =~ "global_table" ]] || false
}
@test "global: a transaction that tries to update multiple branches fails as expected" {
run dolt sql <<SQL
CREATE TABLE aliased_table (pk char(8) PRIMARY KEY);
CALL DOLT_CHECKOUT('-b', 'other');
CREATE TABLE local_table (pk char(8) PRIMARY KEY);
INSERT INTO dolt_global_tables(table_name, target_ref, ref_table, options) VALUES
("global_table", "main", "aliased_table", "immediate");
set autocommit = 0;
INSERT INTO local_table VALUES ("amzmapqt");
INSERT INTO global_table VALUES ("eesekkgo");
COMMIT;
SQL
[ "$status" -eq 1 ]
[[ "$output" =~ "Cannot commit changes on more than one branch / database" ]] || false
}
@test "global: test foreign keys" {
# Currently, foreign keys cannot be added to global tables
dolt checkout -b other
run dolt sql <<SQL
CALL DOLT_CHECKOUT('main');
CREATE TABLE aliased_table (pk char(8) PRIMARY KEY);
INSERT INTO dolt_global_tables(table_name, target_ref, ref_table, options) VALUES
("global_table", "main", "aliased_table", "immediate");
set autocommit = 0;
INSERT INTO global_table VALUES ("eesekkgo");
SQL
run dolt sql <<SQL
CREATE TABLE local_table (pk char(8) PRIMARY KEY, FOREIGN KEY (pk) REFERENCES global_table(pk));
SQL
[ "$status" -eq 1 ]
[[ "$output" =~ "Cannot commit changes on more than one branch / database" ]] || false
}
@test "global: adding an existing table to global tables errors" {
dolt checkout -b other
run dolt sql <<SQL
CREATE TABLE global_table (pk char(8) PRIMARY KEY);
INSERT INTO dolt_global_tables(table_name, target_ref, options) VALUES
("global_table", "main", "immediate");
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "cannot make global table global_table, table already exists on branch other" ]] || false
}
@test "global: global tables appear in show_tables" {
dolt checkout -b other
dolt sql <<SQL
CALL dolt_checkout('main');
CREATE TABLE aliased_table (pk char(8) PRIMARY KEY);
INSERT INTO aliased_table VALUES ("amzmapqt");
CALL dolt_checkout('other');
INSERT INTO dolt_global_tables(table_name, target_ref, ref_table, options) VALUES
("table_alias_branch", "main", "aliased_table", "immediate");
SQL
# Global tables should appear in "show tables"
run dolt sql -q "show tables"
[ "$status" -eq 0 ]
[[ "$output" =~ "table_alias_branch" ]] || false
}
@test "global: trying to dolt_add a global table returns the appropriate warning" {
dolt checkout -b other
dolt sql <<SQL
CALL DOLT_CHECKOUT('main');
CREATE TABLE global_table (pk char(8) PRIMARY KEY);
SQL
dolt sql <<SQL
INSERT INTO dolt_global_tables(table_name, target_ref, options) VALUES
("global_table", "main", "immediate");
INSERT INTO global_table values ('ghdsgerg');
CALL DOLT_ADD('global_table');
SQL
run dolt status
[ "$status" -eq 0 ]
! [[ "$output" =~ "global_table" ]] || false
run dolt sql -q "CALL DOLT_CHECKOUT('main'); SELECT * FROM dolt_status"
[ "$status" -eq 0 ]
! [[ "$output" =~ "global_table" ]] || false
exit 1
}
@test "global: dolt_add('.') doesn't add global tables" {
dolt checkout -b other
dolt sql <<SQL
CALL DOLT_CHECKOUT('main');
CREATE TABLE test_table (pk char(8) PRIMARY KEY);
SQL
dolt sql <<SQL
INSERT INTO dolt_global_tables(table_name, target_ref, options) VALUES
("test_table", "main", "immediate");
INSERT INTO test_table values ('ghdsgerg');
CALL DOLT_ADD('.');
SQL
run dolt sql -q "SELECT * FROM dolt_status"
[ "$status" -eq 0 ]
echo "$output"
! [[ "$output" =~ "test_table" ]] || false
run dolt sql -q "CALL DOLT_CHECKOUT('main'); SELECT * FROM dolt_status"
[ "$status" -eq 0 ]
[[ "$output" =~ "test_table | false" ]] || false
}
@test "global: self-referrential global tables in the same branch as their target are effectively ignored" {
dolt sql <<SQL
CREATE TABLE global_table (pk char(8) PRIMARY KEY);
INSERT INTO dolt_global_tables(table_name, target_ref, options) VALUES
("global_table", "main", "immediate");
INSERT INTO global_table values ('ghdsgerg');
SQL
dolt add global_table
run dolt sql -q "select * from dolt_status"
[ "$status" -eq 0 ]
echo "$output"
[[ "$output" =~ "global_table | true" ]] || false
# Unstage global_table but keep it in the working set
dolt reset HEAD
dolt add .
run dolt sql -q "select * from dolt_status"
[ "$status" -eq 0 ]
[[ "$output" =~ "global_table | true" ]] || false
}
@test "global: invalid options detected" {
dolt sql <<SQL
INSERT INTO dolt_global_tables(table_name, target_ref, options) VALUES
("global_table", "main", "invalid");
SQL
run sql -q "select * from global_table;"
[ "$status" -eq 1 ]
echo "$output"
[[ "$output" =~ "Invalid global table options" ]] || false
}