Merge pull request #9395 from dolthub/tim/dolt_history_dolt_procedures

Implement `dolt_history_dolt_procedures`
This commit is contained in:
Tim Sehn
2025-06-26 12:05:46 -07:00
committed by GitHub
4 changed files with 545 additions and 0 deletions

View File

@@ -409,6 +409,17 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds
}
return DoltSchemasHistoryTable(db.ddb, head, db), true, nil
case lwrName == doltdb.DoltHistoryTablePrefix+doltdb.ProceduresTableName:
// Special handling for dolt_history_dolt_procedures
if head == nil {
var err error
head, err = ds.GetHeadCommit(ctx, db.RevisionQualifiedName())
if err != nil {
return nil, false, err
}
}
return DoltProceduresHistoryTable(db.ddb, head, db), true, nil
case strings.HasPrefix(lwrName, doltdb.DoltHistoryTablePrefix):
baseTableName := tblName[len(doltdb.DoltHistoryTablePrefix):]
baseTable, ok, err := db.getTable(ctx, root, baseTableName)

View File

@@ -0,0 +1,257 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqle
import (
"io"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/go-mysql-server/sql/types"
"github.com/dolthub/vitess/go/sqltypes"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
)
// doltProceduresHistoryTable implements the dolt_history_dolt_procedures system table
type doltProceduresHistoryTable struct {
name string
ddb *doltdb.DoltDB
head *doltdb.Commit
db Database // Add database reference for DoltTable creation
}
var _ sql.Table = (*doltProceduresHistoryTable)(nil)
var _ sql.PrimaryKeyTable = (*doltProceduresHistoryTable)(nil)
// DoltProceduresHistoryTable creates a dolt_procedures history table instance
func DoltProceduresHistoryTable(ddb *doltdb.DoltDB, head *doltdb.Commit, db Database) sql.Table {
return &doltProceduresHistoryTable{
name: doltdb.DoltHistoryTablePrefix + doltdb.ProceduresTableName,
ddb: ddb,
head: head,
db: db,
}
}
// Name implements sql.Table
func (dpht *doltProceduresHistoryTable) Name() string {
return dpht.name
}
// String implements sql.Table
func (dpht *doltProceduresHistoryTable) String() string {
return dpht.name
}
// Schema implements sql.Table
func (dpht *doltProceduresHistoryTable) Schema() sql.Schema {
// Base schema from dolt_procedures table
baseSch := sql.Schema{
&sql.Column{Name: doltdb.ProceduresTableNameCol, Type: types.MustCreateString(sqltypes.VarChar, 64, sql.Collation_utf8mb4_0900_ai_ci), Nullable: false, PrimaryKey: true, Source: dpht.name},
&sql.Column{Name: doltdb.ProceduresTableCreateStmtCol, Type: types.MustCreateString(sqltypes.VarChar, 4096, sql.Collation_utf8mb4_0900_ai_ci), Nullable: false, Source: dpht.name},
&sql.Column{Name: doltdb.ProceduresTableCreatedAtCol, Type: types.Timestamp, Nullable: false, Source: dpht.name},
&sql.Column{Name: doltdb.ProceduresTableModifiedAtCol, Type: types.Timestamp, Nullable: false, Source: dpht.name},
&sql.Column{Name: doltdb.ProceduresTableSqlModeCol, Type: types.MustCreateString(sqltypes.VarChar, 256, sql.Collation_utf8mb4_0900_ai_ci), Nullable: true, Source: dpht.name},
}
// Add commit history columns
historySch := make(sql.Schema, len(baseSch), len(baseSch)+3)
copy(historySch, baseSch)
historySch = append(historySch,
&sql.Column{Name: CommitHashCol, Type: CommitHashColType, Nullable: false, PrimaryKey: true, Source: dpht.name},
&sql.Column{Name: CommitterCol, Type: CommitterColType, Nullable: false, Source: dpht.name},
&sql.Column{Name: CommitDateCol, Type: types.Datetime, Nullable: false, Source: dpht.name},
)
return historySch
}
// Collation implements sql.Table
func (dpht *doltProceduresHistoryTable) Collation() sql.CollationID {
return sql.Collation_Default
}
// Partitions implements sql.Table
func (dpht *doltProceduresHistoryTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) {
// Use the same commit iterator pattern as HistoryTable
cmItr := doltdb.CommitItrForRoots[*sql.Context](dpht.ddb, dpht.head)
return &commitPartitioner{cmItr: cmItr}, nil
}
// PartitionRows implements sql.Table
func (dpht *doltProceduresHistoryTable) PartitionRows(ctx *sql.Context, partition sql.Partition) (sql.RowIter, error) {
cp := partition.(*commitPartition)
return &doltProceduresHistoryRowIter{
ctx: ctx,
ddb: dpht.ddb,
commit: cp.cm,
history: dpht,
}, nil
}
// PrimaryKeySchema implements sql.PrimaryKeyTable
func (dpht *doltProceduresHistoryTable) PrimaryKeySchema() sql.PrimaryKeySchema {
return sql.PrimaryKeySchema{
Schema: dpht.Schema(),
PkOrdinals: []int{0, 5}, // name, commit_hash
}
}
// doltProceduresHistoryRowIter iterates through dolt_procedures rows for a single commit
type doltProceduresHistoryRowIter struct {
ctx *sql.Context
ddb *doltdb.DoltDB
commit *doltdb.Commit
history *doltProceduresHistoryTable // Add reference to parent table
rows []sql.Row
idx int
}
// Next implements sql.RowIter
func (dphri *doltProceduresHistoryRowIter) Next(ctx *sql.Context) (sql.Row, error) {
if dphri.rows == nil {
// Initialize rows from the commit's dolt_procedures table
err := dphri.loadRows()
if err != nil {
return nil, err
}
}
if dphri.idx >= len(dphri.rows) {
return nil, io.EOF
}
row := dphri.rows[dphri.idx]
dphri.idx++
return row, nil
}
func (dphri *doltProceduresHistoryRowIter) loadRows() error {
root, err := dphri.commit.GetRootValue(dphri.ctx)
if err != nil {
return err
}
// Get the table at this commit
tbl, ok, err := root.GetTable(dphri.ctx, doltdb.TableName{Name: doltdb.ProceduresTableName})
if err != nil {
return err
}
if !ok {
// No dolt_procedures table in this commit, return empty rows
dphri.rows = make([]sql.Row, 0)
return nil
}
// Get commit metadata
commitHash, err := dphri.commit.HashOf()
if err != nil {
return err
}
commitMeta, err := dphri.commit.GetCommitMeta(dphri.ctx)
if err != nil {
return err
}
// Convert commit metadata to SQL values
commitHashStr := commitHash.String()
committerStr := commitMeta.Name + " <" + commitMeta.Email + ">"
commitDate := commitMeta.Time()
// Get the schema
sch, err := tbl.GetSchema(dphri.ctx)
if err != nil {
return err
}
// Create a DoltTable using the database reference we now have
doltTable, err := NewDoltTable(doltdb.ProceduresTableName, sch, tbl, dphri.history.db, editor.Options{})
if err != nil {
return err
}
// Lock the table to this specific commit's root
lockedTable, err := doltTable.LockedToRoot(dphri.ctx, root)
if err != nil {
return err
}
// Get partitions and read rows
partitions, err := lockedTable.Partitions(dphri.ctx)
if err != nil {
return err
}
var baseRows []sql.Row
for {
partition, err := partitions.Next(dphri.ctx)
if err == io.EOF {
break
}
if err != nil {
return err
}
rowIter, err := lockedTable.PartitionRows(dphri.ctx, partition)
if err != nil {
return err
}
for {
row, err := rowIter.Next(dphri.ctx)
if err == io.EOF {
break
}
if err != nil {
return err
}
baseRows = append(baseRows, row)
}
err = rowIter.Close(dphri.ctx)
if err != nil {
return err
}
}
err = partitions.Close(dphri.ctx)
if err != nil {
return err
}
// Add commit metadata to each row
rows := make([]sql.Row, 0, len(baseRows))
for _, baseRow := range baseRows {
// Append commit columns to the base row
sqlRow := make(sql.Row, len(baseRow)+3)
copy(sqlRow, baseRow)
sqlRow[len(baseRow)] = commitHashStr
sqlRow[len(baseRow)+1] = committerStr
sqlRow[len(baseRow)+2] = commitDate
rows = append(rows, sqlRow)
}
dphri.rows = rows
return nil
}
// Close implements sql.RowIter
func (dphri *doltProceduresHistoryRowIter) Close(ctx *sql.Context) error {
return nil
}

View File

@@ -0,0 +1,209 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration_test
import (
"context"
"testing"
"github.com/dolthub/go-mysql-server/sql"
"github.com/stretchr/testify/require"
cmd "github.com/dolthub/dolt/go/cmd/dolt/commands"
"github.com/dolthub/dolt/go/libraries/doltcore/dtestutils"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle"
)
func TestDoltProceduresHistoryTable(t *testing.T) {
SkipByDefaultInCI(t)
ctx := context.Background()
dEnv := setupDoltProceduresHistoryTests(t)
defer dEnv.DoltDB(ctx).Close()
for _, test := range doltProceduresHistoryTableTests() {
t.Run(test.name, func(t *testing.T) {
testDoltProceduresHistoryTable(t, test, dEnv)
})
}
}
type doltProceduresTableTest struct {
name string
setup []testCommand
query string
rows []sql.Row
}
// Global variables to store commit hashes for test validation
var (
DOLT_PROCEDURES_HEAD string
DOLT_PROCEDURES_HEAD_1 string
DOLT_PROCEDURES_HEAD_2 string
DOLT_PROCEDURES_INIT string
)
var setupDoltProceduresCommon = []testCommand{
// Create initial procedure
{cmd.SqlCmd{}, args{"-q", "CREATE PROCEDURE test_proc1() SELECT 1"}},
{cmd.AddCmd{}, args{"."}},
{cmd.CommitCmd{}, args{"-m", "first commit: added test_proc1"}},
// Create a second procedure
{cmd.SqlCmd{}, args{"-q", "CREATE PROCEDURE test_proc2(x INT) SELECT x * 2"}},
{cmd.AddCmd{}, args{"."}},
{cmd.CommitCmd{}, args{"-m", "second commit: added test_proc2"}},
// Modify the first procedure
{cmd.SqlCmd{}, args{"-q", "DROP PROCEDURE test_proc1"}},
{cmd.SqlCmd{}, args{"-q", "CREATE PROCEDURE test_proc1() SELECT 'modified'"}},
{cmd.AddCmd{}, args{"."}},
{cmd.CommitCmd{}, args{"-m", "third commit: modified test_proc1"}},
// Add a third procedure
{cmd.SqlCmd{}, args{"-q", "CREATE PROCEDURE test_proc3() SELECT 'hello world' as result"}},
{cmd.AddCmd{}, args{"."}},
{cmd.CommitCmd{}, args{"-m", "fourth commit: added test_proc3"}},
{cmd.LogCmd{}, args{}},
}
func doltProceduresHistoryTableTests() []doltProceduresTableTest {
return []doltProceduresTableTest{
{
name: "verify dolt_history_dolt_procedures has all required columns",
query: "SELECT COUNT(*) FROM (SELECT name, create_stmt, created_at, modified_at, sql_mode, commit_hash, committer, commit_date FROM dolt_history_dolt_procedures LIMIT 0) AS procedures_check",
rows: []sql.Row{
{int64(0)}, // Should return 0 rows but verify all columns exist
},
},
{
name: "check correct number of history entries",
query: "SELECT COUNT(*) FROM dolt_history_dolt_procedures",
rows: []sql.Row{
{int64(8)}, // test_proc1(3 commits) + test_proc2(3 commits) + test_proc3(1 commit) + initial(1 commit) = 8 total
},
},
{
name: "filter for test_proc1 history only",
query: "SELECT COUNT(*) FROM dolt_history_dolt_procedures WHERE name = 'test_proc1'",
rows: []sql.Row{
{int64(4)}, // test_proc1 appears in all 4 commits
},
},
{
name: "filter for test_proc2 history only",
query: "SELECT COUNT(*) FROM dolt_history_dolt_procedures WHERE name = 'test_proc2'",
rows: []sql.Row{
{int64(3)}, // test_proc2 appears in 3 commits (added in 2nd commit)
},
},
{
name: "filter for test_proc3 history only",
query: "SELECT COUNT(*) FROM dolt_history_dolt_procedures WHERE name = 'test_proc3'",
rows: []sql.Row{
{int64(1)}, // test_proc3 appears in 1 commit (added in 4th commit)
},
},
{
name: "check commit_hash is not null",
query: "SELECT COUNT(*) FROM dolt_history_dolt_procedures WHERE commit_hash IS NOT NULL",
rows: []sql.Row{
{int64(8)}, // Total number of procedure entries across all commits
},
},
{
name: "verify procedure names in latest commit",
query: "SELECT name FROM dolt_history_dolt_procedures WHERE commit_hash = '" + "%s" + "' ORDER BY name",
rows: []sql.Row{
{"test_proc1"},
{"test_proc2"},
{"test_proc3"},
},
},
{
name: "check committer column exists",
query: "SELECT COUNT(*) FROM dolt_history_dolt_procedures WHERE committer IS NOT NULL",
rows: []sql.Row{
{int64(8)}, // All entries should have committer info
},
},
{
name: "verify create_stmt column contains procedure definitions",
query: "SELECT COUNT(*) FROM dolt_history_dolt_procedures WHERE create_stmt LIKE '%PROCEDURE%'",
rows: []sql.Row{
{int64(8)}, // All entries should have CREATE PROCEDURE in create_stmt
},
},
{
name: "check created_at and modified_at are not null",
query: "SELECT COUNT(*) FROM dolt_history_dolt_procedures WHERE created_at IS NOT NULL AND modified_at IS NOT NULL",
rows: []sql.Row{
{int64(8)}, // All entries should have timestamp info
},
},
}
}
func setupDoltProceduresHistoryTests(t *testing.T) *env.DoltEnv {
dEnv := dtestutils.CreateTestEnv()
ctx := context.Background()
cliCtx, verr := cmd.NewArgFreeCliContext(ctx, dEnv, dEnv.FS)
require.NoError(t, verr)
for _, c := range setupDoltProceduresCommon {
exitCode := c.cmd.Exec(ctx, c.cmd.Name(), c.args, dEnv, cliCtx)
require.Equal(t, 0, exitCode)
}
// Get commit hashes for test validation
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
rows, err := sqle.ExecuteSelect(ctx, dEnv, root, "SELECT commit_hash FROM dolt_log ORDER BY date DESC")
require.NoError(t, err)
require.Equal(t, 5, len(rows)) // 4 commits + initial commit
DOLT_PROCEDURES_HEAD = rows[0][0].(string)
DOLT_PROCEDURES_HEAD_1 = rows[1][0].(string)
DOLT_PROCEDURES_HEAD_2 = rows[2][0].(string)
DOLT_PROCEDURES_INIT = rows[4][0].(string) // Skip one to get to the first real commit
return dEnv
}
func testDoltProceduresHistoryTable(t *testing.T, test doltProceduresTableTest, dEnv *env.DoltEnv) {
ctx := context.Background()
cliCtx, verr := cmd.NewArgFreeCliContext(ctx, dEnv, dEnv.FS)
require.NoError(t, verr)
for _, c := range test.setup {
exitCode := c.cmd.Exec(ctx, c.cmd.Name(), c.args, dEnv, cliCtx)
require.Equal(t, 0, exitCode)
}
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
// Replace placeholder in query with actual commit hash
query := test.query
if query == "SELECT name FROM dolt_history_dolt_procedures WHERE commit_hash = '"+"%s"+"' ORDER BY name" {
query = "SELECT name FROM dolt_history_dolt_procedures WHERE commit_hash = '" + DOLT_PROCEDURES_HEAD + "' ORDER BY name"
}
actRows, err := sqle.ExecuteSelect(ctx, dEnv, root, query)
require.NoError(t, err)
require.ElementsMatch(t, test.rows, actRows)
}

View File

@@ -908,3 +908,71 @@ SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "-m <msg>, --message=<msg>".*"Use the given msg as the tag message." ]] || false
}
@test "system-tables: query dolt_history_dolt_procedures system table" {
# Set up test data with procedures across multiple commits
dolt sql -q "CREATE PROCEDURE test_proc1(x INT) SELECT x * 2 as result"
dolt add .
dolt commit -m "add first procedure"
dolt sql -q "CREATE PROCEDURE test_proc2(name VARCHAR(50)) SELECT CONCAT('Hello, ', name) as greeting"
dolt add .
dolt commit -m "add second procedure"
dolt sql -q "DROP PROCEDURE test_proc1"
dolt sql -q "CREATE PROCEDURE test_proc1(x INT, y INT) SELECT x + y as sum" # modified
dolt add .
dolt commit -m "modify first procedure"
# Test that the table exists and has correct schema
run dolt sql -r csv -q 'DESCRIBE dolt_history_dolt_procedures'
[ "$status" -eq 0 ]
[[ "$output" =~ "name,varchar" ]] || false
[[ "$output" =~ "create_stmt,varchar" ]] || false
[[ "$output" =~ "created_at,timestamp" ]] || false
[[ "$output" =~ "modified_at,timestamp" ]] || false
[[ "$output" =~ "sql_mode,varchar" ]] || false
[[ "$output" =~ "commit_hash,char(32)" ]] || false
[[ "$output" =~ "committer,varchar" ]] || false
[[ "$output" =~ "commit_date,datetime" ]] || false
# Test that we have procedure history across commits
run dolt sql -q 'SELECT COUNT(*) FROM dolt_history_dolt_procedures'
[ "$status" -eq 0 ]
# Should have entries for: test_proc1 (3 commits), test_proc2 (2 commits) = 5 total
[[ "$output" =~ "5" ]] || false
# Test filtering by procedure name
run dolt sql -q 'SELECT COUNT(*) FROM dolt_history_dolt_procedures WHERE name = "test_proc1"'
[ "$status" -eq 0 ]
[[ "$output" =~ "3" ]] || false
run dolt sql -q 'SELECT COUNT(*) FROM dolt_history_dolt_procedures WHERE name = "test_proc2"'
[ "$status" -eq 0 ]
[[ "$output" =~ "2" ]] || false
# Test that procedure definitions are captured correctly
run dolt sql -q 'SELECT name FROM dolt_history_dolt_procedures WHERE create_stmt LIKE "%x * 2%"'
[ "$status" -eq 0 ]
[[ "$output" =~ "test_proc1" ]] || false
run dolt sql -q 'SELECT name FROM dolt_history_dolt_procedures WHERE create_stmt LIKE "%x + y%"'
[ "$status" -eq 0 ]
[[ "$output" =~ "test_proc1" ]] || false
# Test commit metadata is present for all entries
run dolt sql -q 'SELECT COUNT(*) FROM dolt_history_dolt_procedures WHERE commit_hash IS NOT NULL AND committer IS NOT NULL'
[ "$status" -eq 0 ]
[[ "$output" =~ "5" ]] || false
# Test distinct procedure names
run dolt sql -q 'SELECT DISTINCT name FROM dolt_history_dolt_procedures ORDER BY name'
[ "$status" -eq 0 ]
[[ "$output" =~ "test_proc1" ]] || false
[[ "$output" =~ "test_proc2" ]] || false
# Test that all entries have created_at and modified_at timestamps
run dolt sql -q 'SELECT COUNT(*) FROM dolt_history_dolt_procedures WHERE created_at IS NOT NULL AND modified_at IS NOT NULL'
[ "$status" -eq 0 ]
[[ "$output" =~ "5" ]] || false
}