Files
dolt/go/libraries/doltcore/schema/statistic.go
T
Maximilian Hoffman b8b2ff1c99 [stats] Rewrite stat management to use single threaded event loop (#8815)
* [stats] event loop

* more progress

* basic scheduler test working

* analyze

* add/drop hooks

* gc

* delete an alter

* drop index and table

* fix other tests

* branch management

* starter for kv

* gc and refactor maintanance

* fix bucket doubling

* delete log

* better bucket counting

* test for disk round trip

* more prolly stats gc tests

* rotate backing stats db

* progress towards swapping old for new, deleting old code

* fix gc bucket overflow

* test for gc overflow

* org and closers

* save progress update

* finally get first two bats running

* startup bound hash issue

* rewrite GC to be synchronous, fix more bugs

* fix session freshness

* fix branch gc

* cache writes and gc are serialized

* fix gc/branch update dropped hashes

* fix gc race, doubling race, jobs race

* fix more races

* docs

* convert bats to script tests

* more tests, purge/stop

* validate

* docs

* some PR cleanup

* more cleanup

* stash for pull

* fix bucket hash conflicts

* Fix more collection bugs.

* bump, timer proc

* more test fixes

* cache bats changes

* Another deadlock

* delete comment

* fmt

* no read replica stats

* fix plan tests

* branch qualified analyze fix

* [no-release-notes] go: statspro/jobqueue: Create a SerialQueue, which can perform asynchronous work on a worker thread.

* go: statspro/jobqueue: A bit of cleanup, fix a flakey test.

* rewrite with GDQ

* prog

* tests run

* fix info and storage

* outline for gc impl

* fix tests and races

* bump

* better error and panic management

* better start/stop/wait

* Add rate limiting

* gc ticker

* docs

* doc

* test prog

* fix more tests

* finish up listener tests

* add comments

* gc concurrency

* enginetests and statspro tests passing

* simplify listeners

* bats progress

* small edits

* tests progress

* bats are running

* fmt

* build

* edits

* fix interface

* fix build

* stats alternate index types

* fix mem test

* build

* fix more tests

* fmt

* more fmt

* copyright

* license

* fix races

* syntax error

* fix windows path

* nil mcv panic

* fix test races

* bump def job interval to 30ms

* deterministic tests

* more tests

* TEMP COMMIT: valctx plus some other stuff...

* shorter concurrency tests

* [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh

* nondeterministic test

* try to make queue tests less racy

* missed one start

* stats granular session locks

* simplify a little

* try to avoid serialq test deadlock

* try to fix flakes

* more races

* bump

* another race

* cleanup

* more cleanup

* revert ctx validation

* most zach comments

* more comments

* more race

* bump

* more race

* bump

* schemas

* skip windows racees

* standardize server config init, use background threads management

* [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh

* default stats noop

* threads management improvements

* undo change

* move stats initialization back to engine

* [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh

---------

Co-authored-by: Aaron Son <aaron@dolthub.com>
Co-authored-by: max-hoffman <max-hoffman@users.noreply.github.com>
2025-03-20 15:56:48 -07:00

115 lines
4.7 KiB
Go

// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package schema
import (
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/go-mysql-server/sql/types"
stypes "github.com/dolthub/dolt/go/store/types"
)
const StatsVersion int64 = 1
const (
StatsDbColName = "database_name"
StatsTableColName = "table_name"
StatsIndexColName = "index_name"
StatsBranchName = "branch"
StatsCommitHashColName = "commit_hash"
StatsPrefixLenName = "prefix_len"
StatsRowCountColName = "row_count"
StatsDistinctCountColName = "distinct_count"
StatsNullCountColName = "null_count"
StatsColumnsColName = "columns"
StatsTypesColName = "types"
StatsUpperBoundColName = "upper_bound"
StatsUpperBoundCntColName = "upper_bound_cnt"
StatsCreatedAtColName = "created_at"
StatsMcv1ColName = "mcv1"
StatsMcv2ColName = "mcv2"
StatsMcv3ColName = "mcv3"
StatsMcv4ColName = "mcv4"
StatsMcvCountsColName = "mcv_counts"
StatsVersionColName = "version"
)
const (
StatsDbTag uint64 = iota
StatsTableTag
StatsIndexTag
StatsPositionTag
StatsVersionTag
StatsPrefixLenTag
StatsCommitHashTag
StatsRowCountTag
StatsDistinctCountTag
StatsNullCountTag
StatsColumnsTag
StatsTypesTag
StatsUpperBoundTag
StatsUpperBoundCntTag
StatsCreatedAtTag
StatsMcv1Tag
StatsMcv2Tag
StatsMcv3Tag
StatsMcv4Tag
StatsMcvCountsTag
)
func StatsTableSqlSchema(dbName string) sql.PrimaryKeySchema {
return sql.PrimaryKeySchema{
Schema: sql.Schema{
&sql.Column{Name: StatsDbColName, Type: types.Text, DatabaseSource: dbName},
&sql.Column{Name: StatsTableColName, Type: types.Text, DatabaseSource: dbName},
&sql.Column{Name: StatsIndexColName, Type: types.Text, DatabaseSource: dbName},
&sql.Column{Name: StatsRowCountColName, Type: types.Int64, DatabaseSource: dbName},
&sql.Column{Name: StatsDistinctCountColName, Type: types.Int64, DatabaseSource: dbName},
&sql.Column{Name: StatsNullCountColName, Type: types.Int64, DatabaseSource: dbName},
&sql.Column{Name: StatsColumnsColName, Type: types.Text, DatabaseSource: dbName},
&sql.Column{Name: StatsTypesColName, Type: types.Text, DatabaseSource: dbName},
&sql.Column{Name: StatsUpperBoundColName, Type: types.Text, DatabaseSource: dbName},
&sql.Column{Name: StatsUpperBoundCntColName, Type: types.Int64, DatabaseSource: dbName},
&sql.Column{Name: StatsCreatedAtColName, Type: types.Datetime, DatabaseSource: dbName},
&sql.Column{Name: StatsMcv1ColName, Type: types.Text, DatabaseSource: dbName},
&sql.Column{Name: StatsMcv2ColName, Type: types.Text, DatabaseSource: dbName},
&sql.Column{Name: StatsMcv3ColName, Type: types.Text, DatabaseSource: dbName},
&sql.Column{Name: StatsMcv4ColName, Type: types.Text, DatabaseSource: dbName},
&sql.Column{Name: StatsMcvCountsColName, Type: types.Text, DatabaseSource: dbName},
},
}
}
var StatsTableDoltSchema = StatsTableDoltSchemaGen()
func StatsTableDoltSchemaGen() Schema {
colColl := NewColCollection(
NewColumn(StatsPrefixLenName, StatsPrefixLenTag, stypes.IntKind, true, NotNullConstraint{}),
NewColumn(StatsCommitHashColName, StatsCommitHashTag, stypes.StringKind, true, NotNullConstraint{}),
NewColumn(StatsVersionColName, StatsVersionTag, stypes.IntKind, false, NotNullConstraint{}),
NewColumn(StatsRowCountColName, StatsRowCountTag, stypes.IntKind, false, NotNullConstraint{}),
NewColumn(StatsDistinctCountColName, StatsDistinctCountTag, stypes.IntKind, false, NotNullConstraint{}),
NewColumn(StatsNullCountColName, StatsNullCountTag, stypes.IntKind, false, NotNullConstraint{}),
NewColumn(StatsUpperBoundColName, StatsUpperBoundTag, stypes.StringKind, false, NotNullConstraint{}),
NewColumn(StatsUpperBoundCntColName, StatsUpperBoundCntTag, stypes.IntKind, false, NotNullConstraint{}),
NewColumn(StatsMcv1ColName, StatsMcv1Tag, stypes.StringKind, false),
NewColumn(StatsMcv2ColName, StatsMcv2Tag, stypes.StringKind, false),
NewColumn(StatsMcv3ColName, StatsMcv3Tag, stypes.StringKind, false),
NewColumn(StatsMcv4ColName, StatsMcv4Tag, stypes.StringKind, false),
NewColumn(StatsMcvCountsColName, StatsMcvCountsTag, stypes.StringKind, false, NotNullConstraint{}),
)
return MustSchemaFromCols(colColl)
}