mirror of
https://github.com/dolthub/dolt.git
synced 2026-01-30 03:20:18 -06:00
Disambiguate the term "root".
datas.Root(Set) -> datas.Commit(Set) DataStore::Roots() -> DataStore::Heads()
This commit is contained in:
@@ -20,14 +20,14 @@ func main() {
|
||||
}
|
||||
|
||||
lastVal := uint64(0)
|
||||
roots := ds.Roots()
|
||||
if roots.Len() > uint64(0) {
|
||||
lastVal = uint64(roots.Any().Value().(types.UInt64))
|
||||
commits := ds.Heads()
|
||||
if commits.Len() > uint64(0) {
|
||||
lastVal = uint64(commits.Any().Value().(types.UInt64))
|
||||
}
|
||||
newVal := lastVal + 1
|
||||
ds.Commit(datas.NewRootSet().Insert(
|
||||
datas.NewRoot().SetParents(
|
||||
roots.NomsValue()).SetValue(
|
||||
ds.Commit(datas.NewCommitSet().Insert(
|
||||
datas.NewCommit().SetParents(
|
||||
commits.NomsValue()).SetValue(
|
||||
types.UInt64(newVal))))
|
||||
|
||||
fmt.Println(newVal)
|
||||
|
||||
@@ -59,9 +59,9 @@ func main() {
|
||||
value = value.Append(m)
|
||||
}
|
||||
|
||||
roots := ds.Roots()
|
||||
ds.Commit(datas.NewRootSet().Insert(
|
||||
datas.NewRoot().SetParents(
|
||||
roots.NomsValue()).SetValue(
|
||||
commits := ds.Heads()
|
||||
ds.Commit(datas.NewCommitSet().Insert(
|
||||
datas.NewCommit().SetParents(
|
||||
commits.NomsValue()).SetValue(
|
||||
value)))
|
||||
}
|
||||
|
||||
@@ -67,9 +67,9 @@ func main() {
|
||||
}
|
||||
|
||||
func getUser() {
|
||||
roots := ds.Roots()
|
||||
if roots.Len() > uint64(0) {
|
||||
user = UserFromVal(roots.Any().Value())
|
||||
commits := ds.Heads()
|
||||
if commits.Len() > uint64(0) {
|
||||
user = UserFromVal(commits.Any().Value())
|
||||
if checkAuth() {
|
||||
return
|
||||
}
|
||||
@@ -262,12 +262,12 @@ func awaitOAuthResponse(l *net.TCPListener, tempCred *oauth.Credentials) error {
|
||||
}
|
||||
|
||||
func commitUser() {
|
||||
roots := ds.Roots()
|
||||
rootSet := datas.NewRootSet().Insert(
|
||||
datas.NewRoot().SetParents(
|
||||
roots.NomsValue()).SetValue(
|
||||
commits := ds.Heads()
|
||||
commitSet := datas.NewCommitSet().Insert(
|
||||
datas.NewCommit().SetParents(
|
||||
commits.NomsValue()).SetValue(
|
||||
user.NomsValue()))
|
||||
ds.Commit(rootSet)
|
||||
ds.Commit(commitSet)
|
||||
}
|
||||
|
||||
func callFlickrAPI(method string, response interface{}, args *map[string]string) error {
|
||||
|
||||
@@ -40,13 +40,13 @@ func main() {
|
||||
log.Fatalln("Error decoding JSON: ", err)
|
||||
}
|
||||
|
||||
roots := ds.Roots()
|
||||
commits := ds.Heads()
|
||||
|
||||
value := util.NomsValueFromDecodedJSON(jsonObject)
|
||||
|
||||
ds.Commit(datas.NewRootSet().Insert(
|
||||
datas.NewRoot().SetParents(
|
||||
roots.NomsValue()).SetValue(
|
||||
ds.Commit(datas.NewCommitSet().Insert(
|
||||
datas.NewCommit().SetParents(
|
||||
commits.NomsValue()).SetValue(
|
||||
value)))
|
||||
|
||||
}
|
||||
|
||||
@@ -138,13 +138,13 @@ func main() {
|
||||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
rootDataStore := datas.NewDataStore(cs, cs.(chunks.RootTracker))
|
||||
inputDataset := dataset.NewDataset(rootDataStore, *inputID)
|
||||
outputDataset := dataset.NewDataset(rootDataStore, *outputID)
|
||||
dataStore := datas.NewDataStore(cs, cs.(chunks.RootTracker))
|
||||
inputDataset := dataset.NewDataset(dataStore, *inputID)
|
||||
outputDataset := dataset.NewDataset(dataStore, *outputID)
|
||||
|
||||
input := inputDataset.Roots().Any().Value().(types.List)
|
||||
input := inputDataset.Heads().Any().Value().(types.List)
|
||||
output := getIndex(input)
|
||||
|
||||
outputDataset.Commit(datas.NewRootSet().Insert(
|
||||
datas.NewRoot().SetParents(outputDataset.Roots().NomsValue()).SetValue(output)))
|
||||
outputDataset.Commit(datas.NewCommitSet().Insert(
|
||||
datas.NewCommit().SetParents(outputDataset.Heads().NomsValue()).SetValue(output)))
|
||||
}
|
||||
|
||||
@@ -69,8 +69,8 @@ func (s server) handleGetRef(w http.ResponseWriter, hashString string) {
|
||||
}
|
||||
|
||||
func (s server) handleGetDataset(w http.ResponseWriter, id string) {
|
||||
rootDataStore := datas.NewDataStore(s.cs, s.cs.(chunks.RootTracker))
|
||||
dataset := mgmt.GetDatasetRoot(mgmt.GetDatasets(rootDataStore), id)
|
||||
dataStore := datas.NewDataStore(s.cs, s.cs.(chunks.RootTracker))
|
||||
dataset := mgmt.GetDatasetRoot(mgmt.GetDatasets(dataStore), id)
|
||||
if dataset == nil {
|
||||
http.Error(w, fmt.Sprintf("Dataset not found: %s", id), http.StatusNotFound)
|
||||
return
|
||||
|
||||
@@ -18,10 +18,10 @@ var datasetID = "testdataset"
|
||||
func createTestStore() chunks.ChunkStore {
|
||||
ms := &chunks.MemoryStore{}
|
||||
datasetDs := dataset.NewDataset(datas.NewDataStore(ms, ms), datasetID)
|
||||
datasetRoot := types.NewString("Root value for " + datasetID)
|
||||
datasetDs = datasetDs.Commit(datas.NewRootSet().Insert(
|
||||
datas.NewRoot().SetParents(
|
||||
types.NewSet()).SetValue(datasetRoot)))
|
||||
datasetValue := types.NewString("Value for " + datasetID)
|
||||
datasetDs = datasetDs.Commit(datas.NewCommitSet().Insert(
|
||||
datas.NewCommit().SetParents(
|
||||
types.NewSet()).SetValue(datasetValue)))
|
||||
return ms
|
||||
}
|
||||
|
||||
@@ -62,7 +62,7 @@ func TestGetRef(t *testing.T) {
|
||||
s := server{ms}
|
||||
s.handle(w, req)
|
||||
assert.Equal(w.Code, http.StatusOK)
|
||||
assert.Equal(`j {"set":[{"ref":"sha1-b432c2dd6d7b6e7e163cab2517d1e6221d5d595c"}]}
|
||||
assert.Equal(`j {"set":[{"ref":"sha1-8cd398c860f50a43898e3f95b266b5fdecb4e1e6"}]}
|
||||
`, w.Body.String())
|
||||
}
|
||||
|
||||
|
||||
@@ -69,9 +69,9 @@ func main() {
|
||||
noms := util.NomsValueFromDecodedJSON(objects)
|
||||
|
||||
if !*noIO {
|
||||
ds.Commit(datas.NewRootSet().Insert(
|
||||
datas.NewRoot().SetParents(
|
||||
ds.Roots().NomsValue()).SetValue(noms)))
|
||||
ds.Commit(datas.NewCommitSet().Insert(
|
||||
datas.NewCommit().SetParents(
|
||||
ds.Heads().NomsValue()).SetValue(noms)))
|
||||
}
|
||||
if *memprofile != "" {
|
||||
f, err := os.Create(*memprofile)
|
||||
|
||||
56
datas/commit_cache.go
Normal file
56
datas/commit_cache.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package datas
|
||||
|
||||
import (
|
||||
"github.com/attic-labs/noms/chunks"
|
||||
"github.com/attic-labs/noms/ref"
|
||||
"github.com/attic-labs/noms/types"
|
||||
)
|
||||
|
||||
// commitCache maintains an in-memory cache of all known commits.
|
||||
type commitCache struct {
|
||||
source chunks.ChunkSource
|
||||
refs map[ref.Ref]bool
|
||||
}
|
||||
|
||||
func (cache *commitCache) updateFromCommit(commit Commit) {
|
||||
if _, ok := cache.refs[commit.Ref()]; ok {
|
||||
return
|
||||
}
|
||||
|
||||
parents := commit.Parents()
|
||||
parents.Iter(func(commit types.Value) (stop bool) {
|
||||
cache.updateFromCommit(CommitFromVal(commit))
|
||||
return
|
||||
})
|
||||
cache.refs[commit.Ref()] = true
|
||||
}
|
||||
|
||||
func (cache *commitCache) Update(currentCommits CommitSet) {
|
||||
if currentCommits.Len() == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
commitsRef := currentCommits.Ref()
|
||||
if _, ok := cache.refs[commitsRef]; ok {
|
||||
return
|
||||
}
|
||||
|
||||
commitSet := CommitSet{types.MustReadValue(commitsRef, cache.source).(types.Set)}
|
||||
commitSet.Iter(func(commit Commit) (stop bool) {
|
||||
cache.updateFromCommit(commit)
|
||||
return
|
||||
})
|
||||
cache.refs[commitsRef] = true
|
||||
}
|
||||
|
||||
func (cache *commitCache) Contains(candidate ref.Ref) bool {
|
||||
_, ok := cache.refs[candidate]
|
||||
return ok
|
||||
}
|
||||
|
||||
func NewCommitCache(source chunks.ChunkSource) *commitCache {
|
||||
return &commitCache{
|
||||
source,
|
||||
make(map[ref.Ref]bool),
|
||||
}
|
||||
}
|
||||
@@ -11,95 +11,95 @@ type DataStore struct {
|
||||
chunks.ChunkStore
|
||||
|
||||
rt chunks.RootTracker
|
||||
rc *rootCache
|
||||
roots RootSet
|
||||
rc *commitCache
|
||||
heads CommitSet
|
||||
}
|
||||
|
||||
func NewDataStore(cs chunks.ChunkStore, rt chunks.RootTracker) DataStore {
|
||||
return newDataStoreInternal(cs, rt, NewRootCache(cs))
|
||||
return newDataStoreInternal(cs, rt, NewCommitCache(cs))
|
||||
}
|
||||
|
||||
func newDataStoreInternal(cs chunks.ChunkStore, rt chunks.RootTracker, rc *rootCache) DataStore {
|
||||
func newDataStoreInternal(cs chunks.ChunkStore, rt chunks.RootTracker, rc *commitCache) DataStore {
|
||||
return DataStore{
|
||||
cs, rt, rc, rootSetFromRef(rt.Root(), cs),
|
||||
cs, rt, rc, commitSetFromRef(rt.Root(), cs),
|
||||
}
|
||||
}
|
||||
|
||||
func rootSetFromRef(rootRef ref.Ref, cs chunks.ChunkSource) RootSet {
|
||||
var roots RootSet
|
||||
if (rootRef == ref.Ref{}) {
|
||||
roots = NewRootSet()
|
||||
func commitSetFromRef(commitRef ref.Ref, cs chunks.ChunkSource) CommitSet {
|
||||
var commits CommitSet
|
||||
if (commitRef == ref.Ref{}) {
|
||||
commits = NewCommitSet()
|
||||
} else {
|
||||
roots = RootSetFromVal(types.MustReadValue(rootRef, cs).(types.Set))
|
||||
commits = CommitSetFromVal(types.MustReadValue(commitRef, cs).(types.Set))
|
||||
}
|
||||
|
||||
return roots
|
||||
return commits
|
||||
}
|
||||
|
||||
func (ds *DataStore) Roots() RootSet {
|
||||
return ds.roots
|
||||
func (ds *DataStore) Heads() CommitSet {
|
||||
return ds.heads
|
||||
}
|
||||
|
||||
func (ds *DataStore) Commit(newRoots RootSet) DataStore {
|
||||
Chk.True(newRoots.Len() > 0)
|
||||
func (ds *DataStore) Commit(newCommits CommitSet) DataStore {
|
||||
Chk.True(newCommits.Len() > 0)
|
||||
// TODO: We probably shouldn't let this go *forever*. Considrer putting a limit and... I know don't...panicing?
|
||||
for !ds.doCommit(newRoots) {
|
||||
for !ds.doCommit(newCommits) {
|
||||
}
|
||||
return newDataStoreInternal(ds.ChunkStore, ds.rt, ds.rc)
|
||||
}
|
||||
|
||||
// doCommit manages concurrent access the single logical piece of mutable state: the current root (rootSet). doCommit is optimistic in that it is attempting to create a new root making the assumption that currentRootRef is the existing root. The call to UpdateRoot below will fail if that assumption fails (e.g. because of a race with another writer) and the entire algorigthm must be tried again.
|
||||
func (ds *DataStore) doCommit(roots RootSet) bool {
|
||||
Chk.True(roots.Len() > 0)
|
||||
// doCommit manages concurrent access the single logical piece of mutable state: the set of current heads. doCommit is optimistic in that it is attempting to update heads making the assumption that currentRootRef is the ref of the current heads. The call to UpdateRoot below will fail if that assumption fails (e.g. because of a race with another writer) and the entire algorigthm must be tried again.
|
||||
func (ds *DataStore) doCommit(commits CommitSet) bool {
|
||||
Chk.True(commits.Len() > 0)
|
||||
|
||||
currentRootRef := ds.rt.Root()
|
||||
|
||||
// Note: |currentRoots| may be different from |ds.roots| and *must* be consistent with |currentRootRef|.
|
||||
var currentRoots RootSet
|
||||
if currentRootRef == ds.roots.Ref() {
|
||||
currentRoots = ds.roots
|
||||
// Note: |currentHeads| may be different from |ds.heads| and *must* be consistent with |currentCommitRef|.
|
||||
var currentHeads CommitSet
|
||||
if currentRootRef == ds.heads.Ref() {
|
||||
currentHeads = ds.heads
|
||||
} else {
|
||||
currentRoots = rootSetFromRef(currentRootRef, ds)
|
||||
currentHeads = commitSetFromRef(currentRootRef, ds)
|
||||
}
|
||||
|
||||
newRoots := roots.Union(currentRoots)
|
||||
newHeads := commits.Union(currentHeads)
|
||||
|
||||
roots.Iter(func(root Root) (stop bool) {
|
||||
if ds.isPrexisting(root, currentRoots) {
|
||||
newRoots = newRoots.Remove(root)
|
||||
commits.Iter(func(commit Commit) (stop bool) {
|
||||
if ds.isPrexisting(commit, currentHeads) {
|
||||
newHeads = newHeads.Remove(commit)
|
||||
} else {
|
||||
newRoots = RootSetFromVal(newRoots.NomsValue().Subtract(root.Parents()))
|
||||
newHeads = CommitSetFromVal(newHeads.NomsValue().Subtract(commit.Parents()))
|
||||
}
|
||||
|
||||
return
|
||||
})
|
||||
|
||||
if newRoots.Len() == 0 || newRoots.Equals(currentRoots) {
|
||||
if newHeads.Len() == 0 || newHeads.Equals(currentHeads) {
|
||||
return true
|
||||
}
|
||||
|
||||
// TODO: This set will be orphaned if this UpdateRoot below fails
|
||||
newRootRef, err := types.WriteValue(newRoots.NomsValue(), ds)
|
||||
// TODO: This set will be orphaned if this UpdateCommit below fails
|
||||
newRootRef, err := types.WriteValue(newHeads.NomsValue(), ds)
|
||||
Chk.NoError(err)
|
||||
|
||||
return ds.rt.UpdateRoot(newRootRef, currentRootRef)
|
||||
}
|
||||
|
||||
func (ds *DataStore) isPrexisting(root Root, currentRoots RootSet) bool {
|
||||
if currentRoots.Has(root) {
|
||||
func (ds *DataStore) isPrexisting(commit Commit, currentHeads CommitSet) bool {
|
||||
if currentHeads.Has(commit) {
|
||||
return true
|
||||
}
|
||||
|
||||
// If a new root directly superceeds an existing current root, it can't have already been committed because its hash would be uncomputable.
|
||||
superceedsCurrentRoot := false
|
||||
root.Parents().Iter(func(parent types.Value) (stop bool) {
|
||||
superceedsCurrentRoot = currentRoots.Has(RootFromVal(parent))
|
||||
return superceedsCurrentRoot
|
||||
// If a new commit directly superceeds an existing current commit, it can't have already been committed because its hash would be uncomputable.
|
||||
superceedsCurrentCommit := false
|
||||
commit.Parents().Iter(func(parent types.Value) (stop bool) {
|
||||
superceedsCurrentCommit = currentHeads.Has(CommitFromVal(parent))
|
||||
return superceedsCurrentCommit
|
||||
})
|
||||
if superceedsCurrentRoot {
|
||||
if superceedsCurrentCommit {
|
||||
return false
|
||||
}
|
||||
|
||||
ds.rc.Update(currentRoots)
|
||||
return ds.rc.Contains(root.Ref())
|
||||
ds.rc.Update(currentHeads)
|
||||
return ds.rc.Contains(commit.Ref())
|
||||
}
|
||||
|
||||
@@ -19,82 +19,82 @@ func TestDataStoreCommit(t *testing.T) {
|
||||
chunks := chunks.NewFileStore(dir, "root")
|
||||
ds := NewDataStore(chunks, chunks)
|
||||
|
||||
roots := ds.Roots()
|
||||
assert.Equal(uint64(0), roots.Len())
|
||||
commits := ds.Heads()
|
||||
assert.Equal(uint64(0), commits.Len())
|
||||
|
||||
// |a|
|
||||
a := NewRoot().SetParents(roots.NomsValue()).SetValue(types.NewString("a"))
|
||||
aSet := NewRootSet().Insert(a)
|
||||
a := NewCommit().SetParents(commits.NomsValue()).SetValue(types.NewString("a"))
|
||||
aSet := NewCommitSet().Insert(a)
|
||||
ds2 := ds.Commit(aSet)
|
||||
|
||||
// The old datastore still still references the old roots.
|
||||
assert.True(ds.Roots().Equals(roots))
|
||||
// The old datastore still still references the old commits.
|
||||
assert.True(ds.Heads().Equals(commits))
|
||||
|
||||
// The new datastore has the new roots.
|
||||
assert.True(ds2.Roots().Equals(aSet))
|
||||
// The new datastore has the new commits.
|
||||
assert.True(ds2.Heads().Equals(aSet))
|
||||
ds = ds2
|
||||
|
||||
// |a| <- |b|
|
||||
b := NewRoot().SetParents(aSet.NomsValue()).SetValue(types.NewString("b"))
|
||||
bSet := NewRootSet().Insert(b)
|
||||
b := NewCommit().SetParents(aSet.NomsValue()).SetValue(types.NewString("b"))
|
||||
bSet := NewCommitSet().Insert(b)
|
||||
ds = ds.Commit(bSet)
|
||||
assert.True(ds.Roots().Equals(bSet))
|
||||
assert.True(ds.Heads().Equals(bSet))
|
||||
|
||||
// |a| <- |b|
|
||||
// \----|c|
|
||||
c := NewRoot().SetParents(aSet.NomsValue()).SetValue(types.NewString("c"))
|
||||
cSet := NewRootSet().Insert(c)
|
||||
c := NewCommit().SetParents(aSet.NomsValue()).SetValue(types.NewString("c"))
|
||||
cSet := NewCommitSet().Insert(c)
|
||||
ds = ds.Commit(cSet)
|
||||
bcSet := bSet.Insert(c)
|
||||
assert.True(ds.Roots().Equals(bcSet))
|
||||
assert.True(ds.Heads().Equals(bcSet))
|
||||
|
||||
// |a| <- |b|
|
||||
// \----|c|
|
||||
// \---|d|
|
||||
d := NewRoot().SetParents(aSet.NomsValue()).SetValue(types.NewString("d"))
|
||||
dSet := NewRootSet().Insert(d)
|
||||
d := NewCommit().SetParents(aSet.NomsValue()).SetValue(types.NewString("d"))
|
||||
dSet := NewCommitSet().Insert(d)
|
||||
types.WriteValue(dSet.NomsValue(), chunks)
|
||||
|
||||
ds = ds.Commit(dSet)
|
||||
bcdSet := bcSet.Insert(d)
|
||||
assert.True(ds.Roots().Equals(bcdSet))
|
||||
assert.True(ds.Heads().Equals(bcdSet))
|
||||
|
||||
// |a| <- |b| <-- |e|
|
||||
// \----|c| <--/
|
||||
// \---|d|
|
||||
e := NewRoot().SetParents(bcSet.NomsValue()).SetValue(types.NewString("e"))
|
||||
eSet := NewRootSet().Insert(e)
|
||||
e := NewCommit().SetParents(bcSet.NomsValue()).SetValue(types.NewString("e"))
|
||||
eSet := NewCommitSet().Insert(e)
|
||||
ds = ds.Commit(eSet)
|
||||
deSet := dSet.Insert(e)
|
||||
assert.True(ds.Roots().Equals(deSet))
|
||||
assert.True(ds.Heads().Equals(deSet))
|
||||
|
||||
// |a| <- |b| <-- |e| <- |f|
|
||||
// \----|c| <--/ /
|
||||
// \---|d| <-------/
|
||||
f := NewRoot().SetParents(deSet.NomsValue()).SetValue(types.NewString("f"))
|
||||
fSet := NewRootSet().Insert(f)
|
||||
f := NewCommit().SetParents(deSet.NomsValue()).SetValue(types.NewString("f"))
|
||||
fSet := NewCommitSet().Insert(f)
|
||||
ds = ds.Commit(fSet)
|
||||
assert.True(ds.Roots().Equals(fSet))
|
||||
assert.True(ds.Heads().Equals(fSet))
|
||||
|
||||
// Attempt to recommit |b|
|
||||
ds = ds.Commit(bSet)
|
||||
assert.True(ds.Roots().Equals(fSet))
|
||||
assert.True(ds.Heads().Equals(fSet))
|
||||
|
||||
// Attempt to recommit |f|
|
||||
ds = ds.Commit(fSet)
|
||||
assert.True(ds.Roots().Equals(fSet))
|
||||
assert.True(ds.Heads().Equals(fSet))
|
||||
|
||||
// Attempt to recommit |c| while committing |g|
|
||||
// |a| <- |b| <-- |e| <- |f| <- |g|
|
||||
// \----|c| <--/ / /
|
||||
// \---|d| <-------/------/
|
||||
fdSet := fSet.Insert(d)
|
||||
g := NewRoot().SetParents(fdSet.NomsValue()).SetValue(types.NewString("g"))
|
||||
gSet := NewRootSet().Insert(g)
|
||||
g := NewCommit().SetParents(fdSet.NomsValue()).SetValue(types.NewString("g"))
|
||||
gSet := NewCommitSet().Insert(g)
|
||||
gdSet := gSet.Insert(c)
|
||||
|
||||
ds = ds.Commit(gdSet)
|
||||
assert.True(ds.Roots().Equals(gSet))
|
||||
assert.True(ds.Heads().Equals(gSet))
|
||||
|
||||
// / -|h|
|
||||
// / |
|
||||
@@ -102,12 +102,12 @@ func TestDataStoreCommit(t *testing.T) {
|
||||
// \----|c| <--/ / /
|
||||
// \---|d| <-------/------/
|
||||
abSet := aSet.Insert(b)
|
||||
h := NewRoot().SetParents(abSet.NomsValue()).SetValue(types.NewString("h"))
|
||||
hSet := NewRootSet().Insert(h)
|
||||
h := NewCommit().SetParents(abSet.NomsValue()).SetValue(types.NewString("h"))
|
||||
hSet := NewCommitSet().Insert(h)
|
||||
|
||||
ds = ds.Commit(hSet)
|
||||
hgSet := hSet.Insert(g)
|
||||
assert.True(ds.Roots().Equals(hgSet))
|
||||
assert.True(ds.Heads().Equals(hgSet))
|
||||
}
|
||||
|
||||
func TestDataStoreConcurrency(t *testing.T) {
|
||||
@@ -116,20 +116,20 @@ func TestDataStoreConcurrency(t *testing.T) {
|
||||
defer os.Remove(dir)
|
||||
assert.NoError(err)
|
||||
|
||||
chunks := chunks.NewFileStore(dir, "root")
|
||||
chunks := chunks.NewFileStore(dir, "commit")
|
||||
ds := NewDataStore(chunks, chunks)
|
||||
|
||||
// Setup:
|
||||
// |a| <- |b|
|
||||
// \----|c|
|
||||
a := NewRoot().SetParents(ds.Roots().NomsValue()).SetValue(types.NewString("a"))
|
||||
aSet := NewRootSet().Insert(a)
|
||||
a := NewCommit().SetParents(ds.Heads().NomsValue()).SetValue(types.NewString("a"))
|
||||
aSet := NewCommitSet().Insert(a)
|
||||
ds = ds.Commit(aSet)
|
||||
b := NewRoot().SetParents(aSet.NomsValue()).SetValue(types.NewString("b"))
|
||||
bSet := NewRootSet().Insert(b)
|
||||
b := NewCommit().SetParents(aSet.NomsValue()).SetValue(types.NewString("b"))
|
||||
bSet := NewCommitSet().Insert(b)
|
||||
ds = ds.Commit(bSet)
|
||||
c := NewRoot().SetParents(aSet.NomsValue()).SetValue(types.NewString("c"))
|
||||
cSet := NewRootSet().Insert(c)
|
||||
c := NewCommit().SetParents(aSet.NomsValue()).SetValue(types.NewString("c"))
|
||||
cSet := NewCommitSet().Insert(c)
|
||||
ds = ds.Commit(cSet)
|
||||
bcSet := bSet.Insert(c)
|
||||
|
||||
@@ -139,21 +139,21 @@ func TestDataStoreConcurrency(t *testing.T) {
|
||||
// Change 1:
|
||||
// |a| <- |b| <- |d|
|
||||
// \----|c| --/
|
||||
d := NewRoot().SetParents(bcSet.NomsValue()).SetValue(types.NewString("d"))
|
||||
dSet := NewRootSet().Insert(d)
|
||||
d := NewCommit().SetParents(bcSet.NomsValue()).SetValue(types.NewString("d"))
|
||||
dSet := NewCommitSet().Insert(d)
|
||||
types.WriteValue(dSet.NomsValue(), chunks)
|
||||
ds = ds.Commit(dSet)
|
||||
|
||||
// Change 2:
|
||||
// |a| <- |b| <- |e|
|
||||
// \----|c| --/
|
||||
e := NewRoot().SetParents(bcSet.NomsValue()).SetValue(types.NewString("e"))
|
||||
eSet := NewRootSet().Insert(e)
|
||||
e := NewCommit().SetParents(bcSet.NomsValue()).SetValue(types.NewString("e"))
|
||||
eSet := NewCommitSet().Insert(e)
|
||||
types.WriteValue(eSet.NomsValue(), chunks)
|
||||
ds2 = ds2.Commit(eSet)
|
||||
|
||||
// The chunkstore should have tracked that two conflicting commits happened and both |d| and |e| are now roots
|
||||
// The chunkstore should have tracked that two conflicting commits happened and both |d| and |e| are now commits
|
||||
deSet := dSet.Insert(e)
|
||||
finalRoots := RootSetFromVal(types.MustReadValue(chunks.Root(), chunks).(types.Set))
|
||||
assert.True(finalRoots.Equals(deSet))
|
||||
finalCommits := CommitSetFromVal(types.MustReadValue(chunks.Root(), chunks).(types.Set))
|
||||
assert.True(finalCommits.Equals(deSet))
|
||||
}
|
||||
|
||||
@@ -17,21 +17,21 @@ func main() {
|
||||
return
|
||||
}
|
||||
|
||||
root := types.NewMap(
|
||||
commit := types.NewMap(
|
||||
types.NewString("$type"), types.NewString("noms.StructDef"),
|
||||
types.NewString("$name"), types.NewString("Root"),
|
||||
types.NewString("$name"), types.NewString("Commit"),
|
||||
types.NewString("value"), types.NewString("value"),
|
||||
// grump... circular definition :(
|
||||
types.NewString("parents"), types.NewString("set"),
|
||||
)
|
||||
|
||||
rootSet := types.NewMap(
|
||||
commitSet := types.NewMap(
|
||||
types.NewString("$type"), types.NewString("noms.SetDef"),
|
||||
types.NewString("elem"), root)
|
||||
types.NewString("elem"), commit)
|
||||
|
||||
f, err := os.OpenFile(*outFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
defer f.Close()
|
||||
Chk.NoError(err)
|
||||
ng := nomgen.New(f)
|
||||
ng.WriteGo(rootSet, "datas")
|
||||
ng.WriteGo(commitSet, "datas")
|
||||
}
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
package datas
|
||||
|
||||
import (
|
||||
"github.com/attic-labs/noms/chunks"
|
||||
"github.com/attic-labs/noms/ref"
|
||||
"github.com/attic-labs/noms/types"
|
||||
)
|
||||
|
||||
// rootCache maintains an in-memory cache of all known roots.
|
||||
type rootCache struct {
|
||||
source chunks.ChunkSource
|
||||
refs map[ref.Ref]bool
|
||||
}
|
||||
|
||||
func (cache *rootCache) updateFromCommit(root Root) {
|
||||
if _, ok := cache.refs[root.Ref()]; ok {
|
||||
return
|
||||
}
|
||||
|
||||
parents := root.Parents()
|
||||
parents.Iter(func(commit types.Value) (stop bool) {
|
||||
cache.updateFromCommit(RootFromVal(commit))
|
||||
return
|
||||
})
|
||||
cache.refs[root.Ref()] = true
|
||||
}
|
||||
|
||||
func (cache *rootCache) Update(currentRoots RootSet) {
|
||||
if currentRoots.Len() == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
rootsRef := currentRoots.Ref()
|
||||
if _, ok := cache.refs[rootsRef]; ok {
|
||||
return
|
||||
}
|
||||
|
||||
rootSet := RootSet{types.MustReadValue(rootsRef, cache.source).(types.Set)}
|
||||
rootSet.Iter(func(commit Root) (stop bool) {
|
||||
cache.updateFromCommit(commit)
|
||||
return
|
||||
})
|
||||
cache.refs[rootsRef] = true
|
||||
}
|
||||
|
||||
func (cache *rootCache) Contains(candidate ref.Ref) bool {
|
||||
_, ok := cache.refs[candidate]
|
||||
return ok
|
||||
}
|
||||
|
||||
func NewRootCache(source chunks.ChunkSource) *rootCache {
|
||||
return &rootCache{
|
||||
source,
|
||||
make(map[ref.Ref]bool),
|
||||
}
|
||||
}
|
||||
@@ -8,73 +8,73 @@ import (
|
||||
"github.com/attic-labs/noms/types"
|
||||
)
|
||||
|
||||
// RootSet
|
||||
// CommitSet
|
||||
|
||||
type RootSet struct {
|
||||
type CommitSet struct {
|
||||
s types.Set
|
||||
}
|
||||
|
||||
type RootSetIterCallback (func(p Root) (stop bool))
|
||||
type CommitSetIterCallback (func(p Commit) (stop bool))
|
||||
|
||||
func NewRootSet() RootSet {
|
||||
return RootSet{types.NewSet()}
|
||||
func NewCommitSet() CommitSet {
|
||||
return CommitSet{types.NewSet()}
|
||||
}
|
||||
|
||||
func RootSetFromVal(p types.Value) RootSet {
|
||||
return RootSet{p.(types.Set)}
|
||||
func CommitSetFromVal(p types.Value) CommitSet {
|
||||
return CommitSet{p.(types.Set)}
|
||||
}
|
||||
|
||||
func (s RootSet) NomsValue() types.Set {
|
||||
func (s CommitSet) NomsValue() types.Set {
|
||||
return s.s
|
||||
}
|
||||
|
||||
func (s RootSet) Equals(p RootSet) bool {
|
||||
func (s CommitSet) Equals(p CommitSet) bool {
|
||||
return s.s.Equals(p.s)
|
||||
}
|
||||
|
||||
func (s RootSet) Ref() ref.Ref {
|
||||
func (s CommitSet) Ref() ref.Ref {
|
||||
return s.s.Ref()
|
||||
}
|
||||
|
||||
func (s RootSet) Empty() bool {
|
||||
func (s CommitSet) Empty() bool {
|
||||
return s.s.Empty()
|
||||
}
|
||||
|
||||
func (s RootSet) Len() uint64 {
|
||||
func (s CommitSet) Len() uint64 {
|
||||
return s.s.Len()
|
||||
}
|
||||
|
||||
func (s RootSet) Has(p Root) bool {
|
||||
func (s CommitSet) Has(p Commit) bool {
|
||||
return s.s.Has(p.NomsValue())
|
||||
}
|
||||
|
||||
func (s RootSet) Iter(cb RootSetIterCallback) {
|
||||
func (s CommitSet) Iter(cb CommitSetIterCallback) {
|
||||
s.s.Iter(func(v types.Value) bool {
|
||||
return cb(RootFromVal(v))
|
||||
return cb(CommitFromVal(v))
|
||||
})
|
||||
}
|
||||
|
||||
func (s RootSet) Insert(p ...Root) RootSet {
|
||||
return RootSet{s.s.Insert(s.fromElemSlice(p)...)}
|
||||
func (s CommitSet) Insert(p ...Commit) CommitSet {
|
||||
return CommitSet{s.s.Insert(s.fromElemSlice(p)...)}
|
||||
}
|
||||
|
||||
func (s RootSet) Remove(p ...Root) RootSet {
|
||||
return RootSet{s.s.Remove(s.fromElemSlice(p)...)}
|
||||
func (s CommitSet) Remove(p ...Commit) CommitSet {
|
||||
return CommitSet{s.s.Remove(s.fromElemSlice(p)...)}
|
||||
}
|
||||
|
||||
func (s RootSet) Union(others ...RootSet) RootSet {
|
||||
return RootSet{s.s.Union(s.fromStructSlice(others)...)}
|
||||
func (s CommitSet) Union(others ...CommitSet) CommitSet {
|
||||
return CommitSet{s.s.Union(s.fromStructSlice(others)...)}
|
||||
}
|
||||
|
||||
func (s RootSet) Subtract(others ...RootSet) RootSet {
|
||||
return RootSet{s.s.Subtract(s.fromStructSlice(others)...)}
|
||||
func (s CommitSet) Subtract(others ...CommitSet) CommitSet {
|
||||
return CommitSet{s.s.Subtract(s.fromStructSlice(others)...)}
|
||||
}
|
||||
|
||||
func (s RootSet) Any() Root {
|
||||
return RootFromVal(s.s.Any())
|
||||
func (s CommitSet) Any() Commit {
|
||||
return CommitFromVal(s.s.Any())
|
||||
}
|
||||
|
||||
func (s RootSet) fromStructSlice(p []RootSet) []types.Set {
|
||||
func (s CommitSet) fromStructSlice(p []CommitSet) []types.Set {
|
||||
r := make([]types.Set, len(p))
|
||||
for i, v := range p {
|
||||
r[i] = v.s
|
||||
@@ -82,7 +82,7 @@ func (s RootSet) fromStructSlice(p []RootSet) []types.Set {
|
||||
return r
|
||||
}
|
||||
|
||||
func (s RootSet) fromElemSlice(p []Root) []types.Value {
|
||||
func (s CommitSet) fromElemSlice(p []Commit) []types.Value {
|
||||
r := make([]types.Value, len(p))
|
||||
for i, v := range p {
|
||||
r[i] = v.NomsValue()
|
||||
@@ -90,46 +90,46 @@ func (s RootSet) fromElemSlice(p []Root) []types.Value {
|
||||
return r
|
||||
}
|
||||
|
||||
// Root
|
||||
// Commit
|
||||
|
||||
type Root struct {
|
||||
type Commit struct {
|
||||
m types.Map
|
||||
}
|
||||
|
||||
func NewRoot() Root {
|
||||
return Root{types.NewMap()}
|
||||
func NewCommit() Commit {
|
||||
return Commit{types.NewMap()}
|
||||
}
|
||||
|
||||
func RootFromVal(v types.Value) Root {
|
||||
return Root{v.(types.Map)}
|
||||
func CommitFromVal(v types.Value) Commit {
|
||||
return Commit{v.(types.Map)}
|
||||
}
|
||||
|
||||
// TODO: This was going to be called Value() but it collides with root.value. We need some other place to put the built-in fields like Value() and Equals().
|
||||
func (s Root) NomsValue() types.Map {
|
||||
func (s Commit) NomsValue() types.Map {
|
||||
return s.m
|
||||
}
|
||||
|
||||
func (s Root) Equals(p Root) bool {
|
||||
func (s Commit) Equals(p Commit) bool {
|
||||
return s.m.Equals(p.m)
|
||||
}
|
||||
|
||||
func (s Root) Ref() ref.Ref {
|
||||
func (s Commit) Ref() ref.Ref {
|
||||
return s.m.Ref()
|
||||
}
|
||||
|
||||
func (s Root) Parents() types.Set {
|
||||
func (s Commit) Parents() types.Set {
|
||||
return types.SetFromVal(s.m.Get(types.NewString("parents")))
|
||||
}
|
||||
|
||||
func (s Root) SetParents(p types.Set) Root {
|
||||
return RootFromVal(s.m.Set(types.NewString("parents"), p))
|
||||
func (s Commit) SetParents(p types.Set) Commit {
|
||||
return CommitFromVal(s.m.Set(types.NewString("parents"), p))
|
||||
}
|
||||
|
||||
func (s Root) Value() types.Value {
|
||||
func (s Commit) Value() types.Value {
|
||||
return (s.m.Get(types.NewString("value")))
|
||||
}
|
||||
|
||||
func (s Root) SetValue(p types.Value) Root {
|
||||
return RootFromVal(s.m.Set(types.NewString("value"), p))
|
||||
func (s Commit) SetValue(p types.Value) Commit {
|
||||
return CommitFromVal(s.m.Set(types.NewString("value"), p))
|
||||
}
|
||||
|
||||
|
||||
@@ -15,12 +15,12 @@ type Dataset struct {
|
||||
datas.DataStore
|
||||
}
|
||||
|
||||
func NewDataset(rootStore datas.DataStore, datasetID string) Dataset {
|
||||
return Dataset{datas.NewDataStore(rootStore, &datasetRootTracker{rootStore, datasetID})}
|
||||
func NewDataset(parentStore datas.DataStore, datasetID string) Dataset {
|
||||
return Dataset{datas.NewDataStore(parentStore, &datasetRootTracker{parentStore, datasetID})}
|
||||
}
|
||||
|
||||
func (ds *Dataset) Commit(newRoots datas.RootSet) Dataset {
|
||||
return Dataset{ds.DataStore.Commit(newRoots)}
|
||||
func (ds *Dataset) Commit(newCommits datas.CommitSet) Dataset {
|
||||
return Dataset{ds.DataStore.Commit(newCommits)}
|
||||
}
|
||||
|
||||
type datasetFlags struct {
|
||||
@@ -45,20 +45,20 @@ func (f datasetFlags) CreateDataset() *Dataset {
|
||||
}
|
||||
|
||||
// Blech, kinda sucks to typecast to RootTracker, but we know that all the implementations of ChunkStore that implement it.
|
||||
rootDataStore := datas.NewDataStore(cs, cs.(chunks.RootTracker))
|
||||
commitDataStore := datas.NewDataStore(cs, cs.(chunks.RootTracker))
|
||||
|
||||
ds := NewDataset(rootDataStore, *f.datasetID)
|
||||
ds := NewDataset(commitDataStore, *f.datasetID)
|
||||
return &ds
|
||||
}
|
||||
|
||||
// TODO: Move to separate file
|
||||
type datasetRootTracker struct {
|
||||
rootStore datas.DataStore
|
||||
datasetID string
|
||||
parentStore datas.DataStore
|
||||
datasetID string
|
||||
}
|
||||
|
||||
func (rt *datasetRootTracker) Root() ref.Ref {
|
||||
dataset := mgmt.GetDatasetRoot(mgmt.GetDatasets(rt.rootStore), rt.datasetID)
|
||||
dataset := mgmt.GetDatasetRoot(mgmt.GetDatasets(rt.parentStore), rt.datasetID)
|
||||
if dataset == nil {
|
||||
return ref.Ref{}
|
||||
} else {
|
||||
@@ -71,7 +71,7 @@ func (rt *datasetRootTracker) UpdateRoot(current, last ref.Ref) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
datasetRoot := types.MustReadValue(current, rt.rootStore)
|
||||
rt.rootStore = mgmt.CommitDatasets(rt.rootStore, mgmt.SetDatasetRoot(mgmt.GetDatasets(rt.rootStore), rt.datasetID, datasetRoot))
|
||||
datasetCommit := types.MustReadValue(current, rt.parentStore)
|
||||
rt.parentStore = mgmt.CommitDatasets(rt.parentStore, mgmt.SetDatasetRoot(mgmt.GetDatasets(rt.parentStore), rt.datasetID, datasetCommit))
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -9,30 +9,30 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestDatasetRootTracker(t *testing.T) {
|
||||
func TestDatasetCommitTracker(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
datasetId1 := "testdataset"
|
||||
datasetId2 := "othertestdataset"
|
||||
ms := &chunks.MemoryStore{}
|
||||
|
||||
datasetDs1 := NewDataset(datas.NewDataStore(ms, ms), datasetId1)
|
||||
datasetRoot1 := types.NewString("Root value for " + datasetId1)
|
||||
datasetDs1 = datasetDs1.Commit(datas.NewRootSet().Insert(
|
||||
datas.NewRoot().SetParents(
|
||||
types.NewSet()).SetValue(datasetRoot1)))
|
||||
datasetCommit1 := types.NewString("Commit value for " + datasetId1)
|
||||
datasetDs1 = datasetDs1.Commit(datas.NewCommitSet().Insert(
|
||||
datas.NewCommit().SetParents(
|
||||
types.NewSet()).SetValue(datasetCommit1)))
|
||||
|
||||
datasetDs2 := NewDataset(datas.NewDataStore(ms, ms), datasetId2)
|
||||
datasetRoot2 := types.NewString("Root value for " + datasetId2)
|
||||
datasetDs2 = datasetDs2.Commit(datas.NewRootSet().Insert(
|
||||
datas.NewRoot().SetParents(
|
||||
types.NewSet()).SetValue(datasetRoot2)))
|
||||
datasetCommit2 := types.NewString("Commit value for " + datasetId2)
|
||||
datasetDs2 = datasetDs2.Commit(datas.NewCommitSet().Insert(
|
||||
datas.NewCommit().SetParents(
|
||||
types.NewSet()).SetValue(datasetCommit2)))
|
||||
|
||||
assert.EqualValues(1, datasetDs2.Roots().Len())
|
||||
assert.EqualValues(1, datasetDs1.Roots().Len())
|
||||
assert.EqualValues(datasetRoot1, datasetDs1.Roots().Any().Value())
|
||||
assert.EqualValues(datasetRoot2, datasetDs2.Roots().Any().Value())
|
||||
assert.False(datasetDs2.Roots().Any().Value().Equals(datasetRoot1))
|
||||
assert.False(datasetDs1.Roots().Any().Value().Equals(datasetRoot2))
|
||||
assert.EqualValues(1, datasetDs2.Heads().Len())
|
||||
assert.EqualValues(1, datasetDs1.Heads().Len())
|
||||
assert.EqualValues(datasetCommit1, datasetDs1.Heads().Any().Value())
|
||||
assert.EqualValues(datasetCommit2, datasetDs2.Heads().Any().Value())
|
||||
assert.False(datasetDs2.Heads().Any().Value().Equals(datasetCommit1))
|
||||
assert.False(datasetDs1.Heads().Any().Value().Equals(datasetCommit2))
|
||||
|
||||
assert.Equal(ms.Root().String(), "sha1-183d248d05e639b41054d76076444991b560cdb2")
|
||||
assert.Equal("sha1-9b9fcfcd7e41ff727e6bea0edfa26f71def178a5", ms.Root().String())
|
||||
}
|
||||
|
||||
@@ -8,19 +8,19 @@ import (
|
||||
)
|
||||
|
||||
func GetDatasets(ds datas.DataStore) DatasetSet {
|
||||
if ds.Roots().Empty() {
|
||||
if ds.Heads().Empty() {
|
||||
return NewDatasetSet()
|
||||
} else {
|
||||
// BUG 13: We don't ever want to branch the datasets database. Currently we can't avoid that, but we should change DataStore::Commit() to support that mode of operation.
|
||||
Chk.EqualValues(1, ds.Roots().Len())
|
||||
return DatasetSetFromVal(ds.Roots().Any().Value())
|
||||
Chk.EqualValues(1, ds.Heads().Len())
|
||||
return DatasetSetFromVal(ds.Heads().Any().Value())
|
||||
}
|
||||
}
|
||||
|
||||
func CommitDatasets(ds datas.DataStore, datasets DatasetSet) datas.DataStore {
|
||||
return ds.Commit(datas.NewRootSet().Insert(
|
||||
datas.NewRoot().SetParents(
|
||||
ds.Roots().NomsValue()).SetValue(
|
||||
return ds.Commit(datas.NewCommitSet().Insert(
|
||||
datas.NewCommit().SetParents(
|
||||
ds.Heads().NomsValue()).SetValue(
|
||||
datasets.NomsValue())))
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user