Add walk package

The walk package contains walk.Some() and walk.All() which let you walk the Chunk graph starting at a given Ref. I also added GetReachabilitySetDiff(), which will determine which refs in a given ChunkSource can be reached from one given ref, but not the other. Towards issue #82
2026-05-19 02:59:21 -05:00 · 2015-07-23 13:23:40 -07:00
parent 1ce5e3763b
commit 0ba565fed6
4 changed files with 240 additions and 0 deletions
@@ -0,0 +1,21 @@
+package walk
+
+import (
+	"github.com/attic-labs/noms/chunks"
+	"github.com/attic-labs/noms/ref"
+)
+
+// GetReachabilitySetDiff returns the refs of the chunks reachable from 'big' that cannot be reached from 'small'
+func GetReachabilitySetDiff(small, big ref.Ref, cs chunks.ChunkSource) (refs []ref.Ref) {
+	smallRefs := map[ref.Ref]bool{}
+	All(small, cs, func(r ref.Ref) {
+		smallRefs[r] = true
+	})
+	Some(big, cs, func(r ref.Ref) (skip bool) {
+		if skip = smallRefs[r]; !skip {
+			refs = append(refs, r)
+		}
+		return
+	})
+	return
+}
@@ -0,0 +1,42 @@
+package walk
+
+import (
+	"testing"
+
+	"github.com/attic-labs/noms/chunks"
+	"github.com/attic-labs/noms/ref"
+	"github.com/attic-labs/noms/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetReachabilitySetDiff(t *testing.T) {
+	assert := assert.New(t)
+	cs := &chunks.TestStore{}
+
+	storeAndRef := func(v types.Value) (r ref.Ref) {
+		r, err := types.WriteValue(v, cs)
+		assert.NoError(err)
+		return
+	}
+
+	// {"string": "string",
+	//  "map": {"nested": "string"}
+	//  "mtlist": []
+	// }
+	small := types.NewMap(
+		types.NewString("string"), types.NewString("string"),
+		types.NewString("map"), types.NewMap(types.NewString("nested"), types.NewString("string")),
+		types.NewString("mtlist"), types.NewList())
+
+	setVal := types.NewSet(types.Int32(7))
+	big := small.Set(types.NewString("set"), setVal)
+
+	var hashes []string
+	for _, r := range GetReachabilitySetDiff(storeAndRef(small), storeAndRef(big), cs) {
+		hashes = append(hashes, r.String())
+	}
+
+	assert.Contains(hashes, setVal.Ref().String())
+
+	assert.Empty(GetReachabilitySetDiff(small.Ref(), small.Ref(), cs))
+}
@@ -0,0 +1,43 @@
+package walk
+
+import (
+	"github.com/attic-labs/noms/chunks"
+	"github.com/attic-labs/noms/dbg"
+	"github.com/attic-labs/noms/ref"
+	"github.com/attic-labs/noms/types"
+)
+
+// SomeCallback takes a ref.Ref and returns a bool indicating whether
+// the current walk should skip the tree descending from value.
+type SomeCallback func(r ref.Ref) bool
+
+// AllCallback takes a ref and processes it.
+type AllCallback func(r ref.Ref)
+
+// Some recursively walks over all ref.Refs reachable from r and calls cb on them.
+// If cb ever returns true, the walk will stop recursing on the current ref.
+func Some(r ref.Ref, cs chunks.ChunkSource, cb SomeCallback) {
+	v, err := types.ReadValue(r, cs)
+	dbg.Chk.NoError(err)
+	doTreeWalk(v, cs, cb)
+}
+
+func All(r ref.Ref, cs chunks.ChunkSource, cb AllCallback) {
+	v, err := types.ReadValue(r, cs)
+	dbg.Chk.NoError(err)
+	doTreeWalk(v, cs, func(r ref.Ref) (skip bool) {
+		cb(r)
+		return
+	})
+}
+
+func doTreeWalk(v types.Value, cs chunks.ChunkSource, cb SomeCallback) {
+	if cb(v.Ref()) {
+		return
+	}
+	for _, cf := range v.Chunks() {
+		cv, err := cf.Deref(cs)
+		dbg.Chk.NoError(err)
+		doTreeWalk(cv, cs, cb)
+	}
+}
@@ -0,0 +1,134 @@
+package walk
+
+import (
+	"testing"
+
+	"github.com/attic-labs/noms/chunks"
+	"github.com/attic-labs/noms/dbg"
+	"github.com/attic-labs/noms/ref"
+	"github.com/attic-labs/noms/types"
+	"github.com/stretchr/testify/suite"
+)
+
+func TestWalkTestSuite(t *testing.T) {
+	suite.Run(t, &WalkAllTestSuite{})
+	//suite.Run(t, &WalkTestSuite{})
+}
+
+type WalkAllTestSuite struct {
+	suite.Suite
+	cs *chunks.TestStore
+}
+
+func (suite *WalkAllTestSuite) SetupTest() {
+	suite.cs = &chunks.TestStore{}
+}
+
+func (suite *WalkAllTestSuite) walkWorker(r ref.Ref, expected int) {
+	actual := 0
+	All(r, suite.cs, func(r ref.Ref) {
+		actual++
+	})
+	suite.Equal(expected, actual)
+}
+
+func (suite *WalkAllTestSuite) storeAndRef(v types.Value) (r ref.Ref) {
+	r, err := types.WriteValue(v, suite.cs)
+	dbg.Chk.NoError(err)
+	return
+}
+
+func (suite *WalkAllTestSuite) TestWalkPrimitives() {
+	suite.walkWorker(suite.storeAndRef(types.Float64(0.0)), 1)
+	suite.walkWorker(suite.storeAndRef(types.NewString("hello")), 1)
+}
+
+func (suite *WalkAllTestSuite) TestWalkComposites() {
+	suite.walkWorker(suite.storeAndRef(types.NewList()), 1)
+	suite.walkWorker(suite.storeAndRef(types.NewList(types.Bool(false), types.Int32(8))), 1)
+	suite.walkWorker(suite.storeAndRef(types.NewSet()), 1)
+	suite.walkWorker(suite.storeAndRef(types.NewSet(types.Bool(false), types.Int32(8))), 1)
+	suite.walkWorker(suite.storeAndRef(types.NewMap()), 1)
+	suite.walkWorker(suite.storeAndRef(types.NewMap(types.Int32(8), types.Bool(true), types.Int32(0), types.Bool(false))), 1)
+}
+
+func (suite *WalkAllTestSuite) TestWalkNestedComposites() {
+	suite.walkWorker(suite.storeAndRef(types.NewList(types.NewSet(), types.Int32(8))), 2)
+	suite.walkWorker(suite.storeAndRef(types.NewSet(types.NewList(), types.NewSet())), 3)
+	// {"string": "string",
+	//  "list": [false true],
+	//  "map": {"nested": "string"}
+	//  "mtlist": []
+	//  "set": [5 7 8]
+	//  []: "wow"
+	// }
+	nested := types.NewMap(
+		types.NewString("string"), types.NewString("string"),
+		types.NewString("list"), types.NewList(types.Bool(false), types.Bool(true)),
+		types.NewString("map"), types.NewMap(types.NewString("nested"), types.NewString("string")),
+		types.NewString("mtlist"), types.NewList(),
+		types.NewString("set"), types.NewSet(types.Int32(5), types.Int32(7), types.Int32(8)),
+		types.NewList(), types.NewString("wow"))
+	suite.walkWorker(suite.storeAndRef(nested), 6)
+}
+
+type WalkTestSuite struct {
+	WalkAllTestSuite
+	shouldSee types.Value
+	mustSkip  types.List
+	deadValue types.Value
+}
+
+func (suite *WalkTestSuite) SetupTest() {
+	suite.shouldSee = types.NewString("zzz")
+	suite.deadValue = types.UInt64(0xDEADBEEF)
+	suite.mustSkip = types.NewList(suite.deadValue)
+	suite.cs = &chunks.TestStore{}
+}
+
+func (suite *WalkTestSuite) TestStopWalkImmediately() {
+	actual := 0
+	Some(suite.storeAndRef(types.NewList(types.NewSet(), types.NewList())), suite.cs, func(r ref.Ref) bool {
+		actual++
+		return true
+	})
+	suite.Equal(1, actual)
+}
+
+func (suite *WalkTestSuite) skipWorker(composite types.Value) (reached []ref.Ref) {
+	Some(suite.storeAndRef(composite), suite.cs, func(r ref.Ref) bool {
+		suite.NotEqual(r, suite.deadValue.Ref(), "Should never have reached %+v", suite.deadValue)
+		reached = append(reached, r)
+		return r == suite.mustSkip.Ref()
+	})
+	return reached
+}
+
+// Skipping a sub-tree must allow other items in the list to be processed.
+func (suite *WalkTestSuite) TestSkipListElement() {
+	wholeList := types.NewList(suite.mustSkip, suite.shouldSee, suite.shouldSee)
+	reached := suite.skipWorker(wholeList)
+	for _, r := range []ref.Ref{wholeList.Ref(), suite.mustSkip.Ref(), suite.shouldSee.Ref(), suite.shouldSee.Ref()} {
+		suite.Contains(reached, r)
+	}
+	suite.Len(reached, 4)
+}
+
+func (suite *WalkTestSuite) TestSkipSetElement() {
+	wholeSet := types.NewSet(suite.mustSkip, suite.shouldSee).Insert(suite.shouldSee)
+	reached := suite.skipWorker(wholeSet)
+	suite.EqualValues(types.NewList(wholeSet, suite.mustSkip, suite.shouldSee), reached)
+}
+
+func (suite *WalkTestSuite) TestSkipMapValue() {
+	shouldAlsoSee := types.NewString("Also good")
+	wholeMap := types.NewMap(suite.shouldSee, suite.mustSkip, shouldAlsoSee, suite.shouldSee)
+	reached := suite.skipWorker(wholeMap)
+	suite.EqualValues(types.NewList(wholeMap, shouldAlsoSee, suite.shouldSee, suite.shouldSee, suite.mustSkip), reached)
+}
+
+func (suite *WalkTestSuite) TestSkipMapKey() {
+	wholeMap := types.NewMap(suite.mustSkip, suite.shouldSee, suite.shouldSee, suite.shouldSee)
+	reached := suite.skipWorker(wholeMap)
+	suite.EqualValues(types.NewList(wholeMap, suite.mustSkip, suite.shouldSee, suite.shouldSee, suite.shouldSee), reached)
+}