implementing map/set/orderedSequence diff (#1521)

resolving a portion of issue #1075

js/src/ordered-sequence-diff.js

@@ -89,6 +89,7 @@ async function doFastForward(allowPastEnd: boolean,
       await Promise.all([syncWithIdx(a, aHasMore), syncWithIdx(b, bHasMore)]);
     } else {
       if (a.canAdvanceLocal() && b.canAdvanceLocal()) {
+        // Performance optimisation: allowing non-async resolution of leaf elements
         aHasMore = a.advanceLocal(allowPastEnd);
         bHasMore = b.advanceLocal(allowPastEnd);
       } else {

js/src/sequence.js

@@ -108,6 +108,7 @@ export class SequenceCursor<T, S: Sequence> {
 
   /**
    * Advances the cursor within the current chunk.
+   * Performance optimisation: allowing non-async resolution of leaf elements
    *
    * Returns true if the cursor advanced to a valid position within this chunk, false if not.
    *
@@ -133,6 +134,7 @@ export class SequenceCursor<T, S: Sequence> {
 
   /**
    * Returns true if the cursor can advance within the current chunk to a valid position.
+   * Performance optimisation: allowing non-async resolution of leaf elements
    */
   canAdvanceLocal(): boolean {
     return this.idx < this.length - 1;

types/blob.go

@@ -38,6 +38,20 @@ func (b Blob) Reader() io.ReadSeeker {
 	return &BlobReader{b.seq, cursor, nil, 0}
 }
 
+// Collection interface
+func (b Blob) Len() uint64 {
+	return b.seq.numLeaves()
+}
+
+func (b Blob) Empty() bool {
+	return b.Len() == 0
+}
+
+func (b Blob) sequence() sequence {
+	return b.seq
+}
+
+// Value interface
 func (b Blob) Equals(other Value) bool {
 	return other != nil && b.Ref() == other.Ref()
 }
@@ -50,30 +64,18 @@ func (b Blob) Ref() ref.Ref {
 	return EnsureRef(b.ref, b)
 }
 
-func (b Blob) Len() uint64 {
-	return b.seq.numLeaves()
-}
-
-func (b Blob) Empty() bool {
-	return b.Len() == 0
+func (b Blob) ChildValues() []Value {
+	return []Value{}
 }
 
 func (b Blob) Chunks() []Ref {
 	return b.seq.Chunks()
 }
 
-func (b Blob) ChildValues() []Value {
-	return []Value{}
-}
-
 func (b Blob) Type() *Type {
 	return b.seq.Type()
 }
 
-func (b Blob) sequence() sequence {
-	return b.seq
-}
-
 type BlobReader struct {
 	seq           indexedSequence
 	cursor        *sequenceCursor

types/blob_leaf_sequence.go

@@ -9,6 +9,11 @@ func newBlobLeafSequence(vr ValueReader, data []byte) indexedSequence {
 	return blobLeafSequence{data, vr}
 }
 
+func (bl blobLeafSequence) getOffset(idx int) uint64 {
+	return uint64(idx)
+}
+
+// sequence interface
 func (bl blobLeafSequence) getItem(idx int) sequenceItem {
 	return bl.data[idx]
 }
@@ -21,8 +26,8 @@ func (bl blobLeafSequence) numLeaves() uint64 {
 	return uint64(len(bl.data))
 }
 
-func (bl blobLeafSequence) getOffset(idx int) uint64 {
-	return uint64(idx)
+func (bl blobLeafSequence) valueReader() ValueReader {
+	return bl.vr
 }
 
 func (bl blobLeafSequence) Chunks() []Ref {
@@ -32,7 +37,3 @@ func (bl blobLeafSequence) Chunks() []Ref {
 func (bl blobLeafSequence) Type() *Type {
 	return BlobType
 }
-
-func (bl blobLeafSequence) valueReader() ValueReader {
-	return bl.vr
-}

types/list.go

@@ -45,10 +45,20 @@ func NewStreamingList(vrw ValueReadWriter, values <-chan Value) <-chan List {
 	return out
 }
 
-func (l List) Type() *Type {
-	return l.seq.Type()
+// Collection interface
+func (l List) Len() uint64 {
+	return l.seq.numLeaves()
 }
 
+func (l List) Empty() bool {
+	return l.Len() == 0
+}
+
+func (l List) sequence() sequence {
+	return l.seq
+}
+
+// Value interface
 func (l List) Equals(other Value) bool {
 	return other != nil && l.Ref() == other.Ref()
 }
@@ -61,14 +71,6 @@ func (l List) Ref() ref.Ref {
 	return EnsureRef(l.ref, l)
 }
 
-func (l List) Len() uint64 {
-	return l.seq.numLeaves()
-}
-
-func (l List) Empty() bool {
-	return l.Len() == 0
-}
-
 func (l List) ChildValues() (values []Value) {
 	l.IterAll(func(v Value, idx uint64) {
 		values = append(values, v)
@@ -80,8 +82,8 @@ func (l List) Chunks() []Ref {
 	return l.seq.Chunks()
 }
 
-func (l List) sequence() sequence {
-	return l.seq
+func (l List) Type() *Type {
+	return l.seq.Type()
 }
 
 func (l List) Get(idx uint64) Value {

types/list_leaf_sequence.go

@@ -15,6 +15,11 @@ func newListLeafSequence(vr ValueReader, v ...Value) indexedSequence {
 	return listLeafSequence{v, t, vr}
 }
 
+func (ll listLeafSequence) getOffset(idx int) uint64 {
+	return uint64(idx)
+}
+
+// sequence interface
 func (ll listLeafSequence) getItem(idx int) sequenceItem {
 	return ll.values[idx]
 }
@@ -27,8 +32,8 @@ func (ll listLeafSequence) numLeaves() uint64 {
 	return uint64(len(ll.values))
 }
 
-func (ll listLeafSequence) getOffset(idx int) uint64 {
-	return uint64(idx)
+func (ll listLeafSequence) valueReader() ValueReader {
+	return ll.vr
 }
 
 func (ll listLeafSequence) Chunks() (chunks []Ref) {
@@ -41,7 +46,3 @@ func (ll listLeafSequence) Chunks() (chunks []Ref) {
 func (ll listLeafSequence) Type() *Type {
 	return ll.t
 }
-
-func (ll listLeafSequence) valueReader() ValueReader {
-	return ll.vr
-}

types/map.go

@@ -33,10 +33,24 @@ func NewMap(kv ...Value) Map {
 	return seq.Done().(Map)
 }
 
-func (m Map) Type() *Type {
-	return m.seq.Type()
+func (m Map) Diff(last Map) (added []Value, removed []Value, modified []Value) {
+	return orderedSequenceDiff(last.sequence().(orderedSequence), m.sequence().(orderedSequence))
 }
 
+// Collection interface
+func (m Map) Len() uint64 {
+	return m.seq.numLeaves()
+}
+
+func (m Map) Empty() bool {
+	return m.Len() == 0
+}
+
+func (m Map) sequence() sequence {
+	return m.seq
+}
+
+// Value interface
 func (m Map) Equals(other Value) bool {
 	return other != nil && m.Ref() == other.Ref()
 }
@@ -49,14 +63,6 @@ func (m Map) Ref() ref.Ref {
 	return EnsureRef(m.ref, m)
 }
 
-func (m Map) Len() uint64 {
-	return m.seq.numLeaves()
-}
-
-func (m Map) Empty() bool {
-	return m.Len() == 0
-}
-
 func (m Map) ChildValues() (values []Value) {
 	m.IterAll(func(k, v Value) {
 		values = append(values, k, v)
@@ -68,8 +74,8 @@ func (m Map) Chunks() []Ref {
 	return m.seq.Chunks()
 }
 
-func (m Map) sequence() sequence {
-	return m.seq
+func (m Map) Type() *Type {
+	return m.seq.Type()
 }
 
 func (m Map) First() (Value, Value) {

types/map_leaf_sequence.go

@@ -22,6 +22,7 @@ func newMapLeafSequence(vr ValueReader, data ...mapEntry) orderedSequence {
 	return mapLeafSequence{data, t, vr}
 }
 
+// sequence interface
 func (ml mapLeafSequence) getItem(idx int) sequenceItem {
 	return ml.data[idx]
 }
@@ -34,16 +35,8 @@ func (ml mapLeafSequence) numLeaves() uint64 {
 	return uint64(len(ml.data))
 }
 
-func (ml mapLeafSequence) getKey(idx int) Value {
-	return ml.data[idx].key
-}
-
-func (ml mapLeafSequence) Len() uint64 {
-	return uint64(len(ml.data))
-}
-
-func (ml mapLeafSequence) Empty() bool {
-	return ml.Len() == uint64(0)
+func (ml mapLeafSequence) valueReader() ValueReader {
+	return ml.vr
 }
 
 func (ml mapLeafSequence) Chunks() (chunks []Ref) {
@@ -58,6 +51,22 @@ func (ml mapLeafSequence) Type() *Type {
 	return ml.t
 }
 
-func (ml mapLeafSequence) valueReader() ValueReader {
-	return ml.vr
+// orderedSequence interface
+func (ml mapLeafSequence) getKey(idx int) Value {
+	return ml.data[idx].key
+}
+
+func (ml mapLeafSequence) equalsAt(idx int, other interface{}) bool {
+	entry := ml.data[idx]
+	otherEntry := other.(mapEntry)
+	return entry.key.Equals(otherEntry.key) && entry.value.Equals(otherEntry.value)
+}
+
+// Collection interface
+func (ml mapLeafSequence) Len() uint64 {
+	return uint64(len(ml.data))
+}
+
+func (ml mapLeafSequence) Empty() bool {
+	return ml.Len() == uint64(0)
 }

types/map_test.go

@@ -46,15 +46,58 @@ func (tm testMap) Remove(from, to int) testMap {
 	return testMap{entries, tm.tr, tm.knownBadKey}
 }
 
-func (tm testMap) toMap() Map {
-	keyvals := []Value{}
+func (tm testMap) MaybeGet(key Value) (v Value, ok bool) {
 	for _, entry := range tm.entries {
-		keyvals = append(keyvals, entry.key, entry.value)
+		if entry.key.Equals(key) {
+			return entry.value, true
+		}
 	}
-	return NewMap(keyvals...)
+	return nil, false
 }
 
-func (tm testMap) toCompoundMap() Map {
+func (tm testMap) Diff(last testMap) (added []Value, removed []Value, modified []Value) {
+	// Note: this could be use tm.toMap/last.toMap and then tmMap.Diff(lastMap) but the
+	// purpose of this method is to be redundant.
+	added = make([]Value, 0)
+	removed = make([]Value, 0)
+	modified = make([]Value, 0)
+
+	if len(tm.entries) == 0 && len(last.entries) == 0 {
+		return // nothing changed
+	}
+	if len(tm.entries) == 0 {
+		// everything removed
+		for _, entry := range last.entries {
+			removed = append(removed, entry.key)
+		}
+		return
+	}
+	if len(last.entries) == 0 {
+		// everything added
+		for _, entry := range tm.entries {
+			added = append(added, entry.key)
+		}
+		return
+	}
+
+	for _, entry := range tm.entries {
+		otherValue, exists := last.MaybeGet(entry.key)
+		if !exists {
+			added = append(added, entry.key)
+		} else if !entry.value.Equals(otherValue) {
+			modified = append(modified, entry.key)
+		}
+	}
+	for _, entry := range last.entries {
+		_, exists := tm.MaybeGet(entry.key)
+		if !exists {
+			removed = append(removed, entry.key)
+		}
+	}
+	return
+}
+
+func (tm testMap) toMap() Map {
 	keyvals := []Value{}
 	for _, entry := range tm.entries {
 		keyvals = append(keyvals, entry.key, entry.value)
@@ -84,6 +127,14 @@ func newTestMap(length int) testMap {
 	return testMap{entries, MakeMapType(NumberType, NumberType), Number(length + 2)}
 }
 
+func newTestMapFromMap(m Map) testMap {
+	entries := make([]mapEntry, 0, m.Len())
+	m.IterAll(func(key, value Value) {
+		entries = append(entries, mapEntry{key, value})
+	})
+	return testMap{entries, m.Type(), Number(-0)}
+}
+
 func newTestMapWithGen(length int, gen testMapGenFn, tr *Type) testMap {
 	s := rand.NewSource(4242)
 	used := map[int64]bool{}
@@ -193,6 +244,41 @@ func getTestRefToValueOrderMap(scale int, vw ValueWriter) testMap {
 	}, refType)
 }
 
+func diffMapTest(assert *assert.Assertions, m1 Map, m2 Map, numAddsExpected int, numRemovesExpected int, numModifiedExpected int) (added []Value, removed []Value, modified []Value) {
+	added, removed, modified = m1.Diff(m2)
+	assert.Equal(numAddsExpected, len(added), "num added is not as expected")
+	assert.Equal(numRemovesExpected, len(removed), "num removed is not as expected")
+	assert.Equal(numModifiedExpected, len(modified), "num modified is not as expected")
+
+	tm1 := newTestMapFromMap(m1)
+	tm2 := newTestMapFromMap(m2)
+	tmAdded, tmRemoved, tmModified := tm1.Diff(tm2)
+	assert.Equal(numAddsExpected, len(tmAdded), "num added is not as expected")
+	assert.Equal(numRemovesExpected, len(tmRemoved), "num removed is not as expected")
+	assert.Equal(numModifiedExpected, len(tmModified), "num modified is not as expected")
+
+	assert.Equal(added, tmAdded, "map added != tmMap added")
+	assert.Equal(removed, tmRemoved, "map removed != tmMap removed")
+	assert.Equal(modified, tmModified, "map modified != tmMap modified")
+	return
+}
+
+func TestMapDiff(t *testing.T) {
+	testMap1 := newTestMapWithGen(int(mapPattern)*2, func(v Number) Value {
+		return v
+	}, NumberType)
+	testMap2 := newTestMapWithGen(int(mapPattern)*2, func(v Number) Value {
+		return v
+	}, NumberType)
+	testMapAdded, testMapRemoved, testMapModified := testMap1.Diff(testMap2)
+	map1 := testMap1.toMap()
+	map2 := testMap2.toMap()
+	mapDiffAdded, mapDiffRemoved, mapDiffModified := map1.Diff(map2)
+	assert.Equal(t, testMapAdded, mapDiffAdded, "testMap.diff != map.diff")
+	assert.Equal(t, testMapRemoved, mapDiffRemoved, "testMap.diff != map.diff")
+	assert.Equal(t, testMapModified, mapDiffModified, "testMap.diff != map.diff")
+}
+
 func TestNewMap(t *testing.T) {
 	assert := assert.New(t)
 	m := NewMap()
@@ -247,6 +333,7 @@ func TestMapHas(t *testing.T) {
 			assert.True(m2.Has(k))
 			assert.True(m2.Get(k).Equals(v))
 		}
+		diffMapTest(assert, m, m2, 0, 0, 0)
 	}
 
 	doTest(getTestNativeOrderMap(16))
@@ -281,6 +368,7 @@ func TestMapRemove(t *testing.T) {
 			actual := whole.Remove(tm.entries[i].key)
 			assert.Equal(expected.Len(), actual.Len())
 			assert.True(expected.Equals(actual))
+			diffMapTest(assert, expected, actual, 0, 0, 0)
 		}
 		for i := 0; i < len(tm.entries); i += incr {
 			run(i)
@@ -376,6 +464,7 @@ func TestMapSet(t *testing.T) {
 			actual := tm.Remove(from, to).toMap().SetM(tm.Flatten(from, to)...)
 			assert.Equal(expected.Len(), actual.Len())
 			assert.True(expected.Equals(actual))
+			diffMapTest(assert, expected, actual, 0, 0, 0)
 		}
 		for i := 0; i < len(tm.entries)-offset; i += incr {
 			run(i, i+offset)
@@ -411,6 +500,7 @@ func TestMapSetExistingKeyToNewValue(t *testing.T) {
 	assert.Equal(expected.Len(), actual.Len())
 	assert.True(expected.Equals(actual))
 	assert.False(original.Equals(actual))
+	diffMapTest(assert, expected, actual, 0, 0, 0)
 }
 
 func TestMapSetM(t *testing.T) {
@@ -564,6 +654,12 @@ func TestMapEquals(t *testing.T) {
 	assert.True(m2.Equals(m1))
 	assert.True(m3.Equals(m2))
 	assert.True(m2.Equals(m3))
+	diffMapTest(assert, m1, m2, 0, 0, 0)
+	diffMapTest(assert, m1, m3, 0, 0, 0)
+	diffMapTest(assert, m2, m1, 0, 0, 0)
+	diffMapTest(assert, m2, m3, 0, 0, 0)
+	diffMapTest(assert, m3, m1, 0, 0, 0)
+	diffMapTest(assert, m3, m2, 0, 0, 0)
 
 	m1 = NewMap(NewString("foo"), Number(0.0), NewString("bar"), NewList())
 	m2 = m2.SetM(NewString("foo"), Number(0.0), NewString("bar"), NewList())
@@ -571,6 +667,12 @@ func TestMapEquals(t *testing.T) {
 	assert.True(m2.Equals(m1))
 	assert.False(m2.Equals(m3))
 	assert.False(m3.Equals(m2))
+	diffMapTest(assert, m1, m2, 0, 0, 0)
+	diffMapTest(assert, m1, m3, 2, 0, 0)
+	diffMapTest(assert, m2, m1, 0, 0, 0)
+	diffMapTest(assert, m2, m3, 2, 0, 0)
+	diffMapTest(assert, m3, m1, 0, 2, 0)
+	diffMapTest(assert, m3, m2, 0, 2, 0)
 }
 
 func TestMapNotStringKeys(t *testing.T) {

types/meta_sequence.go

@@ -61,6 +61,11 @@ type metaSequenceObject struct {
 	vr     ValueReader
 }
 
+func (ms metaSequenceObject) data() metaSequenceData {
+	return ms.tuples
+}
+
+// sequence interface
 func (ms metaSequenceObject) getItem(idx int) sequenceItem {
 	return ms.tuples[idx]
 }
@@ -69,31 +74,10 @@ func (ms metaSequenceObject) seqLen() int {
 	return len(ms.tuples)
 }
 
-func (ms metaSequenceObject) getChildSequence(idx int) sequence {
-	mt := ms.tuples[idx]
-	if mt.child != nil {
-		return mt.child.sequence()
-	}
-
-	return mt.childRef.TargetValue(ms.vr).(Collection).sequence()
-}
-
 func (ms metaSequenceObject) valueReader() ValueReader {
 	return ms.vr
 }
 
-func (ms metaSequenceObject) data() metaSequenceData {
-	return ms.tuples
-}
-
-func (ms metaSequenceObject) ChildValues() []Value {
-	vals := make([]Value, len(ms.tuples))
-	for i, mt := range ms.tuples {
-		vals[i] = mt.childRef
-	}
-	return vals
-}
-
 func (ms metaSequenceObject) Chunks() (chunks []Ref) {
 	for _, tuple := range ms.tuples {
 		chunks = append(chunks, tuple.ChildRef())
@@ -105,6 +89,25 @@ func (ms metaSequenceObject) Type() *Type {
 	return ms.t
 }
 
+// Value interface
+func (ms metaSequenceObject) ChildValues() []Value {
+	vals := make([]Value, len(ms.tuples))
+	for i, mt := range ms.tuples {
+		vals[i] = mt.childRef
+	}
+	return vals
+}
+
+// metaSequence interface
+func (ms metaSequenceObject) getChildSequence(idx int) sequence {
+	mt := ms.tuples[idx]
+	if mt.child != nil {
+		return mt.child.sequence()
+	}
+
+	return mt.childRef.TargetValue(ms.vr).(Collection).sequence()
+}
+
 // Creates a sequenceCursor pointing to the first metaTuple in a metaSequence, and returns that cursor plus the leaf Value referenced from that metaTuple.
 func newMetaSequenceCursor(root metaSequence, vr ValueReader) (*sequenceCursor, Value) {
 	d.Chk.NotNil(root)

types/ordered_sequences.go

@@ -11,6 +11,7 @@ import (
 type orderedSequence interface {
 	sequence
 	getKey(idx int) Value
+	equalsAt(idx int, other interface{}) bool
 }
 
 type orderedMetaSequence struct {
@@ -61,6 +62,10 @@ func (oms orderedMetaSequence) getKey(idx int) Value {
 	return oms.tuples[idx].value
 }
 
+func (oms orderedMetaSequence) equalsAt(idx int, other interface{}) bool {
+	return oms.tuples[idx].childRef.Equals(other.(metaTuple).childRef)
+}
+
 func newCursorAtKey(seq orderedSequence, key Value, forInsertion bool, last bool) *sequenceCursor {
 	var cur *sequenceCursor
 	for {
@@ -126,6 +131,13 @@ func seekTo(cur *sequenceCursor, key Value, lastPositionIfNotFound bool) bool {
 	return cur.idx < seq.seqLen()
 }
 
+// Gets the key used for ordering the sequence at current index.
+func getCurrentKey(cur *sequenceCursor) Value {
+	seq, ok := cur.seq.(orderedSequence)
+	d.Chk.True(ok, "need an ordered sequence here")
+	return seq.getKey(cur.idx)
+}
+
 func newOrderedMetaSequenceBoundaryChecker() boundaryChecker {
 	return newBuzHashBoundaryChecker(orderedSequenceWindowSize, sha1.Size, objectPattern, func(item sequenceItem) []byte {
 		digest := item.(metaTuple).ChildRef().TargetRef().Digest()

types/ordered_sequences_diff.go

@@ -0,0 +1,97 @@
+package types
+
+import (
+	"github.com/attic-labs/noms/d"
+)
+
+//
+// Returns a 3-tuple [added, removed, modified] sorted keys.
+//
+func orderedSequenceDiff(last orderedSequence, current orderedSequence) (added []Value, removed []Value, modified []Value) {
+	added = make([]Value, 0)
+	removed = make([]Value, 0)
+	modified = make([]Value, 0)
+	lastCur := newCursorAtKey(last, nil, false, false)
+	currentCur := newCursorAtKey(current, nil, false, false)
+
+	for lastCur.valid() && currentCur.valid() {
+		fastForward(lastCur, currentCur)
+
+		for lastCur.valid() && currentCur.valid() &&
+			!lastCur.seq.(orderedSequence).equalsAt(lastCur.idx, currentCur.current()) {
+			lastKey := getCurrentKey(lastCur)
+			currentKey := getCurrentKey(currentCur)
+			if lastKey.Equals(currentKey) {
+				modified = append(modified, lastKey)
+				lastCur.advance()
+				currentCur.advance()
+			} else if lastKey.Less(currentKey) {
+				removed = append(removed, lastKey)
+				lastCur.advance()
+			} else {
+				added = append(added, currentKey)
+				currentCur.advance()
+			}
+		}
+	}
+
+	for lastCur.valid() {
+		removed = append(removed, getCurrentKey(lastCur))
+		lastCur.advance()
+	}
+	for currentCur.valid() {
+		added = append(added, getCurrentKey(currentCur))
+		currentCur.advance()
+	}
+
+	return added, removed, modified
+}
+
+/**
+ * Advances |a| and |b| past their common sequence of equal values.
+ */
+func fastForward(a *sequenceCursor, b *sequenceCursor) {
+	if a.valid() && b.valid() {
+		doFastForward(true, a, b)
+	}
+}
+
+func syncWithIdx(cur *sequenceCursor, hasMore bool, allowPastEnd bool) {
+	cur.sync()
+	if hasMore {
+		cur.idx = 0
+	} else if allowPastEnd {
+		cur.idx = cur.length()
+	} else {
+		cur.idx = cur.length() - 1
+	}
+}
+
+/*
+ * Returns an array matching |a| and |b| respectively to whether that cursor has more values.
+ */
+func doFastForward(allowPastEnd bool, a *sequenceCursor, b *sequenceCursor) (aHasMore bool, bHasMore bool) {
+	d.Chk.True(a.valid())
+	d.Chk.True(b.valid())
+	aHasMore = true
+	bHasMore = true
+
+	for aHasMore && bHasMore && isCurrentEqual(a, b) {
+		if nil != a.parent && nil != b.parent && isCurrentEqual(a.parent, b.parent) {
+			// Key optimisation: if the sequences have common parents, then entire chunks can be
+			// fast-forwarded without reading unnecessary data.
+			aHasMore, bHasMore = doFastForward(false, a.parent, b.parent)
+			syncWithIdx(a, aHasMore, allowPastEnd)
+			syncWithIdx(b, bHasMore, allowPastEnd)
+		} else {
+			aHasMore = a.advanceMaybeAllowPastEnd(allowPastEnd)
+			bHasMore = b.advanceMaybeAllowPastEnd(allowPastEnd)
+		}
+	}
+	return aHasMore, bHasMore
+}
+
+func isCurrentEqual(a *sequenceCursor, b *sequenceCursor) bool {
+	aSeq := a.seq.(orderedSequence)
+	return aSeq.equalsAt(a.idx, b.current())
+}

types/ordered_sequences_diff_test.go

@@ -0,0 +1,113 @@
+package types
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+)
+
+const (
+	lengthOfNumbersTest = 1000
+)
+
+type diffTestSuite struct {
+	suite.Suite
+	col1                Collection
+	col2                Collection
+	numAddsExpected     int
+	numRemovesExpected  int
+	numModifiedExpected int
+	added               []Value
+	removed             []Value
+	modified            []Value
+}
+
+func newDiffTestSuiteSet(v1 []Value, v2 []Value, numAddsExpected int, numRemovesExpected int, numModifiedExpected int) *diffTestSuite {
+	set1 := NewSet(v1...)
+	set2 := NewSet(v2...)
+	return &diffTestSuite{col1: set1, col2: set2,
+		numAddsExpected:     numAddsExpected,
+		numRemovesExpected:  numRemovesExpected,
+		numModifiedExpected: numModifiedExpected,
+	}
+}
+
+func newDiffTestSuiteMap(v1 []Value, v2 []Value, numAddsExpected int, numRemovesExpected int, numModifiedExpected int) *diffTestSuite {
+	map1 := NewMap(v1...)
+	map2 := NewMap(v2...)
+	return &diffTestSuite{col1: map1, col2: map2,
+		numAddsExpected:     numAddsExpected,
+		numRemovesExpected:  numRemovesExpected,
+		numModifiedExpected: numModifiedExpected,
+	}
+}
+
+// Called from testify suite.Run()
+func (suite *diffTestSuite) TestDiff() {
+	suite.added, suite.removed, suite.modified = orderedSequenceDiff(
+		suite.col1.sequence().(orderedSequence),
+		suite.col2.sequence().(orderedSequence))
+	suite.Equal(suite.numAddsExpected, len(suite.added), "num added is not as expected")
+	suite.Equal(suite.numRemovesExpected, len(suite.removed), "num removed is not as expected")
+	suite.Equal(suite.numModifiedExpected, len(suite.modified), "num modified is not as expected")
+}
+
+// Called from "go test"
+func TestOrderedSequencesIdenticalSet(t *testing.T) {
+	v1 := generateNumbersAsValuesFromToBy(0, lengthOfNumbersTest, 1)
+	v2 := generateNumbersAsValuesFromToBy(0, lengthOfNumbersTest, 1)
+	ts := newDiffTestSuiteSet(v1, v2, 0, 0, 0)
+	suite.Run(t, ts)
+}
+
+func TestOrderedSequencesIdenticalMap(t *testing.T) {
+	v1 := generateNumbersAsValuesFromToBy(0, lengthOfNumbersTest*2, 1)
+	v2 := generateNumbersAsValuesFromToBy(0, lengthOfNumbersTest*2, 1)
+	tm := newDiffTestSuiteMap(v1, v2, 0, 0, 0)
+	suite.Run(t, tm)
+}
+
+func TestOrderedSequencesSubsetSet(t *testing.T) {
+	v1 := generateNumbersAsValuesFromToBy(0, lengthOfNumbersTest, 1)
+	v2 := generateNumbersAsValuesFromToBy(0, lengthOfNumbersTest/2, 1)
+	ts1 := newDiffTestSuiteSet(v1, v2, 0, lengthOfNumbersTest/2, 0)
+	ts2 := newDiffTestSuiteSet(v2, v1, lengthOfNumbersTest/2, 0, 0)
+	suite.Run(t, ts1)
+	suite.Run(t, ts2)
+	ts1.Equal(ts1.added, ts2.removed, "added and removed in reverse order diff")
+	ts1.Equal(ts1.removed, ts2.added, "removed and added in reverse order diff")
+}
+
+func TestOrderedSequencesSubsetMap(t *testing.T) {
+	v1 := generateNumbersAsValuesFromToBy(0, lengthOfNumbersTest*2, 1)
+	v2 := generateNumbersAsValuesFromToBy(0, lengthOfNumbersTest, 1)
+	tm1 := newDiffTestSuiteMap(v1, v2, 0, lengthOfNumbersTest/2, 0)
+	tm2 := newDiffTestSuiteMap(v2, v1, lengthOfNumbersTest/2, 0, 0)
+	suite.Run(t, tm1)
+	suite.Run(t, tm2)
+	tm1.Equal(tm1.added, tm2.removed, "added and removed in reverse order diff")
+	tm1.Equal(tm1.removed, tm2.added, "removed and added in reverse order diff")
+}
+
+func TestOrderedSequencesDisjointSet(t *testing.T) {
+	v1 := generateNumbersAsValuesFromToBy(0, lengthOfNumbersTest, 2)
+	v2 := generateNumbersAsValuesFromToBy(1, lengthOfNumbersTest, 2)
+	ts1 := newDiffTestSuiteSet(v1, v2, lengthOfNumbersTest/2, lengthOfNumbersTest/2, 0)
+	ts2 := newDiffTestSuiteSet(v2, v1, lengthOfNumbersTest/2, lengthOfNumbersTest/2, 0)
+	suite.Run(t, ts1)
+	suite.Run(t, ts2)
+
+	ts1.Equal(ts1.added, ts2.removed, "added and removed in disjoint diff")
+	ts1.Equal(ts1.removed, ts2.added, "removed and added in disjoint diff")
+}
+
+func TestOrderedSequencesDisjointMap(t *testing.T) {
+	v1 := generateNumbersAsValuesFromToBy(0, lengthOfNumbersTest*2, 2)
+	v2 := generateNumbersAsValuesFromToBy(1, lengthOfNumbersTest*2, 2)
+	tm1 := newDiffTestSuiteMap(v1, v2, lengthOfNumbersTest/2, lengthOfNumbersTest/2, 0)
+	tm2 := newDiffTestSuiteMap(v2, v1, lengthOfNumbersTest/2, lengthOfNumbersTest/2, 0)
+	suite.Run(t, tm1)
+	suite.Run(t, tm2)
+	tm1.Equal(tm1.added, tm2.removed, "added and removed in disjoint diff")
+	tm1.Equal(tm1.removed, tm2.added, "removed and added in disjoint diff")
+}

types/ref.go

@@ -34,22 +34,6 @@ func maxChunkHeight(v Value) (max uint64) {
 	return
 }
 
-func (r Ref) Equals(other Value) bool {
-	return other != nil && r.t.Equals(other.Type()) && r.Ref() == other.Ref()
-}
-
-func (r Ref) Ref() ref.Ref {
-	return EnsureRef(r.ref, r)
-}
-
-func (r Ref) Chunks() (chunks []Ref) {
-	return append(chunks, r)
-}
-
-func (r Ref) ChildValues() []Value {
-	return nil
-}
-
 func (r Ref) TargetRef() ref.Ref {
 	return r.target
 }
@@ -58,14 +42,31 @@ func (r Ref) Height() uint64 {
 	return r.height
 }
 
-func (r Ref) Type() *Type {
-	return r.t
+func (r Ref) TargetValue(vr ValueReader) Value {
+	return vr.ReadValue(r.target)
+}
+
+// Value interface
+func (r Ref) Equals(other Value) bool {
+	return other != nil && r.t.Equals(other.Type()) && r.Ref() == other.Ref()
 }
 
 func (r Ref) Less(other Value) bool {
 	return valueLess(r, other)
 }
 
-func (r Ref) TargetValue(vr ValueReader) Value {
-	return vr.ReadValue(r.target)
+func (r Ref) Ref() ref.Ref {
+	return EnsureRef(r.ref, r)
+}
+
+func (r Ref) ChildValues() []Value {
+	return nil
+}
+
+func (r Ref) Chunks() (chunks []Ref) {
+	return append(chunks, r)
+}
+
+func (r Ref) Type() *Type {
+	return r.t
 }

types/sequence_cursor_test.go

@@ -11,7 +11,7 @@ type testSequence struct {
 	items []interface{}
 }
 
-// sequence
+// sequence interface
 func (ts testSequence) getItem(idx int) sequenceItem {
 	return ts.items[idx]
 }
@@ -24,29 +24,40 @@ func (ts testSequence) numLeaves() uint64 {
 	return uint64(len(ts.items))
 }
 
+func (ts testSequence) valueReader() ValueReader {
+	panic("not reached")
+}
+
+// metaSequence interface
 func (ts testSequence) getChildSequence(idx int) sequence {
 	child := ts.items[idx]
 	return testSequence{child.([]interface{})}
 }
 
+// Value interface
 func (ts testSequence) Equals(other Value) bool {
 	panic("not reached")
 }
+
+func (ts testSequence) Less(other Value) bool {
+	panic("not reached")
+}
+
 func (ts testSequence) Ref() ref.Ref {
 	panic("not reached")
 }
+
 func (ts testSequence) ChildValues() []Value {
 	panic("not reached")
 }
+
 func (ts testSequence) Chunks() []Ref {
 	panic("not reached")
 }
+
 func (ts testSequence) Type() *Type {
 	panic("not reached")
 }
-func (ts testSequence) valueReader() ValueReader {
-	panic("not reached")
-}
 
 func newTestSequenceCursor(items []interface{}) *sequenceCursor {
 	parent := newSequenceCursor(nil, testSequence{items}, 0)

types/set.go

@@ -33,10 +33,27 @@ func NewSet(v ...Value) Set {
 	return seq.Done().(Set)
 }
 
-func (s Set) Type() *Type {
-	return s.seq.Type()
+func (s Set) Diff(last Set) (added []Value, removed []Value) {
+	// Set diff shouldn't return modified since it's not possible a value in a set of "changes".
+	// Elements can only enter and exit a set
+	added, removed, _ = orderedSequenceDiff(last.sequence().(orderedSequence), s.sequence().(orderedSequence))
+	return
 }
 
+// Collection interface
+func (s Set) Len() uint64 {
+	return s.seq.numLeaves()
+}
+
+func (s Set) Empty() bool {
+	return s.Len() == 0
+}
+
+func (s Set) sequence() sequence {
+	return s.seq
+}
+
+// Value interface
 func (s Set) Equals(other Value) bool {
 	return other != nil && s.Ref() == other.Ref()
 }
@@ -49,27 +66,19 @@ func (s Set) Ref() ref.Ref {
 	return EnsureRef(s.ref, s)
 }
 
-func (s Set) Len() uint64 {
-	return s.seq.numLeaves()
-}
-
-func (s Set) Empty() bool {
-	return s.Len() == 0
+func (s Set) ChildValues() (values []Value) {
+	s.IterAll(func(v Value) {
+		values = append(values, v)
+	})
+	return
 }
 
 func (s Set) Chunks() []Ref {
 	return s.seq.Chunks()
 }
 
-func (s Set) sequence() sequence {
-	return s.seq
-}
-
-func (s Set) ChildValues() (values []Value) {
-	s.IterAll(func(v Value) {
-		values = append(values, v)
-	})
-	return
+func (s Set) Type() *Type {
+	return s.seq.Type()
 }
 
 func (s Set) First() Value {

types/set_leaf_sequence.go

@@ -15,6 +15,7 @@ func newSetLeafSequence(vr ValueReader, v ...Value) orderedSequence {
 	return setLeafSequence{v, t, vr}
 }
 
+// sequence interface
 func (sl setLeafSequence) getItem(idx int) sequenceItem {
 	return sl.data[idx]
 }
@@ -27,8 +28,8 @@ func (sl setLeafSequence) numLeaves() uint64 {
 	return uint64(len(sl.data))
 }
 
-func (sl setLeafSequence) getKey(idx int) Value {
-	return sl.data[idx]
+func (sl setLeafSequence) valueReader() ValueReader {
+	return sl.vr
 }
 
 func (sl setLeafSequence) Chunks() (chunks []Ref) {
@@ -42,6 +43,12 @@ func (sl setLeafSequence) Type() *Type {
 	return sl.t
 }
 
-func (sl setLeafSequence) valueReader() ValueReader {
-	return sl.vr
+// orderedSequence interface
+func (sl setLeafSequence) getKey(idx int) Value {
+	return sl.data[idx]
+}
+
+func (sl setLeafSequence) equalsAt(idx int, other interface{}) bool {
+	entry := sl.data[idx]
+	return entry.Equals(other.(Value))
 }

types/set_test.go

@@ -37,6 +37,50 @@ func (ts testSet) Remove(from, to int) testSet {
 	return testSet{values, ts.tr}
 }
 
+func (ts testSet) Has(key Value) bool {
+	for _, entry := range ts.values {
+		if entry.Equals(key) {
+			return true
+		}
+	}
+	return false
+}
+
+func (ts testSet) Diff(last testSet) (added []Value, removed []Value) {
+	// Note: this could be use ts.toSet/last.toSet and then tsSet.Diff(lastSet) but the
+	// purpose of this method is to be redundant.
+	added = make([]Value, 0)
+	removed = make([]Value, 0)
+	if len(ts.values) == 0 && len(last.values) == 0 {
+		return // nothing changed
+	}
+	if len(ts.values) == 0 {
+		// everything removed
+		for _, entry := range last.values {
+			removed = append(removed, entry)
+		}
+		return
+	}
+	if len(last.values) == 0 {
+		// everything added
+		for _, entry := range ts.values {
+			added = append(added, entry)
+		}
+		return
+	}
+	for _, entry := range ts.values {
+		if !last.Has(entry) {
+			added = append(added, entry)
+		}
+	}
+	for _, entry := range last.values {
+		if !ts.Has(entry) {
+			removed = append(removed, entry)
+		}
+	}
+	return
+}
+
 func (ts testSet) toSet() Set {
 	return NewSet(ts.values...)
 }
@@ -50,6 +94,14 @@ func newTestSet(length int) testSet {
 	return testSet{values, MakeSetType(NumberType)}
 }
 
+func newTestSetFromSet(s Set) testSet {
+	values := make([]Value, 0, s.Len())
+	s.IterAll(func(v Value) {
+		values = append(values, v)
+	})
+	return testSet{values, s.Type()}
+}
+
 func newTestSetWithGen(length int, gen testSetGenFn, tr *Type) testSet {
 	s := rand.NewSource(4242)
 	used := map[int64]bool{}
@@ -146,6 +198,22 @@ func getTestRefToValueOrderSet(scale int, vw ValueWriter) testSet {
 	}, refType)
 }
 
+func diffSetTest(assert *assert.Assertions, s1 Set, s2 Set, numAddsExpected int, numRemovesExpected int) (added []Value, removed []Value) {
+	added, removed = s1.Diff(s2)
+	assert.Equal(numAddsExpected, len(added), "num added is not as expected")
+	assert.Equal(numRemovesExpected, len(removed), "num removed is not as expected")
+
+	ts1 := newTestSetFromSet(s1)
+	ts2 := newTestSetFromSet(s2)
+	tsAdded, tsRemoved := ts1.Diff(ts2)
+	assert.Equal(numAddsExpected, len(tsAdded), "num added is not as expected")
+	assert.Equal(numRemovesExpected, len(tsRemoved), "num removed is not as expected")
+
+	assert.Equal(added, tsAdded, "set added != tsSet added")
+	assert.Equal(removed, tsRemoved, "set removed != tsSet removed")
+	return
+}
+
 func TestNewSet(t *testing.T) {
 	assert := assert.New(t)
 	s := NewSet()
@@ -168,10 +236,20 @@ func TestSetLen(t *testing.T) {
 	assert.Equal(uint64(0), s0.Len())
 	s1 := NewSet(Bool(true), Number(1), NewString("hi"))
 	assert.Equal(uint64(3), s1.Len())
+	diffSetTest(assert, s0, s1, 0, 3)
+	diffSetTest(assert, s1, s0, 3, 0)
+
 	s2 := s1.Insert(Bool(false))
 	assert.Equal(uint64(4), s2.Len())
+	diffSetTest(assert, s0, s2, 0, 4)
+	diffSetTest(assert, s2, s0, 4, 0)
+	diffSetTest(assert, s1, s2, 0, 1)
+	diffSetTest(assert, s2, s1, 1, 0)
+
 	s3 := s2.Remove(Bool(true))
 	assert.Equal(uint64(3), s3.Len())
+	diffSetTest(assert, s2, s3, 1, 0)
+	diffSetTest(assert, s3, s2, 0, 1)
 }
 
 func TestSetEmpty(t *testing.T) {
@@ -261,6 +339,7 @@ func TestSetHas2(t *testing.T) {
 			assert.True(set.Has(v))
 			assert.True(set2.Has(v))
 		}
+		diffSetTest(assert, set, set2, 0, 0)
 	}
 
 	doTest(getTestNativeOrderSet(16))
@@ -303,6 +382,7 @@ func TestSetInsert2(t *testing.T) {
 			actual := ts.Remove(from, to).toSet().Insert(ts.values[from:to]...)
 			assert.Equal(expected.Len(), actual.Len())
 			assert.True(expected.Equals(actual))
+			diffSetTest(assert, expected, actual, 0, 0)
 		}
 		for i := 0; i < len(ts.values)-offset; i += incr {
 			run(i, i+offset)
@@ -364,6 +444,7 @@ func TestSetRemove2(t *testing.T) {
 			actual := whole.Remove(ts.values[from:to]...)
 			assert.Equal(expected.Len(), actual.Len())
 			assert.True(expected.Equals(actual))
+			diffSetTest(assert, expected, actual, 0, 0)
 		}
 		for i := 0; i < len(ts.values)-offset; i += incr {
 			run(i, i+offset)

types/struct.go

@@ -41,6 +41,7 @@ func NewStructWithType(t *Type, data structData) Struct {
 	return newStructFromData(newData, t)
 }
 
+// Value interface
 func (s Struct) Equals(other Value) bool {
 	return other != nil && s.t.Equals(other.Type()) && s.Ref() == other.Ref()
 }
@@ -53,6 +54,16 @@ func (s Struct) Ref() ref.Ref {
 	return EnsureRef(s.ref, s)
 }
 
+func (s Struct) ChildValues() (res []Value) {
+	res = append(res, s.t)
+	for name := range s.desc().Fields {
+		v, ok := s.data[name]
+		d.Chk.True(ok)
+		res = append(res, v)
+	}
+	return
+}
+
 func (s Struct) Chunks() (chunks []Ref) {
 	chunks = append(chunks, s.t.Chunks()...)
 	for name := range s.desc().Fields {
@@ -64,16 +75,6 @@ func (s Struct) Chunks() (chunks []Ref) {
 	return
 }
 
-func (s Struct) ChildValues() (res []Value) {
-	res = append(res, s.t)
-	for name := range s.desc().Fields {
-		v, ok := s.data[name]
-		d.Chk.True(ok)
-		res = append(res, v)
-	}
-	return
-}
-
 func (s Struct) Type() *Type {
 	return s.t
 }

types/type.go

@@ -21,6 +21,8 @@ type Type struct {
 	ref  *ref.Ref
 }
 
+var typeForType = makePrimitiveType(TypeKind)
+
 // Describe generate text that should parse into the struct being described.
 func (t *Type) Describe() (out string) {
 	return EncodedValue(t)
@@ -36,10 +38,7 @@ func (t *Type) Name() string {
 	return t.Desc.(StructDesc).Name
 }
 
-func (t *Type) Ref() ref.Ref {
-	return EnsureRef(t.ref, t)
-}
-
+// Value interface
 func (t *Type) Equals(other Value) (res bool) {
 	return other != nil && t.Ref() == other.Ref()
 }
@@ -48,8 +47,8 @@ func (t *Type) Less(other Value) (res bool) {
 	return valueLess(t, other)
 }
 
-func (t *Type) Chunks() (chunks []Ref) {
-	return
+func (t *Type) Ref() ref.Ref {
+	return EnsureRef(t.ref, t)
 }
 
 func (t *Type) ChildValues() (res []Value) {
@@ -70,7 +69,9 @@ func (t *Type) ChildValues() (res []Value) {
 	return
 }
 
-var typeForType = makePrimitiveType(TypeKind)
+func (t *Type) Chunks() (chunks []Ref) {
+	return
+}
 
 func (t *Type) Type() *Type {
 	return typeForType

types/util_test.go

@@ -7,8 +7,14 @@ import (
 
 var generateNumbersAsValues = func(n int) []Value {
 	d.Chk.True(n > 0, "must be an integer greater than zero")
+	return generateNumbersAsValuesFromToBy(0, n, 1)
+}
+
+var generateNumbersAsValuesFromToBy = func(from int, to int, by int) []Value {
+	d.Chk.True(to > from, "to must be greater than from")
+	d.Chk.True(by > 0, "must be an integer greater than zero")
 	nums := []Value{}
-	for i := 0; i < n; i++ {
+	for i := from; i < to; i += by {
 		nums = append(nums, Number(i))
 	}
 	return nums