From 1d52617eb5951267d53489fe593af2bd4ee9571d Mon Sep 17 00:00:00 2001 From: cmasone-attic Date: Mon, 24 Apr 2017 14:45:54 -0700 Subject: [PATCH] NBS table names now just hash of suffix block (#3421) Used to be that an NBS table was named by hashing the hashes of every chunk present in the table, in hash order. That means that to generate the name of a table you'd need to iterate the prefix map and load every associated suffix. That would be expensive when e.g. compacting multiple tables. This is waaay cheaper and only slightly more likely to wind up with a name collision. Toward #3411 --- go/nbs/store.go | 2 +- go/nbs/table_writer.go | 3 +-- ...5ke2so64df3 => 7s84n5m4b7i2n1r0vr0ksaemr9qjnhdl} | Bin samples/go/hr/test-data/manifest | 2 +- ...7q926oqn2dn => ullneu8fijlfnhhmq82dtco4n60gupc2} | Bin 5 files changed, 3 insertions(+), 4 deletions(-) rename samples/go/hr/test-data/{m6j2e6jd69tbfk7d4hqf05ke2so64df3 => 7s84n5m4b7i2n1r0vr0ksaemr9qjnhdl} (100%) rename samples/go/hr/test-data/{e9v26bl5mov3mtp2vdvpb7q926oqn2dn => ullneu8fijlfnhhmq82dtco4n60gupc2} (100%) diff --git a/go/nbs/store.go b/go/nbs/store.go index 4825346869..1a6d7ef1d3 100644 --- a/go/nbs/store.go +++ b/go/nbs/store.go @@ -30,7 +30,7 @@ import ( const ( // StorageVersion is the version of the on-disk Noms Chunks Store data format. - StorageVersion = "3" + StorageVersion = "4" defaultMemTableSize uint64 = (1 << 20) * 128 // 128MB defaultAWSReadLimit = 1024 diff --git a/go/nbs/table_writer.go b/go/nbs/table_writer.go index c566eedee4..f0f831daf6 100644 --- a/go/nbs/table_writer.go +++ b/go/nbs/table_writer.go @@ -128,8 +128,6 @@ func (tw *tableWriter) writeIndex() { suffixesOffset := lengthsOffset + numRecords*lengthSize // skip size for each record for _, pi := range tw.prefixes { binary.BigEndian.PutUint64(pfxScratch[:], pi.prefix) - tw.blockHash.Write(pfxScratch[:]) - tw.blockHash.Write(pi.suffix) // hash prefix n := uint64(copy(tw.buff[tw.pos:], pfxScratch[:])) @@ -149,6 +147,7 @@ func (tw *tableWriter) writeIndex() { n = uint64(copy(tw.buff[offset:], pi.suffix)) d.Chk.True(n == addrSuffixSize) } + tw.blockHash.Write(tw.buff[suffixesOffset : suffixesOffset+numRecords*addrSuffixSize]) tw.pos = suffixesOffset + numRecords*addrSuffixSize } diff --git a/samples/go/hr/test-data/m6j2e6jd69tbfk7d4hqf05ke2so64df3 b/samples/go/hr/test-data/7s84n5m4b7i2n1r0vr0ksaemr9qjnhdl similarity index 100% rename from samples/go/hr/test-data/m6j2e6jd69tbfk7d4hqf05ke2so64df3 rename to samples/go/hr/test-data/7s84n5m4b7i2n1r0vr0ksaemr9qjnhdl diff --git a/samples/go/hr/test-data/manifest b/samples/go/hr/test-data/manifest index d921e2b2ac..81b593efd4 100644 --- a/samples/go/hr/test-data/manifest +++ b/samples/go/hr/test-data/manifest @@ -1 +1 @@ -3:7.9:2i6mkbcajmnkkguethlmo929rif79r8r:c1uoqa08f12o0abqgv2lvavmppuc3kg4:m6j2e6jd69tbfk7d4hqf05ke2so64df3:2:e9v26bl5mov3mtp2vdvpb7q926oqn2dn:2 \ No newline at end of file +4:7.9:nh54p8hlk0c6c5q9mf8gb33pt9r9poc0:c1uoqa08f12o0abqgv2lvavmppuc3kg4:7s84n5m4b7i2n1r0vr0ksaemr9qjnhdl:2:ullneu8fijlfnhhmq82dtco4n60gupc2:2 \ No newline at end of file diff --git a/samples/go/hr/test-data/e9v26bl5mov3mtp2vdvpb7q926oqn2dn b/samples/go/hr/test-data/ullneu8fijlfnhhmq82dtco4n60gupc2 similarity index 100% rename from samples/go/hr/test-data/e9v26bl5mov3mtp2vdvpb7q926oqn2dn rename to samples/go/hr/test-data/ullneu8fijlfnhhmq82dtco4n60gupc2