From 0e5b65f27f5460d9a9cc0ca1db0ab687381b9040 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 24 Apr 2024 16:26:04 +0200 Subject: [PATCH] feat(nats-js-kv): bump nats-js-kv Signed-off-by: jkoberg --- .golangci.yml | 1 - changelog/unreleased/bump-natsjskv.md | 5 ++ go.mod | 4 +- go.sum | 4 +- .../plugins/v4/store/nats-js-kv/keys.go | 46 +++++++++++-------- .../plugins/v4/store/nats-js-kv/nats.go | 15 ++++-- .../plugins/v4/store/nats-js-kv/options.go | 8 ++++ vendor/modules.txt | 3 +- 8 files changed, 55 insertions(+), 31 deletions(-) create mode 100644 changelog/unreleased/bump-natsjskv.md diff --git a/.golangci.yml b/.golangci.yml index 1479d22c2..8ed29e034 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -47,7 +47,6 @@ linters-settings: min-complexity: 35 # there are some func unforuntately who need this - should we refactor them? gomoddirectives: replace-allow-list: - - github.com/go-micro/plugins/v4/store/nats-js-kv - github.com/studio-b12/gowebdav - github.com/egirna/icap-client interfacebloat: diff --git a/changelog/unreleased/bump-natsjskv.md b/changelog/unreleased/bump-natsjskv.md new file mode 100644 index 000000000..3ab839211 --- /dev/null +++ b/changelog/unreleased/bump-natsjskv.md @@ -0,0 +1,5 @@ +Enhancement: Bump nats-js-kv pkg + +Uses official nats-js-kv package now. Moves away from custom fork. + +https://github.com/owncloud/ocis/pull/8953 diff --git a/go.mod b/go.mod index 2ee841b67..f9e7380eb 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/go-micro/plugins/v4/registry/nats v1.2.2-0.20230723205323-1ada01245674 github.com/go-micro/plugins/v4/server/grpc v1.2.0 github.com/go-micro/plugins/v4/server/http v1.2.2 - github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-00010101000000-000000000000 + github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-20231226212146-94a49ba3e06e github.com/go-micro/plugins/v4/wrapper/breaker/gobreaker v1.2.0 github.com/go-micro/plugins/v4/wrapper/monitoring/prometheus v1.2.0 github.com/go-micro/plugins/v4/wrapper/trace/opentelemetry v1.2.0 @@ -352,8 +352,6 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348 - replace github.com/studio-b12/gowebdav => github.com/aduffeck/gowebdav v0.0.0-20231215102054-212d4a4374f6 replace github.com/egirna/icap-client => github.com/fschade/icap-client v0.0.0-20240123094924-5af178158eaf diff --git a/go.sum b/go.sum index af258dbcd..a084df8a1 100644 --- a/go.sum +++ b/go.sum @@ -1215,6 +1215,8 @@ github.com/go-micro/plugins/v4/server/http v1.2.2 h1:UK2/09AU0zV3wHELuR72TZzVU2v github.com/go-micro/plugins/v4/server/http v1.2.2/go.mod h1:YuAjaSPxcn3LI8j2FUsqx0Rxunrj4YwDV41Ax76rLl0= github.com/go-micro/plugins/v4/store/nats-js v1.2.1-0.20231129143103-d72facc652f0 h1:Qa1EBQ9UyCGecFAJQovl/MHGnvbcvDaM3qUoAG5Lnvk= github.com/go-micro/plugins/v4/store/nats-js v1.2.1-0.20231129143103-d72facc652f0/go.mod h1:aCRl8JQmqIaonOl88nFPY/BOQnHPVHY9ngStzLkXnYk= +github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-20231226212146-94a49ba3e06e h1:hwH0qXT0J3UFYRi0UD+e3ItL92oW+jdPFA+3o/j6ASg= +github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-20231226212146-94a49ba3e06e/go.mod h1:Goi4eJ9SrKkxE6NsAVqBVNxfQFbwb7UbyII6743ldgM= github.com/go-micro/plugins/v4/store/redis v1.2.1 h1:d9kwr9bSpoK9vkHkqcv+isQUbgBCHpfwCV57pcAPS6c= github.com/go-micro/plugins/v4/store/redis v1.2.1/go.mod h1:MbCG0YiyPqETTtm7uHFmxQNCaW1o9hBoYtFwhbVjLUg= github.com/go-micro/plugins/v4/transport/grpc v1.1.0 h1:mXfDYfFQLnVDzjGY3o84oe4prfux9h8txsnA19dKsj8= @@ -1603,8 +1605,6 @@ github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348 h1:Czv6AW9Suj6npWd5BLZjobdD78c2RdzBeKBgkq3jYZk= -github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348/go.mod h1:Goi4eJ9SrKkxE6NsAVqBVNxfQFbwb7UbyII6743ldgM= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= diff --git a/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/keys.go b/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/keys.go index 8fd6f1aa9..8eb2869ca 100644 --- a/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/keys.go +++ b/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/keys.go @@ -6,24 +6,24 @@ import ( ) // NatsKey is a convenience function to create a key for the nats kv store. -func NatsKey(table, microkey string) string { - return NewKey(table, microkey, "").NatsKey() +func (n *natsStore) NatsKey(table, microkey string) string { + return n.NewKey(table, microkey, "").NatsKey() } // MicroKey is a convenience function to create a key for the micro interface. -func MicroKey(table, natskey string) string { - return NewKey(table, "", natskey).MicroKey() +func (n *natsStore) MicroKey(table, natskey string) string { + return n.NewKey(table, "", natskey).MicroKey() } // MicroKeyFilter is a convenience function to create a key for the micro interface. // It returns false if the key does not match the table, prefix or suffix. -func MicroKeyFilter(table, natskey string, prefix, suffix string) (string, bool) { - k := NewKey(table, "", natskey) +func (n *natsStore) MicroKeyFilter(table, natskey string, prefix, suffix string) (string, bool) { + k := n.NewKey(table, "", natskey) return k.MicroKey(), k.Check(table, prefix, suffix) } // Key represents a key in the store. -// They are used to convert nats keys (base64 encoded) to micro keys (plain text - no table prefix) and vice versa. +// They are used to convert nats keys (base32 encoded) to micro keys (plain text - no table prefix) and vice versa. type Key struct { // Plain is the plain key as requested by the go-micro interface. Plain string @@ -34,7 +34,7 @@ type Key struct { } // NewKey creates a new key. Either plain or encoded must be set. -func NewKey(table string, plain, encoded string) *Key { +func (n *natsStore) NewKey(table string, plain, encoded string) *Key { k := &Key{ Plain: plain, Encoded: encoded, @@ -43,9 +43,9 @@ func NewKey(table string, plain, encoded string) *Key { switch { case k.Plain != "": k.Full = getKey(k.Plain, table) - k.Encoded = encode(k.Full) + k.Encoded = encode(k.Full, n.encoding) case k.Encoded != "": - k.Full = decode(k.Encoded) + k.Full = decode(k.Encoded, n.encoding) k.Plain = trimKey(k.Full, table) } @@ -79,17 +79,27 @@ func (k *Key) Check(table, prefix, suffix string) bool { return true } -func encode(s string) string { - return base32.StdEncoding.EncodeToString([]byte(s)) -} - -func decode(s string) string { - b, err := base32.StdEncoding.DecodeString(s) - if err != nil { +func encode(s string, alg string) string { + switch alg { + case "base32": + return base32.StdEncoding.EncodeToString([]byte(s)) + default: return s } +} - return string(b) +func decode(s string, alg string) string { + switch alg { + case "base32": + b, err := base32.StdEncoding.DecodeString(s) + if err != nil { + return s + } + + return string(b) + default: + return s + } } func getKey(key, table string) string { diff --git a/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/nats.go b/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/nats.go index aa7d50453..13ae81d28 100644 --- a/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/nats.go +++ b/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/nats.go @@ -30,6 +30,7 @@ type natsStore struct { sync.Once sync.RWMutex + encoding string ttl time.Duration storageType nats.StorageType description string @@ -143,6 +144,10 @@ func (n *natsStore) setOption(opts ...store.Option) { n.description = text } + if encoding, ok := n.opts.Context.Value(keyEncodeOptionsKey{}).(string); ok { + n.encoding = encoding + } + // Assign store option server addresses to nats options if len(n.opts.Nodes) > 0 { n.nopts.Url = "" @@ -238,8 +243,8 @@ func (n *natsStore) Write(rec *store.Record, opts ...store.WriteOption) error { return errors.Wrap(err, "Failed to marshal object") } - if _, err := store.Put(NatsKey(opt.Table, rec.Key), b); err != nil { - return errors.Wrapf(err, "Failed to store data in bucket '%s'", NatsKey(opt.Table, rec.Key)) + if _, err := store.Put(n.NatsKey(opt.Table, rec.Key), b); err != nil { + return errors.Wrapf(err, "Failed to store data in bucket '%s'", n.NatsKey(opt.Table, rec.Key)) } return nil @@ -280,7 +285,7 @@ func (n *natsStore) Delete(key string, opts ...store.DeleteOption) error { return ErrBucketNotFound } - if err := store.Delete(NatsKey(opt.Table, key)); err != nil { + if err := store.Delete(n.NatsKey(opt.Table, key)); err != nil { return errors.Wrap(err, "Failed to delete data") } @@ -415,7 +420,7 @@ func (n *natsStore) getRecord(bucket nats.KeyValue, key string) (*store.Record, func (n *natsStore) natsKeys(bucket nats.KeyValue, table, key string, prefix, suffix bool) ([]string, error) { if !suffix && !prefix { - return []string{NatsKey(table, key)}, nil + return []string{n.NatsKey(table, key)}, nil } toS := func(s string, b bool) string { @@ -449,7 +454,7 @@ func (n *natsStore) getKeys(bucket nats.KeyValue, table string, prefix, suffix s microKeys := make([]string, 0, len(names)) for _, k := range names { - mkey, ok := MicroKeyFilter(table, k, prefix, suffix) + mkey, ok := n.MicroKeyFilter(table, k, prefix, suffix) if !ok { continue } diff --git a/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/options.go b/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/options.go index 424ce947f..a9c9ca610 100644 --- a/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/options.go +++ b/vendor/github.com/go-micro/plugins/v4/store/nats-js-kv/options.go @@ -14,6 +14,7 @@ type kvOptionsKey struct{} type ttlOptionsKey struct{} type memoryOptionsKey struct{} type descriptionOptionsKey struct{} +type keyEncodeOptionsKey struct{} // NatsOptions accepts nats.Options. func NatsOptions(opts nats.Options) store.Option { @@ -61,6 +62,13 @@ func DefaultDescription(text string) store.Option { return setStoreOption(descriptionOptionsKey{}, text) } +// EncodeKeys will "base32" encode the keys. +// This is to work around limited characters usable as keys for the natsjs kv store. +// See details here: https://docs.nats.io/nats-concepts/subjects#characters-allowed-for-subject-names +func EncodeKeys() store.Option { + return setStoreOption(keyEncodeOptionsKey{}, "base32") +} + // DeleteBucket will use the key passed to Delete as a bucket (database) name, // // and delete the bucket. diff --git a/vendor/modules.txt b/vendor/modules.txt index 87a99c3ab..61a5546c5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -959,7 +959,7 @@ github.com/go-micro/plugins/v4/server/http # github.com/go-micro/plugins/v4/store/nats-js v1.2.1-0.20231129143103-d72facc652f0 ## explicit; go 1.21 github.com/go-micro/plugins/v4/store/nats-js -# github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-00010101000000-000000000000 => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348 +# github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-20231226212146-94a49ba3e06e ## explicit; go 1.21 github.com/go-micro/plugins/v4/store/nats-js-kv # github.com/go-micro/plugins/v4/store/redis v1.2.1 @@ -2347,6 +2347,5 @@ stash.kopano.io/kgol/oidc-go # stash.kopano.io/kgol/rndm v1.1.2 ## explicit; go 1.13 stash.kopano.io/kgol/rndm -# github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348 # github.com/studio-b12/gowebdav => github.com/aduffeck/gowebdav v0.0.0-20231215102054-212d4a4374f6 # github.com/egirna/icap-client => github.com/fschade/icap-client v0.0.0-20240123094924-5af178158eaf