Switch to xxhash64

internal/consistenthash/consistenthash.go

@@ -18,29 +18,28 @@ limitations under the License.
 package consistenthash
 
 import (
-	"hash/crc32"
 	"sort"
 	"strconv"
 
+	"github.com/cespare/xxhash"
 	"github.com/go-redis/redis/v7/internal"
 )
 
-
 type Map struct {
 	hash     internal.Hash
 	replicas int
-	keys     []int // Sorted
+	keys     []uint64 // Sorted
-	hashMap  map[int]string
+	hashMap  map[uint64]string
 }
 
 func New(replicas int, fn internal.Hash) *Map {
 	m := &Map{
 		replicas: replicas,
 		hash:     fn,
-		hashMap:  make(map[int]string),
+		hashMap:  make(map[uint64]string),
 	}
 	if m.hash == nil {
-		m.hash = crc32.ChecksumIEEE
+		m.hash = xxhash.Sum64
 	}
 	return m
 }
@@ -54,12 +53,14 @@ func (m *Map) IsEmpty() bool {
 func (m *Map) Add(keys ...string) {
 	for _, key := range keys {
 		for i := 0; i < m.replicas; i++ {
-			hash := int(m.hash([]byte(strconv.Itoa(i) + key)))
+			hash := m.hash([]byte(strconv.Itoa(i) + key))
 			m.keys = append(m.keys, hash)
 			m.hashMap[hash] = key
 		}
 	}
-	sort.Ints(m.keys)
+	sort.Slice(m.keys, func(i, j int) bool {
+		return m.keys[i] < m.keys[j]
+	})
 }
 
 // Gets the closest item in the hash to the provided key.
@@ -68,7 +69,7 @@ func (m *Map) Get(key string) string {
 		return ""
 	}
 
-	hash := int(m.hash([]byte(key)))
+	hash := m.hash([]byte(key))
 
 	// Binary search for appropriate replica.
 	idx := sort.Search(len(m.keys), func(i int) bool { return m.keys[i] >= hash })

internal/consistenthash/consistenthash_test.go

@@ -26,12 +26,12 @@ func TestHashing(t *testing.T) {
 
 	// Override the hash function to return easier to reason about values. Assumes
 	// the keys can be converted to an integer.
-	hash := New(3, func(key []byte) uint32 {
+	hash := New(3, func(key []byte) uint64 {
-		i, err := strconv.Atoi(string(key))
+		i, err := strconv.ParseUint(string(key), 10, 64)
 		if err != nil {
 			panic(err)
 		}
-		return uint32(i)
+		return i
 	})
 
 	// Given the above hash function, this will give replicas with "hashes":
@@ -76,6 +76,7 @@ func TestConsistency(t *testing.T) {
 		t.Errorf("Fetching 'Ben' from both hashes should be the same")
 	}
 
+	hash1.Add("Ben", "Becky", "Bobby")
 	hash2.Add("Becky", "Ben", "Bobby")
 
 	if hash1.Get("Ben") != hash2.Get("Ben") ||
@@ -108,3 +109,23 @@ func benchmarkGet(b *testing.B, shards int) {
 		hash.Get(buckets[i&(shards-1)])
 	}
 }
+
+func TestDistribution(t *testing.T) {
+	hash := New(1000, nil)
+	hash.Add("1", "2", "3", "4", "5", "6")
+
+	results := make(map[string]int, 10)
+
+	for i := 0; i < 1000000; i++ {
+		key := strconv.Itoa(i)
+
+		site := hash.Get(key)
+		if val, ok := results[site]; ok {
+			results[site] = val + 1
+		} else {
+			results[site] = 1
+		}
+	}
+
+	fmt.Println(results)
+}

internal/hash.go

@@ -1,3 +1,3 @@
 package internal
 
-type Hash func(data []byte) uint32
+type Hash func(data []byte) uint64

internal/rendezvoushash/rendezvoushash.go

@@ -1,23 +1,21 @@
 package rendezvoushash
 
 import (
-	"crypto/sha1"
+	"github.com/cespare/xxhash"
-	"hash/crc32"
-
 	"github.com/go-redis/redis/v7/internal"
 )
 
 type Map struct {
-	hash     internal.Hash
+	hash  internal.Hash
-	sites    []string
+	sites []string
 }
 
 func New(fn internal.Hash) *Map {
 	m := &Map{
-		hash:     fn,
+		hash: fn,
 	}
 	if m.hash == nil {
-		m.hash = crc32.ChecksumIEEE
+		m.hash = xxhash.Sum64
 	}
 	return m
 }
@@ -41,12 +39,15 @@ func (m *Map) Get(key string) string {
 	}
 
 	// find the site that, when hashed with the key, yields the largest weight
-	maxWeight := uint32(0)
+	var targetSite string
-	targetSite := ""
+
+	var maxWeight uint64
+	buf := make([]byte, len(key), 2*len(key))
+	copy(buf, key)
 	for _, site := range m.sites {
-		hasher := sha1.New()
+		buf = buf[:len(key)]
-		hasher.Write([]byte(site + key))
+		buf = append(buf, site...)
-		siteWeight := m.hash(hasher.Sum(nil))
+		siteWeight := m.hash(buf)
 
 		if siteWeight > maxWeight {
 			maxWeight = siteWeight

internal/rendezvoushash/rendezvoushash_test.go

@@ -2,12 +2,12 @@ package rendezvoushash
 
 import (
 	"fmt"
-	"hash/crc32"
+	"strconv"
 	"testing"
 )
 
 func TestHashing(t *testing.T) {
-	hash := New(crc32.ChecksumIEEE)
+	hash := New(nil)
 	hash.Add("site1", "site2", "site3")
 
 	verifyFn := func(cases map[string]string) {
@@ -20,15 +20,15 @@ func TestHashing(t *testing.T) {
 	}
 
 	testCases := map[string]string{
-		"key1": "site2",
+		"key1":  "site2",
-		"key2": "site1",
+		"key2":  "site1",
-		"key3": "site2",
+		"key3":  "site2",
-		"key4": "site1",
+		"key4":  "site1",
-		"key5": "site2",
+		"key5":  "site2",
-		"key6": "site3",
+		"key6":  "site3",
-		"key7": "site1",
+		"key7":  "site1",
-		"key8": "site1",
+		"key8":  "site1",
-		"key9": "site3",
+		"key9":  "site3",
 		"key10": "site2",
 		"key11": "site3",
 		"key12": "site1",
@@ -84,3 +84,23 @@ func benchmarkGet(b *testing.B, shards int) {
 		hash.Get(buckets[i&(shards-1)])
 	}
 }
+
+func TestDistribution(t *testing.T) {
+	hash := New(nil)
+	hash.Add("1", "2", "3", "4", "5", "6")
+
+	results := make(map[string]int, 10)
+
+	for i := 0; i < 1000000; i++ {
+		key := strconv.Itoa(i)
+
+		site := hash.Get(key)
+		if val, ok := results[site]; ok {
+			results[site] = val + 1
+		} else {
+			results[site] = 1
+		}
+	}
+
+	fmt.Println(results)
+}