Bob Jenkins的哈希性能变差了。

huangapple go评论153阅读模式
英文:

Bob Jenkins' Hash getting bad performance

问题

我正在构建一个布隆过滤器,并研究要使用的哈希函数。Bob Jenkins的哈希函数似乎是一个不错的选择,因为它的分布均匀性很好。
我将给定的C++代码改写成了Go代码(可能有错误,但似乎可以工作)。

我进行了哈希函数性能测试,并发现Go标准库中的SHA1哈希函数要快得多。

PASS
BenchmarkJenkins	 1000000	      2649 ns/op
BenchmarkSHA256	 1000000	      1218 ns/op
BenchmarkSHA1	 5000000	       462 ns/op

当我读到在这种情况下不应该使用加密哈希函数时,我是否被误导了?
还是标准库中的代码比我的代码更优化?

以下是jenkins包中的代码:

package jenkins

import (
	"bytes"
	"encoding/gob"
)

// 从 http://bretmulvey.com/hash/7.html 改编
func ComputeHash(key interface{}) (uint64, error) {
	var buf bytes.Buffer
	enc := gob.NewEncoder(&buf)
	err := enc.Encode(key)
	if err != nil {
		return 0, err
	}
	data := buf.Bytes()

	var a, b, c uint64
	a, b = 0x9e3779b9, 0x9e3779b9
	c = 0
	i := 0

	for i = 0; i < len(data)-12; {
		a += uint64(data[i]) | uint64(data[i+1]<<8) | uint64(data[i+2]<<16) | uint64(data[i+3]<<24)
		i += 4
		b += uint64(data[i]) | uint64(data[i+1]<<8) | uint64(data[i+2]<<16) | uint64(data[i+3]<<24)
		i += 4
		c += uint64(data[i]) | uint64(data[i+1]<<8) | uint64(data[i+2]<<16) | uint64(data[i+3]<<24)

		a, b, c = mix(a, b, c)
	}

	c += uint64(len(data))

	if i < len(data) {
		a += uint64(data[i])
		i++
	}
	if i < len(data) {
		a += uint64(data[i]) << 8
		i++
	}
	if i < len(data) {
		a += uint64(data[i]) << 16
		i++
	}
	if i < len(data) {
		a += uint64(data[i]) << 24
		i++
	}

	if i < len(data) {
		b += uint64(data[i])
		i++
	}
	if i < len(data) {
		b += uint64(data[i]) << 8
		i++
	}
	if i < len(data) {
		b += uint64(data[i]) << 16
		i++
	}
	if i < len(data) {
		b += uint64(data[i]) << 24
		i++
	}

	if i < len(data) {
		c += uint64(data[i]) << 8
		i++
	}
	if i < len(data) {
		c += uint64(data[i]) << 16
		i++
	}
	if i < len(data) {
		c += uint64(data[i]) << 24
		i++
	}

	a, b, c = mix(a, b, c)
	return c, nil
}

func mix(a, b, c uint64) (uint64, uint64, uint64) {
	a -= b
	a -= c
	a ^= (c >> 13)
	b -= c
	b -= a
	b ^= (a << 8)
	c -= a
	c -= b
	c ^= (b >> 13)
	a -= b
	a -= c
	a ^= (c >> 12)
	b -= c
	b -= a
	b ^= (a << 16)
	c -= a
	c -= b
	c ^= (b >> 5)
	a -= b
	a -= c
	a ^= (c >> 3)
	b -= c
	b -= a
	b ^= (a << 10)
	c -= a
	c -= b
	c ^= (b >> 15)
	return a, b, c
}

编辑:

性能测试代码如下:

package bloom

import (
	"testing"

	"crypto/sha1"
	"crypto/sha256"
)

func BenchmarkJenkins(b *testing.B) {
	j := jenkinsHash{}

	for i := 0; i < b.N; i++ {
		j.ComputeHash(i)
	}
}

func BenchmarkSHA1(b *testing.B) {
	for i := 0; i < b.N; i++ {
		sha1.Sum([]byte{byte(i)})
	}
}


func BenchmarkSHA256(b *testing.B) {
	for i := 0; i < b.N; i++ {
		sha256.Sum256([]byte{byte(i)})
	}
}
英文:

I was building a Bloom filter and looked into what hashes to use and the Bob Jenkins' hash seemed like a good choice because of the evenness of the distribution.
I adapted the given C++ code to Go (possibly making a mistake but it seems to work).

I got around to benchmarking the cost of the hash and found that the SHA1 hash in the Go std library was much faster.

PASS
BenchmarkJenkins	 1000000	      2649 ns/op
BenchmarkSHA256	 1000000	      1218 ns/op
BenchmarkSHA1	 5000000	       462 ns/op

Was I misled when I read that you shouldn't use cryptographic hashes in this use case?
Or is the standard library code much more optimized than mine?

package jenkins
import (
&quot;bytes&quot;
&quot;encoding/gob&quot;
)
// adapted from http://bretmulvey.com/hash/7.html
func ComputeHash(key interface{}) (uint64, error) {
var buf bytes.Buffer
enc := gob.NewEncoder(&amp;buf)
err := enc.Encode(key)
if err != nil {
return 0, err
}
data := buf.Bytes()
var a, b, c uint64
a, b = 0x9e3779b9, 0x9e3779b9
c = 0
i := 0
for i = 0; i &lt; len(data)-12; {
a += uint64(data[i]) | uint64(data[i+1]&lt;&lt;8) | uint64(data[i+2]&lt;&lt;16) | uint64(data[i+3]&lt;&lt;24)
i += 4
b += uint64(data[i]) | uint64(data[i+1]&lt;&lt;8) | uint64(data[i+2]&lt;&lt;16) | uint64(data[i+3]&lt;&lt;24)
i += 4
c += uint64(data[i]) | uint64(data[i+1]&lt;&lt;8) | uint64(data[i+2]&lt;&lt;16) | uint64(data[i+3]&lt;&lt;24)
a, b, c = mix(a, b, c)
}
c += uint64(len(data))
if i &lt; len(data) {
a += uint64(data[i])
i++
}
if i &lt; len(data) {
a += uint64(data[i]) &lt;&lt; 8
i++
}
if i &lt; len(data) {
a += uint64(data[i]) &lt;&lt; 16
i++
}
if i &lt; len(data) {
a += uint64(data[i]) &lt;&lt; 24
i++
}
if i &lt; len(data) {
b += uint64(data[i])
i++
}
if i &lt; len(data) {
b += uint64(data[i]) &lt;&lt; 8
i++
}
if i &lt; len(data) {
b += uint64(data[i]) &lt;&lt; 16
i++
}
if i &lt; len(data) {
b += uint64(data[i]) &lt;&lt; 24
i++
}
if i &lt; len(data) {
c += uint64(data[i]) &lt;&lt; 8
i++
}
if i &lt; len(data) {
c += uint64(data[i]) &lt;&lt; 16
i++
}
if i &lt; len(data) {
c += uint64(data[i]) &lt;&lt; 24
i++
}
a, b, c = mix(a, b, c)
return c, nil
}
func mix(a, b, c uint64) (uint64, uint64, uint64) {
a -= b
a -= c
a ^= (c &gt;&gt; 13)
b -= c
b -= a
b ^= (a &lt;&lt; 8)
c -= a
c -= b
c ^= (b &gt;&gt; 13)
a -= b
a -= c
a ^= (c &gt;&gt; 12)
b -= c
b -= a
b ^= (a &lt;&lt; 16)
c -= a
c -= b
c ^= (b &gt;&gt; 5)
a -= b
a -= c
a ^= (c &gt;&gt; 3)
b -= c
b -= a
b ^= (a &lt;&lt; 10)
c -= a
c -= b
c ^= (b &gt;&gt; 15)
return a, b, c
}

EDIT:

Benchmarking code:

package bloom
import (
&quot;testing&quot;
&quot;crypto/sha1&quot;
&quot;crypto/sha256&quot;
)
func BenchmarkJenkins(b *testing.B) {
j := jenkinsHash{}
for i := 0; i &lt; b.N; i++ {
j.ComputeHash(i)
}
}
func BenchmarkSHA1(b *testing.B) {
for i := 0; i &lt; b.N; i++ {
sha1.Sum([]byte{byte(i)})
}
}
func BenchmarkSHA256(b *testing.B) {
for i := 0; i &lt; b.N; i++ {
sha256.Sum256([]byte{byte(i)})
}
}

答案1

得分: 3

我打算在优化方面下注;Bob Jenkin的哈希函数应该比SHA等加密风格的哈希函数快得多。我敢打赌标准库在这方面调用了高度优化的C代码(甚至是汇编代码),这就是为什么它比你的未优化的Go代码更快。

在Go语言中,有一个高效的Murmur3实现可用,可以在https://github.com/reusee/mmh3找到(我还没有尝试过)。你可以尝试使用这个实现,或者调用C/C++来实现你的Bob Jenkins哈希函数。

英文:

I'm going to lay bets on optimization; Bob Jenkin's hash should be substantially faster than any crypto-style hash like SHA. I would bet that the standard library is calling out into heavily-optimized C (or even assembly) for that, which is why it beats your unoptimized Go.

There appears to be an efficient Murmur3 available for Go at https://github.com/reusee/mmh3 (I haven't tried it). You might have better luck with that, or by calling out into C/C++ for your Bob Jenkins implementation.

答案2

得分: 3

go sha1哈希是用汇编语言编写的,并且经过了大量的优化(我贡献了代码的ARM版本)。

根据我看来,你的哈希函数的复杂度与sha1差不多,所以你的运行时间并不让我感到惊讶。

你可以尝试使用md5哈希,它应该适用于你的目的,并且可能更快(它也是用汇编语言编写的)。

如果你只需要一个短的哈希结果(int64),你可以尝试使用Go的CRC函数之一。

英文:

The go sha1 hash is written in assembler and has been heavily optimised (I contributed the ARM version of the code).

Your hash function looks of about equivalent complexity to sha1 to me so I'm not surprised by your run times.

You could try the md5 hash which should do for your purpose and may be faster still (it is also in assembler).

If you only need a short hash result (int64) you could try one of Go's CRC functions.

答案3

得分: 1

@JensG 是正确的。
调用 gob 将密钥编码为二进制占据了绝大部分的成本。
当我改为传入字节数组时,基准测试开始得到我期望的结果。
感谢您的帮助!

基准测试结果:

BenchmarkJenkins	100000000	        20.4 ns/op
BenchmarkSHA1	 5000000	       463 ns/op
BenchmarkSHA256	 1000000	      1223 ns/op

基准测试代码:

package bloom
import (
"testing"
"crypto/sha1"
"crypto/sha256"
)
func BenchmarkJenkins(b *testing.B) {
j := jenkinsHash{}
for i := 0; i < b.N; i++ {
j.ComputeHash([]byte{byte(i)})
}
}
func BenchmarkSHA1(b *testing.B) {
for i := 0; i < b.N; i++ {
sha1.Sum([]byte{byte(i)})
}
}
func BenchmarkSHA256(b *testing.B) {
for i := 0; i < b.N; i++ {
sha256.Sum256([]byte{byte(i)})
}
}

修改后的代码:

package bloom
type jenkinsHash struct {
}
// adapted from http://bretmulvey.com/hash/7.html
func (_ jenkinsHash) ComputeHash(data []byte) (uint64, error) {    
var a, b, c uint64
a, b = 0x9e3779b9, 0x9e3779b9
c = 0
i := 0
for i = 0; i < len(data)-12; {
a += uint64(data[i]) | uint64(data[i+1]<<8) | uint64(data[i+2]<<16) | uint64(data[i+3]<<24)
i += 4
b += uint64(data[i]) | uint64(data[i+1]<<8) | uint64(data[i+2]<<16) | uint64(data[i+3]<<24)
i += 4
c += uint64(data[i]) | uint64(data[i+1]<<8) | uint64(data[i+2]<<16) | uint64(data[i+3]<<24)
a, b, c = mix(a, b, c)
}
c += uint64(len(data))
if i < len(data) {
a += uint64(data[i])
i++
}
if i < len(data) {
a += uint64(data[i]) << 8
i++
}
if i < len(data) {
a += uint64(data[i]) << 16
i++
}
if i < len(data) {
a += uint64(data[i]) << 24
i++
}
if i < len(data) {
b += uint64(data[i])
i++
}
if i < len(data) {
b += uint64(data[i]) << 8
i++
}
if i < len(data) {
b += uint64(data[i]) << 16
i++
}
if i < len(data) {
b += uint64(data[i]) << 24
i++
}
if i < len(data) {
c += uint64(data[i]) << 8
i++
}
if i < len(data) {
c += uint64(data[i]) << 16
i++
}
if i < len(data) {
c += uint64(data[i]) << 24
i++
}
a, b, c = mix(a, b, c)
return c, nil
}
func mix(a, b, c uint64) (uint64, uint64, uint64) {
a -= b
a -= c
a ^= (c >> 13)
b -= c
b -= a
b ^= (a << 8)
c -= a
c -= b
c ^= (b >> 13)
a -= b
a -= c
a ^= (c >> 12)
b -= c
b -= a
b ^= (a << 16)
c -= a
c -= b
c ^= (b >> 5)
a -= b
a -= c
a ^= (c >> 3)
b -= c
b -= a
b ^= (a << 10)
c -= a
c -= b
c ^= (b >> 15)
return a, b, c
}
英文:

@JensG was on the right track.
The calls to gob to encode the key in binary made up the vast majority of the cost.
When I transitioned to passing in byte arrays the benchmark started getting the results I was expecting.
Thanks for the help!

BenchmarkJenkins	100000000	        20.4 ns/op
BenchmarkSHA1	 5000000	       463 ns/op
BenchmarkSHA256	 1000000	      1223 ns/op

Benchmark code:

package bloom
import (
&quot;testing&quot;
&quot;crypto/sha1&quot;
&quot;crypto/sha256&quot;
)
func BenchmarkJenkins(b *testing.B) {
j := jenkinsHash{}
for i := 0; i &lt; b.N; i++ {
j.ComputeHash([]byte{byte(i)})
}
}
func BenchmarkSHA1(b *testing.B) {
for i := 0; i &lt; b.N; i++ {
sha1.Sum([]byte{byte(i)})
}
}
func BenchmarkSHA256(b *testing.B) {
for i := 0; i &lt; b.N; i++ {
sha256.Sum256([]byte{byte(i)})
}
}

Altered code:

package bloom
type jenkinsHash struct {
}
// adapted from http://bretmulvey.com/hash/7.html
func (_ jenkinsHash) ComputeHash(data []byte) (uint64, error) {    
var a, b, c uint64
a, b = 0x9e3779b9, 0x9e3779b9
c = 0
i := 0
for i = 0; i &lt; len(data)-12; {
a += uint64(data[i]) | uint64(data[i+1]&lt;&lt;8) | uint64(data[i+2]&lt;&lt;16) | uint64(data[i+3]&lt;&lt;24)
i += 4
b += uint64(data[i]) | uint64(data[i+1]&lt;&lt;8) | uint64(data[i+2]&lt;&lt;16) | uint64(data[i+3]&lt;&lt;24)
i += 4
c += uint64(data[i]) | uint64(data[i+1]&lt;&lt;8) | uint64(data[i+2]&lt;&lt;16) | uint64(data[i+3]&lt;&lt;24)
a, b, c = mix(a, b, c)
}
c += uint64(len(data))
if i &lt; len(data) {
a += uint64(data[i])
i++
}
if i &lt; len(data) {
a += uint64(data[i]) &lt;&lt; 8
i++
}
if i &lt; len(data) {
a += uint64(data[i]) &lt;&lt; 16
i++
}
if i &lt; len(data) {
a += uint64(data[i]) &lt;&lt; 24
i++
}
if i &lt; len(data) {
b += uint64(data[i])
i++
}
if i &lt; len(data) {
b += uint64(data[i]) &lt;&lt; 8
i++
}
if i &lt; len(data) {
b += uint64(data[i]) &lt;&lt; 16
i++
}
if i &lt; len(data) {
b += uint64(data[i]) &lt;&lt; 24
i++
}
if i &lt; len(data) {
c += uint64(data[i]) &lt;&lt; 8
i++
}
if i &lt; len(data) {
c += uint64(data[i]) &lt;&lt; 16
i++
}
if i &lt; len(data) {
c += uint64(data[i]) &lt;&lt; 24
i++
}
a, b, c = mix(a, b, c)
return c, nil
}
func mix(a, b, c uint64) (uint64, uint64, uint64) {
a -= b
a -= c
a ^= (c &gt;&gt; 13)
b -= c
b -= a
b ^= (a &lt;&lt; 8)
c -= a
c -= b
c ^= (b &gt;&gt; 13)
a -= b
a -= c
a ^= (c &gt;&gt; 12)
b -= c
b -= a
b ^= (a &lt;&lt; 16)
c -= a
c -= b
c ^= (b &gt;&gt; 5)
a -= b
a -= c
a ^= (c &gt;&gt; 3)
b -= c
b -= a
b ^= (a &lt;&lt; 10)
c -= a
c -= b
c ^= (b &gt;&gt; 15)
return a, b, c
}

huangapple
  • 本文由 发表于 2014年5月3日 04:24:17
  • 转载请务必保留本文链接:https://go.coder-hub.com/23436358.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定