diff options
author | Eric Wong <normalperson@yhbt.net> | 2010-12-10 19:49:12 -0800 |
---|---|---|
committer | Eric Wong <normalperson@yhbt.net> | 2010-12-10 19:49:12 -0800 |
commit | 5d6dc5c742f827350490d8f33c4c89b203ae7460 (patch) | |
tree | 9fc79f0ead7a407f7a7c636a92ed372870e5565a /lib/metropolis | |
parent | 8791d27f34d618bc7979b56da7e068b79a79b229 (diff) | |
download | metropolis-5d6dc5c742f827350490d8f33c4c89b203ae7460.tar.gz |
String#hash is not stable across processes :<
We need to implement our own hash functions for splitting databases across multiple files. This was totally fucking up Rainbows!
Diffstat (limited to 'lib/metropolis')
-rw-r--r-- | lib/metropolis/multi_hash.rb | 24 | ||||
-rw-r--r-- | lib/metropolis/multi_hash/digest.rb | 23 | ||||
-rw-r--r-- | lib/metropolis/multi_hash/to_i.rb | 12 | ||||
-rw-r--r-- | lib/metropolis/tc/hdb.rb | 6 | ||||
-rw-r--r-- | lib/metropolis/tc/hdb/ex.rb | 2 | ||||
-rw-r--r-- | lib/metropolis/tc/hdb/ro.rb | 2 | ||||
-rw-r--r-- | lib/metropolis/tdb/multi.rb | 4 |
7 files changed, 68 insertions, 5 deletions
diff --git a/lib/metropolis/multi_hash.rb b/lib/metropolis/multi_hash.rb new file mode 100644 index 0000000..65ed6fe --- /dev/null +++ b/lib/metropolis/multi_hash.rb @@ -0,0 +1,24 @@ +# -*- encoding: binary -*- +module Metropolis::MultiHash + autoload :Digest, 'metropolis/multi_hash/digest' + autoload :ToI, 'metropolis/multi_hash/to_i' + + def self.extended(obj) + sym = obj.instance_eval { + case @multi_hash.to_s + when /\Ato_i/ + extend Metropolis::MultiHash::ToI + when /\Adigest_/ + extend Metropolis::MultiHash::Digest + when /\Atdb_hash_/ + extend TDB::HashFunctions + end + @multi_hash + } + obj.respond_to?(sym) or + raise ArgumentError, "multi_hash=#{sym} not supported" + (class << obj; self; end).instance_eval do + alias_method :multi_hash, sym + end + end +end diff --git a/lib/metropolis/multi_hash/digest.rb b/lib/metropolis/multi_hash/digest.rb new file mode 100644 index 0000000..974d7ac --- /dev/null +++ b/lib/metropolis/multi_hash/digest.rb @@ -0,0 +1,23 @@ +# -*- encoding: binary -*- +require 'digest' +module Metropolis::MultiHash::Digest + def digest_sha1(key) + ::Digest::SHA1.digest(key)[0,4].unpack("N")[0] + end + + def digest_md5(key) + ::Digest::MD5.digest(key)[0,4].unpack("N")[0] + end + + def digest_sha256(key) + ::Digest::SHA256.digest(key)[0,4].unpack("N")[0] + end + + def digest_sha384(key) + ::Digest::SHA384.digest(key)[0,4].unpack("N")[0] + end + + def digest_sha512(key) + ::Digest::SHA512.digest(key)[0,4].unpack("N")[0] + end +end diff --git a/lib/metropolis/multi_hash/to_i.rb b/lib/metropolis/multi_hash/to_i.rb new file mode 100644 index 0000000..69b2819 --- /dev/null +++ b/lib/metropolis/multi_hash/to_i.rb @@ -0,0 +1,12 @@ +# -*- encoding: binary -*- +# simple "hashing" method which converts keys to integers, +# this may be useful for databases that only store numeric keys +module Metropolis::MultiHash::ToI + def to_i(key) + key.to_i + end + + def to_i_16(key) + key.to_i(16) + end +end diff --git a/lib/metropolis/tc/hdb.rb b/lib/metropolis/tc/hdb.rb index d0833f8..e63b015 100644 --- a/lib/metropolis/tc/hdb.rb +++ b/lib/metropolis/tc/hdb.rb @@ -68,6 +68,8 @@ module Metropolis::TC::HDB end [ hdb, path ] end + @multi_hash ||= :digest_sha1 + extend Metropolis::MultiHash extend(RO) if @readonly extend(EX) if @exclusive end @@ -77,7 +79,7 @@ module Metropolis::TC::HDB end def writer(key, &block) - hdb, path = @dbv[key.hash % @nr_slots] + hdb, path = @dbv[multi_hash(key) % @nr_slots] hdb.open(path, @wr_flags) or ex!(:open, hdb) yield hdb ensure @@ -85,7 +87,7 @@ module Metropolis::TC::HDB end def reader(key) - hdb, path = @dbv[key.hash % @nr_slots] + hdb, path = @dbv[multi_hash(key) % @nr_slots] hdb.open(path, @rd_flags) or ex!(:open, hdb) yield hdb ensure diff --git a/lib/metropolis/tc/hdb/ex.rb b/lib/metropolis/tc/hdb/ex.rb index 5bc7f39..d205a76 100644 --- a/lib/metropolis/tc/hdb/ex.rb +++ b/lib/metropolis/tc/hdb/ex.rb @@ -11,7 +11,7 @@ module Metropolis::TC::HDB::EX end def reader(key) - yield @ex_dbv[key.hash % @nr_slots] + yield @ex_dbv[multi_hash(key) % @nr_slots] end alias_method :writer, :reader diff --git a/lib/metropolis/tc/hdb/ro.rb b/lib/metropolis/tc/hdb/ro.rb index 62ededc..72fa968 100644 --- a/lib/metropolis/tc/hdb/ro.rb +++ b/lib/metropolis/tc/hdb/ro.rb @@ -14,6 +14,6 @@ module Metropolis::TC::HDB::RO end def reader(key) - yield @ro_dbv[key.hash % @nr_slots] + yield @ro_dbv[multi_hash(key) % @nr_slots] end end diff --git a/lib/metropolis/tdb/multi.rb b/lib/metropolis/tdb/multi.rb index b22e63b..68c8bf5 100644 --- a/lib/metropolis/tdb/multi.rb +++ b/lib/metropolis/tdb/multi.rb @@ -2,6 +2,8 @@ module Metropolis::TDB::Multi def self.extended(obj) obj.instance_eval do + @multi_hash ||= :tdb_hash_murmur2 + extend Metropolis::MultiHash @dbv = (0...@nr_slots).to_a.map do |slot| path = sprintf(@path_pattern, slot) ::TDB.new(path, @tdb_opts) @@ -10,7 +12,7 @@ module Metropolis::TDB::Multi end def db(key, &block) - yield @dbv[key.hash % @nr_slots] + yield @dbv[multi_hash(key) % @nr_slots] end def close! |