diff options
author | Eric Wong <normalperson@yhbt.net> | 2012-10-08 22:44:03 +0000 |
---|---|---|
committer | Eric Wong <normalperson@yhbt.net> | 2012-10-09 00:17:16 +0000 |
commit | 436f57af99152d62c8c56fda394cbb2159747fe9 (patch) | |
tree | 3f01ff22e6b4d9c6ca52ec9cd9bb275bb087f1cd | |
parent | ef987fde476bd9d36a6514acd50cf94fc8081063 (diff) | |
download | mogilefs-client-436f57af99152d62c8c56fda394cbb2159747fe9.tar.gz |
This allows fast listing of keys and metadata (length, checksum, devcount, class).
-rw-r--r-- | lib/mogilefs/mogilefs.rb | 51 | ||||
-rw-r--r-- | test/test_mogilefs_integration_list_keys.rb | 16 |
2 files changed, 67 insertions, 0 deletions
diff --git a/lib/mogilefs/mogilefs.rb b/lib/mogilefs/mogilefs.rb index eccca78..426217e 100644 --- a/lib/mogilefs/mogilefs.rb +++ b/lib/mogilefs/mogilefs.rb @@ -88,6 +88,57 @@ class MogileFS::MogileFS < MogileFS::Client nil end + # Enumerates keys and yields a +file_info+ hash for each key matched by + # +prefix+ + def each_file_info(prefix = "", args = nil) + # FIXME: there's a lot of duplicate code from list_keys_verbose here... + raise ArgumentError, "need block" unless block_given? + ordered = ready = nil + on_file_info = lambda do |info| + Hash === info or raise info + file_info_cleanup(info) + + # deal with trackers with multiple queryworkers responding out-of-order + ready[info["key"]] = info + while info = ready.delete(ordered[-1]) + ordered.pop + yield info + end + end + + nr = 0 + opts = { :domain => @domain } + opts[:devices] = 1 if args && args[:devices] + after = args ? args[:after] : nil + limit = args ? args[:limit] : nil + + begin + keys, after = list_keys(prefix, after, limit || 1000) + return nr unless keys && keys[0] + ordered = keys.reverse + ready = {} + nr += keys.size + limit -= keys.size if limit + + keys.each do |key| + opts[:key] = key + @backend.pipeline_dispatch(:file_info, opts, &on_file_info) + end + @backend.pipeline_wait + rescue MogileFS::PipelineError, SystemCallError, + MogileFS::RequestTruncatedError, + MogileFS::UnreadableSocketError, + MogileFS::InvalidResponseError, # truncated response + MogileFS::Timeout + @backend.shutdown + keys = ordered - ready.keys + retry + end while limit == nil || limit > 0 + rescue + @backend.shutdown + raise + end + # Retrieves the contents of +key+. If +dst+ is specified, +dst+ # should be an IO-like object capable of receiving the +write+ method # or a path name. +copy_length+ may be specified to limit the number of diff --git a/test/test_mogilefs_integration_list_keys.rb b/test/test_mogilefs_integration_list_keys.rb index 6c62e6b..1de6cda 100644 --- a/test/test_mogilefs_integration_list_keys.rb +++ b/test/test_mogilefs_integration_list_keys.rb @@ -36,5 +36,21 @@ class TestMogileFSIntegrationListKeys < TestMogIntegration assert_equal "ek_#{n.to_s}", key n += 1 end + assert_equal 9, n + end + + def test_each_file_info + 9.times { |i| @client.store_content("ek_#{i}", nil, i.to_s) } + n = 0 + @client.each_file_info do |info| + assert_equal @client.domain, info["domain"] + assert_equal n.to_s.size, info["length"] + assert_kind_of Integer, info["fid"] + assert_kind_of Integer, info["devcount"] + assert_equal "default", info["class"] + assert_equal "ek_#{n}", info["key"] + n += 1 + end + assert_equal 9, n end end |