examples: add usage_fetcher example

This was written a while ago for a large cluster, so I
figured I might as well include it here.

1 files changed, 44 insertions, 0 deletions

diff --git a/examples/usage_fetcher.rb b/examples/usage_fetcher.rb
new file mode 100644
index 0000000..43a3b50
--- /dev/null
+++ b/examples/usage_fetcher.rb
@@ -0,0 +1,44 @@
+# example to fetch all usage files into the current directory
+# This will download dev*/usage files to the current files with the template:
+#   ${HOSTNAME}_${PORT}_dev$DEVID.txt
+$stderr.sync = $stdout.sync = Thread.abort_on_exception = true
+ARGV.empty? and abort "Usage: #$0 TRACKERS"
+
+require 'net/http/persistent' # gem install net-http-persistent
+require 'uri'
+require 'thread'
+require 'mogilefs'
+queue = Queue.new
+nhp = Net::HTTP::Persistent.new
+adm = MogileFS::Admin.new(hosts: ARGV)
+
+by_hostid = {}
+adm.get_hosts.each { |host| by_hostid[host["hostid"]] = host }
+by_hostid.freeze # by_hostid is read-only at this point
+
+nr = by_hostid.size
+thr = nr.times.map do
+  Thread.new do
+    while dev = queue.shift
+      host = by_hostid[dev["hostid"]]
+      port = host["http_get_port"] || host["http_port"]
+      devid = dev["devid"]
+      url = "http://#{host["hostip"]}:#{port}/dev#{devid}/usage"
+      uri = URI(url)
+      response = nhp.request(uri)
+      body = response.body
+      File.open("#{host["hostname"]}_#{port}_dev#{devid}.txt", "w") do |fp|
+        fp.write(body)
+      end
+    end
+  end
+end
+
+adm.get_devices.each do |dev|
+  case dev["status"]
+  when "alive", "readonly", "drain"
+    queue.push(dev)
+  end
+end
+nr.times { queue.push(nil) } # terminate the threads
+thr.each(&:join)