examples: add stale_fid_checker script

This logic may go upstream into the tracker Fsck worker itself,
but until then, we can try this from the client side.

1 files changed, 89 insertions, 0 deletions

diff --git a/examples/stale_fid_checker.rb b/examples/stale_fid_checker.rb
new file mode 100644
index 0000000..77cb23d
--- /dev/null
+++ b/examples/stale_fid_checker.rb
@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+# This requires:
+# * net-http-persistent RubyGem
+# * Ruby 1.9.2+
+# * upstream MogileFS::Server 2.45 or later
+$stdout.sync = $stderr.sync = true
+require 'uri'
+require 'optparse'
+require 'mogilefs'
+require 'net/http/persistent'
+Thread.abort_on_exception = true
+
+trackers = []
+ARGV.options do |x|
+  x.banner = "Usage: #$0 -t TRACKERS"
+  x.separator ''
+  x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]',
+       Array, 'hostnames/IP addresses of trackers') do |args|
+    trackers = args
+  end
+  x.parse!
+end
+
+adm = MogileFS::Admin.new(:hosts => trackers)
+NHP = Net::HTTP::Persistent.new(File.basename($0))
+
+def start_perdev_thread(pfx)
+  todo = Queue.new
+  done = Queue.new
+  Thread.new do
+    while fid_path = todo.shift
+      path = "#{pfx}#{fid_path}"
+      uri = URI.parse(path)
+      begin
+        resp = NHP.request(uri, Net::HTTP::Head.new(uri.path))
+        done << [ path, resp ]
+      rescue => err
+        done << [ path, err ]
+      end
+    end
+  end
+  [ todo, done ]
+end
+
+def setup_devices(dev_map, adm)
+  hosts = {}
+  adm.get_hosts.each do |host|
+    hosts[host["hostid"]] = "http://#{host['hostip']}:#{host['http_port']}/"
+  end
+
+  adm.get_devices.each do |device|
+    pfx = hosts[device["hostid"]] + "dev#{device['devid']}"
+    todo, done = start_perdev_thread(pfx)
+    dev_map[todo] = done
+  end
+end
+
+def check(bad, curfid, rv)
+  path, resp = rv
+  case resp
+  when Net::HTTPNotFound # good
+  when Net::HTTPOK
+    bad << "#{curfid} #{resp.content_length} #{path}\n"
+  else
+    warn "E: #{resp.inspect} (#{resp.class}) #{path}"
+  end
+end
+
+dev_map = {}
+setup_devices(dev_map, adm)
+next_fid = 0
+adm.each_fid do |fid|
+  fidid = fid["fid"]
+
+  if fidid != next_fid
+    (next_fid..(fidid - 1)).each do |curfid|
+      nfid = sprintf("%010u", curfid)
+      /\A(\d)(\d{3})(\d{3})(?:\d{3})\z/ =~ nfid
+      fid_path = "/#$1/#$2/#$3/#{nfid}.fid"
+      bad = []
+      dev_map.each_key { |todo| todo << fid_path }
+      dev_map.each_value { |done| check(bad, curfid, done.shift) }
+      next if bad.empty?
+
+      puts bad.join
+    end
+  end
+  next_fid = fidid + 1
+end