1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
| | #!/usr/bin/env ruby
# This requires:
# * net-http-persistent RubyGem
# * Ruby 1.9.2+
# * upstream MogileFS::Server 2.45 or later
$stdout.sync = $stderr.sync = true
usage = <<EOF
Usage: #$0 -t TRACKERS"
The output of this script can be piped to awk + curl to DELETE the files:
#$0 -t TRACKERS | awk '{system("curl -XDELETE "$3)}'
EOF
require 'uri'
require 'optparse'
require 'mogilefs'
require 'net/http/persistent'
Thread.abort_on_exception = true
MogileFS::VERSION <= "3.0.0" and
abort "Upgrade mogilefs-client (to a version that distributes this script)" \
"MogileFS::Admin#each_fid is probably broken in this version"
trackers = []
ARGV.options do |x|
x.banner = usage.strip
x.separator ''
x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]',
Array, 'hostnames/IP addresses of trackers') do |args|
trackers = args
end
x.on('-h', '--help', 'Show this help message.') { puts x; exit }
x.parse!
end
adm = MogileFS::Admin.new(:hosts => trackers)
NHP = Net::HTTP::Persistent.new(File.basename($0))
client = MogileFS::MogileFS.new(:hosts => trackers, :domain => "none")
def start_perdev_thread(pfx)
todo = Queue.new
done = Queue.new
Thread.new do
while fid_path = todo.shift
path = "#{pfx}#{fid_path}"
uri = URI.parse(path)
begin
resp = NHP.request(uri, Net::HTTP::Head.new(uri.path))
done << [ path, resp ]
rescue => err
done << [ path, err ]
end
end
end
[ todo, done ]
end
def setup_devices(dev_map, adm)
hosts = {}
adm.get_hosts.each do |host|
hosts[host["hostid"]] = "http://#{host['hostip']}:#{host['http_port']}/"
end
adm.get_devices.each do |device|
pfx = hosts[device["hostid"]] + "dev#{device['devid']}"
todo, done = start_perdev_thread(pfx)
dev_map[todo] = done
end
end
def check(bad, curfid, rv)
path, resp = rv
case resp
when Net::HTTPNotFound # good
when Net::HTTPOK
bad << "#{curfid} #{resp.content_length} #{path}\n"
else
warn "E: #{resp.inspect} (#{resp.class}) #{path}"
end
end
dev_map = {}
setup_devices(dev_map, adm)
next_fid = 0
adm.each_fid do |fid|
fidid = fid["fid"]
if fidid != next_fid
(next_fid..(fidid - 1)).each do |curfid|
nfid = sprintf("%010u", curfid)
/\A(\d)(\d{3})(\d{3})(?:\d{3})\z/ =~ nfid
fid_path = "/#$1/#$2/#$3/#{nfid}.fid"
bad = []
dev_map.each_key { |todo| todo << fid_path }
dev_map.each_value { |done| check(bad, curfid, done.shift) }
next if bad.empty?
begin
info = client.file_debug(curfid)
abort "BUG: #{info.inspect} found!" if info["fid_dkey"]
rescue MogileFS::Backend::UnknownFidError
end
puts bad.join
end
end
next_fid = fidid + 1
end
|