1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
| | # -*- encoding: binary -*-
require "./test/fresh"
class TestMogFresh < Test::Unit::TestCase
include TestFreshSetup
alias setup setup_mogilefs
alias teardown teardown_mogilefs
def test_change_device_weight
add_host_device_domain
assert_equal true, @admin.change_device_weight("me", 1, 50)
assert_equal 50, @admin.get_devices(1)[0]["weight"]
end
def test_list_keys_invalid_domain
add_host_device_domain
domain = @domain + ".non-existent"
client = MogileFS::MogileFS.new :hosts => @hosts, :domain => domain
assert_raises(MogileFS::Backend::UnregDomainError) do
client.list_keys
end
end
def test_invalid_key_exists
add_host_device_domain
domain = @domain + ".non-existent"
client = MogileFS::MogileFS.new :hosts => @hosts, :domain => domain
assert_raises(MogileFS::Backend::UnregDomainError) do
client.exist?("FOO")
end
client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
assert_equal false, client.exist?("non-existent")
end
def test_new_file_info(checksum = nil)
add_host_device_domain unless checksum
@client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
info = {}
opts = { :info => info }
key = "new_file_info"
content = "ZZZZ"
if checksum
opts[:content_md5] = [ Digest::MD5.digest(content) ].pack('m').rstrip
opts[:class] = "check"
end
@client.new_file(key, opts) do |http_file|
http_file << content
end
uris = info.delete(:uris)
assert_kind_of Array, uris
assert_equal(uris, (@client.get_uris(key) & uris))
expect_info = @client.file_info(key, :devices => true)
match_keys = %w(class fid key domain length)
match_keys << "checksum" if checksum
match_keys.each do |field|
assert_equal expect_info.delete(field), info.delete(field)
end
assert_operator expect_info.delete("devcount"), :>=, info.delete("devcount")
devids = info.delete("devids")
assert_equal(devids, (expect_info.delete("devids") & devids))
assert info.empty?, info.inspect
assert expect_info.empty?, expect_info.inspect
ensure
@client.delete(key)
end
def test_new_file_info_checksum
add_host_device_domain
opts = @admin.get_domains[@domain]["default"]
opts["hashtype"] = "MD5"
@admin.create_class(@domain, "check", opts)
yield_for_monitor_update do
tmp = @admin.get_domains[@domain]["check"]
if tmp
case tmp["hashtype"]
when "MD5"
break
when nil
warn "skipping checksum test, MogileFS server too old"
return
else
raise "Unhandled hashtype: #{tmp['hashtype']}"
end
end
end
test_new_file_info(:md5)
ensure
@admin.delete_class @domain, "check"
end
def test_create_open_close_opts
add_host_device_domain
client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
socket = client.backend.socket
args = {
:create_open_args => { :hello => "world" },
:create_close_args => { :farewell => "goodnight" },
}
io = client.new_file("foo", args)
socket.write "!recent\n"
buf = ""
buf << socket.readpartial(666) until buf =~ /\.\r?\n\z/
line = buf.split(/\r?\n/).grep(/\screate_open\s/)[0]
assert_equal 0, buf.split(/\r?\n/).grep(/\screate_close\s/).size
assert_equal 0, buf.split(/\r?\n/).grep(/\sfarewell\s/).size
assert_match(/\bhello=world\b/, line)
assert_equal 1, io.write('.')
assert_nil io.close
socket.write "!recent\n"
buf = ""
buf << socket.readpartial(666) until buf =~ /\.\r?\n\z/
line = buf.split(/\r?\n/).grep(/\screate_close\s/)[0]
assert_match(/\bfarewell=goodnight\b/, line)
end
def test_get_file_data_range
add_host_device_domain
client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
data = "data"
client.store_content("key", "default", data)
assert_equal data, client.get_file_data("key")
# ensure offset/length matches IO.copy_stream
src = Tempfile.new("tmp")
src.write(data)
src.flush
[ [1,nil], [1,2], [3,1] ].each do |range|
dst2 = StringIO.new
client.get_file_data("key", dst2, *range)
src.rewind
if IO.respond_to?(:copy_stream)
# ensure we match IO.copy_stream semantics
dst = StringIO.new
IO.copy_stream(src, dst, *range)
assert_equal dst.string, dst2.string
assert_equal dst.string, client.get_file_data("key", nil, *range)
end
assert_equal dst2.string, client.get_file_data("key", nil, *range)
end
src.close!
end
def test_new_file_copy_stream_known_length
add_host_device_domain
opts = { :content_length => 666 }
input = StringIO.new("short")
client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
assert_raises(MogileFS::SizeMismatchError) do
client.new_file("copy_stream", opts) do |io|
IO.copy_stream(input, io)
end
end
assert_raises(MogileFS::SizeMismatchError) do
client.new_file("copy_stream", opts) do |io|
IO.copy_stream(input, io, 666)
end
end
end if IO.respond_to?(:copy_stream)
def test_single_tracker_restart
add_host_device_domain
client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
data = "data"
client.store_content("key", "default", data)
listing = client.list_keys
assert_instance_of Array, listing
# restart the tracker
s = TCPSocket.new(@test_host, @tracker_port)
s.write "!shutdown\r\n"
s.flush # just in case, MRI (at least) syncs by default
assert_nil s.gets
start_tracker
# transparent retry
listing2 = client.list_keys
assert_instance_of Array, listing2
assert_equal listing, listing2
assert_equal([['key'], 'key'], listing)
# kill the tracker
s = TCPSocket.new(@test_host, @tracker_port)
s.write "!shutdown\r\n"
s.flush # just in case, MRI (at least) syncs by default
assert_nil s.gets
@mogilefsd_pid = nil
assert_raises(MogileFS::UnreachableBackendError) { client.list_keys }
end
end
|