1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
| | #!/usr/bin/env ruby
# -*- encoding: binary -*-
# Copyright (C) 2012-2020 all contributors <cmogstored-public@yhbt.net>
# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
require 'test/test_helper'
require 'net/http'
require 'timeout'
class TestUpgrade < Test::Unit::TestCase
def setup
@start_pid = $$
@tmpdir = Dir.mktmpdir('cmogstored-upgrade-test')
@to_close = []
@host = TEST_HOST
http = TCPServer.new(@host, 0)
@http_port = http.addr[1]
mgmt = TCPServer.new(@host, 0)
@mgmt_port = mgmt.addr[1]
@err = Tempfile.new("stderr")
pid = Tempfile.new(%w(upgrade .pid))
@pid_path = pid.path
@to_close << @err
@old = "#@pid_path.oldbin"
ensure
mgmt.close if mgmt
http.close if http
end
def teardown
return if $$ != @start_pid
if @pid_path && File.exist?(@pid_path)
warn "#@pid_path exists"
pid = File.read(@pid_path).to_i rescue 0
if pid > 0 && Process.kill(0, pid)
warn "Failed to kill #{pid}, Nuking"
Process.kill(:KILL, pid)
wait_for_death(pid)
end
end
w = File.read(@err.path).strip
warn(w) if w.size > 0
@to_close.each { |io| io.close unless io.closed? }
FileUtils.rm_rf(@tmpdir)
end
def upgrade_prepare_full(wp = nil)
cmd = [ "cmogstored", "--docroot=#@tmpdir", "--pidfile=#@pid_path",
"--daemonize", "--maxconns=500",
"--mgmtlisten=#@host:#@mgmt_port",
"--httplisten=#@host:#@http_port" ]
cmd << "--worker-processes=#{wp}" if wp
tmp_pid = fork do
$stderr.reopen(@err.path)
exec(*cmd)
end
_, status = Process.waitpid2(tmp_pid)
assert status.success?, status.inspect
wait_for_pidfile(@pid_path)
assert_http_running
old_pid = assert_pidfile_valid(@pid_path)
# start the upgrade
Process.kill(:USR2, old_pid)
wait_for_pidfile(@old)
wait_for_pidfile(@pid_path)
# both old and new should be running
first_pid = assert_pidfile_valid(@old)
assert_equal old_pid, first_pid
assert File.exist?(@pid_path), "#@pid_path exists"
new_pid = assert_pidfile_valid(@pid_path)
assert new_pid != old_pid
[ old_pid, new_pid ]
end
def _test_upgrade_kill(new_sig = :QUIT, wp = nil)
old_pid, new_pid = upgrade_prepare_full(wp)
Process.kill(new_sig, new_pid)
wait_for_death(new_pid)
stop = Time.now + 30
sleep(0.01) while File.exist?(@old) && Time.now < stop
raise "Timed out waiting for #@old to disappear" if File.exist?(@old)
wait_for_pidfile(@pid_path)
orig_pid = assert_pidfile_valid(@pid_path)
assert_equal old_pid, orig_pid
Process.kill(:QUIT, orig_pid)
wait_for_death(orig_pid)
end
def test_upgrade_kill
_test_upgrade_kill
end
def test_upgrade_kill_KILL
_test_upgrade_kill_KILL
end
def test_upgrade_kill_ABRT
_test_upgrade_kill_ABRT
end
def _test_upgrade_kill_KILL(wp = nil)
_test_upgrade_kill(:KILL, wp)
end
def _test_upgrade_kill_ABRT(wp = nil)
_test_upgrade_kill(:ABRT, wp)
end
def _test_upgrade_normal(wp)
old_pid, new_pid = upgrade_prepare_full(wp)
Process.kill(:QUIT, old_pid)
wait_for_death(old_pid)
Process.kill(0, new_pid)
assert_http_running
mgmt = TCPSocket.new(TEST_HOST, @mgmt_port)
mgmt.write "shutdown\n"
Timeout.timeout(30) { assert_nil mgmt.gets }
wait_for_death(new_pid)
end
def test_upgrade_kill_KILL_worker_process
_test_upgrade_kill_KILL(1)
end
def test_upgrade_kill_ABRT_worker_process
_test_upgrade_kill_ABRT(1)
end
def test_upgrade_kill_QUIT_worker_process
_test_upgrade_kill(:QUIT, 1)
end
def test_upgrade_normal_worker_process
_test_upgrade_normal(1)
end
def test_upgrade_normal
_test_upgrade_normal(nil)
end
def wait_for_death(pid, seconds = 30)
stop = Time.now + seconds
begin
Process.kill(0, pid)
sleep(0.01)
rescue Errno::ESRCH
return
end while Time.now < stop
raise "Timed out waiting for #{pid} to die"
end
def wait_for_pidfile(pidf, seconds = 30)
stop = Time.now + seconds
begin
if File.exist?(pidf)
nr = File.read(pidf)
return if nr.to_i > 0
end
sleep 0.01
rescue Errno::ENOENT
sleep 0.01
end while Time.now < stop
raise "Timed out waiting for #{pidf} to be useful"
end
def assert_http_running
# make sure process is running and signals are ready
Net::HTTP.start(@host, @http_port) do |http|
req = Net::HTTP::Get.new("/")
resp = http.request(req)
assert_kind_of Net::HTTPOK, resp
end
end
def assert_pidfile_valid(path)
pid = File.read(path).to_i
assert_operator pid, :>, 0
pid
end
end
|