Debian Codegraph @

misc/ditg-control-server.py

raw
1 # -*- coding: utf-8 -*-
2 #
3 # ditg-control-server.py
4 #
5 # Author: Toke Høiland-Jørgensen ([email protected])
6 # Date: 28 marts 2014
7 # Copyright (c) 2014-2016, Toke Høiland-Jørgensen
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21
22 from __future__ import absolute_import, division, print_function, unicode_literals
23
24 import calendar
25 import hashlib
26 import hmac
27 import json
28 import optparse
29 import os
30 import random
31 import shutil
32 import signal
33 import string
34 import subprocess
35 import sys
36 import tempfile
37 import time
38 import traceback
39
40 from datetime import datetime
41
42 try:
43 from defusedxml.xmlrpc import monkey_patch
44 monkey_patch()
45 del monkey_patch
46 XML_DEFUSED = True
47 except ImportError:
48 XML_DEFUSED = False
49
50 try:
51 # python 2
52 from SimpleXMLRPCServer import SimpleXMLRPCServer
53 except ImportError:
54 from xmlrpc.server import SimpleXMLRPCServer
55
56 # Nicked from the subprocess module -- for closing open file descriptors
57 try:
58 MAXFD = os.sysconf("SC_OPEN_MAX")
59 except:
60 MAXFD = 256
61
62 ALPHABET = list(string.ascii_lowercase) + list(string.digits)
63
64
65 class AlarmException(Exception):
66 pass
67
68
69 parser = optparse.OptionParser(
70 description="Control server for D-ITG receive component session negotiation.")
71
72 parser.add_option('--insecure-xml', action='store_true', dest='INSECURE_XML',
73 default=False,
74 help="Run even though the defusedxml module is unavailable. "
75 "WARNING: Will leave the server open to entity expansion "
76 "attacks!")
77 parser.add_option('-a', '--address', action='store', type='string',
78 dest='BIND_ADDRESS', default='localhost',
79 help="Address to bind to. Default: localhost.")
80 parser.add_option('-p', '--port', action='store', type='int', dest='BIND_PORT',
81 default='8000', help="Bind port. Default: 8000.")
82 parser.add_option('-A', '--itg-address', action='store', type='string',
83 dest='ITG_ADDRESS', default=None,
84 help="Address to bind ITGRecv to. Default: Same as --address.")
85 parser.add_option('-s', '--start-port', action='store', type='int',
86 dest='START_PORT', default=9000, help="Start port for ITGRecv "
87 "control socket binds (default 9000).")
88 parser.add_option('-t', '--max-test-time', action='store', type='int',
89 dest='MAX_TEST_TIME', default=7200, help="Maximum test time "
90 "allowed. Default 7200 seconds (two hours).")
91 parser.add_option('-m', '--max-instances', action='store', type='int',
92 dest='MAX_INSTANCES', default=100,
93 help="Maximum number of running instances before new requests "
94 "are denied (default 100).")
95 parser.add_option('-S', '--secret', action='store', type='string', dest='SECRET',
96 default="", help="Secret for request authentication. "
97 "Default: ''.")
98 parser.add_option('--debug', action='store_true', dest='DEBUG',
99 default=False, help="Debug mode: Don't delete temporary files.")
100
101
102 class DITGManager(object):
103 datafile_pattern = "%s.json"
104
105 def __init__(self, bind_address, start_port, max_test_time, max_instances,
106 secret, debug=False):
107 self.working_dir = tempfile.mkdtemp(prefix='ditgman-')
108 self.seen = {}
109 self.children = []
110 self.toplevel = True
111 self.bind_address = bind_address
112 self.max_test_time = max_test_time
113 self.max_instances = max_instances
114 self.hmac = hmac.new(secret.encode(), digestmod=hashlib.sha256)
115 self.start_port = self.current_port = start_port
116 self.id_length = 20
117 self.debug = debug
118
119 signal.signal(signal.SIGINT, self._exit)
120 signal.signal(signal.SIGTERM, self._exit)
121 signal.signal(signal.SIGALRM, self._alarm)
122 signal.signal(signal.SIGCHLD, self._collect_garbage)
123
124 def get_test_results(self, test_id):
125 """Get the results of a previously initiated test. The sole parameter is
126 the test_id returned from request_new_test().
127
128 The return parameter is a dictionary with the following entries:
129
130 status: 'OK' if everything went well, 'Error' otherwise.
131
132 message: Set if status is 'Error'; contains an error message.
133
134 data: The output of ITGDec -c with the interval requested at initiation.
135
136 utc_offset: UTC timestamp which the time values in the return data is
137 offset by.
138
139 """
140 self._collect_garbage()
141 test_id = str(test_id)
142 if len(test_id) != self.id_length:
143 return {'status': 'Error',
144 'message': "Invalid test id: '%s'." % test_id}
145 for c in test_id:
146 if c not in ALPHABET:
147 return {'status': 'Error',
148 'message': "Invalid test id: '%s'." % test_id}
149 filename = os.path.join(
150 self.working_dir, self.datafile_pattern % test_id)
151 if not os.path.exists(filename):
152 return {'status': 'Error',
153 'message': "Data for test ID '%s' not found." % test_id}
154 with open(filename, 'rt') as fp:
155 data = json.load(fp)
156 return data
157
158 def request_new_test(self, duration, interval, hmac_hex, raw_data=False):
159 """Request a new test instance.
160
161 This will allocate a new ITGRecv instance and parse the log file it
162 produces. The results of this can then be retrieved with
163 get_test_results() after the test has run.
164
165 The parameters are:
166
167 duration: Requested test duration in seconds. The ITGRecv instance will
168 be killed after this time has passed (+ a grace period of five
169 seconds).
170
171 interval: The requested interval for data points, in milliseconds
172 (passed to ITGDec).
173
174 hmac_hex: A hexadecimal HMAC-SHA256 of the two other parameters computed
175 by concatenating their ASCII representations. The HMAC secret
176 is configured by the operator of the control server instance.
177
178 raw_data: Whether to store and return the raw text log from ITGDec (i.e.
179 the output of ITGDec -l)
180
181 The return value is a dictionary with the following keys:
182
183 status: 'OK' if everything went well, 'Error' otherwise.
184
185 message: Set if status is 'Error'; contains an error message.
186
187 test_id: The assigned test ID, to be passed to get_test_results() after
188 the duration has expired.
189
190 port: The control server port of the ITGRecv instance. The sender is
191 expected to use port+1 for the data connection.
192
193 """
194 self._collect_garbage()
195 duration = int(duration)
196 interval = int(interval)
197 hmac = self.hmac.copy()
198 hmac.update(str(duration).encode())
199 hmac.update(str(interval).encode())
200 if hmac.hexdigest() != hmac_hex:
201 return {'status': 'Error', 'message': "HMAC authentication failure."}
202 if duration <= 0 or interval <= 0:
203 return {'status': 'Error',
204 'message': "Duration and interval must be positive integers."}
205 if duration > self.max_test_time:
206 return {'status': 'Error',
207 'message': "Maximum test time of %d seconds exceeded."
208 % self.max_test_time}
209 if interval > duration * 1000:
210 return {'status': 'Error', 'message': "Interval must be <= duration."}
211 if len(self.children) >= self.max_instances:
212 return {'status': 'Error',
213 'message': "Too many concurrent instances running. "
214 "Try again later."}
215
216 test_id = "".join(random.sample(ALPHABET, self.id_length))
217 # Need one port for control, one for data (if the data stream is TCP).
218 port = self.current_port
219 self.current_port += 2
220 return self._spawn_receiver(test_id, duration, interval, port, raw_data)
221
222 def _clean_fork(self, output=None):
223 pipe_r, pipe_w = os.pipe()
224 pid = os.fork()
225 if pid:
226 self.children.append(pid)
227 os.close(pipe_w)
228 return pipe_r, False
229 else:
230 try:
231 self.children = []
232 self.toplevel = False
233 os.chdir(self.working_dir)
234 sys.stdin.close()
235 # Move the pipe fd to position 3, close everything above it
236 os.dup2(pipe_w, 3)
237 os.closerange(4, MAXFD)
238 pipe_w = 3
239
240 if output is not None:
241 with open(os.path.join(self.working_dir, output), "w") as fp:
242 os.dup2(fp.fileno(), 1)
243 os.dup2(fp.fileno(), 2)
244 else:
245 sys.stdout.close()
246 sys.stderr.close()
247 except:
248 try:
249 traceback.print_exc()
250 finally:
251 os._exit(1)
252 return pipe_w, True
253
254 def _unlink(self, filename):
255 if self.debug:
256 return
257 try:
258 os.unlink(filename)
259 except:
260 pass
261
262 def _spawn_receiver(self, test_id, duration, interval, port, raw_data):
263 stdout = "%s.recv.out" % test_id
264 datafile = self.datafile_pattern % test_id
265 error = False
266 pipe, child = self._clean_fork(stdout)
267 if child:
268 try:
269 ret = self._run_receiver(
270 test_id, duration, interval, port, pipe, raw_data)
271 except Exception as e:
272 traceback.print_exc()
273 ret = {'status': 'Error', 'message': str(e)}
274 error = True
275 try:
276 # Write data to temp file, atomically rename to final file name
277 with open('%s.tmp' % datafile, 'wt') as fp:
278 json.dump(ret, fp)
279 os.rename('%s.tmp' % datafile, datafile)
280 finally:
281 if not error:
282 self._unlink(stdout)
283 os._exit(0)
284 os._exit(1)
285 else:
286 try:
287 val = os.read(pipe, 1024).decode()
288 if val == 'OK':
289 return {'status': 'OK', 'test_id': test_id, 'port': port}
290 else:
291 return {'status': 'Error', 'message': val}
292 finally:
293 os.close(pipe)
294
295 def _run_receiver(self, test_id, duration, interval, port, pipe, raw_data):
296 logfile = '%s.log' % test_id
297 txtlog = '%s.log.txt' % test_id
298 outfile = '%s.dat' % test_id
299 ret = {'status': 'OK'}
300
301 # Run ITGRecv; does not terminate after the end of the test, so wait for
302 # the test duration + a grace time of 5 seconds, then kill the process
303 try:
304 proc = subprocess.Popen(['ITGRecv',
305 '-l', logfile,
306 '-I',
307 '-a', self.bind_address,
308 '-Sp', str(port)])
309
310 # Signal back to the parent whether or not we successfully spawned the
311 # receiver.
312 except OSError as e:
313 os.write(pipe, str(e).encode())
314 else:
315 time.sleep(0.1)
316 try:
317 w = os.waitpid(proc.pid, os.WNOHANG)
318 except OSError as e:
319 w = (0, str(e))
320 if w != (0, 0):
321 ret = {'status': 'Error',
322 'message': 'ITGRecv exited immediately with code %s.'
323 % w[1]}
324 os.write(pipe, ret['message'].encode())
325 else:
326 self.children.append(proc.pid)
327 os.write(pipe, b'OK')
328
329 os.close(pipe)
330
331 # Use an alarm signal for the timeout; means we don't have to wait for the
332 # entire test duration if the ITGRecv process terminates before then
333 signal.alarm(duration + 5)
334 try:
335 retval = proc.wait()
336 if retval > 0:
337 raise Exception("ITGRecv non-zero exit")
338 except AlarmException:
339 proc.terminate()
340 signal.alarm(0)
341
342 # Call ITGDec on the log file output, read in the resulting data
343 if not os.path.exists(logfile):
344 raise Exception('No data file produced')
345
346 try:
347 subprocess.check_call(['ITGDec', logfile,
348 '-c', str(interval), outfile,
349 '-l', txtlog])
350 with open(outfile, 'rt') as fp:
351 ret['data'] = fp.read()
352
353 if not ret['data']:
354 # By not raising an exception, the stdout output file is still
355 # cleared, but an error is reported to the caller. This is to
356 # prevent output files from sticking around for, e.g., users
357 # that request a test, but then cannot get packets through; the
358 # output is not useful to diagnose that anyway.
359 ret = {'status': 'Error', 'message': 'Empty data set.'}
360
361 # Read start of text log file to get timestamp of first received
362 # packet
363 with open(txtlog, 'rt') as fp:
364 if raw_data:
365 ret['raw'] = fp.read()
366 data = ret['raw'][:1024]
367 else:
368 data = fp.read(1024)
369 try:
370 idx_s = data.index('rxTime>') + len('rxTime>')
371 idx_e = data.index('Size>')
372 t, microsec = data[idx_s:idx_e].split(".")
373 h, m, s = t.split(":")
374 now = time.gmtime()
375 ret['utc_offset'] = calendar.timegm(
376 (now[0], now[1], now[2],
377 int(h), int(m), int(s),
378 now[6], now[7], now[8])) + int(microsec) / 10**6
379 except Exception as e:
380 ret['utc_offset'] = None
381
382 return ret
383 finally:
384 # Clean up temporary files and exit
385 for f in logfile, txtlog, outfile:
386 self._unlink(f)
387
388 def _collect_garbage(self, *args):
389 for p in self.children:
390 try:
391 if os.waitpid(p, os.WNOHANG) != (0, 0):
392 self.children.remove(p)
393 except OSError:
394 self.children.remove(p)
395 # When no more children are alive, reset the current port
396 if not self.children:
397 self.current_port = self.start_port
398 if not self.toplevel:
399 return
400
401 try:
402 files = os.listdir(self.working_dir)
403 for f in files:
404 if not f.endswith('.json'):
405 continue
406 p = os.path.join(self.working_dir, f)
407 s = os.stat(p)
408 td = datetime.now() - datetime.fromtimestamp(s.st_mtime)
409 if td.seconds > 300:
410 self._unlink(p)
411 except:
412 traceback.print_exc()
413
414 def _alarm(self, signum, frame):
415 raise AlarmException()
416
417 def _exit(self, signum, frame):
418 for p in self.children:
419 try:
420 os.kill(p, signum)
421 except OSError:
422 pass
423
424 # Do not call normal exit handlers when in a subprocess
425 if not self.toplevel:
426 os._exit(0)
427
428 # SIGTERM is successful exit, others are not
429 if signum == signal.SIGTERM:
430 sys.exit(0)
431 sys.exit(1)
432
433 def __del__(self):
434 if self.toplevel and not self.debug:
435 shutil.rmtree(self.working_dir, ignore_errors=True)
436
437
438 def run():
439 options, args = parser.parse_args()
440 if not XML_DEFUSED and not options.INSECURE_XML:
441 sys.stderr.write(
442 "XML EXPANSION ATTACK VULNERABILITY DETECTED. ABORTING!\n"
443 "Run with --insecure-xml to run anyway (will leave the server "
444 "vulnerable!)\n")
445 sys.exit(1)
446
447 server = SimpleXMLRPCServer(
448 (options.BIND_ADDRESS, options.BIND_PORT), allow_none=True)
449 manager = DITGManager(
450 bind_address=options.ITG_ADDRESS or options.BIND_ADDRESS,
451 start_port=options.START_PORT,
452 max_test_time=options.MAX_TEST_TIME,
453 max_instances=options.MAX_INSTANCES,
454 secret=options.SECRET,
455 debug=options.DEBUG,)
456 server.register_instance(manager)
457 server.register_introspection_functions()
458 server.serve_forever()
459
460
461 if __name__ == "__main__":
462 run()