From ccd316c298464d7d0385a15c872499ebd155aade Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sun, 12 Feb 2017 12:31:09 +1300 Subject: [PATCH 01/50] Detect/pay sounds via mplayer or aplay --- play-iridium-ambe | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/play-iridium-ambe b/play-iridium-ambe index 1c76ee3..efcb48c 100755 --- a/play-iridium-ambe +++ b/play-iridium-ambe @@ -2,4 +2,9 @@ bits_to_dfs.py $1 /tmp/voice.dfs #ambe -w /tmp/voice.dfs ir77_ambe_decode /tmp/voice.dfs /tmp/voice.wav -mplayer /tmp/voice.wav + +if hash mplayer 2>/dev/null; then + mplayer /tmp/voice.wav +else + aplay /tmp/voice.wav +fi From 63921befba5a82465ea1871083c6ee294fab7d12 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Thu, 10 May 2018 16:09:46 +0100 Subject: [PATCH 02/50] Update stats-voc.py to more modern python Start to modernise some of the python scripts. This changes the order of some of the comand line args. --- bits_to_dfs.py | 57 ++++++++++------ play-iridium-ambe | 2 +- stats-voc.py | 162 +++++++++++++++++++++++++++------------------- 3 files changed, 135 insertions(+), 86 deletions(-) diff --git a/bits_to_dfs.py b/bits_to_dfs.py index 5f051e4..3b2ec7d 100755 --- a/bits_to_dfs.py +++ b/bits_to_dfs.py @@ -3,6 +3,7 @@ # vim: set ts=4 sw=4 tw=0 et pm=: import fileinput import sys +import argparse """ VOC: i-1443338945.6543-t1 033399141 1625872817 81% 0.027 179 L:no LCW(0,001111,100000000000000000000 E1) 01111001000100010010010011011011011001111 011000010000100001110101111011110010010111011001010001011101010001100000000110010100000110111110010101110101001111010100111001000110100110001110110 1010101010010010001000001110011000001001001010011110011100110100111110001101110010110101010110011101011100011101011000000000 descr_extra: @@ -37,23 +38,39 @@ def chunks(l, n): for i in xrange(0, len(l), n): yield l[i:i+n] -infile = sys.argv[1] -outfile = open(sys.argv[2], 'wb') - -data = '' - -for line in fileinput.input(infile): - line = line.split() - if line[0] == 'VOC:': - if int(line[6]) < 179: - continue - data = line[10] - if (data[0] == "["): - for pos in xrange(1,len(data),3): - byte=int(data[pos:pos+2],16) - byte=int('{:08b}'.format(byte)[::-1], 2) - outfile.write(chr(byte)) - else: - for bits in chunks(data, 8): - byte = int(bits[::-1],2) - outfile.write(chr(byte)) + +def bits_to_dfs(lines, output): + data = '' + for line in lines: + line = line.split() + if line[0] == 'VOC:': + if int(line[6]) < 179: + continue + data = line[10] + if (data[0] == "["): + for pos in xrange(1,len(data),3): + byte=int(data[pos:pos+2],16) + byte=int('{:08b}'.format(byte)[::-1], 2) + output.write(chr(byte)) + else: + for bits in chunks(data, 8): + byte = int(bits[::-1],2) + output.write(chr(byte)) + +def main(): + parser = argparse.ArgumentParser(description='Convert iridium-parser.py VOC output to DFS') + parser.add_argument('output', metavar='OUTPUT', help='Output file for DFS content. If - stdout is used ') + parser.add_argument('input', metavar='FILE', nargs='*', help='Files to read, if empty or -, stdin is used') + args = parser.parse_args() + + output_file = sys.stdout if args.output == '-' else open(args.output, 'w') + input_files = args.input if len(args.input) > 0 else ['-'] + + bits_to_dfs(fileinput.input(files=input_files), output_file) + + if output_file is not sys.stdout: + output_file.close() + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/play-iridium-ambe b/play-iridium-ambe index efcb48c..1302dbb 100755 --- a/play-iridium-ambe +++ b/play-iridium-ambe @@ -1,5 +1,5 @@ #!/bin/sh -bits_to_dfs.py $1 /tmp/voice.dfs +bits_to_dfs.py /tmp/voice.dfs $1 #ambe -w /tmp/voice.dfs ir77_ambe_decode /tmp/voice.dfs /tmp/voice.wav diff --git a/stats-voc.py b/stats-voc.py index 0588fc0..f105418 100755 --- a/stats-voc.py +++ b/stats-voc.py @@ -5,85 +5,117 @@ import sys import matplotlib.pyplot as plt import os +import subprocess +import fileinput +import logging +import tempfile +from bits_to_dfs import bits_to_dfs -def filter_voc(t_start = None, t_stop = None, f_min = None, f_max = None): +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def filter_voc(lines, t_start = None, t_stop = None, f_min = None, f_max = None): tsl = [] fl = [] - lines = [] - f = open(sys.argv[1]) + filtered_lines = [] - for line in f: + for line in lines: + line_split = line.split() + lcw = line[8] + #ts_base = int(line[1].split('-')[1].split('.')[0]) + ts_base = 0 + ts = ts_base + int(line_split[2])/1000. + f = int(line_split[3])/1000. + if ((not t_start or t_start <= ts) and + (not t_stop or ts <= t_stop) and + (not f_min or f_min <= f) and + (not f_max or f <= f_max)): + tsl.append(ts) + fl.append(f) + filtered_lines.append(line) + return tsl, fl, filtered_lines + + +class OnClickHandler(object): + def __init__(self, lines): + self.lines = lines + self.t_start = None + self.t_stop = None + self.f_min = None + self.f_max = None + + def onclick(self, event): + logger.info('button=%d, x=%d, y=%d, xdata=%f, ydata=%f', + event.button, event.x, event.y, event.xdata, event.ydata) + + if event.button == 1: + self.t_start = event.xdata + self.f_min = event.ydata + self.t_stop = None + self.f_max = None + if event.button == 3: + self.t_stop = event.xdata + self.f_max = event.ydata + + if self.t_start and self.t_stop: + self.cut_convert_play(self.t_start, self.t_stop, self.f_min, self.f_max) + + def cut_convert_play(self, t_start, t_stop, f_min, f_max): + logger.info('Starting to play...') + if t_start and t_stop: + if t_start > t_stop: + tmp = t_stop + t_stop = t_start + t_start = tmp + if f_min > f_max: + tmp = f_max + f_max = f_min + f_min = tmp + + _, _, filtered_lines = filter_voc(self.lines, t_start=t_start, t_stop=t_stop, f_min=f_min, f_max=f_max) + + _, dfs_file_path = tempfile.mkstemp(suffix='.dfs') + _, wav_file_path = tempfile.mkstemp(suffix='.wav') + + logger.info('Making dfs file %s', dfs_file_path) + with open(dfs_file_path, 'w') as dfs_file: + bits_to_dfs(filtered_lines, dfs_file) + + logger.info('Making wav file %s', wav_file_path) + subprocess.check_call(['ir77_ambe_decode', dfs_file_path, wav_file_path]) + + logger.info('Cleaning up dfs') + os.remove(dfs_file_path) + + subprocess.check_call(['aplay', wav_file_path]) + + logger.info('Cleaning up wav') + os.remove(wav_file_path) + + logger.info('Finished Playing') + + +def read_lines(): + lines = [] + for line in fileinput.input(): line = line.strip() if 'VOC: ' in line and not "LCW(0,001111,100000000000000000000" in line: - line_split = line.split() - lcw = line[8] - #ts_base = int(line[1].split('-')[1].split('.')[0]) - ts_base = 0 - ts = ts_base + int(line_split[2])/1000. - f = int(line_split[3])/1000. - if ((not t_start or t_start <= ts) and - (not t_stop or ts <= t_stop) and - (not f_min or f_min <= f) and - (not f_max or f <= f_max)): - tsl.append(ts) - fl.append(f) - lines.append(line) - return tsl, fl, lines - - -def cut_convert_play(t_start, t_stop, f_min, f_max): - if t_start and t_stop: - if t_start > t_stop: - tmp = t_stop - t_stop = t_start - t_start = tmp - if f_min > f_max: - tmp = f_max - f_max = f_min - f_min = tmp - - f_out = open('/tmp/voice.bits', 'w') - _, _, lines = filter_voc(t_start, t_stop, f_min, f_max) - for line in lines: - f_out.write(line + "\n") - f_out.close() - os.system("play-iridium-ambe /tmp/voice.bits") - - -def onclick(event): - global t_start, t_stop, f_min, f_max - print 'button=%d, x=%d, y=%d, xdata=%f, ydata=%f'%( - event.button, event.x, event.y, event.xdata, event.ydata) - if event.button == 1: - t_start = event.xdata - f_min = event.ydata - t_stop = None - f_max = None - if event.button == 3: - t_stop = event.xdata - f_max = event.ydata - - if t_start and t_stop: - cut_convert_play(t_start, t_stop, f_min, f_max) + lines.append(line) + return lines def main(): - tsl, fl, _ = filter_voc() - - print len(tsl) + lines = read_lines() + logger.info('Read %d VOC lines from input', len(lines)) + tsl, fl, _ = filter_voc(lines) fig = plt.figure() ax = fig.add_subplot(111) ax.scatter(x = tsl, y = fl) - t_start = None - t_stop = None - f_min = None - f_max = None - - - cid = fig.canvas.mpl_connect('button_press_event', onclick) + on_click_handler = OnClickHandler(lines) + cid = fig.canvas.mpl_connect('button_press_event', on_click_handler.onclick) - plt.title("Click once left and once rigth to define an area. The script will try to play iridium using the play-iridium-ambe shell script.") + plt.title('Click once left and once right to define an area. The script will try to play iridium using ir77_ambe_decode and aplay.') plt.show() From 066c25cb6f56188e9b3306da9657f4f71884c1b2 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Thu, 10 May 2018 17:29:07 +0100 Subject: [PATCH 03/50] Make play-iridium-ambe streamable --- play-iridium-ambe | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/play-iridium-ambe b/play-iridium-ambe index 1302dbb..3a4b22b 100755 --- a/play-iridium-ambe +++ b/play-iridium-ambe @@ -1,10 +1,2 @@ #!/bin/sh -bits_to_dfs.py /tmp/voice.dfs $1 -#ambe -w /tmp/voice.dfs -ir77_ambe_decode /tmp/voice.dfs /tmp/voice.wav - -if hash mplayer 2>/dev/null; then - mplayer /tmp/voice.wav -else - aplay /tmp/voice.wav -fi +bits_to_dfs.py - $1 | ir77_ambe_decode - - | aplay \ No newline at end of file From c6e6093df3be78c0df314a8e8a6ac8a84910f57f Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Thu, 10 May 2018 21:05:05 +0100 Subject: [PATCH 04/50] Only parse each line once in stats-voc.py --- stats-voc.py | 67 ++++++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/stats-voc.py b/stats-voc.py index f105418..d10bfb5 100755 --- a/stats-voc.py +++ b/stats-voc.py @@ -14,26 +14,15 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -def filter_voc(lines, t_start = None, t_stop = None, f_min = None, f_max = None): - tsl = [] - fl = [] - filtered_lines = [] - - for line in lines: +class VocLine(object): + def __init__(self, line): + self.line = line line_split = line.split() - lcw = line[8] + self.lcw = line[8] #ts_base = int(line[1].split('-')[1].split('.')[0]) ts_base = 0 - ts = ts_base + int(line_split[2])/1000. - f = int(line_split[3])/1000. - if ((not t_start or t_start <= ts) and - (not t_stop or ts <= t_stop) and - (not f_min or f_min <= f) and - (not f_max or f <= f_max)): - tsl.append(ts) - fl.append(f) - filtered_lines.append(line) - return tsl, fl, filtered_lines + self.ts = ts_base + int(line_split[2])/1000. + self.f = int(line_split[3])/1000. class OnClickHandler(object): @@ -60,19 +49,30 @@ def onclick(self, event): if self.t_start and self.t_stop: self.cut_convert_play(self.t_start, self.t_stop, self.f_min, self.f_max) + def filter_voc(self, t_start, t_stop, f_min, f_max): + filtered_lines = [] + + for voc_line in self.lines: + ts = voc_line.ts + f = voc_line.f + if t_start <= ts and ts <= t_stop and \ + f_min <= f and f <= f_max: + filtered_lines.append(voc_line.line) + + return filtered_lines + def cut_convert_play(self, t_start, t_stop, f_min, f_max): logger.info('Starting to play...') - if t_start and t_stop: - if t_start > t_stop: - tmp = t_stop - t_stop = t_start - t_start = tmp - if f_min > f_max: - tmp = f_max - f_max = f_min - f_min = tmp - - _, _, filtered_lines = filter_voc(self.lines, t_start=t_start, t_stop=t_stop, f_min=f_min, f_max=f_max) + if t_start > t_stop: + tmp = t_stop + t_stop = t_start + t_start = tmp + if f_min > f_max: + tmp = f_max + f_max = f_min + f_min = tmp + + filtered_lines = self.filter_voc(t_start, t_stop, f_min, f_max) _, dfs_file_path = tempfile.mkstemp(suffix='.dfs') _, wav_file_path = tempfile.mkstemp(suffix='.wav') @@ -100,13 +100,18 @@ def read_lines(): for line in fileinput.input(): line = line.strip() if 'VOC: ' in line and not "LCW(0,001111,100000000000000000000" in line: - lines.append(line) + lines.append(VocLine(line)) return lines def main(): lines = read_lines() logger.info('Read %d VOC lines from input', len(lines)) - tsl, fl, _ = filter_voc(lines) + + tsl = [] + fl = [] + for voc_line in lines: + tsl.append(voc_line.ts) + fl.append(voc_line.f) fig = plt.figure() ax = fig.add_subplot(111) @@ -115,7 +120,7 @@ def main(): on_click_handler = OnClickHandler(lines) cid = fig.canvas.mpl_connect('button_press_event', on_click_handler.onclick) - plt.title('Click once left and once right to define an area. The script will try to play iridium using ir77_ambe_decode and aplay.') + plt.title('Click once left and once right to define an area.\nThe script will try to play iridium using ir77_ambe_decode and aplay.') plt.show() From 3e48bdffc9c247c65e76b5a9dee0d551debffcfc Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Thu, 10 May 2018 22:11:26 +0100 Subject: [PATCH 05/50] Add clustering to stats-voc.py Use nummpy+scipy to colour clusters of voice calls --- stats-voc.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/stats-voc.py b/stats-voc.py index d10bfb5..64a2867 100755 --- a/stats-voc.py +++ b/stats-voc.py @@ -9,6 +9,10 @@ import fileinput import logging import tempfile + +import numpy as np +import scipy.cluster.hierarchy as hcluster + from bits_to_dfs import bits_to_dfs logging.basicConfig(level=logging.INFO) @@ -105,22 +109,31 @@ def read_lines(): def main(): lines = read_lines() - logger.info('Read %d VOC lines from input', len(lines)) - - tsl = [] - fl = [] - for voc_line in lines: - tsl.append(voc_line.ts) - fl.append(voc_line.f) + number_of_lines = len(lines) + logger.info('Read %d VOC lines from input', number_of_lines) + + tsl = np.empty(number_of_lines) + fl = np.empty(number_of_lines) + plot_data = np.empty((number_of_lines, 2)) + for i, voc_line in enumerate(lines): + tsl[i] = voc_line.ts + fl[i] = voc_line.f + plot_data[i][0] = voc_line.ts + plot_data[i][1] = voc_line.f fig = plt.figure() - ax = fig.add_subplot(111) - ax.scatter(x = tsl, y = fl) + thresh = 1.5 + clusters = hcluster.fclusterdata(plot_data, thresh, criterion="distance") + + ax = fig.add_subplot(1, 1, 1) + ax.scatter(*np.transpose(plot_data), c=clusters) on_click_handler = OnClickHandler(lines) cid = fig.canvas.mpl_connect('button_press_event', on_click_handler.onclick) plt.title('Click once left and once right to define an area.\nThe script will try to play iridium using ir77_ambe_decode and aplay.') + plt.xlabel('time') + plt.ylabel('frequency') plt.show() From e252732d8244eeecf36d7f39806c05f59b28926c Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Thu, 10 May 2018 22:20:35 +0100 Subject: [PATCH 06/50] Detect and fail on raw data in stats-voc.py --- stats-voc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stats-voc.py b/stats-voc.py index 64a2867..2396053 100755 --- a/stats-voc.py +++ b/stats-voc.py @@ -103,6 +103,8 @@ def read_lines(): lines = [] for line in fileinput.input(): line = line.strip() + if 'A:OK' in line and "Message: Couldn't parse:" not in line: + raise RuntimeError('Expected "iridium-parser.py" parsed data. Found raw "iridium-extractor" data.') if 'VOC: ' in line and not "LCW(0,001111,100000000000000000000" in line: lines.append(VocLine(line)) return lines From fd1b7360fe3e9c3f27090525e327e014217bb534 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Thu, 10 May 2018 22:44:21 +0100 Subject: [PATCH 07/50] Add tests for bits_to_dfs.py --- bits_to_dfs.py | 50 ++++++++++++++------------------------------- test_bits_to_dfs.py | 44 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 35 deletions(-) create mode 100644 test_bits_to_dfs.py diff --git a/bits_to_dfs.py b/bits_to_dfs.py index 3b2ec7d..ac05365 100755 --- a/bits_to_dfs.py +++ b/bits_to_dfs.py @@ -10,28 +10,6 @@ """ -def turn_symbols(byte): - out = 0 - if byte & 0x01: - out |= 0x02 - if byte & 0x02: - out |= 0x01 - if byte & 0x04: - out |= 0x08 - if byte & 0x08: - out |= 0x04 - - if byte & 0x10: - out |= 0x20 - if byte & 0x20: - out |= 0x10 - if byte & 0x40: - out |= 0x80 - if byte & 0x80: - out |= 0x40 - - return out - def chunks(l, n): """ Yield successive n-sized chunks from l. """ @@ -43,19 +21,21 @@ def bits_to_dfs(lines, output): data = '' for line in lines: line = line.split() - if line[0] == 'VOC:': - if int(line[6]) < 179: - continue - data = line[10] - if (data[0] == "["): - for pos in xrange(1,len(data),3): - byte=int(data[pos:pos+2],16) - byte=int('{:08b}'.format(byte)[::-1], 2) - output.write(chr(byte)) - else: - for bits in chunks(data, 8): - byte = int(bits[::-1],2) - output.write(chr(byte)) + if line[0] != 'VOC:': + continue + if int(line[6]) < 179: + continue + + data = line[10] + if data[0] == "[": + for pos in xrange(1,len(data),3): + byte=int(data[pos:pos+2],16) + byte=int('{:08b}'.format(byte)[::-1], 2) + output.write(chr(byte)) + else: + for bits in chunks(data, 8): + byte = int(bits[::-1],2) + output.write(chr(byte)) def main(): parser = argparse.ArgumentParser(description='Convert iridium-parser.py VOC output to DFS') diff --git a/test_bits_to_dfs.py b/test_bits_to_dfs.py new file mode 100644 index 0000000..aa0abbe --- /dev/null +++ b/test_bits_to_dfs.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +import unittest +from io import BytesIO + +from bits_to_dfs import chunks, bits_to_dfs + + +class ChunksTest(unittest.TestCase): + def test_simple(self): + result = list(chunks([1, 2, 3, 4, 5, 6], 2)) + self.assertEquals(result, [[1, 2], [3, 4], [5, 6]]) + + +class BitsToDfsTest(unittest.TestCase): + TEST_VOC_LINE = 'VOC: i-1443338945.6543-t1 033399141 1625872817 81% 0.027 179 L:no LCW(0,001111,100000000000000000000 E1) 01111001000100010010010011011011011001111 011000010000100001110101111011110010010111011001010001011101010001100000000110010100000110111110010101110101001111010100111001000110100110001110110 1010101010010010001000001110011000001001001010011110011100110100111110001101110010110101010110011101011100011101011000000000 descr_extra:' + + def test_empty_input(self): + output = BytesIO() + bits_to_dfs([], output) + self.assertEquals(output.getvalue(), '') + + def test_single(self): + output = BytesIO() + bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE], output) + self.assertEquals(output.getvalue(), '\x9e\x88$\xdb\xe6\x01') + + def test_multiple(self): + output = BytesIO() + bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE, BitsToDfsTest.TEST_VOC_LINE, BitsToDfsTest.TEST_VOC_LINE], output) + self.assertEquals(output.getvalue(), '\x9e\x88$\xdb\xe6\x01' * 3) + + def test_filters_non_voc_lines(self): + output = BytesIO() + bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE, 'NOT_VOC:', BitsToDfsTest.TEST_VOC_LINE], output) + self.assertEquals(output.getvalue(), '\x9e\x88$\xdb\xe6\x01' * 2) + + +def main(): + unittest.main() + + +if __name__ == "__main__": + main() From a1c56ec329bf4bca0053028a6cf79c1ef7708718 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Thu, 10 May 2018 23:30:33 +0100 Subject: [PATCH 08/50] Auto calculate good cluster threshold --- stats-voc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stats-voc.py b/stats-voc.py index 2396053..b5e12d0 100755 --- a/stats-voc.py +++ b/stats-voc.py @@ -123,10 +123,11 @@ def main(): plot_data[i][0] = voc_line.ts plot_data[i][1] = voc_line.f - fig = plt.figure() - thresh = 1.5 + distances = hcluster.distance.pdist(plot_data) + thresh = 2 * distances.min() clusters = hcluster.fclusterdata(plot_data, thresh, criterion="distance") + fig = plt.figure() ax = fig.add_subplot(1, 1, 1) ax.scatter(*np.transpose(plot_data), c=clusters) From d372ea26d082407eeca8f479762b4d7daa167b2e Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Fri, 11 May 2018 17:09:15 +0100 Subject: [PATCH 09/50] More bits_to_dfs.py tests --- test_bits_to_dfs.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/test_bits_to_dfs.py b/test_bits_to_dfs.py index aa0abbe..5624ef2 100644 --- a/test_bits_to_dfs.py +++ b/test_bits_to_dfs.py @@ -13,27 +13,33 @@ def test_simple(self): class BitsToDfsTest(unittest.TestCase): - TEST_VOC_LINE = 'VOC: i-1443338945.6543-t1 033399141 1625872817 81% 0.027 179 L:no LCW(0,001111,100000000000000000000 E1) 01111001000100010010010011011011011001111 011000010000100001110101111011110010010111011001010001011101010001100000000110010100000110111110010101110101001111010100111001000110100110001110110 1010101010010010001000001110011000001001001010011110011100110100111110001101110010110101010110011101011100011101011000000000 descr_extra:' + TEST_VOC_LINE_1 = 'VOC: i-1443338945.6543-t1 033399141 1625872817 81% 0.027 179 L:no LCW(0,001111,100000000000000000000 E1) 01111001000100010010010011011011011001111 011000010000100001110101111011110010010111011001010001011101010001100000000110010100000110111110010101110101001111010100111001000110100110001110110 1010101010010010001000001110011000001001001010011110011100110100111110001101110010110101010110011101011100011101011000000000 descr_extra:' + TEST_VOC_LINE_2 = 'VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f]' def test_empty_input(self): output = BytesIO() bits_to_dfs([], output) self.assertEquals(output.getvalue(), '') - def test_single(self): + def test_old_format(self): output = BytesIO() - bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE], output) + bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_1], output) self.assertEquals(output.getvalue(), '\x9e\x88$\xdb\xe6\x01') + def test_new_format(self): + output = BytesIO() + bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_2], output) + self.assertEquals(output.getvalue(), '\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') + def test_multiple(self): output = BytesIO() - bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE, BitsToDfsTest.TEST_VOC_LINE, BitsToDfsTest.TEST_VOC_LINE], output) - self.assertEquals(output.getvalue(), '\x9e\x88$\xdb\xe6\x01' * 3) + bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_1, BitsToDfsTest.TEST_VOC_LINE_1, BitsToDfsTest.TEST_VOC_LINE_2], output) + self.assertEquals(output.getvalue(), ('\x9e\x88$\xdb\xe6\x01' * 2) + '\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') def test_filters_non_voc_lines(self): output = BytesIO() - bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE, 'NOT_VOC:', BitsToDfsTest.TEST_VOC_LINE], output) - self.assertEquals(output.getvalue(), '\x9e\x88$\xdb\xe6\x01' * 2) + bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_1, 'NOT_VOC:', BitsToDfsTest.TEST_VOC_LINE_2], output) + self.assertEquals(output.getvalue(), '\x9e\x88$\xdb\xe6\x01' + '\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') def main(): From bb9c8e4a714aa99bacd5371104e6f6664c7de95c Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Fri, 11 May 2018 17:09:48 +0100 Subject: [PATCH 10/50] Allow filtering stats-voc.py by start time --- requirements.txt | 4 +++ stats-voc.py | 71 ++++++++++++++++++++++++++++++++++-------------- 2 files changed, 55 insertions(+), 20 deletions(-) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..12f7cb6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +numpy>=1.14.0 +scipy>=1.1.0 +six>=1.11.0 +dateparser>=0.7.0 \ No newline at end of file diff --git a/stats-voc.py b/stats-voc.py index b5e12d0..1753594 100755 --- a/stats-voc.py +++ b/stats-voc.py @@ -1,7 +1,5 @@ #!/usr/bin/python # vim: set ts=4 sw=4 tw=0 et fenc=utf8 pm=: -#VOC: i-1430527570.4954-t1 421036605 1625859953 66% 0.008 219 L:no LCW(0,001111,100000000000000000000 E1) 101110110101010100101101111000111111111001011111001011010001000010010001101110011010011001111111011101111100011001001001000111001101001011001011000101111111101110110011111000000001110010001110101101001010011001101001010111101100011100110011110010110110101010110001010000100100101011010010100100100011010110101001 - import sys import matplotlib.pyplot as plt import os @@ -9,24 +7,43 @@ import fileinput import logging import tempfile +from datetime import datetime +import argparse import numpy as np import scipy.cluster.hierarchy as hcluster +import six +import dateparser from bits_to_dfs import bits_to_dfs logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +# Example lines +# VOC: i-1430527570.4954-t1 421036605 1625859953 66% 0.008 219 L:no LCW(0,001111,100000000000000000000 E1) 101110110101010100101101111000111111111001011111001011010001000010010001101110011010011001111111011101111100011001001001000111001101001011001011000101111111101110110011111000000001110010001110101101001010011001101001010111101100011100110011110010110110101010110001010000100100101011010010100100100011010110101001 +# VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f] + class VocLine(object): def __init__(self, line): self.line = line - line_split = line.split() - self.lcw = line[8] - #ts_base = int(line[1].split('-')[1].split('.')[0]) - ts_base = 0 - self.ts = ts_base + int(line_split[2])/1000. - self.f = int(line_split[3])/1000. + try: + line_split = line.split() + + raw_time_base = line_split[1] + ts_base_ms = int(raw_time_base.split('-')[1].split('.')[0]) + + time_offset_ns = int(line_split[2]) + self.ts = ts_base_ms + (time_offset_ns / 1000) + + self.f = int(line_split[3])/1000. + self.lcw = line[8] + except Exception as e: + six.raise_from(Exception('Failed to parse line "{}"'.format(line), e), e) + + def datetime(self): + return datetime.utcfromtimestamp(self.ts) + class OnClickHandler(object): @@ -99,40 +116,54 @@ def cut_convert_play(self, t_start, t_stop, f_min, f_max): logger.info('Finished Playing') -def read_lines(): +def read_lines(input_files, start_time_filter): lines = [] - for line in fileinput.input(): + for line in fileinput.input(files=input_files): line = line.strip() if 'A:OK' in line and "Message: Couldn't parse:" not in line: raise RuntimeError('Expected "iridium-parser.py" parsed data. Found raw "iridium-extractor" data.') if 'VOC: ' in line and not "LCW(0,001111,100000000000000000000" in line: - lines.append(VocLine(line)) + voc_line = VocLine(line) + if start_time_filter and start_time_filter > voc_line.datetime(): + continue + lines.append(voc_line) return lines def main(): - lines = read_lines() + parser = argparse.ArgumentParser(description='Convert iridium-parser.py VOC output to DFS') + parser.add_argument('--start', metavar='DATETIME', default=None, help='Filter events before this time') + parser.add_argument('input', metavar='FILE', nargs='*', help='Files to read, if empty or -, stdin is used') + args = parser.parse_args() + + input_files = args.input if len(args.input) > 0 else ['-'] + start_time_filter = dateparser.parse(args.start) if args.start else None + + lines = read_lines(input_files, start_time_filter) number_of_lines = len(lines) logger.info('Read %d VOC lines from input', number_of_lines) - tsl = np.empty(number_of_lines) - fl = np.empty(number_of_lines) + if number_of_lines == 0: + print('No usable data found') + sys.exit(1) + plot_data = np.empty((number_of_lines, 2)) for i, voc_line in enumerate(lines): - tsl[i] = voc_line.ts - fl[i] = voc_line.f plot_data[i][0] = voc_line.ts - plot_data[i][1] = voc_line.f + plot_data[i][1] = np.float64(voc_line.f) distances = hcluster.distance.pdist(plot_data) thresh = 2 * distances.min() clusters = hcluster.fclusterdata(plot_data, thresh, criterion="distance") fig = plt.figure() + #fig.autofmt_xdate() + on_click_handler = OnClickHandler(lines) + fig.canvas.mpl_connect('button_press_event', on_click_handler.onclick) + ax = fig.add_subplot(1, 1, 1) ax.scatter(*np.transpose(plot_data), c=clusters) - - on_click_handler = OnClickHandler(lines) - cid = fig.canvas.mpl_connect('button_press_event', on_click_handler.onclick) + #ax.xaxis_date() + ax.grid(True) plt.title('Click once left and once right to define an area.\nThe script will try to play iridium using ir77_ambe_decode and aplay.') plt.xlabel('time') From 72cc33a51ead43fa4c6ea7fb6a553db928ce9e7d Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Fri, 11 May 2018 17:13:11 +0100 Subject: [PATCH 11/50] Allow filtering stats-voc.py by end time --- stats-voc.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/stats-voc.py b/stats-voc.py index 1753594..6f09189 100755 --- a/stats-voc.py +++ b/stats-voc.py @@ -116,7 +116,7 @@ def cut_convert_play(self, t_start, t_stop, f_min, f_max): logger.info('Finished Playing') -def read_lines(input_files, start_time_filter): +def read_lines(input_files, start_time_filter, end_time_filter): lines = [] for line in fileinput.input(files=input_files): line = line.strip() @@ -126,19 +126,23 @@ def read_lines(input_files, start_time_filter): voc_line = VocLine(line) if start_time_filter and start_time_filter > voc_line.datetime(): continue + if end_time_filter and end_time_filter < voc_line.datetime(): + continue lines.append(voc_line) return lines def main(): parser = argparse.ArgumentParser(description='Convert iridium-parser.py VOC output to DFS') parser.add_argument('--start', metavar='DATETIME', default=None, help='Filter events before this time') + parser.add_argument('--end', metavar='DATETIME', default=None, help='Filter events after this time') parser.add_argument('input', metavar='FILE', nargs='*', help='Files to read, if empty or -, stdin is used') args = parser.parse_args() input_files = args.input if len(args.input) > 0 else ['-'] start_time_filter = dateparser.parse(args.start) if args.start else None + end_time_filter = dateparser.parse(args.end) if args.end else None - lines = read_lines(input_files, start_time_filter) + lines = read_lines(input_files, start_time_filter, end_time_filter) number_of_lines = len(lines) logger.info('Read %d VOC lines from input', number_of_lines) From 82d47e94ec39e55211642cee36b33d85f894bb92 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Fri, 11 May 2018 18:08:19 +0100 Subject: [PATCH 12/50] First setup.py work --- .gitignore | 108 +++++++++++++++++- .travis.yml | 6 + LICENSE | 9 ++ iridiumtk/__init__.py | 5 + bits_to_dfs.py => iridiumtk/bits_to_dfs.py | 1 + stats-voc.py => iridiumtk/stats_voc.py | 2 +- .../test_bits_to_dfs.py | 2 +- iridiumtk/test_stats_voc.py | 19 +++ requirements.txt | 3 +- setup.cfg | 6 + setup.py | 83 ++++++++++++++ tox.ini | 13 +++ 12 files changed, 252 insertions(+), 5 deletions(-) create mode 100644 .travis.yml create mode 100644 LICENSE create mode 100644 iridiumtk/__init__.py rename bits_to_dfs.py => iridiumtk/bits_to_dfs.py (99%) rename stats-voc.py => iridiumtk/stats_voc.py (100%) rename test_bits_to_dfs.py => iridiumtk/test_bits_to_dfs.py (98%) create mode 100644 iridiumtk/test_stats_voc.py create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 7b570ef..3ddbbb1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,114 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Iridium Tookit *.raw *.peaks -*.pyc *.tar.gz *.swp *.swo .mailmap tracking/iridium.txt epochseconds -__pycache__ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..5cea8bd --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +sudo: false +language: python +python: + - "2.7" +install: pip install tox-travis +script: tox \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5a33c9d --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +Copyright 2014 Stefan `Sec` Zehl and Tobias Schneider + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/iridiumtk/__init__.py b/iridiumtk/__init__.py new file mode 100644 index 0000000..4ead932 --- /dev/null +++ b/iridiumtk/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +__title__ = 'iridiumtk' +__version__ = '0.0.1' +__author__ = 'Stefan `Sec` Zehl and Tobias Schneider ' diff --git a/bits_to_dfs.py b/iridiumtk/bits_to_dfs.py similarity index 99% rename from bits_to_dfs.py rename to iridiumtk/bits_to_dfs.py index ac05365..ddda68b 100755 --- a/bits_to_dfs.py +++ b/iridiumtk/bits_to_dfs.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + # -*- coding: utf-8 -*- # vim: set ts=4 sw=4 tw=0 et pm=: import fileinput diff --git a/stats-voc.py b/iridiumtk/stats_voc.py similarity index 100% rename from stats-voc.py rename to iridiumtk/stats_voc.py index 6f09189..7e79e03 100755 --- a/stats-voc.py +++ b/iridiumtk/stats_voc.py @@ -1,7 +1,6 @@ #!/usr/bin/python # vim: set ts=4 sw=4 tw=0 et fenc=utf8 pm=: import sys -import matplotlib.pyplot as plt import os import subprocess import fileinput @@ -10,6 +9,7 @@ from datetime import datetime import argparse +import matplotlib.pyplot as plt import numpy as np import scipy.cluster.hierarchy as hcluster import six diff --git a/test_bits_to_dfs.py b/iridiumtk/test_bits_to_dfs.py similarity index 98% rename from test_bits_to_dfs.py rename to iridiumtk/test_bits_to_dfs.py index 5624ef2..e10a852 100644 --- a/test_bits_to_dfs.py +++ b/iridiumtk/test_bits_to_dfs.py @@ -3,7 +3,7 @@ import unittest from io import BytesIO -from bits_to_dfs import chunks, bits_to_dfs +from .bits_to_dfs import chunks, bits_to_dfs class ChunksTest(unittest.TestCase): diff --git a/iridiumtk/test_stats_voc.py b/iridiumtk/test_stats_voc.py new file mode 100644 index 0000000..6e6d484 --- /dev/null +++ b/iridiumtk/test_stats_voc.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +import unittest + +from .stats_voc import VocLine + + +class VocLineTest(unittest.TestCase): + + def test_empty_input(self): + with self.assertRaises(Exception): + VocLine('') + +def main(): + unittest.main() + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index 12f7cb6..8971e21 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ numpy>=1.14.0 scipy>=1.1.0 six>=1.11.0 -dateparser>=0.7.0 \ No newline at end of file +dateparser>=0.7.0 +matplotlib>=2.2.0 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..0050f52 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[metadata] +description-file = README.md +license_file = LICENSE + +[aliases] +test=pytest \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e38c402 --- /dev/null +++ b/setup.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +from codecs import open # To use a consistent encoding +from os import path, system +import re +import sys + +# Always prefer setuptools over distutils +from setuptools import find_packages, setup +from setuptools.command.test import test as TestCommand # noqa: N812 + + +class PyTest(TestCommand): + user_options = [('pytest-args=', 'a', "Arguments to pass to pytest")] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + import pytest # import here, cause outside the eggs aren't loaded + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + + +if sys.argv[-1] == 'publish': + system('python setup.py sdist upload') + sys.exit() + +with open('iridiumtk/__init__.py', 'r') as fd: + contents = fd.read() + version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', + contents, re.MULTILINE).group(1) + title = re.search(r'^__title__\s*=\s*[\'"]([^\'"]*)[\'"]', + contents, re.MULTILINE).group(1) + author = re.search(r'^__author__\s*=\s*[\'"]([^\'"]*)[\'"]', + contents, re.MULTILINE).group(1) + +if not version: + raise RuntimeError('Cannot find version information') +if not title: + raise RuntimeError('Cannot find title information') +if not author: + raise RuntimeError('Cannot find author information') + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + +setup( + name=title, + version=version, + description='iridiumtk', # TODO + long_description=long_description, + author=author, + author_email='thebigguy.co.uk@gmail.com', # TODO + url='https://github.com/muccc/iridium-toolkit', + packages=find_packages(exclude=['9601', 'ambe_emu', 'nal-shout', 'rtl-sdr', 'tools', 'tracking']), + entry_points={ + 'console_scripts': [ + 'iridiumtk-stats-voc=iridiumtk.stats_voc:main', + 'iridiumtk-bits-to-dfs=iridiumtk.bits_to_dfs:main', + ], + }, + package_data={'': ['LICENSE', 'README.md']}, + zip_safe=False, + install_requires=[], + tests_require=['pytest'], + cmdclass={'test': PyTest}, + keywords=[], + license='BSD', + classifiers=( + 'Development Status :: 3 - Alpha', + 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: Implementation :: CPython', + ), +) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..eef0705 --- /dev/null +++ b/tox.ini @@ -0,0 +1,13 @@ +[tox] +envlist = py27 +platform = linux2 + +[testenv] +deps = + -r{toxinidir}/requirements.txt + pytest + coverage +commands = + coverage run --parallel-mode -m pytest {posargs} # substitute with tox' positional arguments + coverage combine + coverage report -m From af0d007759855c155848c47f6348b66cef460eaa Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Fri, 11 May 2018 18:14:47 +0100 Subject: [PATCH 13/50] Limit coverage metrics to this package --- .coveragerc | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..629e970 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +branch = True +source = + iridiumtk +omit = + *test_*.py \ No newline at end of file From 46500720236cfb696dfdc6b0889c247c6a51cea1 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Fri, 11 May 2018 18:15:05 +0100 Subject: [PATCH 14/50] Add pypy testing --- .travis.yml | 4 ++++ tox.ini | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5cea8bd..dcc1db3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,10 @@ sudo: false +branches: + only: + - py27 language: python python: - "2.7" + - "pypy2.7" install: pip install tox-travis script: tox \ No newline at end of file diff --git a/tox.ini b/tox.ini index eef0705..f5de569 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27 +envlist = py27,pypy platform = linux2 [testenv] From b2c18ae5b6dc574d4fa03614ca82e8907cc8c144 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Fri, 11 May 2018 18:43:55 +0100 Subject: [PATCH 15/50] Try and get travis working --- .travis.yml | 37 +++++++++++++++++++++++++++++++++---- .travis/install.sh | 17 +++++++++++++++++ tox.ini | 3 ++- 3 files changed, 52 insertions(+), 5 deletions(-) create mode 100755 .travis/install.sh diff --git a/.travis.yml b/.travis.yml index dcc1db3..b20b541 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,39 @@ sudo: false + branches: only: - py27 + language: python -python: - - "2.7" - - "pypy2.7" -install: pip install tox-travis + +matrix: + include: + # Python 2.7 + - python: "2.7" + os: "linux" + dist: trusty + env: TOXENV=py27 + + #- os: "osx" + # language: generic + # env: TOXENV=py27 + + # PyPy 2.7 + - python: "pypy-5.4.1" + os: "linux" + dist: trusty + env: TOXENV=pypy + + allow_failures: + - python: "pypy2.7" + - os: "osx" + +cache: + directories: + - $PWD/wheelhouse # cache wheels so that we don't hit network everytime +env: + global: + - PIP_FIND_LINKS=$PWD/wheelhouse + +install: ./.travis/install.sh script: tox \ No newline at end of file diff --git a/.travis/install.sh b/.travis/install.sh new file mode 100755 index 0000000..deff563 --- /dev/null +++ b/.travis/install.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +if [[ $TRAVIS_OS_NAME == 'osx' ]]; then + + # Install custom requirements on OS X + brew install pyenv-virtualenv + + case "${TOXENV}" in + py27) + # Install some custom Python 3.2 requirements on OS X + echo 'Nothing to do' + ;; + esac +else + # Install custom requirements on Linux + pip install tox-travis +fi \ No newline at end of file diff --git a/tox.ini b/tox.ini index f5de569..2b9861d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,7 @@ [tox] -envlist = py27,pypy +envlist = py27, pypy platform = linux2 +skipsdist = True [testenv] deps = From 88c9c35eb02043ff1d6795f958c004d8782d311b Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Fri, 11 May 2018 19:26:41 +0100 Subject: [PATCH 16/50] Update Readme --- README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 522ce71..df8ec1d 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,19 @@ # Simple toolkit to decode Iridium signals -### Requisites +[![Build Status](https://travis-ci.org/TheBiggerGuy/iridium-toolkit.svg?branch=master)](https://travis-ci.org/TheBiggerGuy/iridium-toolkit) - * Python (2.7) - * NumPy (scipy) +## Installing +### User +```bash +python setup.py install +``` -### License - -Unless otherwise noted in a file, everything here is (c) Sec & schneider and licensed under the 2-Clause BSD License +### Development +```bash +virtualenv venv +source venv/bin/activate +pip install -r requirements.txt +``` ### Example usage Either extract some Iridium frames from the air or a file using [gr-iridium](https://github.com/muccc/gr-iridium) (recommended) or use the legacy code located in the [extractror-python](extractor-python/) directory if you don't want to install GNURadio (not recommended). @@ -60,3 +66,5 @@ Run as `grep ^IRA output.parsed |perl mkkml tracks > output.kml` to display sate Run as `grep ^IRA output.parsed |perl mkkml heatmap > output.kml` to create a heatmap of sat positions and downlink positions +### License +Unless otherwise noted in a file, everything here is (c) Sec & schneider and licensed under the 2-Clause BSD License \ No newline at end of file From b5b08d6309efdb0a63ad8488d79b4f6b520523a9 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Fri, 11 May 2018 20:10:32 +0100 Subject: [PATCH 17/50] Alow pypy builds to fail --- .travis.yml | 2 +- .travis/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b20b541..6780b80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ matrix: env: TOXENV=pypy allow_failures: - - python: "pypy2.7" + - python: "pypy-5.4.1" - os: "osx" cache: diff --git a/.travis/install.sh b/.travis/install.sh index deff563..9b76aa9 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -14,4 +14,4 @@ if [[ $TRAVIS_OS_NAME == 'osx' ]]; then else # Install custom requirements on Linux pip install tox-travis -fi \ No newline at end of file +fi From 0d8f8692ecd3703318253ea74e82f1c107941574 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Fri, 11 May 2018 20:12:31 +0100 Subject: [PATCH 18/50] Alow numpy 1.11.1 the default for pypy --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8971e21..b83213b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy>=1.14.0 +numpy>=1.11.1 scipy>=1.1.0 six>=1.11.0 dateparser>=0.7.0 From b895435ac2b61ac610e4b02dd96cb724e182cde9 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Fri, 11 May 2018 20:14:56 +0100 Subject: [PATCH 19/50] Disable pypy buid due to scipy --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6780b80..eb850a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,10 +19,10 @@ matrix: # env: TOXENV=py27 # PyPy 2.7 - - python: "pypy-5.4.1" - os: "linux" - dist: trusty - env: TOXENV=pypy + #- python: "pypy-5.4.1" + # os: "linux" + # dist: trusty + # env: TOXENV=pypy allow_failures: - python: "pypy-5.4.1" From 7d607ec220d3ed872ee50c6629f592fe2e96bb64 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Fri, 11 May 2018 21:07:30 +0100 Subject: [PATCH 20/50] Reorg VOC line parsing --- README.md | 6 ++++ iridiumtk/bits_to_dfs.py | 37 ++++++-------------- iridiumtk/stats_voc.py | 33 +++--------------- iridiumtk/test_bits_to_dfs.py | 29 ++++++++------- iridiumtk/test_stats_voc.py | 9 ++--- iridiumtk/voc/__init__.py | 5 +++ iridiumtk/voc/test_voc_line.py | 45 ++++++++++++++++++++++++ iridiumtk/voc/voc_line.py | 64 ++++++++++++++++++++++++++++++++++ 8 files changed, 155 insertions(+), 73 deletions(-) create mode 100644 iridiumtk/voc/__init__.py create mode 100644 iridiumtk/voc/test_voc_line.py create mode 100644 iridiumtk/voc/voc_line.py diff --git a/README.md b/README.md index df8ec1d..89b45e9 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,12 @@ virtualenv venv source venv/bin/activate pip install -r requirements.txt ``` +Then run commands via a module path. e.g +```bash +python -m iridiumtk.bits_to_dfs my_data.voice.dfs my_data.bits +# or +python -m iridiumtk.stats_voc my_data.bits +``` ### Example usage Either extract some Iridium frames from the air or a file using [gr-iridium](https://github.com/muccc/gr-iridium) (recommended) or use the legacy code located in the [extractror-python](extractor-python/) directory if you don't want to install GNURadio (not recommended). diff --git a/iridiumtk/bits_to_dfs.py b/iridiumtk/bits_to_dfs.py index ddda68b..8b4254c 100755 --- a/iridiumtk/bits_to_dfs.py +++ b/iridiumtk/bits_to_dfs.py @@ -1,42 +1,25 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- -# vim: set ts=4 sw=4 tw=0 et pm=: import fileinput import sys import argparse -""" -VOC: i-1443338945.6543-t1 033399141 1625872817 81% 0.027 179 L:no LCW(0,001111,100000000000000000000 E1) 01111001000100010010010011011011011001111 011000010000100001110101111011110010010111011001010001011101010001100000000110010100000110111110010101110101001111010100111001000110100110001110110 1010101010010010001000001110011000001001001010011110011100110100111110001101110010110101010110011101011100011101011000000000 descr_extra: -""" - -def chunks(l, n): - """ Yield successive n-sized chunks from l. - """ - for i in xrange(0, len(l), n): - yield l[i:i+n] +from .voc import VocLine def bits_to_dfs(lines, output): - data = '' for line in lines: - line = line.split() - if line[0] != 'VOC:': - continue - if int(line[6]) < 179: + if 'A:OK' in line and "Message: Couldn't parse:" not in line: + raise RuntimeError('Expected "iridium-parser.py" parsed data. Found raw "iridium-extractor" data.') + + if not line.startswith('VOC:'): continue - data = line[10] - if data[0] == "[": - for pos in xrange(1,len(data),3): - byte=int(data[pos:pos+2],16) - byte=int('{:08b}'.format(byte)[::-1], 2) - output.write(chr(byte)) - else: - for bits in chunks(data, 8): - byte = int(bits[::-1],2) - output.write(chr(byte)) + raw_voice_bits = VocLine(line).raw_voice_bits() + if raw_voice_bits is not None: + output.write(raw_voice_bits) + def main(): parser = argparse.ArgumentParser(description='Convert iridium-parser.py VOC output to DFS') @@ -54,4 +37,4 @@ def main(): if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/iridiumtk/stats_voc.py b/iridiumtk/stats_voc.py index 7e79e03..e81b435 100755 --- a/iridiumtk/stats_voc.py +++ b/iridiumtk/stats_voc.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# vim: set ts=4 sw=4 tw=0 et fenc=utf8 pm=: + import sys import os import subprocess @@ -9,41 +9,18 @@ from datetime import datetime import argparse + import matplotlib.pyplot as plt import numpy as np import scipy.cluster.hierarchy as hcluster -import six import dateparser -from bits_to_dfs import bits_to_dfs - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -# Example lines -# VOC: i-1430527570.4954-t1 421036605 1625859953 66% 0.008 219 L:no LCW(0,001111,100000000000000000000 E1) 101110110101010100101101111000111111111001011111001011010001000010010001101110011010011001111111011101111100011001001001000111001101001011001011000101111111101110110011111000000001110010001110101101001010011001101001010111101100011100110011110010110110101010110001010000100100101011010010100100100011010110101001 -# VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f] -class VocLine(object): - def __init__(self, line): - self.line = line - try: - line_split = line.split() +from .voc import VocLine - raw_time_base = line_split[1] - ts_base_ms = int(raw_time_base.split('-')[1].split('.')[0]) - - time_offset_ns = int(line_split[2]) - self.ts = ts_base_ms + (time_offset_ns / 1000) - - self.f = int(line_split[3])/1000. - self.lcw = line[8] - except Exception as e: - six.raise_from(Exception('Failed to parse line "{}"'.format(line), e), e) - - def datetime(self): - return datetime.utcfromtimestamp(self.ts) +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) class OnClickHandler(object): diff --git a/iridiumtk/test_bits_to_dfs.py b/iridiumtk/test_bits_to_dfs.py index e10a852..610e15e 100644 --- a/iridiumtk/test_bits_to_dfs.py +++ b/iridiumtk/test_bits_to_dfs.py @@ -3,33 +3,33 @@ import unittest from io import BytesIO -from .bits_to_dfs import chunks, bits_to_dfs - -class ChunksTest(unittest.TestCase): - def test_simple(self): - result = list(chunks([1, 2, 3, 4, 5, 6], 2)) - self.assertEquals(result, [[1, 2], [3, 4], [5, 6]]) +from .bits_to_dfs import bits_to_dfs class BitsToDfsTest(unittest.TestCase): TEST_VOC_LINE_1 = 'VOC: i-1443338945.6543-t1 033399141 1625872817 81% 0.027 179 L:no LCW(0,001111,100000000000000000000 E1) 01111001000100010010010011011011011001111 011000010000100001110101111011110010010111011001010001011101010001100000000110010100000110111110010101110101001111010100111001000110100110001110110 1010101010010010001000001110011000001001001010011110011100110100111110001101110010110101010110011101011100011101011000000000 descr_extra:' TEST_VOC_LINE_2 = 'VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f]' + TEST_VOC_LINE_3 = 'VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 178 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff]' def test_empty_input(self): output = BytesIO() bits_to_dfs([], output) self.assertEquals(output.getvalue(), '') - def test_old_format(self): + def test_raw_data(self): + output = BytesIO() + with self.assertRaises(RuntimeError): + bits_to_dfs(['RAW: i-1525892321-t1 0001338 1623702528 A:OK I:00000000027 79% 0.001 179 0011000000110000111100110001000000010011100100011000001000101111100010010101110100110101010101010101010101010101010101010101010101011100010111010001010101010101010101110011010011010101010101010101010111000101010101011101001101010101010101010101010101010101010101010101010101010101010101010101010101010101010101110001110101000111001101010101010100110101010101010101010101010101010101'], output) + output = BytesIO() - bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_1], output) - self.assertEquals(output.getvalue(), '\x9e\x88$\xdb\xe6\x01') + bits_to_dfs(['RAW: i-1525892321-t1 000045987 1626110208 83% 0.001 <001100000011000011110011> 1100000000000000 0000000000000000 0000000000000000 1001000000000000 0000000000000000 0000000000000000 0100001000110110 0111100001010110 1011001111101110 1010110101000101 1111111001110101 1011110101110001 1110000001110111 0110001000001010 0100100100111000 1000001111010100 1011001101011110 0100011011100010 1111110001010001 1100110110101111 0011100111100101 1110100100000110 0111011111110010 1111110011001000 1101000011000011 1011110101110111 0000101000000001 1000111010101100 1011000001010011 0111011100011101 0101011000011101 0111110001001101 0100001011001010 1101001110100010 0111011001000101 0100111001010110 0001111110101010 1110001100010111 0001010101100100 1001010100111011 1001111110001101 0110100100010010 0001110111101001 1000011010111100 00011001 ERR:Message: unknown Iridium message type'], output) + self.assertEquals(output.getvalue(), '') - def test_new_format(self): + def test_short_packet(self): output = BytesIO() - bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_2], output) - self.assertEquals(output.getvalue(), '\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') + bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_3], output) + self.assertEquals(output.getvalue(), '') def test_multiple(self): output = BytesIO() @@ -42,6 +42,11 @@ def test_filters_non_voc_lines(self): self.assertEquals(output.getvalue(), '\x9e\x88$\xdb\xe6\x01' + '\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') +class MainTest(unittest.TestCase): + def test_pass(self): + pass + + def main(): unittest.main() diff --git a/iridiumtk/test_stats_voc.py b/iridiumtk/test_stats_voc.py index 6e6d484..d798b2a 100644 --- a/iridiumtk/test_stats_voc.py +++ b/iridiumtk/test_stats_voc.py @@ -2,14 +2,11 @@ import unittest -from .stats_voc import VocLine +class MainTest(unittest.TestCase): + def test_pass(self): + pass -class VocLineTest(unittest.TestCase): - - def test_empty_input(self): - with self.assertRaises(Exception): - VocLine('') def main(): unittest.main() diff --git a/iridiumtk/voc/__init__.py b/iridiumtk/voc/__init__.py new file mode 100644 index 0000000..fba8615 --- /dev/null +++ b/iridiumtk/voc/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/python + +from .voc_line import VocLine + +__all__ = [x.__name__ for x in (VocLine,)] \ No newline at end of file diff --git a/iridiumtk/voc/test_voc_line.py b/iridiumtk/voc/test_voc_line.py new file mode 100644 index 0000000..54ac971 --- /dev/null +++ b/iridiumtk/voc/test_voc_line.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +import unittest +from datetime import datetime + + +from .voc_line import chunks, VocLine + + +class ChunksTest(unittest.TestCase): + def test_simple(self): + result = list(chunks([1, 2, 3, 4, 5, 6], 2)) + self.assertEquals(result, [[1, 2], [3, 4], [5, 6]]) + + +class VocLineTest(unittest.TestCase): + TEST_VOC_LINE_1 = 'VOC: i-1443338945.6543-t1 033399141 1625872817 81% 0.027 179 L:no LCW(0,001111,100000000000000000000 E1) 01111001000100010010010011011011011001111 011000010000100001110101111011110010010111011001010001011101010001100000000110010100000110111110010101110101001111010100111001000110100110001110110 1010101010010010001000001110011000001001001010011110011100110100111110001101110010110101010110011101011100011101011000000000 descr_extra:' + TEST_VOC_LINE_2 = 'VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f]' + TEST_VOC_LINE_3 = 'VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 178 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff]' + + def test_empty_input(self): + with self.assertRaises(Exception): + VocLine('') + + def test_old_format(self): + voc_line = VocLine(VocLineTest.TEST_VOC_LINE_1) + self.assertEquals(voc_line.datetime(), datetime.utcfromtimestamp(1443372344)) + self.assertEquals(voc_line.raw_voice_bits(), '\x9e\x88$\xdb\xe6\x01') + + def test_new_format(self): + voc_line = VocLine(VocLineTest.TEST_VOC_LINE_2) + self.assertEquals(voc_line.datetime(), datetime.utcfromtimestamp(1526039102)) + self.assertEquals(voc_line.raw_voice_bits(), '\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') + + def test_raw_voice_bits_small_packet(self): + voc_line = VocLine(VocLineTest.TEST_VOC_LINE_3) + self.assertEquals(voc_line.raw_voice_bits(), None) + + +def main(): + unittest.main() + + +if __name__ == "__main__": + main() diff --git a/iridiumtk/voc/voc_line.py b/iridiumtk/voc/voc_line.py new file mode 100644 index 0000000..a5e4626 --- /dev/null +++ b/iridiumtk/voc/voc_line.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +import logging +from io import BytesIO +from datetime import datetime + + +import six + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def chunks(l, n): + """ Yield successive n-sized chunks from l. + """ + for i in xrange(0, len(l), n): + yield l[i:i+n] + + +# Example lines +# VOC: i-1430527570.4954-t1 421036605 1625859953 66% 0.008 219 L:no LCW(0,001111,100000000000000000000 E1) 101110110101010100101101111000111111111001011111001011010001000010010001101110011010011001111111011101111100011001001001000111001101001011001011000101111111101110110011111000000001110010001110101101001010011001101001010111101100011100110011110010110110101010110001010000100100101011010010100100100011010110101001 +# VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f] +class VocLine(object): + def __init__(self, line): + try: + line_split = line.split() + + raw_time_base = line_split[1] + ts_base_ms = int(raw_time_base.split('-')[1].split('.')[0]) + + time_offset_ns = int(line_split[2]) + self.ts = ts_base_ms + (time_offset_ns / 1000) + + self.f = int(line_split[3])/1000. + self.lcw = line[8] + + if int(line_split[6]) < 179: + self.data = None + else: + self.data = line_split[10] + except Exception as e: + logger.error('Failed to parse line "%s"', line) + six.raise_from(Exception('Failed to parse line "{}"'.format(line), e), e) + + def datetime(self): + return datetime.utcfromtimestamp(self.ts) + + def raw_voice_bits(self): + data = self.data + if data is None: + return None + byte_stream = BytesIO() + if data[0] == "[": + for pos in xrange(1,len(data),3): + byte=int(data[pos:pos+2],16) + byte=int('{:08b}'.format(byte)[::-1], 2) + byte_stream.write(chr(byte)) + else: + for bits in chunks(data, 8): + byte = int(bits[::-1],2) + byte_stream.write(chr(byte)) + return byte_stream.getvalue() From fe750f2fc022dec48d8e9498d6eb8d7d35b74669 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sat, 12 May 2018 11:57:19 +0100 Subject: [PATCH 21/50] Add more stats_voc tests --- iridiumtk/test_stats_voc.py | 44 +++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/iridiumtk/test_stats_voc.py b/iridiumtk/test_stats_voc.py index d798b2a..f9242df 100644 --- a/iridiumtk/test_stats_voc.py +++ b/iridiumtk/test_stats_voc.py @@ -1,12 +1,52 @@ #!/usr/bin/env python import unittest +import os +import tempfile +from .stats_voc import read_lines + class MainTest(unittest.TestCase): - def test_pass(self): - pass + TEST_VOC_LINE_1 = 'VOC: i-1443338945.6543-t1 033399141 1625872817 81% 0.027 179 L:no LCW(0,001111,100000000000000000000 E1) 01111001000100010010010011011011011001111 011000010000100001110101111011110010010111011001010001011101010001100000000110010100000110111110010101110101001111010100111001000110100110001110110 1010101010010010001000001110011000001001001010011110011100110100111110001101110010110101010110011101011100011101011000000000 descr_extra:' + TEST_VOC_LINE_2 = 'VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f]' + TEST_VOC_LINE_3 = 'VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 178 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff]' + + def setUp(self): + self.tempfiles = [] + + def get_temp_file(self): + _, file_path = tempfile.mkstemp() + self.tempfiles.append(file_path) + return file_path + + def test_read_lines(self): + input_file_path = self.get_temp_file() + with open(input_file_path, 'w') as input_file: + input_file.write(MainTest.TEST_VOC_LINE_1 + '\n') + input_file.write(MainTest.TEST_VOC_LINE_2 + '\n') + input_file.write(MainTest.TEST_VOC_LINE_3 + '\n') + + voc_lines = read_lines(input_file_path, None, None) + self.assertEquals(len(voc_lines), 2) + + def test_test_read_lines_with_raw_data(self): + input_file_path = self.get_temp_file() + with open(input_file_path, 'w') as input_file: + input_file.write('RAW: i-1525892321-t1 0001338 1623702528 A:OK I:00000000027 79% 0.001 1 00\n') + with self.assertRaises(RuntimeError): + read_lines(input_file_path, None, None) + + def test_test_read_lines_with_parsed_errors(self): + input_file_path = self.get_temp_file() + with open(input_file_path, 'w') as input_file: + input_file.write('RAW: i-1525892321-t1 000045987 1626110208 83% 0.001 <001100000011000011110011> 1100000000000000 .... 00011001 ERR:Message: unknown Iridium message type\n') + voc_lines = read_lines(input_file_path, None, None) + self.assertEquals(voc_lines, []) + def tearDown(self): + for file in self.tempfiles: + os.remove(file) def main(): unittest.main() From 970022e740cc8db78bd1d5e63858ef588af347c4 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sat, 12 May 2018 11:58:30 +0100 Subject: [PATCH 22/50] Remove scipy dependency --- iridiumtk/stats_voc.py | 62 +++++++++++++++++++++++++++++++-------- iridiumtk/voc/voc_line.py | 2 +- requirements.txt | 1 - 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/iridiumtk/stats_voc.py b/iridiumtk/stats_voc.py index e81b435..f2c4e24 100755 --- a/iridiumtk/stats_voc.py +++ b/iridiumtk/stats_voc.py @@ -8,11 +8,12 @@ import tempfile from datetime import datetime import argparse +from collections import namedtuple import matplotlib.pyplot as plt +from matplotlib.collections import BrokenBarHCollection import numpy as np -import scipy.cluster.hierarchy as hcluster import dateparser @@ -23,6 +24,35 @@ logger = logging.getLogger(__name__) +KHZ = 1000 +MHZ = KHZ * 1000 + +# https://www.sigidwiki.com/wiki/Iridium +ChannelInfo = namedtuple('ChannelInfo', ['description', 'frequency']) +SIMPLEX_CHANELS = { + ChannelInfo('Guard Channel', 1626.020833 * MHZ), + ChannelInfo('Guard Channel', 1626.062500 * MHZ), + ChannelInfo('Quaternary Messaging', 1626.104167 * MHZ), + ChannelInfo('Tertiary Messaging', 1626.145833 * MHZ), + ChannelInfo('Guard Channel', 1626.187500 * MHZ), + ChannelInfo('Guard Channel', 1626.229167 * MHZ), + ChannelInfo('Ring Alert', 1626.270833 * MHZ), + ChannelInfo('Guard Channel', 1626.312500 * MHZ), + ChannelInfo('Guard Channel', 1626.354167 * MHZ), + ChannelInfo('Secondary Messaging', 1626.395833 * MHZ), + ChannelInfo('Primary Messaging', 1626.437500 * MHZ), + ChannelInfo('Guard Channel', 1626.479167 * MHZ), +} +DUPLEX_CHANELS = frozenset(SIMPLEX_CHANELS) + +DUPLEX_CHANELS = set() +for n in xrange(1, 240): + frequency = (1616 + 0.020833 * (2 * n - 1)) * MHZ + DUPLEX_CHANELS.add(ChannelInfo('Channel {}'.format(n), frequency)) +DUPLEX_CHANELS = frozenset(DUPLEX_CHANELS) + +ALL_CHANELS = frozenset((SIMPLEX_CHANELS | DUPLEX_CHANELS)) + class OnClickHandler(object): def __init__(self, lines): self.lines = lines @@ -127,24 +157,32 @@ def main(): print('No usable data found') sys.exit(1) - plot_data = np.empty((number_of_lines, 2)) + plot_data_time = np.empty(number_of_lines, dtype=np.float64) + plot_data_freq = np.empty(number_of_lines, dtype=np.uint32) for i, voc_line in enumerate(lines): - plot_data[i][0] = voc_line.ts - plot_data[i][1] = np.float64(voc_line.f) - - distances = hcluster.distance.pdist(plot_data) - thresh = 2 * distances.min() - clusters = hcluster.fclusterdata(plot_data, thresh, criterion="distance") + #plot_data_time[i] = np.datetime64(voc_line.datetime().isoformat()) + plot_data_time[i] = np.uint32(voc_line.ts) + plot_data_freq[i] = np.float64(voc_line.f) fig = plt.figure() #fig.autofmt_xdate() on_click_handler = OnClickHandler(lines) fig.canvas.mpl_connect('button_press_event', on_click_handler.onclick) - ax = fig.add_subplot(1, 1, 1) - ax.scatter(*np.transpose(plot_data), c=clusters) - #ax.xaxis_date() - ax.grid(True) + subplot = fig.add_subplot(1, 1, 1) + subplot.scatter(plot_data_time, plot_data_freq) + #subplot.xaxis_date() + + for channel in ALL_CHANELS: + if 'Guard' in channel.description: + color = 'tab:gray' + elif 'Messaging' in channel.description: + color = 'tab:orange' + elif 'Ring' in channel.description: + color = 'tab:red' + else: + color = 'tab:green' + subplot.axhline(channel.frequency, color=color, alpha=0.3, label=channel.description) plt.title('Click once left and once right to define an area.\nThe script will try to play iridium using ir77_ambe_decode and aplay.') plt.xlabel('time') diff --git a/iridiumtk/voc/voc_line.py b/iridiumtk/voc/voc_line.py index a5e4626..d523793 100644 --- a/iridiumtk/voc/voc_line.py +++ b/iridiumtk/voc/voc_line.py @@ -33,7 +33,7 @@ def __init__(self, line): time_offset_ns = int(line_split[2]) self.ts = ts_base_ms + (time_offset_ns / 1000) - self.f = int(line_split[3])/1000. + self.f = int(line_split[3]) self.lcw = line[8] if int(line_split[6]) < 179: diff --git a/requirements.txt b/requirements.txt index b83213b..ae8ca5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ numpy>=1.11.1 -scipy>=1.1.0 six>=1.11.0 dateparser>=0.7.0 matplotlib>=2.2.0 \ No newline at end of file From bcd7d5a3ff467233595cc318109b39290fcf76b2 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sat, 12 May 2018 14:18:00 +0100 Subject: [PATCH 23/50] Update stats.py and rename graphing tools --- README.md | 3 +- iridiumtk/bits_to_dfs.py | 8 +- iridiumtk/graph_by_type.py | 95 +++++++++++++++++++ iridiumtk/{stats_voc.py => graph_voc.py} | 17 ++-- iridiumtk/line_parser/__init__.py | 5 + iridiumtk/line_parser/base_line.py | 55 +++++++++++ iridiumtk/line_parser/test_base_line.py | 47 +++++++++ .../{voc => line_parser}/test_voc_line.py | 10 +- iridiumtk/{voc => line_parser}/voc_line.py | 25 ++--- .../{test_stats_voc.py => test_graph_voc.py} | 0 iridiumtk/voc/__init__.py | 5 - setup.py | 3 +- stats.py | 87 ----------------- 13 files changed, 233 insertions(+), 127 deletions(-) create mode 100755 iridiumtk/graph_by_type.py rename iridiumtk/{stats_voc.py => graph_voc.py} (94%) create mode 100644 iridiumtk/line_parser/__init__.py create mode 100644 iridiumtk/line_parser/base_line.py create mode 100644 iridiumtk/line_parser/test_base_line.py rename iridiumtk/{voc => line_parser}/test_voc_line.py (77%) rename iridiumtk/{voc => line_parser}/voc_line.py (79%) rename iridiumtk/{test_stats_voc.py => test_graph_voc.py} (100%) delete mode 100644 iridiumtk/voc/__init__.py delete mode 100755 stats.py diff --git a/README.md b/README.md index 89b45e9..f5650de 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ Then run commands via a module path. e.g ```bash python -m iridiumtk.bits_to_dfs my_data.voice.dfs my_data.bits # or -python -m iridiumtk.stats_voc my_data.bits +python -m iridiumtk.graph_by_type my_data.bits +python -m iridiumtk.graph_voc my_data.bits ``` ### Example usage diff --git a/iridiumtk/bits_to_dfs.py b/iridiumtk/bits_to_dfs.py index 8b4254c..2a64145 100755 --- a/iridiumtk/bits_to_dfs.py +++ b/iridiumtk/bits_to_dfs.py @@ -5,7 +5,7 @@ import argparse -from .voc import VocLine +from .line_parser import VocLine def bits_to_dfs(lines, output): @@ -16,9 +16,9 @@ def bits_to_dfs(lines, output): if not line.startswith('VOC:'): continue - raw_voice_bits = VocLine(line).raw_voice_bits() - if raw_voice_bits is not None: - output.write(raw_voice_bits) + voice_bits = VocLine(line).voice_bits + if voice_bits is not None: + output.write(voice_bits) def main(): diff --git a/iridiumtk/graph_by_type.py b/iridiumtk/graph_by_type.py new file mode 100755 index 0000000..0cfc3db --- /dev/null +++ b/iridiumtk/graph_by_type.py @@ -0,0 +1,95 @@ +#!/usr/bin/python +import sys +import matplotlib.pyplot as plt +import argparse +import collections +import fileinput +from datetime import datetime +import logging + + +import dateparser +import numpy as np + + +from .line_parser import BaseLine +from .stats_voc import ALL_CHANELS + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def read_lines(input_files, start_time_filter, end_time_filter): + lines = [] + for line in fileinput.input(files=input_files): + line = line.strip() + if 'A:OK' in line and "Message: Couldn't parse:" not in line: + raise RuntimeError('Expected "iridium-parser.py" parsed data. Found raw "iridium-extractor" data.') + if line.startswith('ERR: '): + continue + + base_line = BaseLine(line) + if start_time_filter and start_time_filter > base_line.datetime: + continue + if end_time_filter and end_time_filter < base_line.datetime: + continue + yield base_line + + +def main(): + parser = argparse.ArgumentParser(description='Visualise ????') # TODO + parser.add_argument('--start', metavar='DATETIME', default=None, help='Filter events before this time') + parser.add_argument('--end', metavar='DATETIME', default=None, help='Filter events after this time') + parser.add_argument('input', metavar='FILE', nargs='*', help='Files to read, if empty or -, stdin is used') + args = parser.parse_args() + + input_files = args.input if len(args.input) > 0 else ['-'] + start_time_filter = dateparser.parse(args.start) if args.start else None + end_time_filter = dateparser.parse(args.end) if args.end else None + + stats = {} + number_of_lines = 0 + for base_line in read_lines(input_files, start_time_filter, end_time_filter): + number_of_lines += 1 + stats.setdefault(base_line.frame_type, []).append(base_line) + + logger.info('Read %d lines from input', number_of_lines) + for frame_type, frames in stats.iteritems(): + logger.info(' - Read %d\t%s lines from input', len(frames), frame_type) + + fig = plt.figure() + + subplot = fig.add_subplot(1, 1, 1) + for frame_type, frames in stats.iteritems(): + number_of_frames = len(frames) + plot_data_time = np.empty(number_of_frames, dtype=np.float64) + plot_data_freq = np.empty(number_of_frames, dtype=np.uint32) + for i, base_line in enumerate(frames): + plot_data_time[i] = np.uint32(base_line.datetime_unix) + plot_data_freq[i] = np.float64(base_line.frequency) + subplot.scatter(plot_data_time, plot_data_freq, label=frame_type) + + for channel in ALL_CHANELS: + if 'Guard' in channel.description: + color = 'tab:gray' + elif 'Messaging' in channel.description: + color = 'tab:orange' + elif 'Ring' in channel.description: + color = 'tab:red' + else: + color = 'tab:green' + subplot.axhline(channel.frequency, color=color, alpha=0.3, label=channel.description) + + + # Shrink current axis's height on the bottom + handles, labels = zip(*[(h, l) for (h, l) in zip(*subplot.get_legend_handles_labels()) if l in stats]) + subplot.legend(handles, labels, bbox_to_anchor=(1.04,1), loc="upper left") + + plt.title('frequency vs time color coded by frame type') + plt.xlabel('time') + plt.ylabel('frequency') + plt.show() + +if __name__ == "__main__": + main() diff --git a/iridiumtk/stats_voc.py b/iridiumtk/graph_voc.py similarity index 94% rename from iridiumtk/stats_voc.py rename to iridiumtk/graph_voc.py index f2c4e24..5e17573 100755 --- a/iridiumtk/stats_voc.py +++ b/iridiumtk/graph_voc.py @@ -17,7 +17,8 @@ import dateparser -from .voc import VocLine +from .line_parser import VocLine +from .bits_to_dfs import bits_to_dfs logging.basicConfig(level=logging.INFO) @@ -81,11 +82,11 @@ def filter_voc(self, t_start, t_stop, f_min, f_max): filtered_lines = [] for voc_line in self.lines: - ts = voc_line.ts - f = voc_line.f + ts = voc_line.datetime_unix + f = voc_line.frequency if t_start <= ts and ts <= t_stop and \ f_min <= f and f <= f_max: - filtered_lines.append(voc_line.line) + filtered_lines.append(voc_line.raw_line) return filtered_lines @@ -131,9 +132,9 @@ def read_lines(input_files, start_time_filter, end_time_filter): raise RuntimeError('Expected "iridium-parser.py" parsed data. Found raw "iridium-extractor" data.') if 'VOC: ' in line and not "LCW(0,001111,100000000000000000000" in line: voc_line = VocLine(line) - if start_time_filter and start_time_filter > voc_line.datetime(): + if start_time_filter and start_time_filter > voc_line.datetime: continue - if end_time_filter and end_time_filter < voc_line.datetime(): + if end_time_filter and end_time_filter < voc_line.datetime: continue lines.append(voc_line) return lines @@ -161,8 +162,8 @@ def main(): plot_data_freq = np.empty(number_of_lines, dtype=np.uint32) for i, voc_line in enumerate(lines): #plot_data_time[i] = np.datetime64(voc_line.datetime().isoformat()) - plot_data_time[i] = np.uint32(voc_line.ts) - plot_data_freq[i] = np.float64(voc_line.f) + plot_data_time[i] = np.uint32(voc_line.datetime_unix) + plot_data_freq[i] = np.float64(voc_line.frequency) fig = plt.figure() #fig.autofmt_xdate() diff --git a/iridiumtk/line_parser/__init__.py b/iridiumtk/line_parser/__init__.py new file mode 100644 index 0000000..a680dc5 --- /dev/null +++ b/iridiumtk/line_parser/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/python + +from .voc_line import BaseLine, VocLine + +__all__ = [x.__name__ for x in (BaseLine, VocLine)] \ No newline at end of file diff --git a/iridiumtk/line_parser/base_line.py b/iridiumtk/line_parser/base_line.py new file mode 100644 index 0000000..43ccdfd --- /dev/null +++ b/iridiumtk/line_parser/base_line.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +import logging +from io import BytesIO +from datetime import datetime + + +import six + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +# Example lines +# VOC: i-1430527570.4954-t1 421036605 1625859953 66% 0.008 219 L:no LCW(0,001111,100000000000000000000 E1) 101110110101010100101101111000111111111001011111001011010001000010010001101110011010011001111111011101111100011001001001000111001101001011001011000101111111101110110011111000000001110010001110101101001010011001101001010111101100011100110011110010110110101010110001010000100100101011010010100100100011010110101001 +# VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f] +class BaseLine(object): + def __init__(self, line): + try: + self._raw_line = line + line_split = line.split() + + self._frame_type = line_split[0][:-1] + + raw_time_base = line_split[1] + ts_base_ms = int(raw_time_base.split('-')[1].split('.')[0]) + + time_offset_ns = int(line_split[2]) + self._timestamp = ts_base_ms + (time_offset_ns / 1000) + + self._frequnecy = int(line_split[3]) + except Exception as e: + logger.error('Failed to parse line "%s"', line) + six.raise_from(Exception('Failed to parse line "{}"'.format(line), e), e) + + @property + def raw_line(self): + return self._raw_line + + @property + def frame_type(self): + return self._frame_type + + @property + def frequency(self): + return self._frequnecy + + @property + def datetime(self): + return datetime.utcfromtimestamp(self._timestamp) + + @property + def datetime_unix(self): + return self._timestamp diff --git a/iridiumtk/line_parser/test_base_line.py b/iridiumtk/line_parser/test_base_line.py new file mode 100644 index 0000000..6c4a48f --- /dev/null +++ b/iridiumtk/line_parser/test_base_line.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +import unittest +from datetime import datetime + + +from .base_line import BaseLine + + +class BaseLineTest(unittest.TestCase): + TEST_VOC_LINE_1 = 'VOC: i-1443338945.6543-t1 033399141 1625872817 81% 0.027 179 L:no LCW(0,001111,100000000000000000000 E1) 01111001000100010010010011011011011001111 011000010000100001110101111011110010010111011001010001011101010001100000000110010100000110111110010101110101001111010100111001000110100110001110110 1010101010010010001000001110011000001001001010011110011100110100111110001101110010110101010110011101011100011101011000000000 descr_extra:' + TEST_VOC_LINE_2 = 'VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f]' + + def test_empty_input(self): + with self.assertRaises(Exception): + BaseLine('') + + def test_old_format_datetime(self): + base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) + self.assertEquals(base_line.datetime_unix, 1443372344) + self.assertEquals(base_line.datetime, datetime.utcfromtimestamp(1443372344)) + + def test_new_format_datetime(self): + base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) + self.assertEquals(base_line.datetime_unix, 1526039102) + self.assertEquals(base_line.datetime, datetime.utcfromtimestamp(1526039102)) + + def test_frequency(self): + base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) + self.assertEquals(base_line.frequency, 1620359296) + + def test_frame_type(self): + base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) + self.assertEquals(base_line.frame_type, 'VOC') + + def test_raw_line(self): + for line in [BaseLineTest.TEST_VOC_LINE_1, BaseLineTest.TEST_VOC_LINE_2]: + base_line = BaseLine(line) + self.assertEquals(base_line.raw_line, line) + + +def main(): + unittest.main() + + +if __name__ == "__main__": + main() diff --git a/iridiumtk/voc/test_voc_line.py b/iridiumtk/line_parser/test_voc_line.py similarity index 77% rename from iridiumtk/voc/test_voc_line.py rename to iridiumtk/line_parser/test_voc_line.py index 54ac971..ee6097e 100644 --- a/iridiumtk/voc/test_voc_line.py +++ b/iridiumtk/line_parser/test_voc_line.py @@ -24,17 +24,15 @@ def test_empty_input(self): def test_old_format(self): voc_line = VocLine(VocLineTest.TEST_VOC_LINE_1) - self.assertEquals(voc_line.datetime(), datetime.utcfromtimestamp(1443372344)) - self.assertEquals(voc_line.raw_voice_bits(), '\x9e\x88$\xdb\xe6\x01') + self.assertEquals(voc_line.voice_bits, '\x9e\x88$\xdb\xe6\x01') def test_new_format(self): voc_line = VocLine(VocLineTest.TEST_VOC_LINE_2) - self.assertEquals(voc_line.datetime(), datetime.utcfromtimestamp(1526039102)) - self.assertEquals(voc_line.raw_voice_bits(), '\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') + self.assertEquals(voc_line.voice_bits, '\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') - def test_raw_voice_bits_small_packet(self): + def test_voice_bits_small_packet(self): voc_line = VocLine(VocLineTest.TEST_VOC_LINE_3) - self.assertEquals(voc_line.raw_voice_bits(), None) + self.assertEquals(voc_line.voice_bits, None) def main(): diff --git a/iridiumtk/voc/voc_line.py b/iridiumtk/line_parser/voc_line.py similarity index 79% rename from iridiumtk/voc/voc_line.py rename to iridiumtk/line_parser/voc_line.py index d523793..5b36165 100644 --- a/iridiumtk/voc/voc_line.py +++ b/iridiumtk/line_parser/voc_line.py @@ -8,6 +8,9 @@ import six +from .base_line import BaseLine + + logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -22,33 +25,25 @@ def chunks(l, n): # Example lines # VOC: i-1430527570.4954-t1 421036605 1625859953 66% 0.008 219 L:no LCW(0,001111,100000000000000000000 E1) 101110110101010100101101111000111111111001011111001011010001000010010001101110011010011001111111011101111100011001001001000111001101001011001011000101111111101110110011111000000001110010001110101101001010011001101001010111101100011100110011110010110110101010110001010000100100101011010010100100100011010110101001 # VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f] -class VocLine(object): +class VocLine(BaseLine): def __init__(self, line): + super(VocLine, self).__init__(line) try: line_split = line.split() - raw_time_base = line_split[1] - ts_base_ms = int(raw_time_base.split('-')[1].split('.')[0]) - - time_offset_ns = int(line_split[2]) - self.ts = ts_base_ms + (time_offset_ns / 1000) - - self.f = int(line_split[3]) self.lcw = line[8] if int(line_split[6]) < 179: - self.data = None + self._voice_data = None else: - self.data = line_split[10] + self._voice_data = line_split[10] except Exception as e: logger.error('Failed to parse line "%s"', line) six.raise_from(Exception('Failed to parse line "{}"'.format(line), e), e) - def datetime(self): - return datetime.utcfromtimestamp(self.ts) - - def raw_voice_bits(self): - data = self.data + @property + def voice_bits(self): + data = self._voice_data if data is None: return None byte_stream = BytesIO() diff --git a/iridiumtk/test_stats_voc.py b/iridiumtk/test_graph_voc.py similarity index 100% rename from iridiumtk/test_stats_voc.py rename to iridiumtk/test_graph_voc.py diff --git a/iridiumtk/voc/__init__.py b/iridiumtk/voc/__init__.py deleted file mode 100644 index fba8615..0000000 --- a/iridiumtk/voc/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/python - -from .voc_line import VocLine - -__all__ = [x.__name__ for x in (VocLine,)] \ No newline at end of file diff --git a/setup.py b/setup.py index e38c402..d3d0104 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,8 @@ def run_tests(self): packages=find_packages(exclude=['9601', 'ambe_emu', 'nal-shout', 'rtl-sdr', 'tools', 'tracking']), entry_points={ 'console_scripts': [ - 'iridiumtk-stats-voc=iridiumtk.stats_voc:main', + 'iridiumtk-graph-voc=iridiumtk.graph_voc:main', + 'iridiumtk-graph-by-type=iridiumtk.graph_by_type:main', 'iridiumtk-bits-to-dfs=iridiumtk.bits_to_dfs:main', ], }, diff --git a/stats.py b/stats.py deleted file mode 100755 index aed408c..0000000 --- a/stats.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/python -# vim: set ts=4 sw=4 tw=0 et fenc=utf8 pm=: -import sys -import matplotlib.pyplot as plt -import collections - -f = open(sys.argv[1]) - -f.readline() - -max_f = None -min_f = None -min_ts = None -max_ts = None - -colors =['#cab2d6','#33a02c','#fdbf6f','#ffff99','#6a3d9a','#e31a1c','#ff7f00','#fb9a99','#b2df8a','#1f78b4','#aaaaaa', '#a6cee3'] - -frames = collections.OrderedDict() -frames['IMS'] = [colors[0], 'x', [], [], []] -frames['MSG'] = [colors[1], 'o', [], [], []] - -frames['IRA'] = [colors[2], 'x', [], [], []] - -frames['ISY'] = [colors[3], 'o', [], []] - -frames['IBC'] = [colors[4], 'o', [], []] - -frames['IU3'] = [colors[5], 'o', [], []] - -frames['IDA'] = [colors[6], 'o', [], []] - -frames['IIU'] = [colors[7], 'o', [], [], []] -frames['IIR'] = [colors[10], 'o', [], [], []] -frames['IIP'] = [colors[9], 'o', [], [], []] -frames['IIQ'] = [colors[8], 'o', [], [], []] - -frames['VOC'] = [colors[11], 'o', [], []] -frames['VOD'] = [colors[1], 'x', [], []] -frames['VDA'] = [colors[2], 'o', [], []] - -#frames['IRI'] = ['purple', 'x', [], []] -#frames['RAW'] = ['grey', 'x', [], []] - -for line in f: - line = line.strip().split() - type = line[0][:-1] - #ts_base = int(line[1].split('-')[1].split('.')[0]) - ts_base = 0 - ts = ts_base + float(line[2])/1000. - f = int(line[3]) - #len = int(line[6]) - #strength = float(line[5]) - - if max_f == None or max_f < f: - max_f = f - if min_f == None or min_f > f: - min_f = f - if max_ts == None or max_ts < ts: - max_ts = ts - if min_ts == None or min_ts > ts: - min_ts = ts - - if type in frames: - frames[type][2].append(ts) - frames[type][3].append(f) - -for t in frames: - f = frames[t] - plt.scatter(y=f[3], x=f[2], c=f[0], label=t, alpha=1, edgecolors=f[0], marker=f[1], s=20) - -#plt.colorbar() -#plt.ylim([min_f, max_f]) -#plt.ylim([1624.95e6, 1626.5e6]) -#plt.ylim([1616e6, 1627e6]) -plt.ylim([1618e6, 1626.7e6]) -#plt.xlim([1618e6, 1626.7e6]) -#ax = plt.gca() -#ax.ticklabel_format(useOffset=False) -#ax.set_axis_bgcolor('white') - - -plt.xlim([min_ts, max_ts]) -#plt.ylim([min_ts, max_ts]) - -plt.legend() -plt.show() - From dd46e51331a76f66e188103736b7809dc7abe6bc Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sat, 12 May 2018 14:30:52 +0100 Subject: [PATCH 24/50] Small clean up and missing renames --- iridiumtk/graph_by_type.py | 2 +- iridiumtk/graph_voc.py | 6 ++---- iridiumtk/test_graph_voc.py | 8 ++++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/iridiumtk/graph_by_type.py b/iridiumtk/graph_by_type.py index 0cfc3db..d7adf6a 100755 --- a/iridiumtk/graph_by_type.py +++ b/iridiumtk/graph_by_type.py @@ -13,7 +13,7 @@ from .line_parser import BaseLine -from .stats_voc import ALL_CHANELS +from .graph_voc import ALL_CHANELS logging.basicConfig(level=logging.INFO) diff --git a/iridiumtk/graph_voc.py b/iridiumtk/graph_voc.py index 5e17573..575e91e 100755 --- a/iridiumtk/graph_voc.py +++ b/iridiumtk/graph_voc.py @@ -125,7 +125,6 @@ def cut_convert_play(self, t_start, t_stop, f_min, f_max): def read_lines(input_files, start_time_filter, end_time_filter): - lines = [] for line in fileinput.input(files=input_files): line = line.strip() if 'A:OK' in line and "Message: Couldn't parse:" not in line: @@ -136,8 +135,7 @@ def read_lines(input_files, start_time_filter, end_time_filter): continue if end_time_filter and end_time_filter < voc_line.datetime: continue - lines.append(voc_line) - return lines + yield voc_line def main(): parser = argparse.ArgumentParser(description='Convert iridium-parser.py VOC output to DFS') @@ -150,7 +148,7 @@ def main(): start_time_filter = dateparser.parse(args.start) if args.start else None end_time_filter = dateparser.parse(args.end) if args.end else None - lines = read_lines(input_files, start_time_filter, end_time_filter) + lines = list(read_lines(input_files, start_time_filter, end_time_filter)) number_of_lines = len(lines) logger.info('Read %d VOC lines from input', number_of_lines) diff --git a/iridiumtk/test_graph_voc.py b/iridiumtk/test_graph_voc.py index f9242df..157d78a 100644 --- a/iridiumtk/test_graph_voc.py +++ b/iridiumtk/test_graph_voc.py @@ -5,7 +5,7 @@ import tempfile -from .stats_voc import read_lines +from .graph_voc import read_lines class MainTest(unittest.TestCase): TEST_VOC_LINE_1 = 'VOC: i-1443338945.6543-t1 033399141 1625872817 81% 0.027 179 L:no LCW(0,001111,100000000000000000000 E1) 01111001000100010010010011011011011001111 011000010000100001110101111011110010010111011001010001011101010001100000000110010100000110111110010101110101001111010100111001000110100110001110110 1010101010010010001000001110011000001001001010011110011100110100111110001101110010110101010110011101011100011101011000000000 descr_extra:' @@ -27,7 +27,7 @@ def test_read_lines(self): input_file.write(MainTest.TEST_VOC_LINE_2 + '\n') input_file.write(MainTest.TEST_VOC_LINE_3 + '\n') - voc_lines = read_lines(input_file_path, None, None) + voc_lines = list(read_lines(input_file_path, None, None)) self.assertEquals(len(voc_lines), 2) def test_test_read_lines_with_raw_data(self): @@ -35,13 +35,13 @@ def test_test_read_lines_with_raw_data(self): with open(input_file_path, 'w') as input_file: input_file.write('RAW: i-1525892321-t1 0001338 1623702528 A:OK I:00000000027 79% 0.001 1 00\n') with self.assertRaises(RuntimeError): - read_lines(input_file_path, None, None) + list(read_lines(input_file_path, None, None)) def test_test_read_lines_with_parsed_errors(self): input_file_path = self.get_temp_file() with open(input_file_path, 'w') as input_file: input_file.write('RAW: i-1525892321-t1 000045987 1626110208 83% 0.001 <001100000011000011110011> 1100000000000000 .... 00011001 ERR:Message: unknown Iridium message type\n') - voc_lines = read_lines(input_file_path, None, None) + voc_lines = list(read_lines(input_file_path, None, None)) self.assertEquals(voc_lines, []) def tearDown(self): From 0a4342e43b248c47f3e08afc125bf777a3945f55 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sat, 12 May 2018 18:05:47 +0100 Subject: [PATCH 25/50] Fix play-iridium-ambe script --- play-iridium-ambe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/play-iridium-ambe b/play-iridium-ambe index 3a4b22b..5362f06 100755 --- a/play-iridium-ambe +++ b/play-iridium-ambe @@ -1,2 +1,2 @@ #!/bin/sh -bits_to_dfs.py - $1 | ir77_ambe_decode - - | aplay \ No newline at end of file +python -m iridiumtk.bits_to_dfs - $1 | ir77_ambe_decode - - | aplay From 1fe78faccdb07ab0e9920bf5bcbda54d2fe6d598 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sat, 12 May 2018 18:27:52 +0100 Subject: [PATCH 26/50] Remove multiple file writes when playing audio --- iridiumtk/graph_voc.py | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/iridiumtk/graph_voc.py b/iridiumtk/graph_voc.py index 575e91e..9e94621 100755 --- a/iridiumtk/graph_voc.py +++ b/iridiumtk/graph_voc.py @@ -5,10 +5,10 @@ import subprocess import fileinput import logging -import tempfile from datetime import datetime import argparse from collections import namedtuple +import subprocess import matplotlib.pyplot as plt @@ -79,16 +79,12 @@ def onclick(self, event): self.cut_convert_play(self.t_start, self.t_stop, self.f_min, self.f_max) def filter_voc(self, t_start, t_stop, f_min, f_max): - filtered_lines = [] - for voc_line in self.lines: ts = voc_line.datetime_unix f = voc_line.frequency if t_start <= ts and ts <= t_stop and \ f_min <= f and f <= f_max: - filtered_lines.append(voc_line.raw_line) - - return filtered_lines + yield voc_line.raw_line def cut_convert_play(self, t_start, t_stop, f_min, f_max): logger.info('Starting to play...') @@ -101,25 +97,18 @@ def cut_convert_play(self, t_start, t_stop, f_min, f_max): f_max = f_min f_min = tmp - filtered_lines = self.filter_voc(t_start, t_stop, f_min, f_max) - _, dfs_file_path = tempfile.mkstemp(suffix='.dfs') - _, wav_file_path = tempfile.mkstemp(suffix='.wav') + ir77_ambe_decode = subprocess.Popen(['ir77_ambe_decode'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + aplay = subprocess.Popen(['aplay'], stdin=ir77_ambe_decode.stdout, stderr=subprocess.PIPE) - logger.info('Making dfs file %s', dfs_file_path) - with open(dfs_file_path, 'w') as dfs_file: - bits_to_dfs(filtered_lines, dfs_file) - - logger.info('Making wav file %s', wav_file_path) - subprocess.check_call(['ir77_ambe_decode', dfs_file_path, wav_file_path]) - - logger.info('Cleaning up dfs') - os.remove(dfs_file_path) + filtered_lines = self.filter_voc(t_start, t_stop, f_min, f_max) + bits_to_dfs(filtered_lines, ir77_ambe_decode.stdin) + ir77_ambe_decode.stdin.close() - subprocess.check_call(['aplay', wav_file_path]) + logger.info('aplay "%s"', aplay.communicate()[1].strip()) - logger.info('Cleaning up wav') - os.remove(wav_file_path) + ir77_ambe_decode.wait() + aplay.wait() logger.info('Finished Playing') From 072396b51b2fee5a89b1ebc373656e742bb1aaac Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sun, 13 May 2018 10:32:08 +0100 Subject: [PATCH 27/50] Clean up audio playing in graph_voc.py --- iridiumtk/graph_voc.py | 45 ++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/iridiumtk/graph_voc.py b/iridiumtk/graph_voc.py index 9e94621..d6a8dfa 100755 --- a/iridiumtk/graph_voc.py +++ b/iridiumtk/graph_voc.py @@ -12,13 +12,11 @@ import matplotlib.pyplot as plt -from matplotlib.collections import BrokenBarHCollection import numpy as np import dateparser from .line_parser import VocLine -from .bits_to_dfs import bits_to_dfs logging.basicConfig(level=logging.INFO) @@ -53,6 +51,30 @@ DUPLEX_CHANELS = frozenset(DUPLEX_CHANELS) ALL_CHANELS = frozenset((SIMPLEX_CHANELS | DUPLEX_CHANELS)) + + +class VoiceDataPlayer(object): + def __init__(self): + self.ir77_ambe_decode = subprocess.Popen(['ir77_ambe_decode'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.aplay = subprocess.Popen(['aplay'], stdin=self.ir77_ambe_decode.stdout, stderr=subprocess.PIPE) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def play_bits(self, bits): + self.ir77_ambe_decode.stdin.write(bits) + + def close(self): + self.ir77_ambe_decode.stdin.close() + logger.info('aplay "%s"', self.aplay.communicate()[1].strip()) + + logger.info('Waiting for ir77_ambe_decode/aplay') + self.ir77_ambe_decode.wait() + self.aplay.wait() + logger.info('ir77_ambe_decode/aplay closed') class OnClickHandler(object): def __init__(self, lines): @@ -84,7 +106,7 @@ def filter_voc(self, t_start, t_stop, f_min, f_max): f = voc_line.frequency if t_start <= ts and ts <= t_stop and \ f_min <= f and f <= f_max: - yield voc_line.raw_line + yield voc_line def cut_convert_play(self, t_start, t_stop, f_min, f_max): logger.info('Starting to play...') @@ -97,18 +119,11 @@ def cut_convert_play(self, t_start, t_stop, f_min, f_max): f_max = f_min f_min = tmp - - ir77_ambe_decode = subprocess.Popen(['ir77_ambe_decode'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - aplay = subprocess.Popen(['aplay'], stdin=ir77_ambe_decode.stdout, stderr=subprocess.PIPE) - - filtered_lines = self.filter_voc(t_start, t_stop, f_min, f_max) - bits_to_dfs(filtered_lines, ir77_ambe_decode.stdin) - ir77_ambe_decode.stdin.close() - - logger.info('aplay "%s"', aplay.communicate()[1].strip()) - - ir77_ambe_decode.wait() - aplay.wait() + with VoiceDataPlayer() as player: + for voc_frame in self.filter_voc(t_start, t_stop, f_min, f_max): + voice_bits = voc_frame.voice_bits + if voice_bits is not None: + player.play_bits(voice_bits) logger.info('Finished Playing') From 2f3c205a7fa8dd8dc0d61942756e4e4911cec373 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sun, 13 May 2018 10:52:50 +0100 Subject: [PATCH 28/50] Add the first linter, flake8 --- .flake8 | 19 ++++++++++++++++ iridiumtk/bits_to_dfs.py | 2 +- iridiumtk/graph_by_type.py | 16 +++++--------- iridiumtk/graph_voc.py | 29 ++++++++++++------------- iridiumtk/line_parser/__init__.py | 2 +- iridiumtk/line_parser/base_line.py | 3 +-- iridiumtk/line_parser/test_base_line.py | 2 +- iridiumtk/line_parser/test_voc_line.py | 1 - iridiumtk/line_parser/voc_line.py | 15 ++++++------- iridiumtk/test_bits_to_dfs.py | 4 ++-- iridiumtk/test_graph_voc.py | 4 +++- setup.py | 4 ++-- tox.ini | 12 ++++++++++ 13 files changed, 69 insertions(+), 44 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..e989b6f --- /dev/null +++ b/.flake8 @@ -0,0 +1,19 @@ +# Flake8 Configuration +[flake8] +ignore = + D100 + # FIXME: E501 line length needs worked on + E501 +exclude = + .tox, + .git, + __pycache__, + build, + dist, + *.pyc, + *.egg-info, + .cache, + .eggs +import-order-style = google +max-line-length = 99 +format = ${cyan}%(path)s${reset}:${yellow_bold}%(row)d${reset}:${green_bold}%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s diff --git a/iridiumtk/bits_to_dfs.py b/iridiumtk/bits_to_dfs.py index 2a64145..1d2fd41 100755 --- a/iridiumtk/bits_to_dfs.py +++ b/iridiumtk/bits_to_dfs.py @@ -1,8 +1,8 @@ #!/usr/bin/env python +import argparse import fileinput import sys -import argparse from .line_parser import VocLine diff --git a/iridiumtk/graph_by_type.py b/iridiumtk/graph_by_type.py index d7adf6a..36e60a9 100755 --- a/iridiumtk/graph_by_type.py +++ b/iridiumtk/graph_by_type.py @@ -1,19 +1,16 @@ #!/usr/bin/python -import sys -import matplotlib.pyplot as plt import argparse -import collections import fileinput -from datetime import datetime import logging import dateparser +import matplotlib.pyplot as plt import numpy as np -from .line_parser import BaseLine from .graph_voc import ALL_CHANELS +from .line_parser import BaseLine logging.basicConfig(level=logging.INFO) @@ -21,7 +18,6 @@ def read_lines(input_files, start_time_filter, end_time_filter): - lines = [] for line in fileinput.input(files=input_files): line = line.strip() if 'A:OK' in line and "Message: Couldn't parse:" not in line: @@ -38,7 +34,7 @@ def read_lines(input_files, start_time_filter, end_time_filter): def main(): - parser = argparse.ArgumentParser(description='Visualise ????') # TODO + parser = argparse.ArgumentParser(description='Visualise ????') # TODO parser.add_argument('--start', metavar='DATETIME', default=None, help='Filter events before this time') parser.add_argument('--end', metavar='DATETIME', default=None, help='Filter events after this time') parser.add_argument('input', metavar='FILE', nargs='*', help='Files to read, if empty or -, stdin is used') @@ -59,7 +55,7 @@ def main(): logger.info(' - Read %d\t%s lines from input', len(frames), frame_type) fig = plt.figure() - + subplot = fig.add_subplot(1, 1, 1) for frame_type, frames in stats.iteritems(): number_of_frames = len(frames) @@ -81,15 +77,15 @@ def main(): color = 'tab:green' subplot.axhline(channel.frequency, color=color, alpha=0.3, label=channel.description) - # Shrink current axis's height on the bottom handles, labels = zip(*[(h, l) for (h, l) in zip(*subplot.get_legend_handles_labels()) if l in stats]) - subplot.legend(handles, labels, bbox_to_anchor=(1.04,1), loc="upper left") + subplot.legend(handles, labels, bbox_to_anchor=(1.04, 1), loc="upper left") plt.title('frequency vs time color coded by frame type') plt.xlabel('time') plt.ylabel('frequency') plt.show() + if __name__ == "__main__": main() diff --git a/iridiumtk/graph_voc.py b/iridiumtk/graph_voc.py index d6a8dfa..1c8c834 100755 --- a/iridiumtk/graph_voc.py +++ b/iridiumtk/graph_voc.py @@ -1,19 +1,16 @@ #!/usr/bin/python -import sys -import os -import subprocess -import fileinput -import logging -from datetime import datetime import argparse from collections import namedtuple +import fileinput +import logging import subprocess +import sys +import dateparser import matplotlib.pyplot as plt import numpy as np -import dateparser from .line_parser import VocLine @@ -75,7 +72,8 @@ def close(self): self.ir77_ambe_decode.wait() self.aplay.wait() logger.info('ir77_ambe_decode/aplay closed') - + + class OnClickHandler(object): def __init__(self, lines): self.lines = lines @@ -86,7 +84,7 @@ def __init__(self, lines): def onclick(self, event): logger.info('button=%d, x=%d, y=%d, xdata=%f, ydata=%f', - event.button, event.x, event.y, event.xdata, event.ydata) + event.button, event.x, event.y, event.xdata, event.ydata) if event.button == 1: self.t_start = event.xdata @@ -96,7 +94,7 @@ def onclick(self, event): if event.button == 3: self.t_stop = event.xdata self.f_max = event.ydata - + if self.t_start and self.t_stop: self.cut_convert_play(self.t_start, self.t_stop, self.f_min, self.f_max) @@ -133,7 +131,7 @@ def read_lines(input_files, start_time_filter, end_time_filter): line = line.strip() if 'A:OK' in line and "Message: Couldn't parse:" not in line: raise RuntimeError('Expected "iridium-parser.py" parsed data. Found raw "iridium-extractor" data.') - if 'VOC: ' in line and not "LCW(0,001111,100000000000000000000" in line: + if 'VOC: ' in line and "LCW(0,001111,100000000000000000000" not in line: voc_line = VocLine(line) if start_time_filter and start_time_filter > voc_line.datetime: continue @@ -141,6 +139,7 @@ def read_lines(input_files, start_time_filter, end_time_filter): continue yield voc_line + def main(): parser = argparse.ArgumentParser(description='Convert iridium-parser.py VOC output to DFS') parser.add_argument('--start', metavar='DATETIME', default=None, help='Filter events before this time') @@ -163,18 +162,18 @@ def main(): plot_data_time = np.empty(number_of_lines, dtype=np.float64) plot_data_freq = np.empty(number_of_lines, dtype=np.uint32) for i, voc_line in enumerate(lines): - #plot_data_time[i] = np.datetime64(voc_line.datetime().isoformat()) + # plot_data_time[i] = np.datetime64(voc_line.datetime().isoformat()) plot_data_time[i] = np.uint32(voc_line.datetime_unix) plot_data_freq[i] = np.float64(voc_line.frequency) fig = plt.figure() - #fig.autofmt_xdate() + # fig.autofmt_xdate() on_click_handler = OnClickHandler(lines) fig.canvas.mpl_connect('button_press_event', on_click_handler.onclick) - + subplot = fig.add_subplot(1, 1, 1) subplot.scatter(plot_data_time, plot_data_freq) - #subplot.xaxis_date() + # subplot.xaxis_date() for channel in ALL_CHANELS: if 'Guard' in channel.description: diff --git a/iridiumtk/line_parser/__init__.py b/iridiumtk/line_parser/__init__.py index a680dc5..6990b39 100644 --- a/iridiumtk/line_parser/__init__.py +++ b/iridiumtk/line_parser/__init__.py @@ -2,4 +2,4 @@ from .voc_line import BaseLine, VocLine -__all__ = [x.__name__ for x in (BaseLine, VocLine)] \ No newline at end of file +__all__ = [x.__name__ for x in (BaseLine, VocLine)] diff --git a/iridiumtk/line_parser/base_line.py b/iridiumtk/line_parser/base_line.py index 43ccdfd..3dc1838 100644 --- a/iridiumtk/line_parser/base_line.py +++ b/iridiumtk/line_parser/base_line.py @@ -1,8 +1,7 @@ #!/usr/bin/env python -import logging -from io import BytesIO from datetime import datetime +import logging import six diff --git a/iridiumtk/line_parser/test_base_line.py b/iridiumtk/line_parser/test_base_line.py index 6c4a48f..024845a 100644 --- a/iridiumtk/line_parser/test_base_line.py +++ b/iridiumtk/line_parser/test_base_line.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -import unittest from datetime import datetime +import unittest from .base_line import BaseLine diff --git a/iridiumtk/line_parser/test_voc_line.py b/iridiumtk/line_parser/test_voc_line.py index ee6097e..2086149 100644 --- a/iridiumtk/line_parser/test_voc_line.py +++ b/iridiumtk/line_parser/test_voc_line.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import unittest -from datetime import datetime from .voc_line import chunks, VocLine diff --git a/iridiumtk/line_parser/voc_line.py b/iridiumtk/line_parser/voc_line.py index 5b36165..59f1e59 100644 --- a/iridiumtk/line_parser/voc_line.py +++ b/iridiumtk/line_parser/voc_line.py @@ -1,8 +1,7 @@ #!/usr/bin/env python -import logging from io import BytesIO -from datetime import datetime +import logging import six @@ -19,7 +18,7 @@ def chunks(l, n): """ Yield successive n-sized chunks from l. """ for i in xrange(0, len(l), n): - yield l[i:i+n] + yield l[i:i + n] # Example lines @@ -48,12 +47,12 @@ def voice_bits(self): return None byte_stream = BytesIO() if data[0] == "[": - for pos in xrange(1,len(data),3): - byte=int(data[pos:pos+2],16) - byte=int('{:08b}'.format(byte)[::-1], 2) + for pos in xrange(1, len(data), 3): + byte = int(data[pos:pos + 2], 16) + byte = int('{:08b}'.format(byte)[::-1], 2) byte_stream.write(chr(byte)) else: for bits in chunks(data, 8): - byte = int(bits[::-1],2) + byte = int(bits[::-1], 2) byte_stream.write(chr(byte)) - return byte_stream.getvalue() + return byte_stream.getvalue() diff --git a/iridiumtk/test_bits_to_dfs.py b/iridiumtk/test_bits_to_dfs.py index 610e15e..86625db 100644 --- a/iridiumtk/test_bits_to_dfs.py +++ b/iridiumtk/test_bits_to_dfs.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -import unittest from io import BytesIO +import unittest from .bits_to_dfs import bits_to_dfs @@ -21,7 +21,7 @@ def test_raw_data(self): output = BytesIO() with self.assertRaises(RuntimeError): bits_to_dfs(['RAW: i-1525892321-t1 0001338 1623702528 A:OK I:00000000027 79% 0.001 179 0011000000110000111100110001000000010011100100011000001000101111100010010101110100110101010101010101010101010101010101010101010101011100010111010001010101010101010101110011010011010101010101010101010111000101010101011101001101010101010101010101010101010101010101010101010101010101010101010101010101010101010101110001110101000111001101010101010100110101010101010101010101010101010101'], output) - + output = BytesIO() bits_to_dfs(['RAW: i-1525892321-t1 000045987 1626110208 83% 0.001 <001100000011000011110011> 1100000000000000 0000000000000000 0000000000000000 1001000000000000 0000000000000000 0000000000000000 0100001000110110 0111100001010110 1011001111101110 1010110101000101 1111111001110101 1011110101110001 1110000001110111 0110001000001010 0100100100111000 1000001111010100 1011001101011110 0100011011100010 1111110001010001 1100110110101111 0011100111100101 1110100100000110 0111011111110010 1111110011001000 1101000011000011 1011110101110111 0000101000000001 1000111010101100 1011000001010011 0111011100011101 0101011000011101 0111110001001101 0100001011001010 1101001110100010 0111011001000101 0100111001010110 0001111110101010 1110001100010111 0001010101100100 1001010100111011 1001111110001101 0110100100010010 0001110111101001 1000011010111100 00011001 ERR:Message: unknown Iridium message type'], output) self.assertEquals(output.getvalue(), '') diff --git a/iridiumtk/test_graph_voc.py b/iridiumtk/test_graph_voc.py index 157d78a..2d7e114 100644 --- a/iridiumtk/test_graph_voc.py +++ b/iridiumtk/test_graph_voc.py @@ -1,12 +1,13 @@ #!/usr/bin/env python -import unittest import os import tempfile +import unittest from .graph_voc import read_lines + class MainTest(unittest.TestCase): TEST_VOC_LINE_1 = 'VOC: i-1443338945.6543-t1 033399141 1625872817 81% 0.027 179 L:no LCW(0,001111,100000000000000000000 E1) 01111001000100010010010011011011011001111 011000010000100001110101111011110010010111011001010001011101010001100000000110010100000110111110010101110101001111010100111001000110100110001110110 1010101010010010001000001110011000001001001010011110011100110100111110001101110010110101010110011101011100011101011000000000 descr_extra:' TEST_VOC_LINE_2 = 'VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f]' @@ -48,6 +49,7 @@ def tearDown(self): for file in self.tempfiles: os.remove(file) + def main(): unittest.main() diff --git a/setup.py b/setup.py index d3d0104..94ade6b 100644 --- a/setup.py +++ b/setup.py @@ -53,10 +53,10 @@ def run_tests(self): setup( name=title, version=version, - description='iridiumtk', # TODO + description='iridiumtk', # TODO long_description=long_description, author=author, - author_email='thebigguy.co.uk@gmail.com', # TODO + author_email='thebigguy.co.uk@gmail.com', # TODO url='https://github.com/muccc/iridium-toolkit', packages=find_packages(exclude=['9601', 'ambe_emu', 'nal-shout', 'rtl-sdr', 'tools', 'tracking']), entry_points={ diff --git a/tox.ini b/tox.ini index 2b9861d..d96101f 100644 --- a/tox.ini +++ b/tox.ini @@ -12,3 +12,15 @@ commands = coverage run --parallel-mode -m pytest {posargs} # substitute with tox' positional arguments coverage combine coverage report -m + +# Linters +[testenv:flake8] +basepython = python2.7 +skip_install = true +deps = + flake8 + flake8-import-order>=0.9 + pep8-naming + flake8-colors +commands = + flake8 iridiumtk/ setup.py \ No newline at end of file From d56a4a1f40f71bb1fcc65d53fe90077d3bd04872 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sun, 13 May 2018 11:10:05 +0100 Subject: [PATCH 29/50] Add pylint to linters --- .pylintrc | 43 +++++++++++++++++++++++++ iridiumtk/graph_voc.py | 3 +- iridiumtk/line_parser/base_line.py | 8 +++-- iridiumtk/line_parser/test_base_line.py | 4 +-- iridiumtk/line_parser/test_voc_line.py | 3 +- iridiumtk/line_parser/voc_line.py | 6 ++-- iridiumtk/test_graph_voc.py | 4 +-- tox.ini | 23 +++++++++++-- 8 files changed, 81 insertions(+), 13 deletions(-) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..e37abdb --- /dev/null +++ b/.pylintrc @@ -0,0 +1,43 @@ +[MASTER] + +# Use multiple processes to speed up Pylint. +jobs=4 + +[MESSAGES CONTROL] + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" + +disable= + missing-docstring, + line-too-long, + bad-continuation, + invalid-name, + bad-whitespace, + too-many-arguments, + too-many-locals, + len-as-condition, + too-few-public-methods, + fixme, + redefined-argument-from-local, + too-many-return-statements, + unsubscriptable-object, + too-many-boolean-expressions, + too-many-branches, + too-many-statements, + unused-variable, + arguments-differ, + method-hidden, + unused-import, + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 diff --git a/iridiumtk/graph_voc.py b/iridiumtk/graph_voc.py index 1c8c834..238dab2 100755 --- a/iridiumtk/graph_voc.py +++ b/iridiumtk/graph_voc.py @@ -1,5 +1,6 @@ #!/usr/bin/python +from __future__ import print_function import argparse from collections import namedtuple import fileinput @@ -156,7 +157,7 @@ def main(): logger.info('Read %d VOC lines from input', number_of_lines) if number_of_lines == 0: - print('No usable data found') + print('No usable data found', file=sys.stderr) sys.exit(1) plot_data_time = np.empty(number_of_lines, dtype=np.float64) diff --git a/iridiumtk/line_parser/base_line.py b/iridiumtk/line_parser/base_line.py index 3dc1838..bf5102a 100644 --- a/iridiumtk/line_parser/base_line.py +++ b/iridiumtk/line_parser/base_line.py @@ -11,6 +11,10 @@ logger = logging.getLogger(__name__) +class LineParseException(Exception): + pass + + # Example lines # VOC: i-1430527570.4954-t1 421036605 1625859953 66% 0.008 219 L:no LCW(0,001111,100000000000000000000 E1) 101110110101010100101101111000111111111001011111001011010001000010010001101110011010011001111111011101111100011001001001000111001101001011001011000101111111101110110011111000000001110010001110101101001010011001101001010111101100011100110011110010110110101010110001010000100100101011010010100100100011010110101001 # VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f] @@ -29,9 +33,9 @@ def __init__(self, line): self._timestamp = ts_base_ms + (time_offset_ns / 1000) self._frequnecy = int(line_split[3]) - except Exception as e: + except (IndexError, ValueError) as e: logger.error('Failed to parse line "%s"', line) - six.raise_from(Exception('Failed to parse line "{}"'.format(line), e), e) + six.raise_from(LineParseException('Failed to parse line "{}"'.format(line), e), e) @property def raw_line(self): diff --git a/iridiumtk/line_parser/test_base_line.py b/iridiumtk/line_parser/test_base_line.py index 024845a..6f8593b 100644 --- a/iridiumtk/line_parser/test_base_line.py +++ b/iridiumtk/line_parser/test_base_line.py @@ -4,7 +4,7 @@ import unittest -from .base_line import BaseLine +from .base_line import BaseLine, LineParseException class BaseLineTest(unittest.TestCase): @@ -12,7 +12,7 @@ class BaseLineTest(unittest.TestCase): TEST_VOC_LINE_2 = 'VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f]' def test_empty_input(self): - with self.assertRaises(Exception): + with self.assertRaises(LineParseException): BaseLine('') def test_old_format_datetime(self): diff --git a/iridiumtk/line_parser/test_voc_line.py b/iridiumtk/line_parser/test_voc_line.py index 2086149..22871ca 100644 --- a/iridiumtk/line_parser/test_voc_line.py +++ b/iridiumtk/line_parser/test_voc_line.py @@ -3,6 +3,7 @@ import unittest +from .base_line import LineParseException from .voc_line import chunks, VocLine @@ -18,7 +19,7 @@ class VocLineTest(unittest.TestCase): TEST_VOC_LINE_3 = 'VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 178 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff]' def test_empty_input(self): - with self.assertRaises(Exception): + with self.assertRaises(LineParseException): VocLine('') def test_old_format(self): diff --git a/iridiumtk/line_parser/voc_line.py b/iridiumtk/line_parser/voc_line.py index 59f1e59..22c1193 100644 --- a/iridiumtk/line_parser/voc_line.py +++ b/iridiumtk/line_parser/voc_line.py @@ -7,7 +7,7 @@ import six -from .base_line import BaseLine +from .base_line import BaseLine, LineParseException logging.basicConfig(level=logging.INFO) @@ -36,9 +36,9 @@ def __init__(self, line): self._voice_data = None else: self._voice_data = line_split[10] - except Exception as e: + except (IndexError, ValueError) as e: logger.error('Failed to parse line "%s"', line) - six.raise_from(Exception('Failed to parse line "{}"'.format(line), e), e) + six.raise_from(LineParseException('Failed to parse line "{}"'.format(line), e), e) @property def voice_bits(self): diff --git a/iridiumtk/test_graph_voc.py b/iridiumtk/test_graph_voc.py index 2d7e114..9f2dc6f 100644 --- a/iridiumtk/test_graph_voc.py +++ b/iridiumtk/test_graph_voc.py @@ -46,8 +46,8 @@ def test_test_read_lines_with_parsed_errors(self): self.assertEquals(voc_lines, []) def tearDown(self): - for file in self.tempfiles: - os.remove(file) + for path in self.tempfiles: + os.remove(path) def main(): diff --git a/tox.ini b/tox.ini index d96101f..794a3ad 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, pypy +envlist = py27 platform = linux2 skipsdist = True @@ -23,4 +23,23 @@ deps = pep8-naming flake8-colors commands = - flake8 iridiumtk/ setup.py \ No newline at end of file + flake8 iridiumtk/ setup.py + +[testenv:pylint] +basepython = python2.7 +skip_install = true +deps = + {[testenv]deps} + pyflakes + pylint +commands = + pylint iridiumtk + +[testenv:linters] +basepython = python2.7 +deps = + {[testenv:flake8]deps} + {[testenv:pylint]deps} +commands = + {[testenv:flake8]commands} + {[testenv:pylint]commands} From 4d421e201f465e0867b32aae634e4e227b67e845 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sun, 13 May 2018 11:13:56 +0100 Subject: [PATCH 30/50] Enable more linter options --- .flake8 | 2 +- .pylintrc | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.flake8 b/.flake8 index e989b6f..448a79e 100644 --- a/.flake8 +++ b/.flake8 @@ -15,5 +15,5 @@ exclude = .cache, .eggs import-order-style = google -max-line-length = 99 +max-line-length = 100 format = ${cyan}%(path)s${reset}:${yellow_bold}%(row)d${reset}:${green_bold}%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s diff --git a/.pylintrc b/.pylintrc index e37abdb..11d426e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -20,7 +20,6 @@ disable= line-too-long, bad-continuation, invalid-name, - bad-whitespace, too-many-arguments, too-many-locals, len-as-condition, @@ -32,10 +31,6 @@ disable= too-many-boolean-expressions, too-many-branches, too-many-statements, - unused-variable, - arguments-differ, - method-hidden, - unused-import, [FORMAT] From 607afe94faa1da1fe3f7e42595f05d930178c0d3 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sun, 13 May 2018 17:35:12 +0100 Subject: [PATCH 31/50] Clean up rx-stats-X-hist.py ready for merging --- rx-stats-freq-hist.py | 19 +++++++++++----- rx-stats-len-hist.py | 21 +++++++++++------ rx-stats.py => rx-stats-time-hist.py | 34 ++++++++++++++++------------ bitutils.py => rx_stats_bitutils.py | 0 4 files changed, 46 insertions(+), 28 deletions(-) rename rx-stats.py => rx-stats-time-hist.py (51%) rename bitutils.py => rx_stats_bitutils.py (100%) diff --git a/rx-stats-freq-hist.py b/rx-stats-freq-hist.py index 7769846..c92ffc6 100755 --- a/rx-stats-freq-hist.py +++ b/rx-stats-freq-hist.py @@ -2,22 +2,27 @@ # vim: set ts=4 sw=4 tw=0 et pm=: # Parses .bits files and displays the distribution -# of the length of received frames +# of the "HIST_DIMENTION_KEY" of received frames import sys import matplotlib.pyplot as plt import getopt -import bitutils +import rx_stats_bitutils as bitutils -options, remainder = getopt.getopt(sys.argv[1:], 'b:c:eo', [ +HIST_DIMENTION = 'frequency' +HIST_DIMENTION_KEY = 'freq' + +options, remainder = getopt.getopt(sys.argv[1:], 'b:c:l:eo', [ 'bin=', 'minimum_confidence=', + 'minimum_length=', 'errors', 'lead_out_required' ]) bin_size = 10000 minimum_confidence = 0 +minimum_length = 0 lead_out_required = False show_errors = False @@ -26,6 +31,8 @@ bin_size = int(arg) elif opt in ('-c', '--minimum_confidence'): minimum_confidence = int(arg) + elif opt in ('-l', '--minimum_length'): + minimum_length = int(arg) elif opt in ('-o', '--lead_out_required'): lead_out_required = True elif opt in ('-e', '--errors'): @@ -36,12 +43,12 @@ messages = bitutils.read_file(remainder) -data = [s['freq'] for s in messages if s['length'] > 100 and s['confidence'] > minimum_confidence and (s['lead_out'] or not lead_out_required) and (s['error'] == show_errors)] +data = [s[HIST_DIMENTION_KEY] for s in messages if s['length'] > minimum_length and s['confidence'] > minimum_confidence and (s['lead_out'] or not lead_out_required) and (s['error'] == show_errors) and s['freq'] < 1.626e9] -bins = (max(data) - min(data))/bin_size +bins = int((max(data) - min(data))/bin_size) filename=["",",".join(remainder)][remainder is None] -title = "File: %s : Distribution of message frequency. Bin Size: %d, Minimum Confidence: %d" % (filename, bin_size, minimum_confidence) +title = "File: %s : Distribution of message %s. Bin Size: %d, Minimum Confidence: %d" % (filename, HIST_DIMENTION, bin_size, minimum_confidence) if lead_out_required: title += ', lead out needs to be present' else: diff --git a/rx-stats-len-hist.py b/rx-stats-len-hist.py index c3ffe46..1386e70 100755 --- a/rx-stats-len-hist.py +++ b/rx-stats-len-hist.py @@ -2,22 +2,27 @@ # vim: set ts=4 sw=4 tw=0 et pm=: # Parses .bits files and displays the distribution -# of the length of received frames +# of the "HIST_DIMENTION_KEY" of received frames import sys import matplotlib.pyplot as plt import getopt -import bitutils +import rx_stats_bitutils as bitutils -options, remainder = getopt.getopt(sys.argv[1:], 'b:c:eo', [ +HIST_DIMENTION = 'length' +HIST_DIMENTION_KEY = 'length' + +options, remainder = getopt.getopt(sys.argv[1:], 'b:c:l:eo', [ 'bin=', 'minimum_confidence=', + 'minimum_length=', 'errors', 'lead_out_required' ]) bin_size = 1 minimum_confidence = 0 +minimum_length = 0 lead_out_required = False show_errors = False @@ -26,6 +31,8 @@ bin_size = int(arg) elif opt in ('-c', '--minimum_confidence'): minimum_confidence = int(arg) + elif opt in ('-l', '--minimum_length'): + minimum_length = int(arg) elif opt in ('-o', '--lead_out_required'): lead_out_required = True elif opt in ('-e', '--errors'): @@ -36,12 +43,12 @@ messages = bitutils.read_file(remainder) -lens = [s['length'] for s in messages if s['length'] > 100 and s['confidence'] > minimum_confidence and (s['lead_out'] or not lead_out_required) and (s['error'] == show_errors) and s['freq'] < 1.626e9] +data = [s[HIST_DIMENTION_KEY] for s in messages if s['length'] > 100 and s['confidence'] > minimum_confidence and (s['lead_out'] or not lead_out_required) and (s['error'] == show_errors) and s['freq'] < 1.626e9] -bins = (max(lens) - min(lens))/bin_size +bins = (max(data) - min(data))/bin_size filename=["",",".join(remainder)][remainder is None] -title = "File: %s : Distribution of message length. Bin Size: %d, Minimum Confidence: %d" % (filename, bin_size, minimum_confidence) +title = "File: %s : Distribution of message %s. Bin Size: %d, Minimum Confidence: %d" % (filename, HIST_DIMENTION, bin_size, minimum_confidence) if lead_out_required: title += ', lead out needs to be present' else: @@ -50,5 +57,5 @@ title += " and having decoding errors" plt.title(title) -plt.hist(lens, bins) +plt.hist(data, bins) plt.show() diff --git a/rx-stats.py b/rx-stats-time-hist.py similarity index 51% rename from rx-stats.py rename to rx-stats-time-hist.py index b9c4917..af74716 100755 --- a/rx-stats.py +++ b/rx-stats-time-hist.py @@ -1,29 +1,36 @@ #!/usr/bin/env python # vim: set ts=4 sw=4 tw=0 et pm=: -# Parses .bits files and displays how many messages have -# been received over time. Usefull to tune a receiving setup +# Parses .bits files and displays the distribution +# of the "HIST_DIMENTION_KEY" of received frames import sys import matplotlib.pyplot as plt import getopt -import bitutils +import rx_stats_bitutils as bitutils -options, remainder = getopt.getopt(sys.argv[1:], 's:l:eo', [ - 'span=', +HIST_DIMENTION = 'time' +HIST_DIMENTION_KEY = 'timestamp' + +options, remainder = getopt.getopt(sys.argv[1:], 'b:c:l:eo', [ + 'bin=', + 'minimum_confidence=', 'minimum_length=', 'errors', 'lead_out_required' ]) -span = 3600 +bin_size = 3600 +minimum_confidence = 0 minimum_length = 0 lead_out_required = False show_errors = False for opt, arg in options: - if opt in ('-s', '--span'): - span = int(arg) + if opt in ('-b', '--bin'): + bin_size = int(arg) + elif opt in ('-c', '--minimum_confidence'): + minimum_confidence = int(arg) elif opt in ('-l', '--minimum_length'): minimum_length = int(arg) elif opt in ('-o', '--lead_out_required'): @@ -36,15 +43,12 @@ messages = bitutils.read_file(remainder) -timestamps = [s['timestamp'] for s in messages if s['length'] > minimum_length and (not lead_out_required or s['lead_out']) and (s['error'] == show_errors)] - -t0 = min(timestamps) -t = max(timestamps) +data = [s[HIST_DIMENTION_KEY] for s in messages if s['length'] > minimum_length and s['confidence'] > minimum_confidence and (s['lead_out'] or not lead_out_required) and (s['error'] == show_errors) and s['freq'] < 1.626e9] -bins = (t - t0)/span +bins = int((max(data) - min(data))/bin_size) filename=["",",".join(remainder)][remainder is None] -title = "File: %s : Messages per %d seconds, longer than %d symbols" % (filename, span, minimum_length) +title = "File: %s : Distribution of message %s. Bin Size: %d, Minimum Confidence: %d" % (filename, HIST_DIMENTION, bin_size, minimum_confidence) if lead_out_required: title += ', lead out needs to be present' else: @@ -53,5 +57,5 @@ title += " and having decoding errors" plt.title(title) -plt.hist(timestamps, bins) +plt.hist(data, bins) plt.show() diff --git a/bitutils.py b/rx_stats_bitutils.py similarity index 100% rename from bitutils.py rename to rx_stats_bitutils.py From 4bf627723ec0a3f942d9f67910aaef8d657c901e Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sun, 13 May 2018 17:36:25 +0100 Subject: [PATCH 32/50] Add template for iridiumtk/test_graph_by_type.py --- iridiumtk/test_graph_by_type.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 iridiumtk/test_graph_by_type.py diff --git a/iridiumtk/test_graph_by_type.py b/iridiumtk/test_graph_by_type.py new file mode 100644 index 0000000..d798b2a --- /dev/null +++ b/iridiumtk/test_graph_by_type.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import unittest + + +class MainTest(unittest.TestCase): + def test_pass(self): + pass + + +def main(): + unittest.main() + + +if __name__ == "__main__": + main() From 8faaee55a15e458949c5e52edd0399ef245cfe22 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sun, 13 May 2018 22:12:35 +0100 Subject: [PATCH 33/50] Fix lint rule --- iridiumtk/graph_voc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/iridiumtk/graph_voc.py b/iridiumtk/graph_voc.py index 238dab2..cf39ec6 100755 --- a/iridiumtk/graph_voc.py +++ b/iridiumtk/graph_voc.py @@ -1,6 +1,7 @@ #!/usr/bin/python from __future__ import print_function + import argparse from collections import namedtuple import fileinput From 4949aee5bc1cbfd611c7b2c7544f658fe827dcbc Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sun, 13 May 2018 22:20:40 +0100 Subject: [PATCH 34/50] Merge common rx_stats tools into rx_stats_hist.py --- iridiumtk/rx_stats_hist.py | 163 ++++++++++++++++++++++++++++++++ iridiumtk/test_rx_stats_hist.py | 16 ++++ rx-stats-freq-hist.py | 61 ------------ rx-stats-len-hist.py | 61 ------------ rx-stats-time-hist.py | 61 ------------ rx_stats_bitutils.py | 66 ------------- setup.py | 1 + 7 files changed, 180 insertions(+), 249 deletions(-) create mode 100755 iridiumtk/rx_stats_hist.py create mode 100644 iridiumtk/test_rx_stats_hist.py delete mode 100755 rx-stats-freq-hist.py delete mode 100755 rx-stats-len-hist.py delete mode 100755 rx-stats-time-hist.py delete mode 100644 rx_stats_bitutils.py diff --git a/iridiumtk/rx_stats_hist.py b/iridiumtk/rx_stats_hist.py new file mode 100755 index 0000000..ae7dd0b --- /dev/null +++ b/iridiumtk/rx_stats_hist.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +# vim: set ts=4 sw=4 tw=0 et pm=: + +# Parses .bits files and displays the distribution +# of the "HIST_DIMENTION_KEY" of received frames + +from __future__ import print_function + +import argparse +from collections import namedtuple +from datetime import datetime +import fileinput +import logging +import re +import sys + + +import dateparser +import matplotlib.pyplot as plt + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +Dimension = namedtuple('Dimension', ['key', 'bin_size']) +HIST_DIMENSIONS = { + 'frequency': Dimension('freq', 10000), + 'length': Dimension('length', 1), + 'time': Dimension('timestamp', 3600), +} + + +def extract_timestamp(filename, dt): + mm = re.match(r'i-(\d+(?:\.\d+)?)-[vbsrtl]1.([a-z])([a-z])', filename) + if mm: + b26 = (ord(mm.group(2)) - ord('a')) * 26 + ord(mm.group(3)) - ord('a') + timestamp = float(mm.group(1)) + float(dt) / 1000 + b26 * 600 + return timestamp + + mm = re.match(r'i-(\d+(?:\.\d+)?)-[vbsrtl]1(?:-o[+-]\d+)?$', filename) + if mm: + timestamp = float(mm.group(1)) + float(dt) / 1000 + return timestamp + + mm = re.match(r'(\d\d)-(\d\d)-(20\d\d)T(\d\d)-(\d\d)-(\d\d)-[sr]1', filename) + if mm: + month, day, year, hour, minute, second = map(int, mm.groups()) + timestamp = datetime(year, month, day, hour, minute, second) + timestamp = (timestamp - datetime(1970, 1, 1)).total_seconds() + timestamp += float(dt) / 1000 + return timestamp + + return 0 + + +def parse_line_to_message(line): + line = line.split() + if not line[0] == 'RX' and ('A:OK' not in line or len(line) < 10): + return None + access = True + lead_out = 'L:OK' in line + name = line[1] + if name == "X": + timestamp = float(line[2]) + else: + timestamp = extract_timestamp(name, line[2]) + freq = int(line[3]) + confidence = int(line[6][:-1]) + strength = float(line[7]) + length = int(line[8]) + if name == "X": + error = line[9] == 'True' + msgtype = line[10] + else: + error = False + msgtype = None + + return { + 'name': name, + 'timestamp': timestamp, + 'freq': freq, + 'access': access, + 'lead_out': lead_out, + 'confidence': confidence, + 'strength': strength, + 'length': length, + 'error': error, + 'msgtype': msgtype, + } + + +def read_lines(input_files, start_time_filter, end_time_filter): + for line in fileinput.input(files=input_files): + try: + message = parse_line_to_message(line) + except (IndexError, ValueError): + continue + if not message: + continue + timestamp = datetime.utcfromtimestamp(message['timestamp']) + if start_time_filter and start_time_filter > timestamp: + continue + if end_time_filter and end_time_filter < timestamp: + continue + yield message + + +def main(): + parser = argparse.ArgumentParser(description='Convert iridium-parser.py VOC output to DFS') + parser.add_argument('--start', metavar='DATETIME', type=str, default=None, help='Filter events before this time') + parser.add_argument('--end', metavar='DATETIME', type=str, default=None, help='Filter events after this time') + + parser.add_argument('--bin-size', metavar='INT', type=int, default=None, help='Size of bins') + parser.add_argument('--minimum-length', metavar='INT', type=int, default=0) + parser.add_argument('--minimum-confidence', metavar='INT', type=int, default=0) + parser.add_argument('--lead-out-required', metavar='INT', type=bool, default=False) + parser.add_argument('--show-errors', metavar='INT', type=bool, default=False) + + parser.add_argument('--dimension', choices=HIST_DIMENSIONS.keys(), required=True) + + parser.add_argument('input', metavar='FILE', nargs='*', help='Files to read, if empty or -, stdin is used') + args = parser.parse_args() + + input_files = args.input if len(args.input) > 0 else ['-'] + start_time_filter = dateparser.parse(args.start) if args.start else None + end_time_filter = dateparser.parse(args.end) if args.end else None + + dimension = HIST_DIMENSIONS[args.dimension] + + bin_size = args.bin_size if args.bin_size else dimension.bin_size + minimum_confidence = args.minimum_confidence + minimum_length = args.minimum_length + lead_out_required = args.lead_out_required + show_errors = args.show_errors + + lines = list(read_lines(input_files, start_time_filter, end_time_filter)) + number_of_lines = len(lines) + logger.info('Read %d lines from input', number_of_lines) + + if number_of_lines == 0: + print('No usable data found', file=sys.stderr) + sys.exit(1) + + data = [s[dimension.key] for s in lines if s['length'] > minimum_length and s['confidence'] > minimum_confidence and (s['lead_out'] or not lead_out_required) and (s['error'] == show_errors) and s['freq'] < 1.626e9] + + bins = int((max(data) - min(data)) / bin_size) + + title = "File: %s : Distribution of message %s. Bin Size: %d, Minimum Confidence: %d" % (input_files, args.dimension, bin_size, minimum_confidence) + if lead_out_required: + title += ', lead out needs to be present' + else: + title += ', lead out does not need to be present' + if show_errors: + title += " and having decoding errors" + + plt.title(title) + plt.hist(data, bins) + plt.show() + + +if __name__ == '__main__': + main() diff --git a/iridiumtk/test_rx_stats_hist.py b/iridiumtk/test_rx_stats_hist.py new file mode 100644 index 0000000..d798b2a --- /dev/null +++ b/iridiumtk/test_rx_stats_hist.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import unittest + + +class MainTest(unittest.TestCase): + def test_pass(self): + pass + + +def main(): + unittest.main() + + +if __name__ == "__main__": + main() diff --git a/rx-stats-freq-hist.py b/rx-stats-freq-hist.py deleted file mode 100755 index c92ffc6..0000000 --- a/rx-stats-freq-hist.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# vim: set ts=4 sw=4 tw=0 et pm=: - -# Parses .bits files and displays the distribution -# of the "HIST_DIMENTION_KEY" of received frames - -import sys -import matplotlib.pyplot as plt -import getopt - -import rx_stats_bitutils as bitutils - -HIST_DIMENTION = 'frequency' -HIST_DIMENTION_KEY = 'freq' - -options, remainder = getopt.getopt(sys.argv[1:], 'b:c:l:eo', [ - 'bin=', - 'minimum_confidence=', - 'minimum_length=', - 'errors', - 'lead_out_required' - ]) -bin_size = 10000 -minimum_confidence = 0 -minimum_length = 0 -lead_out_required = False -show_errors = False - -for opt, arg in options: - if opt in ('-b', '--bin'): - bin_size = int(arg) - elif opt in ('-c', '--minimum_confidence'): - minimum_confidence = int(arg) - elif opt in ('-l', '--minimum_length'): - minimum_length = int(arg) - elif opt in ('-o', '--lead_out_required'): - lead_out_required = True - elif opt in ('-e', '--errors'): - show_errors = True - else: - print opt - raise Exception("unknown argument?") - -messages = bitutils.read_file(remainder) - -data = [s[HIST_DIMENTION_KEY] for s in messages if s['length'] > minimum_length and s['confidence'] > minimum_confidence and (s['lead_out'] or not lead_out_required) and (s['error'] == show_errors) and s['freq'] < 1.626e9] - -bins = int((max(data) - min(data))/bin_size) - -filename=["",",".join(remainder)][remainder is None] -title = "File: %s : Distribution of message %s. Bin Size: %d, Minimum Confidence: %d" % (filename, HIST_DIMENTION, bin_size, minimum_confidence) -if lead_out_required: - title += ', lead out needs to be present' -else: - title += ', lead out does not need to be present' -if show_errors: - title += " and having decoding errors" - -plt.title(title) -plt.hist(data, bins) -plt.show() diff --git a/rx-stats-len-hist.py b/rx-stats-len-hist.py deleted file mode 100755 index 1386e70..0000000 --- a/rx-stats-len-hist.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# vim: set ts=4 sw=4 tw=0 et pm=: - -# Parses .bits files and displays the distribution -# of the "HIST_DIMENTION_KEY" of received frames - -import sys -import matplotlib.pyplot as plt -import getopt - -import rx_stats_bitutils as bitutils - -HIST_DIMENTION = 'length' -HIST_DIMENTION_KEY = 'length' - -options, remainder = getopt.getopt(sys.argv[1:], 'b:c:l:eo', [ - 'bin=', - 'minimum_confidence=', - 'minimum_length=', - 'errors', - 'lead_out_required' - ]) -bin_size = 1 -minimum_confidence = 0 -minimum_length = 0 -lead_out_required = False -show_errors = False - -for opt, arg in options: - if opt in ('-b', '--bin'): - bin_size = int(arg) - elif opt in ('-c', '--minimum_confidence'): - minimum_confidence = int(arg) - elif opt in ('-l', '--minimum_length'): - minimum_length = int(arg) - elif opt in ('-o', '--lead_out_required'): - lead_out_required = True - elif opt in ('-e', '--errors'): - show_errors = True - else: - print opt - raise Exception("unknown argument?") - -messages = bitutils.read_file(remainder) - -data = [s[HIST_DIMENTION_KEY] for s in messages if s['length'] > 100 and s['confidence'] > minimum_confidence and (s['lead_out'] or not lead_out_required) and (s['error'] == show_errors) and s['freq'] < 1.626e9] - -bins = (max(data) - min(data))/bin_size - -filename=["",",".join(remainder)][remainder is None] -title = "File: %s : Distribution of message %s. Bin Size: %d, Minimum Confidence: %d" % (filename, HIST_DIMENTION, bin_size, minimum_confidence) -if lead_out_required: - title += ', lead out needs to be present' -else: - title += ', lead out does not need to be present' -if show_errors: - title += " and having decoding errors" - -plt.title(title) -plt.hist(data, bins) -plt.show() diff --git a/rx-stats-time-hist.py b/rx-stats-time-hist.py deleted file mode 100755 index af74716..0000000 --- a/rx-stats-time-hist.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# vim: set ts=4 sw=4 tw=0 et pm=: - -# Parses .bits files and displays the distribution -# of the "HIST_DIMENTION_KEY" of received frames - -import sys -import matplotlib.pyplot as plt -import getopt - -import rx_stats_bitutils as bitutils - -HIST_DIMENTION = 'time' -HIST_DIMENTION_KEY = 'timestamp' - -options, remainder = getopt.getopt(sys.argv[1:], 'b:c:l:eo', [ - 'bin=', - 'minimum_confidence=', - 'minimum_length=', - 'errors', - 'lead_out_required' - ]) -bin_size = 3600 -minimum_confidence = 0 -minimum_length = 0 -lead_out_required = False -show_errors = False - -for opt, arg in options: - if opt in ('-b', '--bin'): - bin_size = int(arg) - elif opt in ('-c', '--minimum_confidence'): - minimum_confidence = int(arg) - elif opt in ('-l', '--minimum_length'): - minimum_length = int(arg) - elif opt in ('-o', '--lead_out_required'): - lead_out_required = True - elif opt in ('-e', '--errors'): - show_errors = True - else: - print opt - raise Exception("unknown argument?") - -messages = bitutils.read_file(remainder) - -data = [s[HIST_DIMENTION_KEY] for s in messages if s['length'] > minimum_length and s['confidence'] > minimum_confidence and (s['lead_out'] or not lead_out_required) and (s['error'] == show_errors) and s['freq'] < 1.626e9] - -bins = int((max(data) - min(data))/bin_size) - -filename=["",",".join(remainder)][remainder is None] -title = "File: %s : Distribution of message %s. Bin Size: %d, Minimum Confidence: %d" % (filename, HIST_DIMENTION, bin_size, minimum_confidence) -if lead_out_required: - title += ', lead out needs to be present' -else: - title += ', lead out does not need to be present' -if show_errors: - title += " and having decoding errors" - -plt.title(title) -plt.hist(data, bins) -plt.show() diff --git a/rx_stats_bitutils.py b/rx_stats_bitutils.py deleted file mode 100644 index 3e34a61..0000000 --- a/rx_stats_bitutils.py +++ /dev/null @@ -1,66 +0,0 @@ -# Some utility functions to do simple stuff with our .bits files -# Mostly just to do statistics - -import re -import datetime -import fileinput - -def extract_timestamp(filename, dt): - mm=re.match("i-(\d+(?:\.\d+)?)-[vbsrtl]1.([a-z])([a-z])",filename) - if mm: - b26=(ord(mm.group(2))-ord('a'))*26+ ord(mm.group(3))-ord('a') - timestamp=float(mm.group(1))+float(dt)/1000+b26*600 - return timestamp - - mm=re.match("i-(\d+(?:\.\d+)?)-[vbsrtl]1(?:-o[+-]\d+)?$",filename) - if mm: - timestamp=float(mm.group(1))+float(dt)/1000 - return timestamp - - mm=re.match("(\d\d)-(\d\d)-(20\d\d)T(\d\d)-(\d\d)-(\d\d)-[sr]1",filename) - if mm: - month, day, year, hour, minute, second = map(int, mm.groups()) - timestamp=datetime.datetime(year,month,day,hour,minute,second) - timestamp=(timestamp- datetime.datetime(1970,1,1)).total_seconds() - timestamp+=float(dt)/1000 - return timestamp - - return 0 - -def parse_line_to_message(line): - line = line.split() - if not line[0] == 'RX' and ('A:OK' not in line or len(line) < 10): - return None - access = True - lead_out = 'L:OK' in line - name = line[1] - if name == "X": - timestamp = float(line[2]) - else: - timestamp = extract_timestamp(name, line[2]) - freq = int(line[3]) - confidence = int(line[6][:-1]) - strength = float(line[7]) - length = int(line[8]) - if name == "X": - error = line[9]=="True" - msgtype = line[10] - else: - error = False - msgtype = None - - return {'name': name, 'timestamp':timestamp, 'freq':freq, 'access':access, 'lead_out':lead_out, 'confidence':confidence, 'strength': strength, 'length': length, 'error': error, 'msgtype': msgtype} - -def print_message(m): - print "RAW:", m['name'], m['freq'], "%06d"%m['timestamp'], 'A:%s'%m[access], 'L:%s'%m[lead_out], '%03d%%'%m['confidence'], "%03d"%m['length'] - -def read_file(filenames): - messages = [] - for line in fileinput.input(filenames): - try: - message = parse_line_to_message(line) - if message: - messages.append(message) - except: - pass - return messages diff --git a/setup.py b/setup.py index 94ade6b..b6de9af 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,7 @@ def run_tests(self): 'iridiumtk-graph-voc=iridiumtk.graph_voc:main', 'iridiumtk-graph-by-type=iridiumtk.graph_by_type:main', 'iridiumtk-bits-to-dfs=iridiumtk.bits_to_dfs:main', + 'iridiumtk-rx-stats-hist=iridiumtk.rx_stats_hist:main', ], }, package_data={'': ['LICENSE', 'README.md']}, From b2f59fb16a073ef551e89999af90c34a6f3f1855 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Sun, 13 May 2018 22:22:22 +0100 Subject: [PATCH 35/50] Enable linters in travis --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index eb850a8..5d3d44f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,12 @@ matrix: # dist: trusty # env: TOXENV=pypy + # Linters + - python: "2.7" + os: "linux" + dist: trusty + env: TOXENV=linters + allow_failures: - python: "pypy-5.4.1" - os: "osx" From 8bbd587dd9a22dbcdc9e41f7ede4fdc2d5781460 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Mon, 14 May 2018 15:27:00 +0100 Subject: [PATCH 36/50] First support for Python3.6 in graph-voc --- iridiumtk/graph_voc.py | 7 ++++--- iridiumtk/line_parser/base_line.py | 2 +- iridiumtk/line_parser/test_voc_line.py | 4 ++-- iridiumtk/line_parser/voc_line.py | 12 ++++++------ iridiumtk/test_bits_to_dfs.py | 10 +++++----- tox.ini | 2 +- 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/iridiumtk/graph_voc.py b/iridiumtk/graph_voc.py index cf39ec6..d88031a 100755 --- a/iridiumtk/graph_voc.py +++ b/iridiumtk/graph_voc.py @@ -13,6 +13,7 @@ import dateparser import matplotlib.pyplot as plt import numpy as np +from six.moves import range from .line_parser import VocLine @@ -44,7 +45,7 @@ DUPLEX_CHANELS = frozenset(SIMPLEX_CHANELS) DUPLEX_CHANELS = set() -for n in xrange(1, 240): +for n in range(1, 240): frequency = (1616 + 0.020833 * (2 * n - 1)) * MHZ DUPLEX_CHANELS.add(ChannelInfo('Channel {}'.format(n), frequency)) DUPLEX_CHANELS = frozenset(DUPLEX_CHANELS) @@ -54,8 +55,8 @@ class VoiceDataPlayer(object): def __init__(self): - self.ir77_ambe_decode = subprocess.Popen(['ir77_ambe_decode'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - self.aplay = subprocess.Popen(['aplay'], stdin=self.ir77_ambe_decode.stdout, stderr=subprocess.PIPE) + self.ir77_ambe_decode = subprocess.Popen(['ir77_ambe_decode'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=-1) + self.aplay = subprocess.Popen(['aplay'], stdin=self.ir77_ambe_decode.stdout, stderr=subprocess.PIPE, bufsize=-1) def __enter__(self): return self diff --git a/iridiumtk/line_parser/base_line.py b/iridiumtk/line_parser/base_line.py index bf5102a..dbfb0ee 100644 --- a/iridiumtk/line_parser/base_line.py +++ b/iridiumtk/line_parser/base_line.py @@ -30,7 +30,7 @@ def __init__(self, line): ts_base_ms = int(raw_time_base.split('-')[1].split('.')[0]) time_offset_ns = int(line_split[2]) - self._timestamp = ts_base_ms + (time_offset_ns / 1000) + self._timestamp = int(ts_base_ms + (time_offset_ns / 1000)) self._frequnecy = int(line_split[3]) except (IndexError, ValueError) as e: diff --git a/iridiumtk/line_parser/test_voc_line.py b/iridiumtk/line_parser/test_voc_line.py index 22871ca..53e2de8 100644 --- a/iridiumtk/line_parser/test_voc_line.py +++ b/iridiumtk/line_parser/test_voc_line.py @@ -24,11 +24,11 @@ def test_empty_input(self): def test_old_format(self): voc_line = VocLine(VocLineTest.TEST_VOC_LINE_1) - self.assertEquals(voc_line.voice_bits, '\x9e\x88$\xdb\xe6\x01') + self.assertEquals(voc_line.voice_bits, b'\x9e\x88$\xdb\xe6\x01') def test_new_format(self): voc_line = VocLine(VocLineTest.TEST_VOC_LINE_2) - self.assertEquals(voc_line.voice_bits, '\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') + self.assertEquals(voc_line.voice_bits, b'\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') def test_voice_bits_small_packet(self): voc_line = VocLine(VocLineTest.TEST_VOC_LINE_3) diff --git a/iridiumtk/line_parser/voc_line.py b/iridiumtk/line_parser/voc_line.py index 22c1193..ca8df98 100644 --- a/iridiumtk/line_parser/voc_line.py +++ b/iridiumtk/line_parser/voc_line.py @@ -1,10 +1,10 @@ #!/usr/bin/env python -from io import BytesIO import logging import six +from six.moves import range from .base_line import BaseLine, LineParseException @@ -17,7 +17,7 @@ def chunks(l, n): """ Yield successive n-sized chunks from l. """ - for i in xrange(0, len(l), n): + for i in range(0, len(l), n): yield l[i:i + n] @@ -45,14 +45,14 @@ def voice_bits(self): data = self._voice_data if data is None: return None - byte_stream = BytesIO() + byte_stream = six.BytesIO() if data[0] == "[": - for pos in xrange(1, len(data), 3): + for pos in range(1, len(data), 3): byte = int(data[pos:pos + 2], 16) byte = int('{:08b}'.format(byte)[::-1], 2) - byte_stream.write(chr(byte)) + byte_stream.write(six.int2byte(byte)) else: for bits in chunks(data, 8): byte = int(bits[::-1], 2) - byte_stream.write(chr(byte)) + byte_stream.write(six.int2byte(byte)) return byte_stream.getvalue() diff --git a/iridiumtk/test_bits_to_dfs.py b/iridiumtk/test_bits_to_dfs.py index 86625db..41f3c18 100644 --- a/iridiumtk/test_bits_to_dfs.py +++ b/iridiumtk/test_bits_to_dfs.py @@ -15,7 +15,7 @@ class BitsToDfsTest(unittest.TestCase): def test_empty_input(self): output = BytesIO() bits_to_dfs([], output) - self.assertEquals(output.getvalue(), '') + self.assertEquals(output.getvalue(), b'') def test_raw_data(self): output = BytesIO() @@ -24,22 +24,22 @@ def test_raw_data(self): output = BytesIO() bits_to_dfs(['RAW: i-1525892321-t1 000045987 1626110208 83% 0.001 <001100000011000011110011> 1100000000000000 0000000000000000 0000000000000000 1001000000000000 0000000000000000 0000000000000000 0100001000110110 0111100001010110 1011001111101110 1010110101000101 1111111001110101 1011110101110001 1110000001110111 0110001000001010 0100100100111000 1000001111010100 1011001101011110 0100011011100010 1111110001010001 1100110110101111 0011100111100101 1110100100000110 0111011111110010 1111110011001000 1101000011000011 1011110101110111 0000101000000001 1000111010101100 1011000001010011 0111011100011101 0101011000011101 0111110001001101 0100001011001010 1101001110100010 0111011001000101 0100111001010110 0001111110101010 1110001100010111 0001010101100100 1001010100111011 1001111110001101 0110100100010010 0001110111101001 1000011010111100 00011001 ERR:Message: unknown Iridium message type'], output) - self.assertEquals(output.getvalue(), '') + self.assertEquals(output.getvalue(), b'') def test_short_packet(self): output = BytesIO() bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_3], output) - self.assertEquals(output.getvalue(), '') + self.assertEquals(output.getvalue(), b'') def test_multiple(self): output = BytesIO() bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_1, BitsToDfsTest.TEST_VOC_LINE_1, BitsToDfsTest.TEST_VOC_LINE_2], output) - self.assertEquals(output.getvalue(), ('\x9e\x88$\xdb\xe6\x01' * 2) + '\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') + self.assertEquals(output.getvalue(), (b'\x9e\x88$\xdb\xe6\x01' * 2) + b'\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') def test_filters_non_voc_lines(self): output = BytesIO() bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_1, 'NOT_VOC:', BitsToDfsTest.TEST_VOC_LINE_2], output) - self.assertEquals(output.getvalue(), '\x9e\x88$\xdb\xe6\x01' + '\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') + self.assertEquals(output.getvalue(), b'\x9e\x88$\xdb\xe6\x01' + b'\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') class MainTest(unittest.TestCase): diff --git a/tox.ini b/tox.ini index 794a3ad..e82c093 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27 +envlist = py27, py36 platform = linux2 skipsdist = True From e5baf64db421710df7cf0e856e5d074e36dd29ff Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Mon, 14 May 2018 15:28:17 +0100 Subject: [PATCH 37/50] Test Python3.6 in travis --- .travis.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5d3d44f..af31b52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ sudo: false branches: only: - - py27 + - py27_36 language: python @@ -14,6 +14,12 @@ matrix: dist: trusty env: TOXENV=py27 + # Python 3.6 + - python: "3.6" + os: "linux" + dist: trusty + env: TOXENV=py36 + #- os: "osx" # language: generic # env: TOXENV=py27 From bdef5051fa06a5c2d2352a4579eeeadd6e4cdfa1 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Tue, 15 May 2018 14:51:12 +0100 Subject: [PATCH 38/50] Clean up and add py3 support Both graph_by_type and rx_stats_hist are now py3 enabled --- iridiumtk/graph/__init__.py | 5 +++ iridiumtk/graph/iridium_matplotlib.py | 54 +++++++++++++++++++++++++++ iridiumtk/graph_by_type.py | 21 +++-------- iridiumtk/graph_voc.py | 45 ++-------------------- iridiumtk/rx_stats_hist.py | 7 +++- 5 files changed, 74 insertions(+), 58 deletions(-) create mode 100644 iridiumtk/graph/__init__.py create mode 100644 iridiumtk/graph/iridium_matplotlib.py diff --git a/iridiumtk/graph/__init__.py b/iridiumtk/graph/__init__.py new file mode 100644 index 0000000..744184d --- /dev/null +++ b/iridiumtk/graph/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/python + +from .iridium_matplotlib import add_chanel_lines_to_axis, ALL_CHANELS, DUPLEX_CHANELS, SIMPLEX_CHANELS # noqa: F401 + +__all__ = [x if isinstance(x, str) else x.__name__ for x in (add_chanel_lines_to_axis, 'ALL_CHANELS', 'DUPLEX_CHANELS', 'SIMPLEX_CHANELS')] diff --git a/iridiumtk/graph/iridium_matplotlib.py b/iridiumtk/graph/iridium_matplotlib.py new file mode 100644 index 0000000..9771e38 --- /dev/null +++ b/iridiumtk/graph/iridium_matplotlib.py @@ -0,0 +1,54 @@ +#!/usr/bin/python + +from collections import namedtuple + + +from six.moves import range + + +KHZ = 1000 +MHZ = KHZ * 1000 + + +# https://www.sigidwiki.com/wiki/Iridium +ChannelInfo = namedtuple('ChannelInfo', ['description', 'frequency']) +SIMPLEX_CHANELS = { + ChannelInfo('Guard Channel', 1626.020833 * MHZ), + ChannelInfo('Guard Channel', 1626.062500 * MHZ), + ChannelInfo('Quaternary Messaging', 1626.104167 * MHZ), + ChannelInfo('Tertiary Messaging', 1626.145833 * MHZ), + ChannelInfo('Guard Channel', 1626.187500 * MHZ), + ChannelInfo('Guard Channel', 1626.229167 * MHZ), + ChannelInfo('Ring Alert', 1626.270833 * MHZ), + ChannelInfo('Guard Channel', 1626.312500 * MHZ), + ChannelInfo('Guard Channel', 1626.354167 * MHZ), + ChannelInfo('Secondary Messaging', 1626.395833 * MHZ), + ChannelInfo('Primary Messaging', 1626.437500 * MHZ), + ChannelInfo('Guard Channel', 1626.479167 * MHZ), +} +DUPLEX_CHANELS = frozenset(SIMPLEX_CHANELS) + +DUPLEX_CHANELS = set() +for n in range(1, 240): + frequency = (1616 + 0.020833 * (2 * n - 1)) * MHZ + DUPLEX_CHANELS.add(ChannelInfo('Channel {}'.format(n), frequency)) +DUPLEX_CHANELS = frozenset(DUPLEX_CHANELS) + +ALL_CHANELS = frozenset((SIMPLEX_CHANELS | DUPLEX_CHANELS)) + + +def add_chanel_lines_to_axis(axis): + for channel in ALL_CHANELS: + if 'Guard' in channel.description: + color = 'tab:gray' + elif 'Messaging' in channel.description: + color = 'tab:orange' + elif 'Ring' in channel.description: + color = 'tab:red' + else: + color = 'tab:green' + axis.axhline(channel.frequency, color=color, alpha=0.3, label=channel.description) + + +if __name__ == '__main__': + raise RuntimeError('{} can only be used as a module'.format(__name__)) diff --git a/iridiumtk/graph_by_type.py b/iridiumtk/graph_by_type.py index 36e60a9..c36d726 100755 --- a/iridiumtk/graph_by_type.py +++ b/iridiumtk/graph_by_type.py @@ -7,10 +7,11 @@ import dateparser import matplotlib.pyplot as plt import numpy as np +import six -from .graph_voc import ALL_CHANELS from .line_parser import BaseLine +from .graph import add_chanel_lines_to_axis logging.basicConfig(level=logging.INFO) @@ -51,13 +52,13 @@ def main(): stats.setdefault(base_line.frame_type, []).append(base_line) logger.info('Read %d lines from input', number_of_lines) - for frame_type, frames in stats.iteritems(): + for frame_type, frames in six.iteritems(stats): logger.info(' - Read %d\t%s lines from input', len(frames), frame_type) fig = plt.figure() subplot = fig.add_subplot(1, 1, 1) - for frame_type, frames in stats.iteritems(): + for frame_type, frames in six.iteritems(stats): number_of_frames = len(frames) plot_data_time = np.empty(number_of_frames, dtype=np.float64) plot_data_freq = np.empty(number_of_frames, dtype=np.uint32) @@ -66,18 +67,8 @@ def main(): plot_data_freq[i] = np.float64(base_line.frequency) subplot.scatter(plot_data_time, plot_data_freq, label=frame_type) - for channel in ALL_CHANELS: - if 'Guard' in channel.description: - color = 'tab:gray' - elif 'Messaging' in channel.description: - color = 'tab:orange' - elif 'Ring' in channel.description: - color = 'tab:red' - else: - color = 'tab:green' - subplot.axhline(channel.frequency, color=color, alpha=0.3, label=channel.description) - - # Shrink current axis's height on the bottom + add_chanel_lines_to_axis(subplot) + handles, labels = zip(*[(h, l) for (h, l) in zip(*subplot.get_legend_handles_labels()) if l in stats]) subplot.legend(handles, labels, bbox_to_anchor=(1.04, 1), loc="upper left") diff --git a/iridiumtk/graph_voc.py b/iridiumtk/graph_voc.py index d88031a..161269e 100755 --- a/iridiumtk/graph_voc.py +++ b/iridiumtk/graph_voc.py @@ -2,8 +2,8 @@ from __future__ import print_function + import argparse -from collections import namedtuple import fileinput import logging import subprocess @@ -13,9 +13,9 @@ import dateparser import matplotlib.pyplot as plt import numpy as np -from six.moves import range +from .graph import add_chanel_lines_to_axis from .line_parser import VocLine @@ -23,36 +23,6 @@ logger = logging.getLogger(__name__) -KHZ = 1000 -MHZ = KHZ * 1000 - -# https://www.sigidwiki.com/wiki/Iridium -ChannelInfo = namedtuple('ChannelInfo', ['description', 'frequency']) -SIMPLEX_CHANELS = { - ChannelInfo('Guard Channel', 1626.020833 * MHZ), - ChannelInfo('Guard Channel', 1626.062500 * MHZ), - ChannelInfo('Quaternary Messaging', 1626.104167 * MHZ), - ChannelInfo('Tertiary Messaging', 1626.145833 * MHZ), - ChannelInfo('Guard Channel', 1626.187500 * MHZ), - ChannelInfo('Guard Channel', 1626.229167 * MHZ), - ChannelInfo('Ring Alert', 1626.270833 * MHZ), - ChannelInfo('Guard Channel', 1626.312500 * MHZ), - ChannelInfo('Guard Channel', 1626.354167 * MHZ), - ChannelInfo('Secondary Messaging', 1626.395833 * MHZ), - ChannelInfo('Primary Messaging', 1626.437500 * MHZ), - ChannelInfo('Guard Channel', 1626.479167 * MHZ), -} -DUPLEX_CHANELS = frozenset(SIMPLEX_CHANELS) - -DUPLEX_CHANELS = set() -for n in range(1, 240): - frequency = (1616 + 0.020833 * (2 * n - 1)) * MHZ - DUPLEX_CHANELS.add(ChannelInfo('Channel {}'.format(n), frequency)) -DUPLEX_CHANELS = frozenset(DUPLEX_CHANELS) - -ALL_CHANELS = frozenset((SIMPLEX_CHANELS | DUPLEX_CHANELS)) - - class VoiceDataPlayer(object): def __init__(self): self.ir77_ambe_decode = subprocess.Popen(['ir77_ambe_decode'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=-1) @@ -178,16 +148,7 @@ def main(): subplot.scatter(plot_data_time, plot_data_freq) # subplot.xaxis_date() - for channel in ALL_CHANELS: - if 'Guard' in channel.description: - color = 'tab:gray' - elif 'Messaging' in channel.description: - color = 'tab:orange' - elif 'Ring' in channel.description: - color = 'tab:red' - else: - color = 'tab:green' - subplot.axhline(channel.frequency, color=color, alpha=0.3, label=channel.description) + add_chanel_lines_to_axis(subplot) plt.title('Click once left and once right to define an area.\nThe script will try to play iridium using ir77_ambe_decode and aplay.') plt.xlabel('time') diff --git a/iridiumtk/rx_stats_hist.py b/iridiumtk/rx_stats_hist.py index ae7dd0b..a6742bd 100755 --- a/iridiumtk/rx_stats_hist.py +++ b/iridiumtk/rx_stats_hist.py @@ -154,8 +154,13 @@ def main(): if show_errors: title += " and having decoding errors" + fig = plt.figure() + subplot = fig.add_subplot(1, 1, 1) + subplot.hist(data, bins) + plt.title(title) - plt.hist(data, bins) + plt.xlabel(args.dimension) + plt.ylabel('count') plt.show() From 8a8957a366799185c2d15b22a5114f48cf487993 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Tue, 15 May 2018 15:09:56 +0100 Subject: [PATCH 39/50] Move reassembler and add to setup --- reassembler.py => iridiumtk/reassembler.py | 99 +++++++++++----------- setup.py | 1 + 2 files changed, 52 insertions(+), 48 deletions(-) rename reassembler.py => iridiumtk/reassembler.py (67%) diff --git a/reassembler.py b/iridiumtk/reassembler.py similarity index 67% rename from reassembler.py rename to iridiumtk/reassembler.py index 02bc6cc..f6fd9e5 100755 --- a/reassembler.py +++ b/iridiumtk/reassembler.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# vim: set ts=4 sw=4 tw=0 et pm=: import sys import fileinput @@ -82,7 +81,6 @@ def perline(q): q.posx= m.group(3) q.posy= m.group(4) q.alt= int(m.group(5)) -# print "%d %d:"%(q.sat,q.beam) p=re.compile('PAGE\(tmsi:([0-9a-f]+) msc_id:([0-9]+)\)') m=p.findall(m.group(6)) for x in m: @@ -127,51 +125,56 @@ def perline(q): print "Unknown output mode." exit(1) -do_input(input) - -if ofmt == "msg": - buf={} - ricseq={} - wrapmargin=10 - for m in selected: - # msg_seq wraps around after 61, detect it, and fix it. - if m.msg_ric in ricseq: - if (m.msg_seq + wrapmargin) < ricseq[m.msg_ric][1]: # seq wrapped around - ricseq[m.msg_ric][0]+=62 - if (m.msg_seq + wrapmargin - 62) > ricseq[m.msg_ric][1]: # "wrapped back" (out-of-order old message) - ricseq[m.msg_ric][0]-=62 - else: - ricseq[m.msg_ric]=[0,0] - ricseq[m.msg_ric][1]=m.msg_seq - id="%07d %04d"%(m.msg_ric,(m.msg_seq+ricseq[m.msg_ric][0])) - ts=m.time - if id in buf: - if buf[id].msg_checksum != m.msg_checksum: - print "Whoa! Checksum changed? Message %s (1: @%d checksum %d/2: @%d checksum %d)"%(id,buf[id].time,buf[id].msg_checksum,m.time,m.msg_checksum) - # "Wrap around" to not miss the changed packet. - ricseq[m.msg_ric][0]+=62 - id="%07d %04d"%(m.msg_ric,(m.msg_seq+ricseq[m.msg_ric][0])) + +def main(): + do_input(input) + + if ofmt == "msg": + buf={} + ricseq={} + wrapmargin=10 + for m in selected: + # msg_seq wraps around after 61, detect it, and fix it. + if m.msg_ric in ricseq: + if (m.msg_seq + wrapmargin) < ricseq[m.msg_ric][1]: # seq wrapped around + ricseq[m.msg_ric][0]+=62 + if (m.msg_seq + wrapmargin - 62) > ricseq[m.msg_ric][1]: # "wrapped back" (out-of-order old message) + ricseq[m.msg_ric][0]-=62 + else: + ricseq[m.msg_ric]=[0,0] + ricseq[m.msg_ric][1]=m.msg_seq + id="%07d %04d"%(m.msg_ric,(m.msg_seq+ricseq[m.msg_ric][0])) + ts=m.time + if id in buf: + if buf[id].msg_checksum != m.msg_checksum: + print "Whoa! Checksum changed? Message %s (1: @%d checksum %d/2: @%d checksum %d)"%(id,buf[id].time,buf[id].msg_checksum,m.time,m.msg_checksum) + # "Wrap around" to not miss the changed packet. + ricseq[m.msg_ric][0]+=62 + id="%07d %04d"%(m.msg_ric,(m.msg_seq+ricseq[m.msg_ric][0])) + m.msgs=['[MISSING]']*3 + buf[id]=m + else: m.msgs=['[MISSING]']*3 buf[id]=m - else: - m.msgs=['[MISSING]']*3 - buf[id]=m - buf[id].msgs[m.msg_ctr]=m.msg_ascii - - def messagechecksum(msg): - csum=0 - for x in msg: - csum=(csum+ord(x))%128 - return (~csum)%128 - - for b in sorted(buf, key=lambda x: buf[x].time): - msg="".join(buf[b].msgs[:1+buf[b].msg_ctr_max]) - msg=re.sub("(\[3\])+$","",msg) # XXX: should be done differently - cmsg=re.sub("\[10\]","\n",msg) # XXX: should be done differently -# csum="" - csum=messagechecksum(cmsg) - str="Message %s @%s (len:%d)"%(b,datetime.datetime.fromtimestamp(buf[b].time).strftime("%Y-%m-%dT%H:%M:%S"),buf[b].msg_ctr_max) - str+= " %3d"%buf[b].msg_checksum - str+= (" fail"," OK ")[buf[b].msg_checksum == csum] - str+= ": %s"%(msg) - print str + buf[id].msgs[m.msg_ctr]=m.msg_ascii + + def messagechecksum(msg): + csum=0 + for x in msg: + csum=(csum+ord(x))%128 + return (~csum)%128 + + for b in sorted(buf, key=lambda x: buf[x].time): + msg="".join(buf[b].msgs[:1+buf[b].msg_ctr_max]) + msg=re.sub("(\[3\])+$","",msg) # XXX: should be done differently + cmsg=re.sub("\[10\]","\n",msg) # XXX: should be done differently + # csum="" + csum=messagechecksum(cmsg) + str="Message %s @%s (len:%d)"%(b,datetime.datetime.fromtimestamp(buf[b].time).strftime("%Y-%m-%dT%H:%M:%S"),buf[b].msg_ctr_max) + str+= " %3d"%buf[b].msg_checksum + str+= (" fail"," OK ")[buf[b].msg_checksum == csum] + str+= ": %s"%(msg) + print str + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index b6de9af..af3d5e0 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ def run_tests(self): 'iridiumtk-graph-by-type=iridiumtk.graph_by_type:main', 'iridiumtk-bits-to-dfs=iridiumtk.bits_to_dfs:main', 'iridiumtk-rx-stats-hist=iridiumtk.rx_stats_hist:main', + 'iridiumtk-reassembler=iridiumtk.reassembler:main', ], }, package_data={'': ['LICENSE', 'README.md']}, From c55f2efe8912dcaeca12764cc4d90acf67543eda Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Tue, 15 May 2018 14:51:12 +0100 Subject: [PATCH 40/50] Clean up and add py3 support Both graph_by_type and rx_stats_hist are now py3 enabled --- iridiumtk/graph/__init__.py | 5 +++ iridiumtk/graph/iridium_matplotlib.py | 54 +++++++++++++++++++++++++++ iridiumtk/graph_by_type.py | 21 +++-------- iridiumtk/graph_voc.py | 45 ++-------------------- iridiumtk/rx_stats_hist.py | 7 +++- 5 files changed, 74 insertions(+), 58 deletions(-) create mode 100644 iridiumtk/graph/__init__.py create mode 100644 iridiumtk/graph/iridium_matplotlib.py diff --git a/iridiumtk/graph/__init__.py b/iridiumtk/graph/__init__.py new file mode 100644 index 0000000..744184d --- /dev/null +++ b/iridiumtk/graph/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/python + +from .iridium_matplotlib import add_chanel_lines_to_axis, ALL_CHANELS, DUPLEX_CHANELS, SIMPLEX_CHANELS # noqa: F401 + +__all__ = [x if isinstance(x, str) else x.__name__ for x in (add_chanel_lines_to_axis, 'ALL_CHANELS', 'DUPLEX_CHANELS', 'SIMPLEX_CHANELS')] diff --git a/iridiumtk/graph/iridium_matplotlib.py b/iridiumtk/graph/iridium_matplotlib.py new file mode 100644 index 0000000..9771e38 --- /dev/null +++ b/iridiumtk/graph/iridium_matplotlib.py @@ -0,0 +1,54 @@ +#!/usr/bin/python + +from collections import namedtuple + + +from six.moves import range + + +KHZ = 1000 +MHZ = KHZ * 1000 + + +# https://www.sigidwiki.com/wiki/Iridium +ChannelInfo = namedtuple('ChannelInfo', ['description', 'frequency']) +SIMPLEX_CHANELS = { + ChannelInfo('Guard Channel', 1626.020833 * MHZ), + ChannelInfo('Guard Channel', 1626.062500 * MHZ), + ChannelInfo('Quaternary Messaging', 1626.104167 * MHZ), + ChannelInfo('Tertiary Messaging', 1626.145833 * MHZ), + ChannelInfo('Guard Channel', 1626.187500 * MHZ), + ChannelInfo('Guard Channel', 1626.229167 * MHZ), + ChannelInfo('Ring Alert', 1626.270833 * MHZ), + ChannelInfo('Guard Channel', 1626.312500 * MHZ), + ChannelInfo('Guard Channel', 1626.354167 * MHZ), + ChannelInfo('Secondary Messaging', 1626.395833 * MHZ), + ChannelInfo('Primary Messaging', 1626.437500 * MHZ), + ChannelInfo('Guard Channel', 1626.479167 * MHZ), +} +DUPLEX_CHANELS = frozenset(SIMPLEX_CHANELS) + +DUPLEX_CHANELS = set() +for n in range(1, 240): + frequency = (1616 + 0.020833 * (2 * n - 1)) * MHZ + DUPLEX_CHANELS.add(ChannelInfo('Channel {}'.format(n), frequency)) +DUPLEX_CHANELS = frozenset(DUPLEX_CHANELS) + +ALL_CHANELS = frozenset((SIMPLEX_CHANELS | DUPLEX_CHANELS)) + + +def add_chanel_lines_to_axis(axis): + for channel in ALL_CHANELS: + if 'Guard' in channel.description: + color = 'tab:gray' + elif 'Messaging' in channel.description: + color = 'tab:orange' + elif 'Ring' in channel.description: + color = 'tab:red' + else: + color = 'tab:green' + axis.axhline(channel.frequency, color=color, alpha=0.3, label=channel.description) + + +if __name__ == '__main__': + raise RuntimeError('{} can only be used as a module'.format(__name__)) diff --git a/iridiumtk/graph_by_type.py b/iridiumtk/graph_by_type.py index 36e60a9..7a2cbf4 100755 --- a/iridiumtk/graph_by_type.py +++ b/iridiumtk/graph_by_type.py @@ -7,9 +7,10 @@ import dateparser import matplotlib.pyplot as plt import numpy as np +import six -from .graph_voc import ALL_CHANELS +from .graph import add_chanel_lines_to_axis from .line_parser import BaseLine @@ -51,13 +52,13 @@ def main(): stats.setdefault(base_line.frame_type, []).append(base_line) logger.info('Read %d lines from input', number_of_lines) - for frame_type, frames in stats.iteritems(): + for frame_type, frames in six.iteritems(stats): logger.info(' - Read %d\t%s lines from input', len(frames), frame_type) fig = plt.figure() subplot = fig.add_subplot(1, 1, 1) - for frame_type, frames in stats.iteritems(): + for frame_type, frames in six.iteritems(stats): number_of_frames = len(frames) plot_data_time = np.empty(number_of_frames, dtype=np.float64) plot_data_freq = np.empty(number_of_frames, dtype=np.uint32) @@ -66,18 +67,8 @@ def main(): plot_data_freq[i] = np.float64(base_line.frequency) subplot.scatter(plot_data_time, plot_data_freq, label=frame_type) - for channel in ALL_CHANELS: - if 'Guard' in channel.description: - color = 'tab:gray' - elif 'Messaging' in channel.description: - color = 'tab:orange' - elif 'Ring' in channel.description: - color = 'tab:red' - else: - color = 'tab:green' - subplot.axhline(channel.frequency, color=color, alpha=0.3, label=channel.description) - - # Shrink current axis's height on the bottom + add_chanel_lines_to_axis(subplot) + handles, labels = zip(*[(h, l) for (h, l) in zip(*subplot.get_legend_handles_labels()) if l in stats]) subplot.legend(handles, labels, bbox_to_anchor=(1.04, 1), loc="upper left") diff --git a/iridiumtk/graph_voc.py b/iridiumtk/graph_voc.py index d88031a..161269e 100755 --- a/iridiumtk/graph_voc.py +++ b/iridiumtk/graph_voc.py @@ -2,8 +2,8 @@ from __future__ import print_function + import argparse -from collections import namedtuple import fileinput import logging import subprocess @@ -13,9 +13,9 @@ import dateparser import matplotlib.pyplot as plt import numpy as np -from six.moves import range +from .graph import add_chanel_lines_to_axis from .line_parser import VocLine @@ -23,36 +23,6 @@ logger = logging.getLogger(__name__) -KHZ = 1000 -MHZ = KHZ * 1000 - -# https://www.sigidwiki.com/wiki/Iridium -ChannelInfo = namedtuple('ChannelInfo', ['description', 'frequency']) -SIMPLEX_CHANELS = { - ChannelInfo('Guard Channel', 1626.020833 * MHZ), - ChannelInfo('Guard Channel', 1626.062500 * MHZ), - ChannelInfo('Quaternary Messaging', 1626.104167 * MHZ), - ChannelInfo('Tertiary Messaging', 1626.145833 * MHZ), - ChannelInfo('Guard Channel', 1626.187500 * MHZ), - ChannelInfo('Guard Channel', 1626.229167 * MHZ), - ChannelInfo('Ring Alert', 1626.270833 * MHZ), - ChannelInfo('Guard Channel', 1626.312500 * MHZ), - ChannelInfo('Guard Channel', 1626.354167 * MHZ), - ChannelInfo('Secondary Messaging', 1626.395833 * MHZ), - ChannelInfo('Primary Messaging', 1626.437500 * MHZ), - ChannelInfo('Guard Channel', 1626.479167 * MHZ), -} -DUPLEX_CHANELS = frozenset(SIMPLEX_CHANELS) - -DUPLEX_CHANELS = set() -for n in range(1, 240): - frequency = (1616 + 0.020833 * (2 * n - 1)) * MHZ - DUPLEX_CHANELS.add(ChannelInfo('Channel {}'.format(n), frequency)) -DUPLEX_CHANELS = frozenset(DUPLEX_CHANELS) - -ALL_CHANELS = frozenset((SIMPLEX_CHANELS | DUPLEX_CHANELS)) - - class VoiceDataPlayer(object): def __init__(self): self.ir77_ambe_decode = subprocess.Popen(['ir77_ambe_decode'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=-1) @@ -178,16 +148,7 @@ def main(): subplot.scatter(plot_data_time, plot_data_freq) # subplot.xaxis_date() - for channel in ALL_CHANELS: - if 'Guard' in channel.description: - color = 'tab:gray' - elif 'Messaging' in channel.description: - color = 'tab:orange' - elif 'Ring' in channel.description: - color = 'tab:red' - else: - color = 'tab:green' - subplot.axhline(channel.frequency, color=color, alpha=0.3, label=channel.description) + add_chanel_lines_to_axis(subplot) plt.title('Click once left and once right to define an area.\nThe script will try to play iridium using ir77_ambe_decode and aplay.') plt.xlabel('time') diff --git a/iridiumtk/rx_stats_hist.py b/iridiumtk/rx_stats_hist.py index ae7dd0b..a6742bd 100755 --- a/iridiumtk/rx_stats_hist.py +++ b/iridiumtk/rx_stats_hist.py @@ -154,8 +154,13 @@ def main(): if show_errors: title += " and having decoding errors" + fig = plt.figure() + subplot = fig.add_subplot(1, 1, 1) + subplot.hist(data, bins) + plt.title(title) - plt.hist(data, bins) + plt.xlabel(args.dimension) + plt.ylabel('count') plt.show() From 871d9aab20a285c01fd887ad45b0a42e4096e250 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Tue, 15 May 2018 15:11:52 +0100 Subject: [PATCH 41/50] Enable travis for reassembler branch --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index af31b52..f75cfca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,9 @@ sudo: false branches: only: + - py27 - py27_36 + - reassembler language: python From d1409c10f959d0e2aafd65136e34211ad9eca16d Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Tue, 15 May 2018 16:16:01 +0100 Subject: [PATCH 42/50] Add IraLine parser --- iridiumtk/graph_by_type.py | 2 +- iridiumtk/line_parser/base_line.py | 1 + iridiumtk/line_parser/ira_line.py | 71 ++++++++++++++++++++++++++ iridiumtk/line_parser/test_ira_line.py | 32 ++++++++++++ iridiumtk/line_parser/voc_line.py | 1 + iridiumtk/reassembler.py | 29 ++++------- 6 files changed, 117 insertions(+), 19 deletions(-) create mode 100644 iridiumtk/line_parser/ira_line.py create mode 100644 iridiumtk/line_parser/test_ira_line.py diff --git a/iridiumtk/graph_by_type.py b/iridiumtk/graph_by_type.py index c36d726..7a2cbf4 100755 --- a/iridiumtk/graph_by_type.py +++ b/iridiumtk/graph_by_type.py @@ -10,8 +10,8 @@ import six -from .line_parser import BaseLine from .graph import add_chanel_lines_to_axis +from .line_parser import BaseLine logging.basicConfig(level=logging.INFO) diff --git a/iridiumtk/line_parser/base_line.py b/iridiumtk/line_parser/base_line.py index dbfb0ee..9ecf0d9 100644 --- a/iridiumtk/line_parser/base_line.py +++ b/iridiumtk/line_parser/base_line.py @@ -18,6 +18,7 @@ class LineParseException(Exception): # Example lines # VOC: i-1430527570.4954-t1 421036605 1625859953 66% 0.008 219 L:no LCW(0,001111,100000000000000000000 E1) 101110110101010100101101111000111111111001011111001011010001000010010001101110011010011001111111011101111100011001001001000111001101001011001011000101111111101110110011111000000001110010001110101101001010011001101001010111101100011100110011110010110110101010110001010000100100101011010010100100100011010110101001 # VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f] +# IRA: i-1526300857-t1 000159537 1626299264 100% 0.003 130 DL sat:80 beam:30 pos=(+54.57/-001.24) alt=001 RAI:48 ?00 bc_sb:07 PAGE(tmsi:0cf155ab msc_id:03) PAGE(NONE) descr_extra:011010110101111001110011001111100110 class BaseLine(object): def __init__(self, line): try: diff --git a/iridiumtk/line_parser/ira_line.py b/iridiumtk/line_parser/ira_line.py new file mode 100644 index 0000000..4e5f931 --- /dev/null +++ b/iridiumtk/line_parser/ira_line.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +from collections import namedtuple +import logging +import re + + +import six + + +from .base_line import BaseLine, LineParseException + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +IRA_META_REGEX = re.compile(r'.*sat:(\d+) beam:(\d+) pos=\((.[0-9.]+)/(.[0-9.]+)\) alt=([-0-9]+) .* bc_sb:\d+ (.*)') +IRA_PAGE_REGEX = re.compile(r'PAGE\(tmsi:([0-9a-f]+) msc_id:([0-9]+)\)') + + +Coordinates = namedtuple('Coordinates', ['x', 'y']) +Page = namedtuple('Page', ['tmsi', 'msc_id']) + + +# Example lines +# IRA: i-1526300857-t1 000159537 1626299264 100% 0.003 130 DL sat:80 beam:30 pos=(+54.57/-001.24) alt=001 RAI:48 ?00 bc_sb:07 PAGE(tmsi:0cf155ab msc_id:03) PAGE(NONE) descr_extra:011010110101111001110011001111100110 +class IraLine(BaseLine): + def __init__(self, line): + super(IraLine, self).__init__(line) + try: + line_split = line.split() + assert line_split[0] == 'IRA:', 'Non VOC line passed to VocLine' + + data = line.split(None, 8)[8] + matches = IRA_META_REGEX.match(data) + if not matches: + raise ValueError('Failed to parse IRA data section: {}'.format(data)) + + self._satellite = int(matches.group(1)) + self._beam = int(matches.group(2)) + self._position = Coordinates(x=float(matches.group(3)), y=float(matches.group(4))) + self._altitude = int(matches.group(5)) + + self._pages = [] + matches = IRA_PAGE_REGEX.findall(matches.group(6)) + for match in matches: + self._pages.append(Page(tmsi=match[0], msc_id=int(match[1]))) + except (IndexError, ValueError) as e: + logger.error('Failed to parse line "%s"', line) + six.raise_from(LineParseException('Failed to parse line "{}"'.format(line), e), e) + + @property + def satellite(self): + return self._satellite + + @property + def beam(self): + return self._beam + + @property + def position(self): + return self._position + + @property + def altitude(self): + return self._altitude + + @property + def pages(self): + return self._pages diff --git a/iridiumtk/line_parser/test_ira_line.py b/iridiumtk/line_parser/test_ira_line.py new file mode 100644 index 0000000..91382a2 --- /dev/null +++ b/iridiumtk/line_parser/test_ira_line.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +import unittest + + +from .base_line import LineParseException +from .ira_line import IraLine, Page + + +class IraLineTest(unittest.TestCase): + TEST_IRA_LINE_1 = 'IRA: i-1526300857-t1 000159537 1626299264 100% 0.003 130 DL sat:80 beam:30 pos=(+54.57/-001.24) alt=001 RAI:48 ?00 bc_sb:07 PAGE(tmsi:0cf155ab msc_id:03) PAGE(NONE) descr_extra:011010110101111001110011001111100110' + + def test_empty_input(self): + with self.assertRaises(LineParseException): + IraLine('') + + def test_simple(self): + ira_line = IraLine(IraLineTest.TEST_IRA_LINE_1) + + self.assertEquals(ira_line.satellite, 80) + self.assertEquals(ira_line.beam, 30) + self.assertEquals(ira_line.position, (54.57, -1.24)) + self.assertEquals(ira_line.altitude, 1) + self.assertEquals(ira_line.pages, [Page(tmsi='0cf155ab', msc_id=3)]) + + +def main(): + unittest.main() + + +if __name__ == "__main__": + main() diff --git a/iridiumtk/line_parser/voc_line.py b/iridiumtk/line_parser/voc_line.py index ca8df98..c2f357b 100644 --- a/iridiumtk/line_parser/voc_line.py +++ b/iridiumtk/line_parser/voc_line.py @@ -29,6 +29,7 @@ def __init__(self, line): super(VocLine, self).__init__(line) try: line_split = line.split() + assert line_split[0] == 'VOC:', 'Non VOC line passed to VocLine' self.lcw = line[8] diff --git a/iridiumtk/reassembler.py b/iridiumtk/reassembler.py index f6fd9e5..d9ec5a8 100755 --- a/iridiumtk/reassembler.py +++ b/iridiumtk/reassembler.py @@ -7,7 +7,6 @@ import re verbose = False -ifmt= "line" ofmt= "undef" options, remainder = getopt.getopt(sys.argv[1:], 'vi:o:', [ @@ -19,8 +18,6 @@ for opt, arg in options: if opt in ('-v', '--verbose'): verbose = True - elif opt in ('-i', '--input'): - ifmt=arg elif opt in ('-o', '--output'): ofmt=arg else: @@ -28,7 +25,7 @@ class Message(object): def __init__(self,line): - self.typ,self.name,self.time,self.frequency,self.confidence,self.level,self.symbols,self.uldl,self.data=line.split(None,8) + self.typ, self.name, self.time, self.frequency, self.confidence, self.level, self.symbols, self.uldl, self.data = line.split(None,8) def upgrade(self): # return IridiumMessage(self).upgrade() return self @@ -45,19 +42,15 @@ def pretty(self): selected=[] def do_input(type): - if ifmt=="line": - for line in fileinput.input(remainder): - qqq=re.compile('Warning:') - if qqq.match(line): - print "Skip: ",line - continue -# try: - perline(Message(line.strip()).upgrade()) -# except ValueError: -# print >> sys.stderr, "Couldn't parse line",line - else: - print "Unknown input mode." - exit(1) + for line in fileinput.input(remainder): + qqq=re.compile('Warning:') + if qqq.match(line): + print "Skip: ",line + continue + # try: + perline(Message(line.strip()).upgrade()) + # except ValueError: + # print >> sys.stderr, "Couldn't parse line",line def fixtime(n,t): try: @@ -71,7 +64,7 @@ def perline(q): pass elif ofmt == "page": if q.typ=="IRA:": - p=re.compile('.*sat:(\d+) beam:(\d+) pos=\((.[0-9.]+)/(.[0-9.]+)\) alt=([-0-9]+) .* bch:\d+ (.*)') + p=re.compile('.*sat:(\d+) beam:(\d+) pos=\((.[0-9.]+)/(.[0-9.]+)\) alt=([-0-9]+) .* bc_sb:\d+ (.*)') m=p.match(q.data) if(not m): print >> sys.stderr, "Couldn't parse IRA: ",q.data From b6b9cd55dccc640f7518ec62d99b7273b85869c5 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Tue, 15 May 2018 18:11:42 +0100 Subject: [PATCH 43/50] Add MSG parser and impprove base_parser --- iridiumtk/line_parser/base_line.py | 47 +++++++++- iridiumtk/line_parser/msg_line.py | 111 ++++++++++++++++++++++++ iridiumtk/line_parser/test_base_line.py | 29 ++++++- iridiumtk/line_parser/test_msg_line.py | 38 ++++++++ requirements.txt | 3 +- 5 files changed, 222 insertions(+), 6 deletions(-) create mode 100644 iridiumtk/line_parser/msg_line.py create mode 100644 iridiumtk/line_parser/test_msg_line.py diff --git a/iridiumtk/line_parser/base_line.py b/iridiumtk/line_parser/base_line.py index 9ecf0d9..4fc7966 100644 --- a/iridiumtk/line_parser/base_line.py +++ b/iridiumtk/line_parser/base_line.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from datetime import datetime +from enum import Enum import logging @@ -15,6 +16,12 @@ class LineParseException(Exception): pass +class LinkDirection(Enum): + UPLINK = 'ul' + DOWNLINK = 'dl' + NO_DIRECTION = 'L:no' + + # Example lines # VOC: i-1430527570.4954-t1 421036605 1625859953 66% 0.008 219 L:no LCW(0,001111,100000000000000000000 E1) 101110110101010100101101111000111111111001011111001011010001000010010001101110011010011001111111011101111100011001001001000111001101001011001011000101111111101110110011111000000001110010001110101101001010011001101001010111101100011100110011110010110110101010110001010000100100101011010010100100100011010110101001 # VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f] @@ -34,6 +41,16 @@ def __init__(self, line): self._timestamp = int(ts_base_ms + (time_offset_ns / 1000)) self._frequnecy = int(line_split[3]) + self._confidence = int(line_split[4][:-1]) + self._level = float(line_split[5]) + self._symbols = int(line_split[6]) + + if line_split[7] == 'DL': + self._link_direction = LinkDirection.DOWNLINK + elif line_split[7] == 'UL': + self._link_direction = LinkDirection.UPLINK + else: + self._link_direction = LinkDirection.NO_DIRECTION except (IndexError, ValueError) as e: logger.error('Failed to parse line "%s"', line) six.raise_from(LineParseException('Failed to parse line "{}"'.format(line), e), e) @@ -46,10 +63,6 @@ def raw_line(self): def frame_type(self): return self._frame_type - @property - def frequency(self): - return self._frequnecy - @property def datetime(self): return datetime.utcfromtimestamp(self._timestamp) @@ -57,3 +70,29 @@ def datetime(self): @property def datetime_unix(self): return self._timestamp + + @property + def frequency(self): + return self._frequnecy + + @property + def confidence(self): + return self._confidence + + @property + def level(self): + return self._level + + @property + def symbols(self): + return self._symbols + + @property + def link_direction(self): + return self._link_direction + + def is_uplink(self): + return self._link_direction == LinkDirection.UPLINK + + def is_downlink(self): + return self._link_direction == LinkDirection.DOWNLINK diff --git a/iridiumtk/line_parser/msg_line.py b/iridiumtk/line_parser/msg_line.py new file mode 100644 index 0000000..bf9812b --- /dev/null +++ b/iridiumtk/line_parser/msg_line.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python + +import logging +import re + + +import six + + +from .base_line import BaseLine, LineParseException + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# eg .... ric:3525696 fmt:05 seq:18 1101000111 0/0 AgAFACYCgTLIxITKA4x8qpLs5geb4SICAVgVzXq9gdkuxao79yKD7DG5XpZD +1111 +MSG_META_REGEX = re.compile(r'.* ric:(\d+) fmt:(\d+) seq:(\d+) [01]+ (\d)/(\d) ') +MSG_TEXT_REGEX = re.compile(r'.* csum:([0-9a-f][0-9a-f]) msg:([0-9a-f]+)\.([01]*) ') + + +# Example lines +# MSG: i-1526039037-t1 001174920 1626447232 100% 0.006 432 DL 00110011111100110011001111110011 odd:01100000000000000000000001 1:0:03 ric:3525696 fmt:05 seq:18 1101000111 0/0 AgAFACYCgTLIxITKA4x8qpLs5geb4SICAVgVzXq9gdkuxao79yKD7DG5XpZD +1111 +class MsgLine(BaseLine): + def __init__(self, line): + super(MsgLine, self).__init__(line) + try: + line_split = line.split() + assert line_split[0] == 'MSG:', 'Non MSG line passed to MsgLine' + + data = line.split(None, 8)[8] + matches = MSG_META_REGEX.match(data) + if not matches: + raise ValueError('Failed to parse MSG data section: {}'.format(data)) + + self._msg_ric = int(matches.group(1)) + self._format = int(matches.group(2)) + self._msg_seq = int(matches.group(3)) + self._msg_ctr = int(matches.group(4)) + self._msg_ctr_max = int(matches.group(5)) + + matches = MSG_TEXT_REGEX.match(data) + if matches: + self._msg_checksum = int(matches.group(6), 16) + self._msg_hex = matches.group(7) + self._msg_brest = matches.group(8) + + msg_msgdata = ''.join(["{0:08b}".format(int(self._msg_hex[i:i + 2], 16)) for i in range(0, len(self._msg_hex), 2)]) + msg_msgdata += self._msg_brest + matches = re.compile(r'(\d{7})').findall(msg_msgdata) + msg_ascii = '' + for (group) in matches: + character = int(group, 2) + if (character < 32 or character == 127): + msg_ascii += '[%d]' % character + else: + msg_ascii += chr(character) + self._msg_ascii = msg_ascii + + if len(msg_msgdata) % 7: + self._msg_rest = msg_msgdata[-(len(msg_msgdata) % 7):] + else: + self._msg_rest = '' + else: + self._msg_checksum = None + self._msg_hex = None + self._msg_brest = None + self._msg_ascii = None + self._msg_rest = None + except (IndexError, ValueError) as e: + logger.error('Failed to parse line "%s"', line) + six.raise_from(LineParseException('Failed to parse line "{}"'.format(line), e), e) + + @property + def message_ric(self): + return self._msg_ric + + @property + def format(self): + return self._format + + @property + def message_sequence(self): + return self._msg_seq + + @property + def message_ctr(self): + return self._msg_ctr + + @property + def message_ctr_max(self): + return self._msg_ctr_max + + @property + def message_checksum(self): + return self._msg_checksum + + @property + def message_hex(self): + return self._msg_hex + + @property + def message_brest(self): + return self._msg_brest + + @property + def message_ascii(self): + return self._msg_ascii + + @property + def message_rest(self): + return self._msg_rest diff --git a/iridiumtk/line_parser/test_base_line.py b/iridiumtk/line_parser/test_base_line.py index 6f8593b..959b345 100644 --- a/iridiumtk/line_parser/test_base_line.py +++ b/iridiumtk/line_parser/test_base_line.py @@ -4,7 +4,7 @@ import unittest -from .base_line import BaseLine, LineParseException +from .base_line import BaseLine, LineParseException, LinkDirection class BaseLineTest(unittest.TestCase): @@ -33,6 +33,33 @@ def test_frame_type(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) self.assertEquals(base_line.frame_type, 'VOC') + def test_confidence(self): + base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) + self.assertEquals(base_line.confidence, 81) + base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) + self.assertEquals(base_line.confidence, 100) + + def test_level(self): + base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) + self.assertEquals(base_line.level, 0.027) + base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) + self.assertEquals(base_line.level, 0.003) + + def test_symbols(self): + base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) + self.assertEquals(base_line.symbols, 179) + + def test_link_direction(self): + base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) + self.assertEquals(base_line.link_direction, LinkDirection.NO_DIRECTION) + self.assertEquals(base_line.is_downlink(), False) + self.assertEquals(base_line.is_uplink(), False) + + base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) + self.assertEquals(base_line.link_direction, LinkDirection.DOWNLINK) + self.assertEquals(base_line.is_downlink(), True) + self.assertEquals(base_line.is_uplink(), False) + def test_raw_line(self): for line in [BaseLineTest.TEST_VOC_LINE_1, BaseLineTest.TEST_VOC_LINE_2]: base_line = BaseLine(line) diff --git a/iridiumtk/line_parser/test_msg_line.py b/iridiumtk/line_parser/test_msg_line.py new file mode 100644 index 0000000..9e5f392 --- /dev/null +++ b/iridiumtk/line_parser/test_msg_line.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +import unittest + + +from .base_line import LineParseException +from .msg_line import MsgLine + + +class MsgLineTest(unittest.TestCase): + TEST_MSG_LINE_1 = 'MSG: i-1526039037-t1 001174920 1626447232 100% 0.006 432 DL 00110011111100110011001111110011 odd:01100000000000000000000001 1:0:03 ric:3525696 fmt:05 seq:18 1101000111 0/0 AgAFACYCgTLIxITKA4x8qpLs5geb4SICAVgVzXq9gdkuxao79yKD7DG5XpZD +1111' + + def test_empty_input(self): + with self.assertRaises(LineParseException): + MsgLine('') + + def test_simple(self): + msg_line = MsgLine(MsgLineTest.TEST_MSG_LINE_1) + + self.assertEquals(msg_line.message_ric, 3525696) + self.assertEquals(msg_line.format, 5) + self.assertEquals(msg_line.message_sequence, 18) + self.assertEquals(msg_line.message_ctr, 0) + self.assertEquals(msg_line.message_ctr_max, 0) + + self.assertEquals(msg_line.message_checksum, None) + self.assertEquals(msg_line.message_hex, None) + self.assertEquals(msg_line.message_brest, None) + self.assertEquals(msg_line.message_ascii, None) + self.assertEquals(msg_line.message_rest, None) + + +def main(): + unittest.main() + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index ae8ca5e..d4edbd2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ numpy>=1.11.1 six>=1.11.0 dateparser>=0.7.0 -matplotlib>=2.2.0 \ No newline at end of file +matplotlib>=2.2.0 +enum34>=1.1.0 \ No newline at end of file From d2f222c0bacfc870a79091e2bf5d022e18ae9fa8 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Tue, 29 May 2018 19:48:35 +0100 Subject: [PATCH 44/50] Open output file as binary --- iridiumtk/bits_to_dfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iridiumtk/bits_to_dfs.py b/iridiumtk/bits_to_dfs.py index 1d2fd41..e734dfa 100755 --- a/iridiumtk/bits_to_dfs.py +++ b/iridiumtk/bits_to_dfs.py @@ -27,7 +27,7 @@ def main(): parser.add_argument('input', metavar='FILE', nargs='*', help='Files to read, if empty or -, stdin is used') args = parser.parse_args() - output_file = sys.stdout if args.output == '-' else open(args.output, 'w') + output_file = sys.stdout if args.output == '-' else open(args.output, 'wb') input_files = args.input if len(args.input) > 0 else ['-'] bits_to_dfs(fileinput.input(files=input_files), output_file) From aafb11d437de71503c91f5686e52a77e65dc57fe Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Tue, 29 May 2018 19:50:55 +0100 Subject: [PATCH 45/50] Add extra type filter to graph_by_type --- iridiumtk/graph_by_type.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/iridiumtk/graph_by_type.py b/iridiumtk/graph_by_type.py index 7a2cbf4..c51fc4b 100755 --- a/iridiumtk/graph_by_type.py +++ b/iridiumtk/graph_by_type.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) -def read_lines(input_files, start_time_filter, end_time_filter): +def read_lines(input_files, start_time_filter, end_time_filter, type_filter): for line in fileinput.input(files=input_files): line = line.strip() if 'A:OK' in line and "Message: Couldn't parse:" not in line: @@ -31,6 +31,8 @@ def read_lines(input_files, start_time_filter, end_time_filter): continue if end_time_filter and end_time_filter < base_line.datetime: continue + if type_filter and base_line.frame_type not in type_filter: + continue yield base_line @@ -38,16 +40,18 @@ def main(): parser = argparse.ArgumentParser(description='Visualise ????') # TODO parser.add_argument('--start', metavar='DATETIME', default=None, help='Filter events before this time') parser.add_argument('--end', metavar='DATETIME', default=None, help='Filter events after this time') + parser.add_argument('--filter-to-types', metavar='TYPE', default=None, help='Filter events by type (coma separated)') parser.add_argument('input', metavar='FILE', nargs='*', help='Files to read, if empty or -, stdin is used') args = parser.parse_args() input_files = args.input if len(args.input) > 0 else ['-'] start_time_filter = dateparser.parse(args.start) if args.start else None end_time_filter = dateparser.parse(args.end) if args.end else None + type_filter = [t.strip().upper() for t in args.filter_to_types.split(',')] if args.filter_to_types else None stats = {} number_of_lines = 0 - for base_line in read_lines(input_files, start_time_filter, end_time_filter): + for base_line in read_lines(input_files, start_time_filter, end_time_filter, type_filter): number_of_lines += 1 stats.setdefault(base_line.frame_type, []).append(base_line) @@ -72,6 +76,8 @@ def main(): handles, labels = zip(*[(h, l) for (h, l) in zip(*subplot.get_legend_handles_labels()) if l in stats]) subplot.legend(handles, labels, bbox_to_anchor=(1.04, 1), loc="upper left") + del stats + plt.title('frequency vs time color coded by frame type') plt.xlabel('time') plt.ylabel('frequency') From 41a2bd770b500b80e50293c669b0c43d63152a43 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Tue, 29 May 2018 19:54:26 +0100 Subject: [PATCH 46/50] Work to clean up reassembler.py --- iridiumtk/line_parser/__init__.py | 8 +- iridiumtk/line_parser/base_line.py | 28 ++- iridiumtk/line_parser/msg_line.py | 49 +---- iridiumtk/line_parser/test_base_line.py | 15 ++ iridiumtk/line_parser/test_msg_line.py | 22 ++- iridiumtk/reassembler.py | 241 ++++++++---------------- requirements.txt | 2 +- 7 files changed, 149 insertions(+), 216 deletions(-) diff --git a/iridiumtk/line_parser/__init__.py b/iridiumtk/line_parser/__init__.py index 6990b39..c0ede78 100644 --- a/iridiumtk/line_parser/__init__.py +++ b/iridiumtk/line_parser/__init__.py @@ -1,5 +1,9 @@ #!/usr/bin/python -from .voc_line import BaseLine, VocLine +from .base_line import BaseLine +from .voc_line import VocLine +from .ira_line import IraLine +from .msg_line import MsgLine -__all__ = [x.__name__ for x in (BaseLine, VocLine)] + +__all__ = [x.__name__ for x in (BaseLine, VocLine, IraLine, MsgLine)] diff --git a/iridiumtk/line_parser/base_line.py b/iridiumtk/line_parser/base_line.py index 4fc7966..5d42502 100644 --- a/iridiumtk/line_parser/base_line.py +++ b/iridiumtk/line_parser/base_line.py @@ -3,6 +3,7 @@ from datetime import datetime from enum import Enum import logging +import time import six @@ -27,7 +28,9 @@ class LinkDirection(Enum): # VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f] # IRA: i-1526300857-t1 000159537 1626299264 100% 0.003 130 DL sat:80 beam:30 pos=(+54.57/-001.24) alt=001 RAI:48 ?00 bc_sb:07 PAGE(tmsi:0cf155ab msc_id:03) PAGE(NONE) descr_extra:011010110101111001110011001111100110 class BaseLine(object): - def __init__(self, line): + __slots__ = ['_raw_line', '_frame_type', '_timestamp', '_frequnecy', '_confidence', '_level', '_symbols', '_link_direction'] + + def __init__(self, line, now=datetime.utcnow()): try: self._raw_line = line line_split = line.split() @@ -35,7 +38,11 @@ def __init__(self, line): self._frame_type = line_split[0][:-1] raw_time_base = line_split[1] - ts_base_ms = int(raw_time_base.split('-')[1].split('.')[0]) + try: + ts_base_ms = int(raw_time_base.split('-')[1].split('.')[0]) + except ValueError as e: + logger.warn('No base datetime found. Using now instead') + ts_base_ms = time.mktime(now.timetuple()) time_offset_ns = int(line_split[2]) self._timestamp = int(ts_base_ms + (time_offset_ns / 1000)) @@ -43,14 +50,19 @@ def __init__(self, line): self._frequnecy = int(line_split[3]) self._confidence = int(line_split[4][:-1]) self._level = float(line_split[5]) - self._symbols = int(line_split[6]) - if line_split[7] == 'DL': - self._link_direction = LinkDirection.DOWNLINK - elif line_split[7] == 'UL': - self._link_direction = LinkDirection.UPLINK + if self._frame_type != 'RAW': + self._symbols = int(line_split[6]) + + if line_split[7] == 'DL': + self._link_direction = LinkDirection.DOWNLINK + elif line_split[7] == 'UL': + self._link_direction = LinkDirection.UPLINK + else: + self._link_direction = LinkDirection.NO_DIRECTION else: - self._link_direction = LinkDirection.NO_DIRECTION + self._symbols = None + self._link_direction = None except (IndexError, ValueError) as e: logger.error('Failed to parse line "%s"', line) six.raise_from(LineParseException('Failed to parse line "{}"'.format(line), e), e) diff --git a/iridiumtk/line_parser/msg_line.py b/iridiumtk/line_parser/msg_line.py index bf9812b..11b7298 100644 --- a/iridiumtk/line_parser/msg_line.py +++ b/iridiumtk/line_parser/msg_line.py @@ -14,8 +14,7 @@ logger = logging.getLogger(__name__) # eg .... ric:3525696 fmt:05 seq:18 1101000111 0/0 AgAFACYCgTLIxITKA4x8qpLs5geb4SICAVgVzXq9gdkuxao79yKD7DG5XpZD +1111 -MSG_META_REGEX = re.compile(r'.* ric:(\d+) fmt:(\d+) seq:(\d+) [01]+ (\d)/(\d) ') -MSG_TEXT_REGEX = re.compile(r'.* csum:([0-9a-f][0-9a-f]) msg:([0-9a-f]+)\.([01]*) ') +MSG_META_REGEX = re.compile(r'.* ric:(\d+) fmt:(\d+) seq:(\d+) [01]+ (\d)/(\d) (.{65,}) \+([01]{0,6})') # Example lines @@ -38,34 +37,8 @@ def __init__(self, line): self._msg_ctr = int(matches.group(4)) self._msg_ctr_max = int(matches.group(5)) - matches = MSG_TEXT_REGEX.match(data) - if matches: - self._msg_checksum = int(matches.group(6), 16) - self._msg_hex = matches.group(7) - self._msg_brest = matches.group(8) - - msg_msgdata = ''.join(["{0:08b}".format(int(self._msg_hex[i:i + 2], 16)) for i in range(0, len(self._msg_hex), 2)]) - msg_msgdata += self._msg_brest - matches = re.compile(r'(\d{7})').findall(msg_msgdata) - msg_ascii = '' - for (group) in matches: - character = int(group, 2) - if (character < 32 or character == 127): - msg_ascii += '[%d]' % character - else: - msg_ascii += chr(character) - self._msg_ascii = msg_ascii - - if len(msg_msgdata) % 7: - self._msg_rest = msg_msgdata[-(len(msg_msgdata) % 7):] - else: - self._msg_rest = '' - else: - self._msg_checksum = None - self._msg_hex = None - self._msg_brest = None - self._msg_ascii = None - self._msg_rest = None + self._msg_data_escaped = matches.group(6).strip() + self._msg_rest = matches.group(7) except (IndexError, ValueError) as e: logger.error('Failed to parse line "%s"', line) six.raise_from(LineParseException('Failed to parse line "{}"'.format(line), e), e) @@ -91,20 +64,12 @@ def message_ctr_max(self): return self._msg_ctr_max @property - def message_checksum(self): - return self._msg_checksum + def message_data_escaped(self): + return self._msg_data_escaped @property - def message_hex(self): - return self._msg_hex - - @property - def message_brest(self): - return self._msg_brest - - @property - def message_ascii(self): - return self._msg_ascii + def message_data(self): + return bytearray(re.sub(r'\[[0-9]{1,3}\]', lambda matchobj: chr(int(matchobj.group(0)[1:-1])), self._msg_data_escaped), 'ascii') @property def message_rest(self): diff --git a/iridiumtk/line_parser/test_base_line.py b/iridiumtk/line_parser/test_base_line.py index 959b345..c75636d 100644 --- a/iridiumtk/line_parser/test_base_line.py +++ b/iridiumtk/line_parser/test_base_line.py @@ -10,6 +10,8 @@ class BaseLineTest(unittest.TestCase): TEST_VOC_LINE_1 = 'VOC: i-1443338945.6543-t1 033399141 1625872817 81% 0.027 179 L:no LCW(0,001111,100000000000000000000 E1) 01111001000100010010010011011011011001111 011000010000100001110101111011110010010111011001010001011101010001100000000110010100000110111110010101110101001111010100111001000110100110001110110 1010101010010010001000001110011000001001001010011110011100110100111110001101110010110101010110011101011100011101011000000000 descr_extra:' TEST_VOC_LINE_2 = 'VOC: i-1526039037-t1 000065686 1620359296 100% 0.003 179 DL LCW(0,T:maint,C:maint[2][lqi:3,power:0,f_dtoa:0,f_dfoa:127](3),786686 E0) [df.ff.f3.fc.10.33.c3.1f.0c.83.c3.cc.cc.30.ff.f3.ef.00.bc.0c.b4.0f.dc.d0.1a.cc.9c.c5.0c.fc.28.01.cc.38.c2.33.e0.ff.4f]' + TEST_RAW_LINE_1 = 'RAW: i-1526300857-t1 000001404 1619597184 79% 0.001 <001100000011000011110011> 0110000000100011 0000100000001011 0001100100000010 1010101010101010 1000111010101010 0011101010001011 0011101010100010 1010101011101000 1010101010101010 1010001110101010 1010101010101010 1010110010101010 1010101010101010 1010001110101000 1011101010101010 1010101010101010 1010101010101010 1011001010101010 1010101010101010 1010101010101010 1010101010101010 1010101010101010 101010 ERR:Message: unknown Iridium message type' + TEST_IMS_LINE_1 = 'IMS: capture-s1 000003681 1626472922 100% 0.043 127 DL 00110011111100110011001111110011 odd:100001 9:A:22 1 c=03829 00000000 00000000000000000000 00000000000000000000 00000000000000000000 descr_extra:011010110101111001110011001111' def test_empty_input(self): with self.assertRaises(LineParseException): @@ -25,6 +27,12 @@ def test_new_format_datetime(self): self.assertEquals(base_line.datetime_unix, 1526039102) self.assertEquals(base_line.datetime, datetime.utcfromtimestamp(1526039102)) + def test_filename_datetime(self): + now = datetime.utcfromtimestamp(1526300857) + base_line = BaseLine(BaseLineTest.TEST_IMS_LINE_1, now=now) + self.assertEquals(base_line.datetime_unix, 1526297260) + self.assertEquals(base_line.datetime, datetime.utcfromtimestamp(1526297260)) + def test_frequency(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) self.assertEquals(base_line.frequency, 1620359296) @@ -48,6 +56,8 @@ def test_level(self): def test_symbols(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) self.assertEquals(base_line.symbols, 179) + base_line = BaseLine(BaseLineTest.TEST_RAW_LINE_1) + self.assertEquals(base_line.symbols, None) def test_link_direction(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) @@ -60,6 +70,11 @@ def test_link_direction(self): self.assertEquals(base_line.is_downlink(), True) self.assertEquals(base_line.is_uplink(), False) + base_line = BaseLine(BaseLineTest.TEST_RAW_LINE_1) + self.assertEquals(base_line.link_direction,None) + self.assertEquals(base_line.is_downlink(), False) + self.assertEquals(base_line.is_uplink(), False) + def test_raw_line(self): for line in [BaseLineTest.TEST_VOC_LINE_1, BaseLineTest.TEST_VOC_LINE_2]: base_line = BaseLine(line) diff --git a/iridiumtk/line_parser/test_msg_line.py b/iridiumtk/line_parser/test_msg_line.py index 9e5f392..b3b21bb 100644 --- a/iridiumtk/line_parser/test_msg_line.py +++ b/iridiumtk/line_parser/test_msg_line.py @@ -9,6 +9,8 @@ class MsgLineTest(unittest.TestCase): TEST_MSG_LINE_1 = 'MSG: i-1526039037-t1 001174920 1626447232 100% 0.006 432 DL 00110011111100110011001111110011 odd:01100000000000000000000001 1:0:03 ric:3525696 fmt:05 seq:18 1101000111 0/0 AgAFACYCgTLIxITKA4x8qpLs5geb4SICAVgVzXq9gdkuxao79yKD7DG5XpZD +1111' + TEST_MSG_LINE_2 = 'MSG: i-1526039037-t1 002353427 1626432128 90% 0.002 432 DL 00110011111100110011001111110011 odd:01100000000000000000000001 1:3:05 ric:3525696 fmt:05 seq:24 0010000101 0/0 AgAFAiZzt1VegmFKoMZzD/Bb.M![127][25][19]+/tuE3QMEXmzPe433ff0L2RchgTp2z +1111' + TEST_MSG_LINE_3 = 'MSG: i-1526039037-t1 000083205 1626415820 100% 0.038 358 DL 00110011111100110011001111110011 odd:10000110000000000001 2:A:34 1 c=09297 00000000 01110000000000000000 00000100000000000000 00000000000000000000 ric:3221191 fmt:05 seq:07 0010101111 0/0 Estab comms 99 @ 2030 for sitrep + descr_extra:01101011010111100111001100111101101110000011' def test_empty_input(self): with self.assertRaises(LineParseException): @@ -23,11 +25,21 @@ def test_simple(self): self.assertEquals(msg_line.message_ctr, 0) self.assertEquals(msg_line.message_ctr_max, 0) - self.assertEquals(msg_line.message_checksum, None) - self.assertEquals(msg_line.message_hex, None) - self.assertEquals(msg_line.message_brest, None) - self.assertEquals(msg_line.message_ascii, None) - self.assertEquals(msg_line.message_rest, None) + self.assertEquals(msg_line.message_data_escaped, 'AgAFACYCgTLIxITKA4x8qpLs5geb4SICAVgVzXq9gdkuxao79yKD7DG5XpZD') + self.assertEquals(msg_line.message_data, b'AgAFACYCgTLIxITKA4x8qpLs5geb4SICAVgVzXq9gdkuxao79yKD7DG5XpZD') + self.assertEquals(msg_line.message_rest, '1111') + + def test_escaped_message(self): + msg_line = MsgLine(MsgLineTest.TEST_MSG_LINE_2) + + self.assertEquals(msg_line.message_data_escaped, 'AgAFAiZzt1VegmFKoMZzD/Bb.M![127][25][19]+/tuE3QMEXmzPe433ff0L2RchgTp2z') + self.assertEquals(msg_line.message_data, b'AgAFAiZzt1VegmFKoMZzD/Bb.M!\x7f\x19\x13+/tuE3QMEXmzPe433ff0L2RchgTp2z') + self.assertEquals(msg_line.message_rest, '1111') + + def test_no_msg_reset_chars(self): + msg_line = MsgLine(MsgLineTest.TEST_MSG_LINE_3) + + self.assertEquals(msg_line.message_rest, '') def main(): diff --git a/iridiumtk/reassembler.py b/iridiumtk/reassembler.py index d9ec5a8..6681a4c 100755 --- a/iridiumtk/reassembler.py +++ b/iridiumtk/reassembler.py @@ -1,173 +1,98 @@ #!/usr/bin/env python +import argparse import sys import fileinput import getopt import datetime import re +import logging -verbose = False -ofmt= "undef" - -options, remainder = getopt.getopt(sys.argv[1:], 'vi:o:', [ - 'verbose', - 'input=', - 'output=', - ]) - -for opt, arg in options: - if opt in ('-v', '--verbose'): - verbose = True - elif opt in ('-o', '--output'): - ofmt=arg - else: - raise Exception("unknown argument?") - -class Message(object): - def __init__(self,line): - self.typ, self.name, self.time, self.frequency, self.confidence, self.level, self.symbols, self.uldl, self.data = line.split(None,8) - def upgrade(self): -# return IridiumMessage(self).upgrade() - return self - def _pretty_header(self): - return "%s %s %09d %010d %3d%% %.3f"%(self.typ,self.filename,self.time,self.frequency,self.confidence,self.level) - def _pretty_trailer(self): - return "" - def pretty(self): - str= "RAW: "+self._pretty_header() - bs=self.bitstream_raw - str+=self._pretty_trailer() - return str - -selected=[] - -def do_input(type): - for line in fileinput.input(remainder): - qqq=re.compile('Warning:') - if qqq.match(line): - print "Skip: ",line + +import dateparser + + +from .line_parser import BaseLine, IraLine, MsgLine + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class MessageTypeProcessor(object): + def __init__(self, base_frames): + self._base_frames = base_frames + + def frames(self): + selected = [] + for base_frame in self._base_frames: + if base_frame.frame_type != 'MSG': + continue + message_frame = MsgLine(base_frame.raw_line) + yield 'Message "{}"'.format(message_frame.message_data_escaped) + + +class PageTypeProcessor(object): + def __init__(self, base_frames): + self._base_frames = base_frames + + def frames(self): + for base_frame in self._base_frames: + if base_frame.frame_type != 'IRA': + continue + ira_frame = IraLine(base_frame.raw_line) + + for page in ira_frame.pages: + yield "%02d %02d %s %s %03d : %s %s" % (ira_frame.satellite, ira_frame.beam, ira_frame.position.x, ira_frame.position.y, ira_frame.altitude, page.tmsi, page.msc_id) + + +FRAME_TYPES = { + 'messages': MessageTypeProcessor, + 'pages': PageTypeProcessor, +} + + +def parse_to_base_frame_and_filter_time(input_files, start_time_filter, end_time_filter): + for line in fileinput.input(files=input_files): + line = line.strip() + if 'A:OK' in line and "Message: Couldn't parse:" not in line: + raise RuntimeError('Expected "iridium-parser.py" parsed data. Found raw "iridium-extractor" data.') + if line.startswith('ERR: '): + continue + if line.startswith('Warning:'): continue - # try: - perline(Message(line.strip()).upgrade()) - # except ValueError: - # print >> sys.stderr, "Couldn't parse line",line - -def fixtime(n,t): - try: - (crap,ts,fnord)=n.split("-",3) - return (float(ts)+int(t)/1000) - except: - return int(t)/1000 - -def perline(q): - if False: - pass - elif ofmt == "page": - if q.typ=="IRA:": - p=re.compile('.*sat:(\d+) beam:(\d+) pos=\((.[0-9.]+)/(.[0-9.]+)\) alt=([-0-9]+) .* bc_sb:\d+ (.*)') - m=p.match(q.data) - if(not m): - print >> sys.stderr, "Couldn't parse IRA: ",q.data - else: - q.sat= int(m.group(1)) - q.beam= int(m.group(2)) - q.posx= m.group(3) - q.posy= m.group(4) - q.alt= int(m.group(5)) - p=re.compile('PAGE\(tmsi:([0-9a-f]+) msc_id:([0-9]+)\)') - m=p.findall(m.group(6)) - for x in m: - print "%02d %02d %s %s %03d : %s %s"%(q.sat,q.beam,q.posx,q.posy,q.alt,x[0],x[1]) - - elif ofmt == "msg": - if q.typ == "MSG:": - #ric:0098049 fmt:05 seq:43 1010010000 1/1 oNEZCOuxvM3PuiQHujzQYd5n0Q8ra0wfMG2WnnhoxAnunT9xzIBSkXyvNP[3] +11111 - p=re.compile('.* ric:(\d+) fmt:(\d+) seq:(\d+) [01]+ (\d)/(\d) csum:([0-9a-f][0-9a-f]) msg:([0-9a-f]+)\.([01]*) ') - m=p.match(q.data) - if(not m): - print >> sys.stderr, "Couldn't parse MSG: ",q.data - else: - q.msg_ric= int(m.group(1)) - q.fmt= int(m.group(2)) - q.msg_seq= int(m.group(3)) - q.msg_ctr= int(m.group(4)) - q.msg_ctr_max= int(m.group(5)) - q.msg_checksum=int(m.group(6),16) - q.msg_hex= m.group(7) - q.msg_brest= m.group(8) - q.time= fixtime(q.name,q.time) - - - q.msg_msgdata = ''.join(["{0:08b}".format(int(q.msg_hex[i:i+2], 16)) for i in range(0, len(q.msg_hex), 2)]) - q.msg_msgdata+=q.msg_brest - m=re.compile('(\d{7})').findall(q.msg_msgdata) - q.msg_ascii="" - for (group) in m: - character = int(group, 2) - if(character<32 or character==127): - q.msg_ascii+="[%d]"%character - else: - q.msg_ascii+=chr(character) - if len(q.msg_msgdata)%7: - q.msg_rest=q.msg_msgdata[-(len(q.msg_msgdata)%7):] - else: - q.msg_rest="" - - selected.append(q) - else: - print "Unknown output mode." - exit(1) + base_line = BaseLine(line) + if start_time_filter and start_time_filter > base_line.datetime: + continue + if end_time_filter and end_time_filter < base_line.datetime: + continue + yield base_line def main(): - do_input(input) - - if ofmt == "msg": - buf={} - ricseq={} - wrapmargin=10 - for m in selected: - # msg_seq wraps around after 61, detect it, and fix it. - if m.msg_ric in ricseq: - if (m.msg_seq + wrapmargin) < ricseq[m.msg_ric][1]: # seq wrapped around - ricseq[m.msg_ric][0]+=62 - if (m.msg_seq + wrapmargin - 62) > ricseq[m.msg_ric][1]: # "wrapped back" (out-of-order old message) - ricseq[m.msg_ric][0]-=62 - else: - ricseq[m.msg_ric]=[0,0] - ricseq[m.msg_ric][1]=m.msg_seq - id="%07d %04d"%(m.msg_ric,(m.msg_seq+ricseq[m.msg_ric][0])) - ts=m.time - if id in buf: - if buf[id].msg_checksum != m.msg_checksum: - print "Whoa! Checksum changed? Message %s (1: @%d checksum %d/2: @%d checksum %d)"%(id,buf[id].time,buf[id].msg_checksum,m.time,m.msg_checksum) - # "Wrap around" to not miss the changed packet. - ricseq[m.msg_ric][0]+=62 - id="%07d %04d"%(m.msg_ric,(m.msg_seq+ricseq[m.msg_ric][0])) - m.msgs=['[MISSING]']*3 - buf[id]=m - else: - m.msgs=['[MISSING]']*3 - buf[id]=m - buf[id].msgs[m.msg_ctr]=m.msg_ascii - - def messagechecksum(msg): - csum=0 - for x in msg: - csum=(csum+ord(x))%128 - return (~csum)%128 - - for b in sorted(buf, key=lambda x: buf[x].time): - msg="".join(buf[b].msgs[:1+buf[b].msg_ctr_max]) - msg=re.sub("(\[3\])+$","",msg) # XXX: should be done differently - cmsg=re.sub("\[10\]","\n",msg) # XXX: should be done differently - # csum="" - csum=messagechecksum(cmsg) - str="Message %s @%s (len:%d)"%(b,datetime.datetime.fromtimestamp(buf[b].time).strftime("%Y-%m-%dT%H:%M:%S"),buf[b].msg_ctr_max) - str+= " %3d"%buf[b].msg_checksum - str+= (" fail"," OK ")[buf[b].msg_checksum == csum] - str+= ": %s"%(msg) - print str + parser = argparse.ArgumentParser(description='Convert iridium-parser.py VOC output to DFS') + parser.add_argument('--start', metavar='DATETIME', type=str, default=None, help='Filter events before this time') + parser.add_argument('--end', metavar='DATETIME', type=str, default=None, help='Filter events after this time') + + parser.add_argument('--type', choices=FRAME_TYPES.keys(), required=True) + + parser.add_argument('input', metavar='FILE', nargs='*', help='Files to read, if empty or -, stdin is used') + args = parser.parse_args() + + input_files = args.input if len(args.input) > 0 else ['-'] + start_time_filter = dateparser.parse(args.start) if args.start else None + end_time_filter = dateparser.parse(args.end) if args.end else None + + type_processor = FRAME_TYPES[args.type] + + base_frames = parse_to_base_frame_and_filter_time(input_files, start_time_filter, end_time_filter) + + logger.info('Staring to parse frames') + for frame in type_processor(base_frames).frames(): + print(frame) + logger.info('Finished parsing frames') + + if __name__ == '__main__': main() diff --git a/requirements.txt b/requirements.txt index d4edbd2..e0dab12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ numpy>=1.11.1 six>=1.11.0 dateparser>=0.7.0 matplotlib>=2.2.0 -enum34>=1.1.0 \ No newline at end of file +enum34>=1.1.0 From fc8b0c19f13cc3fa9cf141e6dc46f03b52372776 Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Tue, 29 May 2018 19:56:13 +0100 Subject: [PATCH 47/50] Enable travis on all branches --- .travis.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index f75cfca..08ca170 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,5 @@ sudo: false -branches: - only: - - py27 - - py27_36 - - reassembler - language: python matrix: From 4d214d73edd64721ff7fe3ef6165cd067cf8a79b Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Wed, 30 May 2018 12:46:06 +0100 Subject: [PATCH 48/50] Move to just Python3.6+ support This cleans up some code and allows work to progress without having to deal with str vs. bytes all the time. --- .travis.yml | 28 +++++++++++++------------ iridiumtk/graph/iridium_matplotlib.py | 3 --- iridiumtk/graph_by_type.py | 5 ++++- iridiumtk/graph_voc.py | 5 ++++- iridiumtk/line_parser/test_voc_line.py | 2 +- iridiumtk/line_parser/voc_line.py | 29 +++++++++++++------------- iridiumtk/rx_stats_hist.py | 5 ++++- iridiumtk/test_bits_to_dfs.py | 4 ++-- requirements-gui.txt | 1 + requirements.txt | 3 --- setup.py | 7 ++++++- tox.ini | 8 +++---- 12 files changed, 56 insertions(+), 44 deletions(-) create mode 100644 requirements-gui.txt diff --git a/.travis.yml b/.travis.yml index 08ca170..b42b264 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,36 +4,38 @@ language: python matrix: include: - # Python 2.7 - - python: "2.7" - os: "linux" - dist: trusty - env: TOXENV=py27 - # Python 3.6 - python: "3.6" os: "linux" dist: trusty env: TOXENV=py36 + # Python 3.7-dev + - python: "3.7-dev" + os: "linux" + dist: trusty + env: TOXENV=py37 + #- os: "osx" # language: generic # env: TOXENV=py27 - # PyPy 2.7 - #- python: "pypy-5.4.1" - # os: "linux" - # dist: trusty - # env: TOXENV=pypy + # PyPy3 v? + - python: "pypy3" + os: "linux" + dist: trusty + env: TOXENV=pypy3 # Linters - - python: "2.7" + - python: "3.6" os: "linux" dist: trusty env: TOXENV=linters allow_failures: - - python: "pypy-5.4.1" + - python: + - "pypy3" + - "3.7-dev" - os: "osx" cache: diff --git a/iridiumtk/graph/iridium_matplotlib.py b/iridiumtk/graph/iridium_matplotlib.py index 9771e38..75e60e9 100644 --- a/iridiumtk/graph/iridium_matplotlib.py +++ b/iridiumtk/graph/iridium_matplotlib.py @@ -3,9 +3,6 @@ from collections import namedtuple -from six.moves import range - - KHZ = 1000 MHZ = KHZ * 1000 diff --git a/iridiumtk/graph_by_type.py b/iridiumtk/graph_by_type.py index c51fc4b..23f1eb9 100755 --- a/iridiumtk/graph_by_type.py +++ b/iridiumtk/graph_by_type.py @@ -5,7 +5,10 @@ import dateparser -import matplotlib.pyplot as plt +try: + import matplotlib.pyplot as plt +except ImportError: + print('Failed to import matplotlib. This prevents any GUI.' , file=sys.stderr) import numpy as np import six diff --git a/iridiumtk/graph_voc.py b/iridiumtk/graph_voc.py index 161269e..207f3dd 100755 --- a/iridiumtk/graph_voc.py +++ b/iridiumtk/graph_voc.py @@ -11,7 +11,10 @@ import dateparser -import matplotlib.pyplot as plt +try: + import matplotlib.pyplot as plt +except ImportError: + print('Failed to import matplotlib. This prevents any GUI.' , file=sys.stderr) import numpy as np diff --git a/iridiumtk/line_parser/test_voc_line.py b/iridiumtk/line_parser/test_voc_line.py index 53e2de8..b19d4f1 100644 --- a/iridiumtk/line_parser/test_voc_line.py +++ b/iridiumtk/line_parser/test_voc_line.py @@ -28,7 +28,7 @@ def test_old_format(self): def test_new_format(self): voc_line = VocLine(VocLineTest.TEST_VOC_LINE_2) - self.assertEquals(voc_line.voice_bits, b'\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') + self.assertEquals(voc_line.voice_bits, b'\xdf\xff\xf3\xfc\x10\x33\xc3\x1f\x0c\x83\xc3\xcc\xcc\x30\xff\xf3\xef\x00\xbc\x0c\xb4\x0f\xdc\xd0\x1a\xcc\x9c\xc5\x0c\xfc\x28\x01\xcc\x38\xc2\x33\xe0\xff\x4f') def test_voice_bits_small_packet(self): voc_line = VocLine(VocLineTest.TEST_VOC_LINE_3) diff --git a/iridiumtk/line_parser/voc_line.py b/iridiumtk/line_parser/voc_line.py index c2f357b..71aa235 100644 --- a/iridiumtk/line_parser/voc_line.py +++ b/iridiumtk/line_parser/voc_line.py @@ -1,10 +1,7 @@ #!/usr/bin/env python import logging - - -import six -from six.moves import range +from io import BytesIO from .base_line import BaseLine, LineParseException @@ -39,21 +36,25 @@ def __init__(self, line): self._voice_data = line_split[10] except (IndexError, ValueError) as e: logger.error('Failed to parse line "%s"', line) - six.raise_from(LineParseException('Failed to parse line "{}"'.format(line), e), e) + raise LineParseException(f'Failed to parse line "{line}"') from e @property def voice_bits(self): data = self._voice_data if data is None: return None - byte_stream = six.BytesIO() if data[0] == "[": - for pos in range(1, len(data), 3): - byte = int(data[pos:pos + 2], 16) - byte = int('{:08b}'.format(byte)[::-1], 2) - byte_stream.write(six.int2byte(byte)) + return bytes.fromhex(data[1:-1].replace('.', ' ')) else: - for bits in chunks(data, 8): - byte = int(bits[::-1], 2) - byte_stream.write(six.int2byte(byte)) - return byte_stream.getvalue() + result = bytearray() + for bits_of_byte in chunks(data, 8): + byte = 0 + for i in range(0, len(bits_of_byte)): + if bits_of_byte[i] == '0': + continue + elif bits_of_byte[i] == '1': + byte = byte | (1 << i) + else: + raise LineParseException(f'Unknown byte format: {data}') + result.append(byte) + return result diff --git a/iridiumtk/rx_stats_hist.py b/iridiumtk/rx_stats_hist.py index a6742bd..3837a58 100755 --- a/iridiumtk/rx_stats_hist.py +++ b/iridiumtk/rx_stats_hist.py @@ -16,7 +16,10 @@ import dateparser -import matplotlib.pyplot as plt +try: + import matplotlib.pyplot as plt +except ImportError: + print('Failed to import matplotlib. This prevents any GUI.' , file=sys.stderr) logging.basicConfig(level=logging.INFO) diff --git a/iridiumtk/test_bits_to_dfs.py b/iridiumtk/test_bits_to_dfs.py index 41f3c18..9a33561 100644 --- a/iridiumtk/test_bits_to_dfs.py +++ b/iridiumtk/test_bits_to_dfs.py @@ -34,12 +34,12 @@ def test_short_packet(self): def test_multiple(self): output = BytesIO() bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_1, BitsToDfsTest.TEST_VOC_LINE_1, BitsToDfsTest.TEST_VOC_LINE_2], output) - self.assertEquals(output.getvalue(), (b'\x9e\x88$\xdb\xe6\x01' * 2) + b'\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') + self.assertEquals(output.getvalue(), (b'\x9e\x88$\xdb\xe6\x01' * 2) + b'\xdf\xff\xf3\xfc\x103\xc3\x1f\x0c\x83\xc3\xcc\xcc0\xff\xf3\xef\x00\xbc\x0c\xb4\x0f\xdc\xd0\x1a\xcc\x9c\xc5\x0c\xfc(\x01\xcc8\xc23\xe0\xffO') def test_filters_non_voc_lines(self): output = BytesIO() bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_1, 'NOT_VOC:', BitsToDfsTest.TEST_VOC_LINE_2], output) - self.assertEquals(output.getvalue(), b'\x9e\x88$\xdb\xe6\x01' + b'\xfb\xff\xcf?\x08\xcc\xc3\xf80\xc1\xc333\x0c\xff\xcf\xf7\x00=0-\xf0;\x0bX39\xa30?\x14\x803\x1cC\xcc\x07\xff\xf2') + self.assertEquals(output.getvalue(), b'\x9e\x88$\xdb\xe6\x01' + b'\xdf\xff\xf3\xfc\x103\xc3\x1f\x0c\x83\xc3\xcc\xcc0\xff\xf3\xef\x00\xbc\x0c\xb4\x0f\xdc\xd0\x1a\xcc\x9c\xc5\x0c\xfc(\x01\xcc8\xc23\xe0\xffO') class MainTest(unittest.TestCase): diff --git a/requirements-gui.txt b/requirements-gui.txt new file mode 100644 index 0000000..075f4c8 --- /dev/null +++ b/requirements-gui.txt @@ -0,0 +1 @@ +matplotlib>=2.2.0 diff --git a/requirements.txt b/requirements.txt index e0dab12..9c3be23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,2 @@ numpy>=1.11.1 -six>=1.11.0 dateparser>=0.7.0 -matplotlib>=2.2.0 -enum34>=1.1.0 diff --git a/setup.py b/setup.py index af3d5e0..217a1df 100644 --- a/setup.py +++ b/setup.py @@ -70,8 +70,11 @@ def run_tests(self): }, package_data={'': ['LICENSE', 'README.md']}, zip_safe=False, + + python_requires='>=3.6', install_requires=[], tests_require=['pytest'], + cmdclass={'test': PyTest}, keywords=[], license='BSD', @@ -80,7 +83,9 @@ def run_tests(self): 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', ), ) diff --git a/tox.ini b/tox.ini index e82c093..ef30c0a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py36 +envlist = py36, py37, pypy3 platform = linux2 skipsdist = True @@ -15,7 +15,7 @@ commands = # Linters [testenv:flake8] -basepython = python2.7 +basepython = python3.6 skip_install = true deps = flake8 @@ -26,7 +26,7 @@ commands = flake8 iridiumtk/ setup.py [testenv:pylint] -basepython = python2.7 +basepython = python3.6 skip_install = true deps = {[testenv]deps} @@ -36,7 +36,7 @@ commands = pylint iridiumtk [testenv:linters] -basepython = python2.7 +basepython = python3.6 deps = {[testenv:flake8]deps} {[testenv:pylint]deps} From 4aee04c120e5c8528349935aa8cbb82ce4b0637c Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Wed, 30 May 2018 13:22:26 +0100 Subject: [PATCH 49/50] Fix timezone and py2->3 conversion issues. --- iridiumtk/graph_by_type.py | 2 +- iridiumtk/graph_voc.py | 4 ++-- iridiumtk/line_parser/base_line.py | 11 ++++------- iridiumtk/line_parser/ira_line.py | 5 +---- iridiumtk/line_parser/msg_line.py | 5 +---- iridiumtk/line_parser/test_base_line.py | 8 ++++---- 6 files changed, 13 insertions(+), 22 deletions(-) diff --git a/iridiumtk/graph_by_type.py b/iridiumtk/graph_by_type.py index 23f1eb9..14b3f1e 100755 --- a/iridiumtk/graph_by_type.py +++ b/iridiumtk/graph_by_type.py @@ -70,7 +70,7 @@ def main(): plot_data_time = np.empty(number_of_frames, dtype=np.float64) plot_data_freq = np.empty(number_of_frames, dtype=np.uint32) for i, base_line in enumerate(frames): - plot_data_time[i] = np.uint32(base_line.datetime_unix) + plot_data_time[i] = np.uint32(base_line.datetime_unix_utc) plot_data_freq[i] = np.float64(base_line.frequency) subplot.scatter(plot_data_time, plot_data_freq, label=frame_type) diff --git a/iridiumtk/graph_voc.py b/iridiumtk/graph_voc.py index 207f3dd..8bd3a48 100755 --- a/iridiumtk/graph_voc.py +++ b/iridiumtk/graph_voc.py @@ -76,7 +76,7 @@ def onclick(self, event): def filter_voc(self, t_start, t_stop, f_min, f_max): for voc_line in self.lines: - ts = voc_line.datetime_unix + ts = voc_line.datetime_unix_utc f = voc_line.frequency if t_start <= ts and ts <= t_stop and \ f_min <= f and f <= f_max: @@ -139,7 +139,7 @@ def main(): plot_data_freq = np.empty(number_of_lines, dtype=np.uint32) for i, voc_line in enumerate(lines): # plot_data_time[i] = np.datetime64(voc_line.datetime().isoformat()) - plot_data_time[i] = np.uint32(voc_line.datetime_unix) + plot_data_time[i] = np.uint32(voc_line.datetime_unix_utc) plot_data_freq[i] = np.float64(voc_line.frequency) fig = plt.figure() diff --git a/iridiumtk/line_parser/base_line.py b/iridiumtk/line_parser/base_line.py index 5d42502..0900486 100644 --- a/iridiumtk/line_parser/base_line.py +++ b/iridiumtk/line_parser/base_line.py @@ -3,10 +3,7 @@ from datetime import datetime from enum import Enum import logging -import time - - -import six +import calendar logging.basicConfig(level=logging.INFO) @@ -42,7 +39,7 @@ def __init__(self, line, now=datetime.utcnow()): ts_base_ms = int(raw_time_base.split('-')[1].split('.')[0]) except ValueError as e: logger.warn('No base datetime found. Using now instead') - ts_base_ms = time.mktime(now.timetuple()) + ts_base_ms = calendar.timegm(now.timetuple()) time_offset_ns = int(line_split[2]) self._timestamp = int(ts_base_ms + (time_offset_ns / 1000)) @@ -65,7 +62,7 @@ def __init__(self, line, now=datetime.utcnow()): self._link_direction = None except (IndexError, ValueError) as e: logger.error('Failed to parse line "%s"', line) - six.raise_from(LineParseException('Failed to parse line "{}"'.format(line), e), e) + raise LineParseException(f'Failed to parse line "{line}"') from e @property def raw_line(self): @@ -80,7 +77,7 @@ def datetime(self): return datetime.utcfromtimestamp(self._timestamp) @property - def datetime_unix(self): + def datetime_unix_utc(self): return self._timestamp @property diff --git a/iridiumtk/line_parser/ira_line.py b/iridiumtk/line_parser/ira_line.py index 4e5f931..45c2644 100644 --- a/iridiumtk/line_parser/ira_line.py +++ b/iridiumtk/line_parser/ira_line.py @@ -5,9 +5,6 @@ import re -import six - - from .base_line import BaseLine, LineParseException @@ -48,7 +45,7 @@ def __init__(self, line): self._pages.append(Page(tmsi=match[0], msc_id=int(match[1]))) except (IndexError, ValueError) as e: logger.error('Failed to parse line "%s"', line) - six.raise_from(LineParseException('Failed to parse line "{}"'.format(line), e), e) + raise LineParseException(f'Failed to parse line "{line}"') from e @property def satellite(self): diff --git a/iridiumtk/line_parser/msg_line.py b/iridiumtk/line_parser/msg_line.py index 11b7298..fb385f7 100644 --- a/iridiumtk/line_parser/msg_line.py +++ b/iridiumtk/line_parser/msg_line.py @@ -4,9 +4,6 @@ import re -import six - - from .base_line import BaseLine, LineParseException @@ -41,7 +38,7 @@ def __init__(self, line): self._msg_rest = matches.group(7) except (IndexError, ValueError) as e: logger.error('Failed to parse line "%s"', line) - six.raise_from(LineParseException('Failed to parse line "{}"'.format(line), e), e) + raise LineParseException(f'Failed to parse line "{line}"') from e @property def message_ric(self): diff --git a/iridiumtk/line_parser/test_base_line.py b/iridiumtk/line_parser/test_base_line.py index c75636d..1069cb5 100644 --- a/iridiumtk/line_parser/test_base_line.py +++ b/iridiumtk/line_parser/test_base_line.py @@ -19,19 +19,19 @@ def test_empty_input(self): def test_old_format_datetime(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) - self.assertEquals(base_line.datetime_unix, 1443372344) + self.assertEquals(base_line.datetime_unix_utc, 1443372344) self.assertEquals(base_line.datetime, datetime.utcfromtimestamp(1443372344)) def test_new_format_datetime(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) - self.assertEquals(base_line.datetime_unix, 1526039102) + self.assertEquals(base_line.datetime_unix_utc, 1526039102) self.assertEquals(base_line.datetime, datetime.utcfromtimestamp(1526039102)) def test_filename_datetime(self): now = datetime.utcfromtimestamp(1526300857) base_line = BaseLine(BaseLineTest.TEST_IMS_LINE_1, now=now) - self.assertEquals(base_line.datetime_unix, 1526297260) - self.assertEquals(base_line.datetime, datetime.utcfromtimestamp(1526297260)) + self.assertEquals(base_line.datetime_unix_utc, 1526300860) + self.assertEquals(base_line.datetime, datetime.utcfromtimestamp(1526300860)) def test_frequency(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) From 6dddc27a45992f049d3aab36d9a4a227ee684e2b Mon Sep 17 00:00:00 2001 From: Guy Taylor Date: Wed, 30 May 2018 13:41:44 +0100 Subject: [PATCH 50/50] Fix lint issues --- .pylintrc | 1 + iridiumtk/graph_by_type.py | 3 +- iridiumtk/graph_voc.py | 2 +- iridiumtk/line_parser/__init__.py | 2 +- iridiumtk/line_parser/base_line.py | 6 ++-- iridiumtk/line_parser/test_base_line.py | 48 ++++++++++++------------- iridiumtk/line_parser/test_ira_line.py | 10 +++--- iridiumtk/line_parser/test_msg_line.py | 24 ++++++------- iridiumtk/line_parser/test_voc_line.py | 8 ++--- iridiumtk/line_parser/voc_line.py | 11 +++--- iridiumtk/reassembler.py | 9 ++--- iridiumtk/rx_stats_hist.py | 5 +-- iridiumtk/test_bits_to_dfs.py | 10 +++--- iridiumtk/test_graph_voc.py | 4 +-- setup.py | 2 +- 15 files changed, 70 insertions(+), 75 deletions(-) diff --git a/.pylintrc b/.pylintrc index 11d426e..b29e755 100644 --- a/.pylintrc +++ b/.pylintrc @@ -31,6 +31,7 @@ disable= too-many-boolean-expressions, too-many-branches, too-many-statements, + too-many-instance-attributes, [FORMAT] diff --git a/iridiumtk/graph_by_type.py b/iridiumtk/graph_by_type.py index 14b3f1e..644b410 100755 --- a/iridiumtk/graph_by_type.py +++ b/iridiumtk/graph_by_type.py @@ -2,13 +2,14 @@ import argparse import fileinput import logging +import sys import dateparser try: import matplotlib.pyplot as plt except ImportError: - print('Failed to import matplotlib. This prevents any GUI.' , file=sys.stderr) + print('Failed to import matplotlib. This prevents any GUI.', file=sys.stderr) import numpy as np import six diff --git a/iridiumtk/graph_voc.py b/iridiumtk/graph_voc.py index 8bd3a48..bb7510f 100755 --- a/iridiumtk/graph_voc.py +++ b/iridiumtk/graph_voc.py @@ -14,7 +14,7 @@ try: import matplotlib.pyplot as plt except ImportError: - print('Failed to import matplotlib. This prevents any GUI.' , file=sys.stderr) + print('Failed to import matplotlib. This prevents any GUI.', file=sys.stderr) import numpy as np diff --git a/iridiumtk/line_parser/__init__.py b/iridiumtk/line_parser/__init__.py index c0ede78..14eb3f2 100644 --- a/iridiumtk/line_parser/__init__.py +++ b/iridiumtk/line_parser/__init__.py @@ -1,9 +1,9 @@ #!/usr/bin/python from .base_line import BaseLine -from .voc_line import VocLine from .ira_line import IraLine from .msg_line import MsgLine +from .voc_line import VocLine __all__ = [x.__name__ for x in (BaseLine, VocLine, IraLine, MsgLine)] diff --git a/iridiumtk/line_parser/base_line.py b/iridiumtk/line_parser/base_line.py index 0900486..95907f7 100644 --- a/iridiumtk/line_parser/base_line.py +++ b/iridiumtk/line_parser/base_line.py @@ -1,9 +1,9 @@ #!/usr/bin/env python +import calendar from datetime import datetime from enum import Enum import logging -import calendar logging.basicConfig(level=logging.INFO) @@ -37,8 +37,8 @@ def __init__(self, line, now=datetime.utcnow()): raw_time_base = line_split[1] try: ts_base_ms = int(raw_time_base.split('-')[1].split('.')[0]) - except ValueError as e: - logger.warn('No base datetime found. Using now instead') + except ValueError: + logger.warning('No base datetime found. Using now instead') ts_base_ms = calendar.timegm(now.timetuple()) time_offset_ns = int(line_split[2]) diff --git a/iridiumtk/line_parser/test_base_line.py b/iridiumtk/line_parser/test_base_line.py index 1069cb5..f5aed05 100644 --- a/iridiumtk/line_parser/test_base_line.py +++ b/iridiumtk/line_parser/test_base_line.py @@ -19,66 +19,66 @@ def test_empty_input(self): def test_old_format_datetime(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) - self.assertEquals(base_line.datetime_unix_utc, 1443372344) - self.assertEquals(base_line.datetime, datetime.utcfromtimestamp(1443372344)) + self.assertEqual(base_line.datetime_unix_utc, 1443372344) + self.assertEqual(base_line.datetime, datetime.utcfromtimestamp(1443372344)) def test_new_format_datetime(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) - self.assertEquals(base_line.datetime_unix_utc, 1526039102) - self.assertEquals(base_line.datetime, datetime.utcfromtimestamp(1526039102)) + self.assertEqual(base_line.datetime_unix_utc, 1526039102) + self.assertEqual(base_line.datetime, datetime.utcfromtimestamp(1526039102)) def test_filename_datetime(self): now = datetime.utcfromtimestamp(1526300857) base_line = BaseLine(BaseLineTest.TEST_IMS_LINE_1, now=now) - self.assertEquals(base_line.datetime_unix_utc, 1526300860) - self.assertEquals(base_line.datetime, datetime.utcfromtimestamp(1526300860)) + self.assertEqual(base_line.datetime_unix_utc, 1526300860) + self.assertEqual(base_line.datetime, datetime.utcfromtimestamp(1526300860)) def test_frequency(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) - self.assertEquals(base_line.frequency, 1620359296) + self.assertEqual(base_line.frequency, 1620359296) def test_frame_type(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) - self.assertEquals(base_line.frame_type, 'VOC') + self.assertEqual(base_line.frame_type, 'VOC') def test_confidence(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) - self.assertEquals(base_line.confidence, 81) + self.assertEqual(base_line.confidence, 81) base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) - self.assertEquals(base_line.confidence, 100) + self.assertEqual(base_line.confidence, 100) def test_level(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) - self.assertEquals(base_line.level, 0.027) + self.assertEqual(base_line.level, 0.027) base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) - self.assertEquals(base_line.level, 0.003) + self.assertEqual(base_line.level, 0.003) def test_symbols(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) - self.assertEquals(base_line.symbols, 179) + self.assertEqual(base_line.symbols, 179) base_line = BaseLine(BaseLineTest.TEST_RAW_LINE_1) - self.assertEquals(base_line.symbols, None) + self.assertEqual(base_line.symbols, None) def test_link_direction(self): base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_1) - self.assertEquals(base_line.link_direction, LinkDirection.NO_DIRECTION) - self.assertEquals(base_line.is_downlink(), False) - self.assertEquals(base_line.is_uplink(), False) + self.assertEqual(base_line.link_direction, LinkDirection.NO_DIRECTION) + self.assertEqual(base_line.is_downlink(), False) + self.assertEqual(base_line.is_uplink(), False) base_line = BaseLine(BaseLineTest.TEST_VOC_LINE_2) - self.assertEquals(base_line.link_direction, LinkDirection.DOWNLINK) - self.assertEquals(base_line.is_downlink(), True) - self.assertEquals(base_line.is_uplink(), False) + self.assertEqual(base_line.link_direction, LinkDirection.DOWNLINK) + self.assertEqual(base_line.is_downlink(), True) + self.assertEqual(base_line.is_uplink(), False) base_line = BaseLine(BaseLineTest.TEST_RAW_LINE_1) - self.assertEquals(base_line.link_direction,None) - self.assertEquals(base_line.is_downlink(), False) - self.assertEquals(base_line.is_uplink(), False) + self.assertEqual(base_line.link_direction, None) + self.assertEqual(base_line.is_downlink(), False) + self.assertEqual(base_line.is_uplink(), False) def test_raw_line(self): for line in [BaseLineTest.TEST_VOC_LINE_1, BaseLineTest.TEST_VOC_LINE_2]: base_line = BaseLine(line) - self.assertEquals(base_line.raw_line, line) + self.assertEqual(base_line.raw_line, line) def main(): diff --git a/iridiumtk/line_parser/test_ira_line.py b/iridiumtk/line_parser/test_ira_line.py index 91382a2..6e49614 100644 --- a/iridiumtk/line_parser/test_ira_line.py +++ b/iridiumtk/line_parser/test_ira_line.py @@ -17,11 +17,11 @@ def test_empty_input(self): def test_simple(self): ira_line = IraLine(IraLineTest.TEST_IRA_LINE_1) - self.assertEquals(ira_line.satellite, 80) - self.assertEquals(ira_line.beam, 30) - self.assertEquals(ira_line.position, (54.57, -1.24)) - self.assertEquals(ira_line.altitude, 1) - self.assertEquals(ira_line.pages, [Page(tmsi='0cf155ab', msc_id=3)]) + self.assertEqual(ira_line.satellite, 80) + self.assertEqual(ira_line.beam, 30) + self.assertEqual(ira_line.position, (54.57, -1.24)) + self.assertEqual(ira_line.altitude, 1) + self.assertEqual(ira_line.pages, [Page(tmsi='0cf155ab', msc_id=3)]) def main(): diff --git a/iridiumtk/line_parser/test_msg_line.py b/iridiumtk/line_parser/test_msg_line.py index b3b21bb..24fa301 100644 --- a/iridiumtk/line_parser/test_msg_line.py +++ b/iridiumtk/line_parser/test_msg_line.py @@ -19,27 +19,27 @@ def test_empty_input(self): def test_simple(self): msg_line = MsgLine(MsgLineTest.TEST_MSG_LINE_1) - self.assertEquals(msg_line.message_ric, 3525696) - self.assertEquals(msg_line.format, 5) - self.assertEquals(msg_line.message_sequence, 18) - self.assertEquals(msg_line.message_ctr, 0) - self.assertEquals(msg_line.message_ctr_max, 0) + self.assertEqual(msg_line.message_ric, 3525696) + self.assertEqual(msg_line.format, 5) + self.assertEqual(msg_line.message_sequence, 18) + self.assertEqual(msg_line.message_ctr, 0) + self.assertEqual(msg_line.message_ctr_max, 0) - self.assertEquals(msg_line.message_data_escaped, 'AgAFACYCgTLIxITKA4x8qpLs5geb4SICAVgVzXq9gdkuxao79yKD7DG5XpZD') - self.assertEquals(msg_line.message_data, b'AgAFACYCgTLIxITKA4x8qpLs5geb4SICAVgVzXq9gdkuxao79yKD7DG5XpZD') - self.assertEquals(msg_line.message_rest, '1111') + self.assertEqual(msg_line.message_data_escaped, 'AgAFACYCgTLIxITKA4x8qpLs5geb4SICAVgVzXq9gdkuxao79yKD7DG5XpZD') + self.assertEqual(msg_line.message_data, b'AgAFACYCgTLIxITKA4x8qpLs5geb4SICAVgVzXq9gdkuxao79yKD7DG5XpZD') + self.assertEqual(msg_line.message_rest, '1111') def test_escaped_message(self): msg_line = MsgLine(MsgLineTest.TEST_MSG_LINE_2) - self.assertEquals(msg_line.message_data_escaped, 'AgAFAiZzt1VegmFKoMZzD/Bb.M![127][25][19]+/tuE3QMEXmzPe433ff0L2RchgTp2z') - self.assertEquals(msg_line.message_data, b'AgAFAiZzt1VegmFKoMZzD/Bb.M!\x7f\x19\x13+/tuE3QMEXmzPe433ff0L2RchgTp2z') - self.assertEquals(msg_line.message_rest, '1111') + self.assertEqual(msg_line.message_data_escaped, 'AgAFAiZzt1VegmFKoMZzD/Bb.M![127][25][19]+/tuE3QMEXmzPe433ff0L2RchgTp2z') + self.assertEqual(msg_line.message_data, b'AgAFAiZzt1VegmFKoMZzD/Bb.M!\x7f\x19\x13+/tuE3QMEXmzPe433ff0L2RchgTp2z') + self.assertEqual(msg_line.message_rest, '1111') def test_no_msg_reset_chars(self): msg_line = MsgLine(MsgLineTest.TEST_MSG_LINE_3) - self.assertEquals(msg_line.message_rest, '') + self.assertEqual(msg_line.message_rest, '') def main(): diff --git a/iridiumtk/line_parser/test_voc_line.py b/iridiumtk/line_parser/test_voc_line.py index b19d4f1..69f7ea8 100644 --- a/iridiumtk/line_parser/test_voc_line.py +++ b/iridiumtk/line_parser/test_voc_line.py @@ -10,7 +10,7 @@ class ChunksTest(unittest.TestCase): def test_simple(self): result = list(chunks([1, 2, 3, 4, 5, 6], 2)) - self.assertEquals(result, [[1, 2], [3, 4], [5, 6]]) + self.assertEqual(result, [[1, 2], [3, 4], [5, 6]]) class VocLineTest(unittest.TestCase): @@ -24,15 +24,15 @@ def test_empty_input(self): def test_old_format(self): voc_line = VocLine(VocLineTest.TEST_VOC_LINE_1) - self.assertEquals(voc_line.voice_bits, b'\x9e\x88$\xdb\xe6\x01') + self.assertEqual(voc_line.voice_bits, b'\x9e\x88$\xdb\xe6\x01') def test_new_format(self): voc_line = VocLine(VocLineTest.TEST_VOC_LINE_2) - self.assertEquals(voc_line.voice_bits, b'\xdf\xff\xf3\xfc\x10\x33\xc3\x1f\x0c\x83\xc3\xcc\xcc\x30\xff\xf3\xef\x00\xbc\x0c\xb4\x0f\xdc\xd0\x1a\xcc\x9c\xc5\x0c\xfc\x28\x01\xcc\x38\xc2\x33\xe0\xff\x4f') + self.assertEqual(voc_line.voice_bits, b'\xdf\xff\xf3\xfc\x10\x33\xc3\x1f\x0c\x83\xc3\xcc\xcc\x30\xff\xf3\xef\x00\xbc\x0c\xb4\x0f\xdc\xd0\x1a\xcc\x9c\xc5\x0c\xfc\x28\x01\xcc\x38\xc2\x33\xe0\xff\x4f') def test_voice_bits_small_packet(self): voc_line = VocLine(VocLineTest.TEST_VOC_LINE_3) - self.assertEquals(voc_line.voice_bits, None) + self.assertEqual(voc_line.voice_bits, None) def main(): diff --git a/iridiumtk/line_parser/voc_line.py b/iridiumtk/line_parser/voc_line.py index 71aa235..f5fa413 100644 --- a/iridiumtk/line_parser/voc_line.py +++ b/iridiumtk/line_parser/voc_line.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import logging -from io import BytesIO from .base_line import BaseLine, LineParseException @@ -45,16 +44,18 @@ def voice_bits(self): return None if data[0] == "[": return bytes.fromhex(data[1:-1].replace('.', ' ')) - else: + elif data[0] in ('0', '1'): result = bytearray() for bits_of_byte in chunks(data, 8): byte = 0 - for i in range(0, len(bits_of_byte)): - if bits_of_byte[i] == '0': + for i, b in enumerate(bits_of_byte): + if b == '0': continue - elif bits_of_byte[i] == '1': + elif b == '1': byte = byte | (1 << i) else: raise LineParseException(f'Unknown byte format: {data}') result.append(byte) return result + else: + raise LineParseException(f'Unknown binary format: {data}') diff --git a/iridiumtk/reassembler.py b/iridiumtk/reassembler.py index 6681a4c..1bc60ae 100755 --- a/iridiumtk/reassembler.py +++ b/iridiumtk/reassembler.py @@ -1,11 +1,7 @@ #!/usr/bin/env python import argparse -import sys import fileinput -import getopt -import datetime -import re import logging @@ -24,7 +20,6 @@ def __init__(self, base_frames): self._base_frames = base_frames def frames(self): - selected = [] for base_frame in self._base_frames: if base_frame.frame_type != 'MSG': continue @@ -41,7 +36,7 @@ def frames(self): if base_frame.frame_type != 'IRA': continue ira_frame = IraLine(base_frame.raw_line) - + for page in ira_frame.pages: yield "%02d %02d %s %s %03d : %s %s" % (ira_frame.satellite, ira_frame.beam, ira_frame.position.x, ira_frame.position.y, ira_frame.altitude, page.tmsi, page.msc_id) @@ -69,6 +64,7 @@ def parse_to_base_frame_and_filter_time(input_files, start_time_filter, end_time continue yield base_line + def main(): parser = argparse.ArgumentParser(description='Convert iridium-parser.py VOC output to DFS') parser.add_argument('--start', metavar='DATETIME', type=str, default=None, help='Filter events before this time') @@ -93,6 +89,5 @@ def main(): logger.info('Finished parsing frames') - if __name__ == '__main__': main() diff --git a/iridiumtk/rx_stats_hist.py b/iridiumtk/rx_stats_hist.py index 3837a58..bb099f4 100755 --- a/iridiumtk/rx_stats_hist.py +++ b/iridiumtk/rx_stats_hist.py @@ -3,9 +3,6 @@ # Parses .bits files and displays the distribution # of the "HIST_DIMENTION_KEY" of received frames - -from __future__ import print_function - import argparse from collections import namedtuple from datetime import datetime @@ -19,7 +16,7 @@ try: import matplotlib.pyplot as plt except ImportError: - print('Failed to import matplotlib. This prevents any GUI.' , file=sys.stderr) + print('Failed to import matplotlib. This prevents any GUI.', file=sys.stderr) logging.basicConfig(level=logging.INFO) diff --git a/iridiumtk/test_bits_to_dfs.py b/iridiumtk/test_bits_to_dfs.py index 9a33561..053d972 100644 --- a/iridiumtk/test_bits_to_dfs.py +++ b/iridiumtk/test_bits_to_dfs.py @@ -15,7 +15,7 @@ class BitsToDfsTest(unittest.TestCase): def test_empty_input(self): output = BytesIO() bits_to_dfs([], output) - self.assertEquals(output.getvalue(), b'') + self.assertEqual(output.getvalue(), b'') def test_raw_data(self): output = BytesIO() @@ -24,22 +24,22 @@ def test_raw_data(self): output = BytesIO() bits_to_dfs(['RAW: i-1525892321-t1 000045987 1626110208 83% 0.001 <001100000011000011110011> 1100000000000000 0000000000000000 0000000000000000 1001000000000000 0000000000000000 0000000000000000 0100001000110110 0111100001010110 1011001111101110 1010110101000101 1111111001110101 1011110101110001 1110000001110111 0110001000001010 0100100100111000 1000001111010100 1011001101011110 0100011011100010 1111110001010001 1100110110101111 0011100111100101 1110100100000110 0111011111110010 1111110011001000 1101000011000011 1011110101110111 0000101000000001 1000111010101100 1011000001010011 0111011100011101 0101011000011101 0111110001001101 0100001011001010 1101001110100010 0111011001000101 0100111001010110 0001111110101010 1110001100010111 0001010101100100 1001010100111011 1001111110001101 0110100100010010 0001110111101001 1000011010111100 00011001 ERR:Message: unknown Iridium message type'], output) - self.assertEquals(output.getvalue(), b'') + self.assertEqual(output.getvalue(), b'') def test_short_packet(self): output = BytesIO() bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_3], output) - self.assertEquals(output.getvalue(), b'') + self.assertEqual(output.getvalue(), b'') def test_multiple(self): output = BytesIO() bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_1, BitsToDfsTest.TEST_VOC_LINE_1, BitsToDfsTest.TEST_VOC_LINE_2], output) - self.assertEquals(output.getvalue(), (b'\x9e\x88$\xdb\xe6\x01' * 2) + b'\xdf\xff\xf3\xfc\x103\xc3\x1f\x0c\x83\xc3\xcc\xcc0\xff\xf3\xef\x00\xbc\x0c\xb4\x0f\xdc\xd0\x1a\xcc\x9c\xc5\x0c\xfc(\x01\xcc8\xc23\xe0\xffO') + self.assertEqual(output.getvalue(), (b'\x9e\x88$\xdb\xe6\x01' * 2) + b'\xdf\xff\xf3\xfc\x103\xc3\x1f\x0c\x83\xc3\xcc\xcc0\xff\xf3\xef\x00\xbc\x0c\xb4\x0f\xdc\xd0\x1a\xcc\x9c\xc5\x0c\xfc(\x01\xcc8\xc23\xe0\xffO') def test_filters_non_voc_lines(self): output = BytesIO() bits_to_dfs([BitsToDfsTest.TEST_VOC_LINE_1, 'NOT_VOC:', BitsToDfsTest.TEST_VOC_LINE_2], output) - self.assertEquals(output.getvalue(), b'\x9e\x88$\xdb\xe6\x01' + b'\xdf\xff\xf3\xfc\x103\xc3\x1f\x0c\x83\xc3\xcc\xcc0\xff\xf3\xef\x00\xbc\x0c\xb4\x0f\xdc\xd0\x1a\xcc\x9c\xc5\x0c\xfc(\x01\xcc8\xc23\xe0\xffO') + self.assertEqual(output.getvalue(), b'\x9e\x88$\xdb\xe6\x01' + b'\xdf\xff\xf3\xfc\x103\xc3\x1f\x0c\x83\xc3\xcc\xcc0\xff\xf3\xef\x00\xbc\x0c\xb4\x0f\xdc\xd0\x1a\xcc\x9c\xc5\x0c\xfc(\x01\xcc8\xc23\xe0\xffO') class MainTest(unittest.TestCase): diff --git a/iridiumtk/test_graph_voc.py b/iridiumtk/test_graph_voc.py index 9f2dc6f..5a5607d 100644 --- a/iridiumtk/test_graph_voc.py +++ b/iridiumtk/test_graph_voc.py @@ -29,7 +29,7 @@ def test_read_lines(self): input_file.write(MainTest.TEST_VOC_LINE_3 + '\n') voc_lines = list(read_lines(input_file_path, None, None)) - self.assertEquals(len(voc_lines), 2) + self.assertEqual(len(voc_lines), 2) def test_test_read_lines_with_raw_data(self): input_file_path = self.get_temp_file() @@ -43,7 +43,7 @@ def test_test_read_lines_with_parsed_errors(self): with open(input_file_path, 'w') as input_file: input_file.write('RAW: i-1525892321-t1 000045987 1626110208 83% 0.001 <001100000011000011110011> 1100000000000000 .... 00011001 ERR:Message: unknown Iridium message type\n') voc_lines = list(read_lines(input_file_path, None, None)) - self.assertEquals(voc_lines, []) + self.assertEqual(voc_lines, []) def tearDown(self): for path in self.tempfiles: diff --git a/setup.py b/setup.py index 217a1df..3390d92 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ def run_tests(self): python_requires='>=3.6', install_requires=[], tests_require=['pytest'], - + cmdclass={'test': PyTest}, keywords=[], license='BSD',