diff options
author | ffki-rz root user <root@freifunk.in-kiel.de> | 2014-09-04 23:43:42 +0200 |
---|---|---|
committer | ffki-rz root user <root@freifunk.in-kiel.de> | 2014-09-04 23:43:42 +0200 |
commit | 162b30f3a61958dbad4b4c4f9439b80082a33153 (patch) | |
tree | c72934f5fc5ce997da1359f7f23ddc2891c9fdf2 | |
parent | c3cf5e4197dac6d469cd1e445b8d241d791c44f7 (diff) | |
parent | 65655a38bbd0773841d60285dc1c8758aaa17e61 (diff) |
Merge remote-tracking branch 'ffnord/master'
-rw-r--r-- | RRD.py | 17 | ||||
-rw-r--r-- | alfred.py | 2 | ||||
-rwxr-xr-x | bat2nodes.py | 17 | ||||
-rwxr-xr-x | batman.py | 2 | ||||
-rw-r--r-- | d3mapbuilder.py | 1 | ||||
-rw-r--r-- | hostid.py | 13 | ||||
-rw-r--r-- | node.py | 2 | ||||
-rw-r--r-- | nodedb.py | 188 | ||||
-rw-r--r-- | rrddb.py (renamed from rrd.py) | 22 |
9 files changed, 80 insertions, 184 deletions
@@ -80,9 +80,9 @@ class RRD: raise FileNotFoundError(self.filename) info = self.info() if set(ds_list) - set(info['ds'].values()) != set(): - if set((ds.name, ds.type) for ds in ds_list) \ - - set((ds.name, ds.type) for ds in info['ds'].values()) != set(): - raise RRDIncompatibleException() + for ds in ds_list: + if ds.name in info['ds'] and ds.type != info['ds'][ds.name].type: + raise RRDIncompatibleException("%s is %s but should be %s" % (ds.name, ds.type, info['ds'][ds.name].type)) else: raise RRDOutdatedException() @@ -177,15 +177,8 @@ class RRD: echo = True dump.stdout.close() restore.stdin.close() - try: - dump.wait(1) - except subprocess.TimeoutExpired: - dump.kill() - try: - restore.wait(2) - except subprocess.TimeoutExpired: - dump.kill() - raise RuntimeError("rrdtool restore process killed") + dump.wait() + restore.wait() os.rename(self.filename + ".new", self.filename) self._cached_info = None @@ -7,7 +7,7 @@ class alfred: self.request_data_type = request_data_type def aliases(self): - output = subprocess.check_output(["alfred-json","-r",str(self.request_data_type),"-f","json"]) + output = subprocess.check_output(["alfred-json","-r",str(self.request_data_type),"-f","json","-z"]) alfred_data = json.loads(output.decode("utf-8")) alias = {} for mac,node in alfred_data.items(): diff --git a/bat2nodes.py b/bat2nodes.py index 6047d95..1e55b3e 100755 --- a/bat2nodes.py +++ b/bat2nodes.py @@ -4,10 +4,11 @@ import json import fileinput import argparse import os +import time from batman import batman from alfred import alfred -from rrd import rrd +from rrddb import rrd from nodedb import NodeDB from d3mapbuilder import D3MapBuilder @@ -30,9 +31,6 @@ parser.add_argument('-a', '--aliases', parser.add_argument('-m', '--mesh', action='append', help='batman mesh interface') -parser.add_argument('-o', '--obscure', action='store_true', - help='obscure client macs') - parser.add_argument('-A', '--alfred', action='store_true', help='retrieve aliases from alfred') @@ -43,7 +41,8 @@ args = parser.parse_args() options = vars(args) -db = NodeDB() +db = NodeDB(int(time.time())) + if options['mesh']: for mesh_interface in options['mesh']: bm = batman(mesh_interface) @@ -64,10 +63,12 @@ if options['alfred']: af = alfred() db.import_aliases(af.aliases()) -db.count_clients() +db.load_state("state.json") + +# remove nodes that have been offline for more than 30 days +db.prune_offline(time.time() - 30*86400) -if options['obscure']: - db.obscure_clients() +db.dump_state("state.json") scriptdir = os.path.dirname(os.path.realpath(__file__)) @@ -32,8 +32,6 @@ class batman: output = subprocess.check_output(["batctl","-m",self.mesh_interface,"vd","json","-n"]) lines = output.splitlines() vds = self.vis_data_helper(lines) - for vd in vds: - vd['legacy'] = True return vds def vis_data_batadv_vis(self): diff --git a/d3mapbuilder.py b/d3mapbuilder.py index ff7589f..8fb1961 100644 --- a/d3mapbuilder.py +++ b/d3mapbuilder.py @@ -13,7 +13,6 @@ class D3MapBuilder: nodes = self._db.get_nodes() output['nodes'] = [{'name': x.name, 'id': x.id, - 'macs': ', '.join(x.macs), 'geo': [float(x) for x in x.gps.split(" ")] if x.gps else None, 'firmware': x.firmware, 'flags': x.flags, diff --git a/hostid.py b/hostid.py deleted file mode 100644 index 2b4038e..0000000 --- a/hostid.py +++ /dev/null @@ -1,13 +0,0 @@ -import re -from functools import reduce - -def mac_to_hostid(mac): - int_mac = list(map(lambda x: int(x, 16), mac.split(":"))) - int_mac[0] ^= 2 - bytes = map(lambda x: "%02x" % x, int_mac[0:3] + [0xff, 0xfe] + int_mac[3:]) - return reduce(lambda a, i: - [a[0] + ("" if i == 0 else ":") + a[1] + a[2]] + a[3:], - range(0, 4), - [""] + list(bytes) - ) - @@ -7,11 +7,11 @@ class Node(): self.flags = dict({ "online": False, "gateway": False, - "client": False }) self.gps = None self.firmware = None self.clientcount = 0 + self.lastseen = 0 def add_mac(self, mac): mac = mac.lower() @@ -5,7 +5,8 @@ from node import Node, Interface from link import Link, LinkConnector class NodeDB: - def __init__(self): + def __init__(self, time=0): + self.time = time self._nodes = [] self._links = [] @@ -18,15 +19,42 @@ class NodeDB: def get_nodes(self): return self._nodes - def maybe_node_by_fuzzy_mac(self, mac): - mac_a = mac.lower() + # remove all offlines nodes with lastseen < timestamp + def prune_offline(self, timestamp): + self._nodes = list(filter(lambda x: x.lastseen >= timestamp, self._nodes)) - for node in self._nodes: - for mac_b in node.macs: - if is_derived_mac(mac_a, mac_b): - return node + # write persistent state to file + def dump_state(self, filename): + obj = [] - raise KeyError + for node in self._nodes: + obj.append({ 'id': node.id + , 'name': node.name + , 'lastseen': node.lastseen + , 'geo': node.gps + }) + + with open(filename, "w") as f: + json.dump(obj, f) + + # load persistent state from file + def load_state(self, filename): + try: + with open(filename, "r") as f: + obj = json.load(f) + for n in obj: + try: + node = self.maybe_node_by_id(n['id']) + except: + node = Node() + node.id = n['id'] + node.name = n['name'] + node.lastseen = n['lastseen'] + node.gps = n['geo'] + self._nodes.append(node) + + except: + pass def maybe_node_by_mac(self, macs): for node in self._nodes: @@ -51,37 +79,28 @@ class NodeDB: node = self.maybe_node_by_mac((x['of'], x['secondary'])) except: node = Node() + node.lastseen = self.time node.flags['online'] = True - if 'legacy' in x: - node.flags['legacy'] = True self._nodes.append(node) node.add_mac(x['of']) node.add_mac(x['secondary']) for x in vis_data: - if 'router' in x: + # TTs will be processed later + if x['label'] == "TT": + continue + try: node = self.maybe_node_by_mac((x['router'], )) except: node = Node() + node.lastseen = self.time node.flags['online'] = True - if 'legacy' in x: - node.flags['legacy'] = True node.add_mac(x['router']) self._nodes.append(node) - # If it's a TT link and the MAC is very similar - # consider this MAC as one of the routers - # MACs - if 'gateway' in x and x['label'] == "TT": - if is_similar(x['router'], x['gateway']): - node.add_mac(x['gateway']) - - # skip processing as regular link - continue - try: if 'neighbor' in x: try: @@ -95,16 +114,17 @@ class NodeDB: node = self.maybe_node_by_mac((x['neighbor'], )) except: node = Node() + node.lastseen = self.time node.flags['online'] = True - if x['label'] == 'TT': - node.flags['client'] = True - node.add_mac(x['neighbor']) self._nodes.append(node) for x in vis_data: - if 'router' in x: + # TTs will be processed later + if x['label'] == "TT": + continue + try: if 'gateway' in x: x['neighbor'] = x['gateway'] @@ -128,13 +148,9 @@ class NodeDB: link.quality = x['label'] link.id = "-".join(sorted((link.source.interface, link.target.interface))) - if x['label'] == "TT": - link.type = "client" - self._links.append(link) for x in vis_data: - if 'primary' in x: try: node = self.maybe_node_by_mac((x['primary'], )) @@ -143,6 +159,16 @@ class NodeDB: node.id = x['primary'] + for x in vis_data: + if 'router' in x and x['label'] == 'TT': + try: + node = self.maybe_node_by_mac((x['router'], )) + node.add_mac(x['gateway']) + if not is_similar(x['router'], x['gateway']): + node.clientcount += 1 + except: + pass + def reduce_links(self): tmp_links = defaultdict(list) @@ -171,13 +197,10 @@ class NodeDB: try: node = self.maybe_node_by_mac([mac]) except: - try: - node = self.maybe_node_by_fuzzy_mac(mac) - except: - # create an offline node - node = Node() - node.add_mac(mac) - self._nodes.append(node) + # create an offline node + node = Node() + node.add_mac(mac) + self._nodes.append(node) if 'name' in alias: node.name = alias['name'] @@ -212,9 +235,6 @@ class NodeDB: while changes > 0: changes = 0 for link in self._links: - if link.type == "client": - continue - source_interface = self._nodes[link.source.id].interfaces[link.source.interface] target_interface = self._nodes[link.target.id].interfaces[link.target.interface] if source_interface.vpn or target_interface.vpn: @@ -225,92 +245,6 @@ class NodeDB: link.type = "vpn" - def count_clients(self): - for link in self._links: - try: - a = self.maybe_node_by_id(link.source.interface) - b = self.maybe_node_by_id(link.target.interface) - - if a.flags['client']: - client = a - node = b - elif b.flags['client']: - client = b - node = a - else: - continue - - node.clientcount += 1 - except: - pass - - def obscure_clients(self): - - globalIdCounter = 0 - nodeCounters = {} - clientIds = {} - - for node in self._nodes: - if node.flags['client']: - node.macs = set() - clientIds[node.id] = None - - for link in self._links: - ids = link.source.interface - idt = link.target.interface - - try: - node_source = self.maybe_node_by_fuzzy_mac(ids) - node_target = self.maybe_node_by_id(idt) - - if not node_source.flags['client'] and not node_target.flags['client']: - # if none of the nodes associated with this link are clients, - # we do not want to obscure - continue - - if ids in clientIds and idt in clientIds: - # This is for corner cases, when a client - # is linked to another client. - clientIds[ids] = str(globalIdCounter) - ids = str(globalIdCounter) - globalIdCounter += 1 - - clientIds[idt] = str(globalIdCounter) - idt = str(globalIdCounter) - globalIdCounter += 1 - - elif ids in clientIds: - newId = generateId(idt) - clientIds[ids] = newId - ids = newId - - link.source.interface = ids; - node_source.id = ids; - - elif idt in clientIds: - newId = generateId(ids,nodeCounters) - clientIds[idt] = newId - idt = newId - - link.target.interface = idt; - node_target.id = idt; - - link.id = ids + "-" + idt - - except KeyError: - pass - -# extends node id by incremented node counter -def generateId(nodeId,nodeCounters): - if nodeId in nodeCounters: - n = nodeCounters[nodeId] - nodeCounters[nodeId] = n + 1 - else: - nodeCounters[nodeId] = 1 - n = 0 - - return nodeId + "_" + str(n) - # compares two MACs and decides whether they are # similar and could be from the same node def is_similar(a, b): @@ -27,27 +27,11 @@ class rrd: os.mkdir(self.imagePath) def update_database(self,db): - nodes = {} - clientCount = 0 - for node in db.get_nodes(): - if node.flags['online']: - if not node.flags['client']: - nodes[node.id] = node - node.clients = 0; - if 'legacy' in node.flags and node.flags['legacy']: - clientCount -= 1 - else: - clientCount += 1 - for link in db.get_links(): - source = link.source.interface - target = link.target.interface - if source in nodes and not target in nodes: - nodes[source].clients += 1 - elif target in nodes and not source in nodes: - nodes[target].clients += 1 + nodes = db.get_nodes() + clientCount = sum(map(lambda d: d.clientcount, nodes)) self.globalDb.update(len(nodes), clientCount) - for node in nodes.values(): + for node in nodes: rrd = NodeRRD( os.path.join(self.dbPath, str(node.id).replace(':', '') + '.rrd'), node |