aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorffki-rz root user <root@freifunk.in-kiel.de>2014-09-04 23:43:42 +0200
committerffki-rz root user <root@freifunk.in-kiel.de>2014-09-04 23:43:42 +0200
commit162b30f3a61958dbad4b4c4f9439b80082a33153 (patch)
treec72934f5fc5ce997da1359f7f23ddc2891c9fdf2
parentc3cf5e4197dac6d469cd1e445b8d241d791c44f7 (diff)
parent65655a38bbd0773841d60285dc1c8758aaa17e61 (diff)
Merge remote-tracking branch 'ffnord/master'
-rw-r--r--RRD.py17
-rw-r--r--alfred.py2
-rwxr-xr-xbat2nodes.py17
-rwxr-xr-xbatman.py2
-rw-r--r--d3mapbuilder.py1
-rw-r--r--hostid.py13
-rw-r--r--node.py2
-rw-r--r--nodedb.py188
-rw-r--r--rrddb.py (renamed from rrd.py)22
9 files changed, 80 insertions, 184 deletions
diff --git a/RRD.py b/RRD.py
index d1ae870..9bb87a0 100644
--- a/RRD.py
+++ b/RRD.py
@@ -80,9 +80,9 @@ class RRD:
raise FileNotFoundError(self.filename)
info = self.info()
if set(ds_list) - set(info['ds'].values()) != set():
- if set((ds.name, ds.type) for ds in ds_list) \
- - set((ds.name, ds.type) for ds in info['ds'].values()) != set():
- raise RRDIncompatibleException()
+ for ds in ds_list:
+ if ds.name in info['ds'] and ds.type != info['ds'][ds.name].type:
+ raise RRDIncompatibleException("%s is %s but should be %s" % (ds.name, ds.type, info['ds'][ds.name].type))
else:
raise RRDOutdatedException()
@@ -177,15 +177,8 @@ class RRD:
echo = True
dump.stdout.close()
restore.stdin.close()
- try:
- dump.wait(1)
- except subprocess.TimeoutExpired:
- dump.kill()
- try:
- restore.wait(2)
- except subprocess.TimeoutExpired:
- dump.kill()
- raise RuntimeError("rrdtool restore process killed")
+ dump.wait()
+ restore.wait()
os.rename(self.filename + ".new", self.filename)
self._cached_info = None
diff --git a/alfred.py b/alfred.py
index 6d926bb..b956026 100644
--- a/alfred.py
+++ b/alfred.py
@@ -7,7 +7,7 @@ class alfred:
self.request_data_type = request_data_type
def aliases(self):
- output = subprocess.check_output(["alfred-json","-r",str(self.request_data_type),"-f","json"])
+ output = subprocess.check_output(["alfred-json","-r",str(self.request_data_type),"-f","json","-z"])
alfred_data = json.loads(output.decode("utf-8"))
alias = {}
for mac,node in alfred_data.items():
diff --git a/bat2nodes.py b/bat2nodes.py
index 6047d95..1e55b3e 100755
--- a/bat2nodes.py
+++ b/bat2nodes.py
@@ -4,10 +4,11 @@ import json
import fileinput
import argparse
import os
+import time
from batman import batman
from alfred import alfred
-from rrd import rrd
+from rrddb import rrd
from nodedb import NodeDB
from d3mapbuilder import D3MapBuilder
@@ -30,9 +31,6 @@ parser.add_argument('-a', '--aliases',
parser.add_argument('-m', '--mesh', action='append',
help='batman mesh interface')
-parser.add_argument('-o', '--obscure', action='store_true',
- help='obscure client macs')
-
parser.add_argument('-A', '--alfred', action='store_true',
help='retrieve aliases from alfred')
@@ -43,7 +41,8 @@ args = parser.parse_args()
options = vars(args)
-db = NodeDB()
+db = NodeDB(int(time.time()))
+
if options['mesh']:
for mesh_interface in options['mesh']:
bm = batman(mesh_interface)
@@ -64,10 +63,12 @@ if options['alfred']:
af = alfred()
db.import_aliases(af.aliases())
-db.count_clients()
+db.load_state("state.json")
+
+# remove nodes that have been offline for more than 30 days
+db.prune_offline(time.time() - 30*86400)
-if options['obscure']:
- db.obscure_clients()
+db.dump_state("state.json")
scriptdir = os.path.dirname(os.path.realpath(__file__))
diff --git a/batman.py b/batman.py
index 05a4380..119a44e 100755
--- a/batman.py
+++ b/batman.py
@@ -32,8 +32,6 @@ class batman:
output = subprocess.check_output(["batctl","-m",self.mesh_interface,"vd","json","-n"])
lines = output.splitlines()
vds = self.vis_data_helper(lines)
- for vd in vds:
- vd['legacy'] = True
return vds
def vis_data_batadv_vis(self):
diff --git a/d3mapbuilder.py b/d3mapbuilder.py
index ff7589f..8fb1961 100644
--- a/d3mapbuilder.py
+++ b/d3mapbuilder.py
@@ -13,7 +13,6 @@ class D3MapBuilder:
nodes = self._db.get_nodes()
output['nodes'] = [{'name': x.name, 'id': x.id,
- 'macs': ', '.join(x.macs),
'geo': [float(x) for x in x.gps.split(" ")] if x.gps else None,
'firmware': x.firmware,
'flags': x.flags,
diff --git a/hostid.py b/hostid.py
deleted file mode 100644
index 2b4038e..0000000
--- a/hostid.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import re
-from functools import reduce
-
-def mac_to_hostid(mac):
- int_mac = list(map(lambda x: int(x, 16), mac.split(":")))
- int_mac[0] ^= 2
- bytes = map(lambda x: "%02x" % x, int_mac[0:3] + [0xff, 0xfe] + int_mac[3:])
- return reduce(lambda a, i:
- [a[0] + ("" if i == 0 else ":") + a[1] + a[2]] + a[3:],
- range(0, 4),
- [""] + list(bytes)
- )
-
diff --git a/node.py b/node.py
index 0fe35fb..fce1866 100644
--- a/node.py
+++ b/node.py
@@ -7,11 +7,11 @@ class Node():
self.flags = dict({
"online": False,
"gateway": False,
- "client": False
})
self.gps = None
self.firmware = None
self.clientcount = 0
+ self.lastseen = 0
def add_mac(self, mac):
mac = mac.lower()
diff --git a/nodedb.py b/nodedb.py
index fa9caed..a45c7a1 100644
--- a/nodedb.py
+++ b/nodedb.py
@@ -5,7 +5,8 @@ from node import Node, Interface
from link import Link, LinkConnector
class NodeDB:
- def __init__(self):
+ def __init__(self, time=0):
+ self.time = time
self._nodes = []
self._links = []
@@ -18,15 +19,42 @@ class NodeDB:
def get_nodes(self):
return self._nodes
- def maybe_node_by_fuzzy_mac(self, mac):
- mac_a = mac.lower()
+ # remove all offlines nodes with lastseen < timestamp
+ def prune_offline(self, timestamp):
+ self._nodes = list(filter(lambda x: x.lastseen >= timestamp, self._nodes))
- for node in self._nodes:
- for mac_b in node.macs:
- if is_derived_mac(mac_a, mac_b):
- return node
+ # write persistent state to file
+ def dump_state(self, filename):
+ obj = []
- raise KeyError
+ for node in self._nodes:
+ obj.append({ 'id': node.id
+ , 'name': node.name
+ , 'lastseen': node.lastseen
+ , 'geo': node.gps
+ })
+
+ with open(filename, "w") as f:
+ json.dump(obj, f)
+
+ # load persistent state from file
+ def load_state(self, filename):
+ try:
+ with open(filename, "r") as f:
+ obj = json.load(f)
+ for n in obj:
+ try:
+ node = self.maybe_node_by_id(n['id'])
+ except:
+ node = Node()
+ node.id = n['id']
+ node.name = n['name']
+ node.lastseen = n['lastseen']
+ node.gps = n['geo']
+ self._nodes.append(node)
+
+ except:
+ pass
def maybe_node_by_mac(self, macs):
for node in self._nodes:
@@ -51,37 +79,28 @@ class NodeDB:
node = self.maybe_node_by_mac((x['of'], x['secondary']))
except:
node = Node()
+ node.lastseen = self.time
node.flags['online'] = True
- if 'legacy' in x:
- node.flags['legacy'] = True
self._nodes.append(node)
node.add_mac(x['of'])
node.add_mac(x['secondary'])
for x in vis_data:
-
if 'router' in x:
+ # TTs will be processed later
+ if x['label'] == "TT":
+ continue
+
try:
node = self.maybe_node_by_mac((x['router'], ))
except:
node = Node()
+ node.lastseen = self.time
node.flags['online'] = True
- if 'legacy' in x:
- node.flags['legacy'] = True
node.add_mac(x['router'])
self._nodes.append(node)
- # If it's a TT link and the MAC is very similar
- # consider this MAC as one of the routers
- # MACs
- if 'gateway' in x and x['label'] == "TT":
- if is_similar(x['router'], x['gateway']):
- node.add_mac(x['gateway'])
-
- # skip processing as regular link
- continue
-
try:
if 'neighbor' in x:
try:
@@ -95,16 +114,17 @@ class NodeDB:
node = self.maybe_node_by_mac((x['neighbor'], ))
except:
node = Node()
+ node.lastseen = self.time
node.flags['online'] = True
- if x['label'] == 'TT':
- node.flags['client'] = True
-
node.add_mac(x['neighbor'])
self._nodes.append(node)
for x in vis_data:
-
if 'router' in x:
+ # TTs will be processed later
+ if x['label'] == "TT":
+ continue
+
try:
if 'gateway' in x:
x['neighbor'] = x['gateway']
@@ -128,13 +148,9 @@ class NodeDB:
link.quality = x['label']
link.id = "-".join(sorted((link.source.interface, link.target.interface)))
- if x['label'] == "TT":
- link.type = "client"
-
self._links.append(link)
for x in vis_data:
-
if 'primary' in x:
try:
node = self.maybe_node_by_mac((x['primary'], ))
@@ -143,6 +159,16 @@ class NodeDB:
node.id = x['primary']
+ for x in vis_data:
+ if 'router' in x and x['label'] == 'TT':
+ try:
+ node = self.maybe_node_by_mac((x['router'], ))
+ node.add_mac(x['gateway'])
+ if not is_similar(x['router'], x['gateway']):
+ node.clientcount += 1
+ except:
+ pass
+
def reduce_links(self):
tmp_links = defaultdict(list)
@@ -171,13 +197,10 @@ class NodeDB:
try:
node = self.maybe_node_by_mac([mac])
except:
- try:
- node = self.maybe_node_by_fuzzy_mac(mac)
- except:
- # create an offline node
- node = Node()
- node.add_mac(mac)
- self._nodes.append(node)
+ # create an offline node
+ node = Node()
+ node.add_mac(mac)
+ self._nodes.append(node)
if 'name' in alias:
node.name = alias['name']
@@ -212,9 +235,6 @@ class NodeDB:
while changes > 0:
changes = 0
for link in self._links:
- if link.type == "client":
- continue
-
source_interface = self._nodes[link.source.id].interfaces[link.source.interface]
target_interface = self._nodes[link.target.id].interfaces[link.target.interface]
if source_interface.vpn or target_interface.vpn:
@@ -225,92 +245,6 @@ class NodeDB:
link.type = "vpn"
- def count_clients(self):
- for link in self._links:
- try:
- a = self.maybe_node_by_id(link.source.interface)
- b = self.maybe_node_by_id(link.target.interface)
-
- if a.flags['client']:
- client = a
- node = b
- elif b.flags['client']:
- client = b
- node = a
- else:
- continue
-
- node.clientcount += 1
- except:
- pass
-
- def obscure_clients(self):
-
- globalIdCounter = 0
- nodeCounters = {}
- clientIds = {}
-
- for node in self._nodes:
- if node.flags['client']:
- node.macs = set()
- clientIds[node.id] = None
-
- for link in self._links:
- ids = link.source.interface
- idt = link.target.interface
-
- try:
- node_source = self.maybe_node_by_fuzzy_mac(ids)
- node_target = self.maybe_node_by_id(idt)
-
- if not node_source.flags['client'] and not node_target.flags['client']:
- # if none of the nodes associated with this link are clients,
- # we do not want to obscure
- continue
-
- if ids in clientIds and idt in clientIds:
- # This is for corner cases, when a client
- # is linked to another client.
- clientIds[ids] = str(globalIdCounter)
- ids = str(globalIdCounter)
- globalIdCounter += 1
-
- clientIds[idt] = str(globalIdCounter)
- idt = str(globalIdCounter)
- globalIdCounter += 1
-
- elif ids in clientIds:
- newId = generateId(idt)
- clientIds[ids] = newId
- ids = newId
-
- link.source.interface = ids;
- node_source.id = ids;
-
- elif idt in clientIds:
- newId = generateId(ids,nodeCounters)
- clientIds[idt] = newId
- idt = newId
-
- link.target.interface = idt;
- node_target.id = idt;
-
- link.id = ids + "-" + idt
-
- except KeyError:
- pass
-
-# extends node id by incremented node counter
-def generateId(nodeId,nodeCounters):
- if nodeId in nodeCounters:
- n = nodeCounters[nodeId]
- nodeCounters[nodeId] = n + 1
- else:
- nodeCounters[nodeId] = 1
- n = 0
-
- return nodeId + "_" + str(n)
-
# compares two MACs and decides whether they are
# similar and could be from the same node
def is_similar(a, b):
diff --git a/rrd.py b/rrddb.py
index 5c3330d..6f48913 100644
--- a/rrd.py
+++ b/rrddb.py
@@ -27,27 +27,11 @@ class rrd:
os.mkdir(self.imagePath)
def update_database(self,db):
- nodes = {}
- clientCount = 0
- for node in db.get_nodes():
- if node.flags['online']:
- if not node.flags['client']:
- nodes[node.id] = node
- node.clients = 0;
- if 'legacy' in node.flags and node.flags['legacy']:
- clientCount -= 1
- else:
- clientCount += 1
- for link in db.get_links():
- source = link.source.interface
- target = link.target.interface
- if source in nodes and not target in nodes:
- nodes[source].clients += 1
- elif target in nodes and not source in nodes:
- nodes[target].clients += 1
+ nodes = db.get_nodes()
+ clientCount = sum(map(lambda d: d.clientcount, nodes))
self.globalDb.update(len(nodes), clientCount)
- for node in nodes.values():
+ for node in nodes:
rrd = NodeRRD(
os.path.join(self.dbPath, str(node.id).replace(':', '') + '.rrd'),
node