Merge remote-tracking branch 'ffnord/master'

9 files changed, 80 insertions, 184 deletions

diff --git a/RRD.py b/RRD.py
index d1ae870..9bb87a0 100644
--- a/RRD.py
+++ b/RRD.py
@@ -80,9 +80,9 @@ class RRD:
             raise FileNotFoundError(self.filename)
         info = self.info()
         if set(ds_list) - set(info['ds'].values()) != set():
-            if set((ds.name, ds.type) for ds in ds_list) \
-             - set((ds.name, ds.type) for ds in info['ds'].values()) != set():
-                raise RRDIncompatibleException()
+            for ds in ds_list:
+                if ds.name in info['ds'] and ds.type != info['ds'][ds.name].type:
+                    raise RRDIncompatibleException("%s is %s but should be %s" % (ds.name, ds.type, info['ds'][ds.name].type))
             else:
                 raise RRDOutdatedException()
 
@@ -177,15 +177,8 @@ class RRD:
                 echo = True
         dump.stdout.close()
         restore.stdin.close()
-        try:
-            dump.wait(1)
-        except subprocess.TimeoutExpired:
-            dump.kill()
-        try:
-            restore.wait(2)
-        except subprocess.TimeoutExpired:
-            dump.kill()
-            raise RuntimeError("rrdtool restore process killed")
+        dump.wait()
+        restore.wait()
 
         os.rename(self.filename + ".new", self.filename)
         self._cached_info = None
diff --git a/alfred.py b/alfred.py
index 6d926bb..b956026 100644
--- a/alfred.py
+++ b/alfred.py
@@ -7,7 +7,7 @@ class alfred:
     self.request_data_type = request_data_type
 
   def aliases(self):
-    output = subprocess.check_output(["alfred-json","-r",str(self.request_data_type),"-f","json"])
+    output = subprocess.check_output(["alfred-json","-r",str(self.request_data_type),"-f","json","-z"])
     alfred_data = json.loads(output.decode("utf-8"))
     alias = {}
     for mac,node in alfred_data.items():
diff --git a/bat2nodes.py b/bat2nodes.py
index 6047d95..1e55b3e 100755
--- a/bat2nodes.py
+++ b/bat2nodes.py
@@ -4,10 +4,11 @@ import json
 import fileinput
 import argparse
 import os
+import time
 
 from batman import batman
 from alfred import alfred
-from rrd import rrd
+from rrddb import rrd
 from nodedb import NodeDB
 from d3mapbuilder import D3MapBuilder
 
@@ -30,9 +31,6 @@ parser.add_argument('-a', '--aliases',
 parser.add_argument('-m', '--mesh', action='append',
                   help='batman mesh interface')
 
-parser.add_argument('-o', '--obscure', action='store_true',
-                  help='obscure client macs')
-
 parser.add_argument('-A', '--alfred', action='store_true',
                   help='retrieve aliases from alfred')
 
@@ -43,7 +41,8 @@ args = parser.parse_args()
 
 options = vars(args)
 
-db = NodeDB()
+db = NodeDB(int(time.time()))
+
 if options['mesh']:
   for mesh_interface in options['mesh']:
     bm = batman(mesh_interface)
@@ -64,10 +63,12 @@ if options['alfred']:
   af = alfred()
   db.import_aliases(af.aliases())
 
-db.count_clients()
+db.load_state("state.json")
+
+# remove nodes that have been offline for more than 30 days
+db.prune_offline(time.time() - 30*86400)
 
-if options['obscure']:
-  db.obscure_clients()
+db.dump_state("state.json")
 
 scriptdir = os.path.dirname(os.path.realpath(__file__))
 
diff --git a/batman.py b/batman.py
index 05a4380..119a44e 100755
--- a/batman.py
+++ b/batman.py
@@ -32,8 +32,6 @@ class batman:
     output = subprocess.check_output(["batctl","-m",self.mesh_interface,"vd","json","-n"])
     lines = output.splitlines()
     vds = self.vis_data_helper(lines)
-    for vd in vds:
-      vd['legacy'] = True
     return vds
 
   def vis_data_batadv_vis(self):
diff --git a/d3mapbuilder.py b/d3mapbuilder.py
index ff7589f..8fb1961 100644
--- a/d3mapbuilder.py
+++ b/d3mapbuilder.py
@@ -13,7 +13,6 @@ class D3MapBuilder:
     nodes = self._db.get_nodes()
 
     output['nodes'] = [{'name': x.name, 'id': x.id,
-                        'macs': ', '.join(x.macs),
                         'geo': [float(x) for x in x.gps.split(" ")] if x.gps else None,
                         'firmware': x.firmware,
                         'flags': x.flags,
diff --git a/hostid.py b/hostid.py
deleted file mode 100644
index 2b4038e..0000000
--- a/hostid.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import re
-from functools import reduce
-
-def mac_to_hostid(mac):
-  int_mac = list(map(lambda x: int(x, 16), mac.split(":")))
-  int_mac[0] ^= 2
-  bytes = map(lambda x: "%02x" % x, int_mac[0:3] + [0xff, 0xfe] + int_mac[3:])
-  return reduce(lambda a, i:
-                  [a[0] + ("" if i == 0 else ":") + a[1] + a[2]] + a[3:],
-                range(0, 4),
-                [""] + list(bytes)
-               )
-
diff --git a/node.py b/node.py
index 0fe35fb..fce1866 100644
--- a/node.py
+++ b/node.py
@@ -7,11 +7,11 @@ class Node():
     self.flags = dict({
       "online": False,
       "gateway": False,
-      "client": False
     })
     self.gps = None
     self.firmware = None
     self.clientcount = 0
+    self.lastseen = 0
 
   def add_mac(self, mac):
     mac = mac.lower()
diff --git a/nodedb.py b/nodedb.py
index fa9caed..a45c7a1 100644
--- a/nodedb.py
+++ b/nodedb.py
@@ -5,7 +5,8 @@ from node import Node, Interface
 from link import Link, LinkConnector
 
 class NodeDB:
-  def __init__(self):
+  def __init__(self, time=0):
+    self.time = time
     self._nodes = []
     self._links = []
 
@@ -18,15 +19,42 @@ class NodeDB:
   def get_nodes(self):
     return self._nodes
 
-  def maybe_node_by_fuzzy_mac(self, mac):
-    mac_a = mac.lower()
+  # remove all offlines nodes with lastseen < timestamp
+  def prune_offline(self, timestamp):
+    self._nodes = list(filter(lambda x: x.lastseen >= timestamp, self._nodes))
 
-    for node in self._nodes:
-      for mac_b in node.macs:
-        if is_derived_mac(mac_a, mac_b):
-          return node
+  # write persistent state to file
+  def dump_state(self, filename):
+    obj = []
 
-    raise KeyError
+    for node in self._nodes:
+      obj.append({ 'id': node.id
+                 , 'name': node.name
+                 , 'lastseen': node.lastseen
+                 , 'geo': node.gps
+                 })
+
+    with open(filename, "w") as f:
+      json.dump(obj, f)
+
+  # load persistent state from file
+  def load_state(self, filename):
+    try:
+      with open(filename, "r") as f:
+        obj = json.load(f)
+        for n in obj:
+          try:
+            node = self.maybe_node_by_id(n['id'])
+          except:
+            node = Node()
+            node.id = n['id']
+            node.name = n['name']
+            node.lastseen = n['lastseen']
+            node.gps = n['geo']
+            self._nodes.append(node)
+
+    except:
+      pass
 
   def maybe_node_by_mac(self, macs):
     for node in self._nodes:
@@ -51,37 +79,28 @@ class NodeDB:
           node = self.maybe_node_by_mac((x['of'], x['secondary']))
         except:
           node = Node()
+          node.lastseen = self.time
           node.flags['online'] = True
-          if 'legacy' in x:
-            node.flags['legacy'] = True
           self._nodes.append(node)
 
         node.add_mac(x['of'])
         node.add_mac(x['secondary'])
 
     for x in vis_data:
-
       if 'router' in x:
+        # TTs will be processed later
+        if x['label'] == "TT":
+          continue
+
         try:
           node = self.maybe_node_by_mac((x['router'], ))
         except:
           node = Node()
+          node.lastseen = self.time
           node.flags['online'] = True
-          if 'legacy' in x:
-            node.flags['legacy'] = True
           node.add_mac(x['router'])
           self._nodes.append(node)
 
-        # If it's a TT link and the MAC is very similar
-        # consider this MAC as one of the routers
-        # MACs
-        if 'gateway' in x and x['label'] == "TT":
-          if is_similar(x['router'], x['gateway']):
-            node.add_mac(x['gateway'])
-
-            # skip processing as regular link
-            continue
-
         try:
           if 'neighbor' in x:
             try:
@@ -95,16 +114,17 @@ class NodeDB:
           node = self.maybe_node_by_mac((x['neighbor'], ))
         except:
           node = Node()
+          node.lastseen = self.time
           node.flags['online'] = True
-          if x['label'] == 'TT':
-            node.flags['client'] = True
-
           node.add_mac(x['neighbor'])
           self._nodes.append(node)
 
     for x in vis_data:
-
       if 'router' in x:
+        # TTs will be processed later
+        if x['label'] == "TT":
+          continue
+
         try:
           if 'gateway' in x:
             x['neighbor'] = x['gateway']
@@ -128,13 +148,9 @@ class NodeDB:
         link.quality = x['label']
         link.id = "-".join(sorted((link.source.interface, link.target.interface)))
 
-        if x['label'] == "TT":
-          link.type = "client"
-
         self._links.append(link)
 
     for x in vis_data:
-
       if 'primary' in x:
         try:
           node = self.maybe_node_by_mac((x['primary'], ))
@@ -143,6 +159,16 @@ class NodeDB:
 
         node.id = x['primary']
 
+    for x in vis_data:
+      if 'router' in x and x['label'] == 'TT':
+        try:
+          node = self.maybe_node_by_mac((x['router'], ))
+          node.add_mac(x['gateway'])
+          if not is_similar(x['router'], x['gateway']):
+            node.clientcount += 1
+        except:
+          pass
+
   def reduce_links(self):
     tmp_links = defaultdict(list)
 
@@ -171,13 +197,10 @@ class NodeDB:
       try:
         node = self.maybe_node_by_mac([mac])
       except:
-        try:
-          node = self.maybe_node_by_fuzzy_mac(mac)
-        except:
-          # create an offline node
-          node = Node()
-          node.add_mac(mac)
-          self._nodes.append(node)
+        # create an offline node
+        node = Node()
+        node.add_mac(mac)
+        self._nodes.append(node)
 
       if 'name' in alias:
         node.name = alias['name']
@@ -212,9 +235,6 @@ class NodeDB:
     while changes > 0:
       changes = 0
       for link in self._links:
-        if link.type == "client":
-          continue
-
         source_interface = self._nodes[link.source.id].interfaces[link.source.interface]
         target_interface = self._nodes[link.target.id].interfaces[link.target.interface]
         if source_interface.vpn or target_interface.vpn:
@@ -225,92 +245,6 @@ class NodeDB:
 
           link.type = "vpn"
 
-  def count_clients(self):
-    for link in self._links:
-      try:
-        a = self.maybe_node_by_id(link.source.interface)
-        b = self.maybe_node_by_id(link.target.interface)
-
-        if a.flags['client']:
-          client = a
-          node = b
-        elif b.flags['client']:
-          client = b
-          node = a
-        else:
-          continue
-
-        node.clientcount += 1
-      except:
-        pass
-
-  def obscure_clients(self):
-
-    globalIdCounter = 0
-    nodeCounters = {}
-    clientIds = {}
-
-    for node in self._nodes:
-      if node.flags['client']:
-        node.macs = set()
-        clientIds[node.id] = None
-
-    for link in self._links:
-      ids = link.source.interface
-      idt = link.target.interface
-
-      try:
-        node_source = self.maybe_node_by_fuzzy_mac(ids)
-        node_target = self.maybe_node_by_id(idt)
-
-        if not node_source.flags['client'] and not node_target.flags['client']:
-          # if none of the nodes associated with this link are clients,
-          # we do not want to obscure
-          continue
-
-        if ids in clientIds and idt in clientIds:
-          # This is for corner cases, when a client
-          # is linked to another client.
-          clientIds[ids] = str(globalIdCounter)
-          ids = str(globalIdCounter)
-          globalIdCounter += 1
-
-          clientIds[idt] = str(globalIdCounter)
-          idt = str(globalIdCounter)
-          globalIdCounter += 1
-
-        elif ids in clientIds:
-          newId = generateId(idt)
-          clientIds[ids] = newId
-          ids = newId
-
-          link.source.interface = ids;
-          node_source.id = ids;
-
-        elif idt in clientIds:
-          newId = generateId(ids,nodeCounters)
-          clientIds[idt] = newId
-          idt = newId
-
-          link.target.interface = idt;
-          node_target.id = idt;
-
-        link.id = ids + "-" + idt
-
-      except KeyError:
-        pass
-
-# extends node id by incremented node counter
-def generateId(nodeId,nodeCounters):
-  if nodeId in nodeCounters:
-    n = nodeCounters[nodeId]
-    nodeCounters[nodeId] = n + 1
-  else:
-    nodeCounters[nodeId] = 1
-    n = 0
-
-  return nodeId + "_" + str(n)
-
 # compares two MACs and decides whether they are
 # similar and could be from the same node
 def is_similar(a, b):
diff --git a/rrd.py b/rrddb.py
index 5c3330d..6f48913 100644
--- a/rrd.py
+++ b/rrddb.py
@@ -27,27 +27,11 @@ class rrd:
       os.mkdir(self.imagePath)
 
   def update_database(self,db):
-    nodes = {}
-    clientCount = 0
-    for node in db.get_nodes():
-      if node.flags['online']:
-        if not node.flags['client']:
-          nodes[node.id] = node
-          node.clients = 0;
-          if 'legacy' in node.flags and node.flags['legacy']:
-            clientCount -= 1
-        else:
-          clientCount += 1
-    for link in db.get_links():
-      source = link.source.interface
-      target = link.target.interface
-      if source in nodes and not target in nodes:
-        nodes[source].clients += 1
-      elif target in nodes and not source in nodes:
-        nodes[target].clients += 1
+    nodes = db.get_nodes()
+    clientCount = sum(map(lambda d: d.clientcount, nodes))
 
     self.globalDb.update(len(nodes), clientCount)
-    for node in nodes.values():
+    for node in nodes:
       rrd = NodeRRD(
         os.path.join(self.dbPath, str(node.id).replace(':', '') + '.rrd'),
         node