Add iwlist scanning & parsing, feed heatmap

.gitignore

@@ -4,3 +4,4 @@ yarn.lock
 node_modules
 .venv
 __pycache__
+data.yml

iwlist.py

+import re
+import subprocess
+import sys
+import time
+from collections import defaultdict
+
+import yaml
+
+ADDRESS_RE = re.compile(r'^Cell \d+ - Address: (?P<address>([0-9A-F]{2}:){5}[0-9A-F]{2})$')
+CHANNEL_RE = re.compile(r'^Channel:(?P<channel>\d+)$')
+FREQUENCY_RE = re.compile(r'^Frequency:(?P<freq>\d\.\d+) GHz( \(Channel \d+\))?$')
+POWER_RE = re.compile(r'^Quality=\d+/\d+  Signal level=(?P<power>-\d+) dBm')
+ESSID_RE = re.compile(r'^ESSID:"(?P<essid>.*)"$')
+
+
+def scan(interface='wlp2s0', retries=5):
+    if retries == 0:
+        print('Giving up...', file=sys.stderr)
+        return None
+    proc = subprocess.run(['iwlist', interface, 'scanning'], stdout=subprocess.PIPE)
+    stdout = (l.strip() for l in proc.stdout.decode('utf8').splitlines())
+    first_line = next(stdout).lower()
+    if 'no scan results' in first_line:
+        print(
+            'Scanning was not successfull... Retrying for {} times'.format(retries),
+            file=sys.stderr,
+        )
+        time.sleep(1)
+        # try again
+        return scan(interface, retries - 1)
+    assert 'scan completed' in first_line
+    cells = {}
+    current_cell = {}
+    for line in stdout:
+        if ADDRESS_RE.match(line):
+            if current_cell:
+                cells[current_cell['address']] = current_cell
+                current_cell = {}
+            current_cell['address'] = ADDRESS_RE.match(line).group('address')
+        elif CHANNEL_RE.match(line):
+            current_cell['channel'] = CHANNEL_RE.match(line).group('channel')
+        elif FREQUENCY_RE.match(line):
+            current_cell['freq'] = float(FREQUENCY_RE.match(line).group('freq'))  # GHz
+        elif POWER_RE.match(line):
+            current_cell['power'] = int(POWER_RE.match(line).group('power'))  # dBm
+        elif ESSID_RE.match(line):
+            current_cell['essid'] = ESSID_RE.match(line).group('essid')
+        # else:
+        #     print(line)
+
+    if current_cell:
+        cells[current_cell['address']] = current_cell
+
+    return cells
+
+
+def make_reliable_measurement(times=5, delay=1):
+    measurements = []
+    for _ in range(times):
+        point = scan()
+        if point:
+            measurements.append(point)
+        time.sleep(delay)
+    if not measurements:
+        return None
+    # average
+    cells = defaultdict(list)
+    for measurement in measurements:
+        for address, cell in measurement.items():
+            cells[address].append(cell)
+    avg_cells = {}
+    for address, cell_measurements in cells.items():
+        assert all(m['channel'] == cell_measurements[0]['channel'] for m in cell_measurements)
+        assert all(m['essid'] == cell_measurements[0]['essid'] for m in cell_measurements)
+        assert all(m['freq'] == cell_measurements[0]['freq'] for m in cell_measurements)
+        avg_cells[address] = {
+            'address': address,
+            'channel': cell_measurements[0]['channel'],
+            'essid': cell_measurements[0]['essid'],
+            'freq': cell_measurements[0]['freq'],
+            'power': sum(m['power'] for m in cell_measurements) / len(cell_measurements),
+        }
+    return avg_cells
+
+
+def append_measurement(lat, lon, acc, filename='data.yml'):
+    t1 = time.time()
+    measurement = make_reliable_measurement()
+    t2 = time.time()
+    timestamp = int((t2 + t1) // 2)
+    with open(filename, 'a') as file:
+        yaml.dump([{
+            'lat': lat,
+            'lon': lon,
+            'acc': acc,
+            'time': timestamp,
+            'data': measurement,
+        }], stream=file)

position.py

 import datetime
+from collections import defaultdict
 
-from flask import Flask, jsonify, render_template, request, send_from_directory
+import yaml
+from flask import (Flask, abort, jsonify, render_template, request,
+                   send_from_directory)
+
+from iwlist import append_measurement
 
 app = Flask(__name__, static_url_path='/static')
 
@@ -20,8 +25,51 @@ def hello():
 def setpos():
     data = request.get_json()
     print(data)
-    time = datetime.datetime.fromtimestamp(data["t"]/1000)
-    time = time.strftime("%H:%M:%S") + ".%.2d" % round(time.microsecond/10000)
+    time = datetime.datetime.fromtimestamp(data["t"] / 1000)
+    time = time.strftime("%H:%M:%S") + ".%.2d" % round(time.microsecond / 10000)
     b = "%.2d%6.2f,%s" % (int(data["lat"]))
     print("$GPGGA,%s,BBBB.BBBB,b,LLLLL.LLLL,l,Q,NN,D.D,H.H,h,G.G,g,A.A,RRRR*PP" % (time))
     return jsonify({'success': True})
+
+@app.route('/snapshot', methods=['POST'])
+def snapshot():
+    data = request.get_json()
+    print('Snapshotting at lat:', data['lat'], 'lon:', data['lon'], 'acc:', data['acc'])
+    append_measurement(lat=data['lat'], lon=data['lon'], acc=data['acc'])
+    return jsonify({'success': True})
+
+@app.route('/data')
+def data():
+    group_by = request.args.get('group_by', None)
+    filter_by = request.args.get('filter_by_address', None)
+    if group_by and group_by not in ['essid', 'address']:
+        abort(400)
+    if filter_by:
+        filter_by = filter_by.split(',')
+    with open('./data.yml', 'r' ) as file:
+        data = yaml.load(file)
+    if group_by == 'address':
+        for measurement in data:
+            by_address = defaultdict(list)
+            if not measurement.get('data', None):
+                continue
+            for address, point in measurement['data'].items():
+                by_address[address].append(point)
+            measurement['data'] = by_address
+    elif group_by == 'essid':
+        for measurement in data:
+            by_address = defaultdict(list)
+            if not measurement.get('data', None):
+                continue
+            for address, point in measurement['data'].items():
+                by_address[point['essid']].append(point)
+            measurement['data'] = by_address
+    else:
+        if filter_by:
+            for measurement in data:
+                measurement['data'] = {
+                    address: point for address, point in measurement['data'].items()
+                    if address in filter_by
+                }
+
+    return jsonify(data)

static/index.js

@@ -31,7 +31,7 @@ $(() => {
     // which field name in your data represents the latitude - default "lat"
     latField: 'lat',
     // which field name in your data represents the longitude - default "lng"
-    lngField: 'lng',
+    lngField: 'lon',
     // which field name in your data represents the data value - default "value"
     valueField: 'val',
   });
@@ -49,16 +49,90 @@ $(() => {
     }
   );
 
-  log('Click start', 'info');
+  log('Click "Follow position" and then "snapshot"', 'info');
 
-  window.polyline = L.polygon([], {color: 'red'}).addTo(window.map);
+  window.polyline = L.polyline([], {color: 'red'}).addTo(window.map);
+
+  // amount of addresses per location
+  // $.getJSON({
+  //   url:'/data',
+  //   success: (data) => {
+  //     var cleaned = data.map((measurement) => ({
+  //       lat: measurement.lat,
+  //       lon: measurement.lon,
+  //       val: measurement.data.length,
+  //     }));
+  //     window.heatmapLayer.setData({
+  //       data: cleaned,
+  //     });
+  //   },
+  //   error: console.error,
+  // });
+  // amount of essids per location
+  // $.getJSON({
+  //   url:'/data',
+  //   data: {group_by: 'essid'},
+  //   success: (data) => {
+  //     console.log(data);
+  //     var cleaned = data.map((measurement) => ({
+  //       lat: measurement.lat,
+  //       lon: measurement.lon,
+  //       val: measurement.data.length,
+  //     }));
+  //     window.heatmapLayer.setData({
+  //       data: cleaned,
+  //     });
+  //   },
+  //   error: console.error,
+  // });
+  // power average for icmp9
+  // $.getJSON({
+  //   url:'/data',
+  //   data: {group_by: 'essid'},
+  //   success: (data) => {
+  //     console.log(data);
+  //     var cleaned = data.map((measurement) => ({
+  //       lat: measurement.lat,
+  //       lon: measurement.lon,
+  //       val: measurement.data["ICMP9"].map((point) => (
+  //         point.power + 120
+  //       )).reduce((a, b) => a + b) / measurement.data["ICMP9"].length,
+  //     }));
+  //     console.log(cleaned);
+  //     window.heatmapLayer.setData({
+  //       min: 41,
+  //       max: 74,
+  //       data: cleaned,
+  //     });
+  //   },
+  //   error: console.error,
+  // });
+  // power by address
+  $.getJSON({
+    url:'/data',
+    data: {filter_by_address: 'B0:4E:26:85:F9:FF'},
+    success: (data) => {
+      var cleaned = data.map((measurement) => ({
+        lat: measurement.lat,
+        lon: measurement.lon,
+        val: measurement.data['B0:4E:26:85:F9:FF'].power + 120,
+      }));
+      console.log(cleaned);
+      window.heatmapLayer.setData({
+        min: 41,
+        max: 69,
+        data: cleaned,
+      });
+    },
+    error: console.error,
+  });
 });
 
 successCounter = 0;
 errorCounter = 0;
 
 function showPosition(position) {
-  $('#toggleBtn').text('Stop Logging');
+  $('#toggleBtn').text('Stop following');
   var location = JSON.stringify({
     lon: position.coords.longitude,
     lat: position.coords.latitude,
@@ -73,27 +147,27 @@ function showPosition(position) {
   );
   $(location, 'info');
 
-  window.heatmapLayer.addData({
-    lat: position.coords.latitude,
-    lng: position.coords.longitude,
-    val: 5,
-  });
-  $.post({
-    url: '/set',
-    data: location,
-    success: () => {
-      successCounter += 1;
-      $('#successCounter').text(successCounter);
-    },
-    fail: (message) => {
-      console.error(message);
-      log(message, 'danger');
-      errorCounter += 1;
-      $('#errorCounter').text(errorCounter);
-    },
-    contentType: "application/json",
-    dataType: 'json',
-  });
+  // window.heatmapLayer.addData({
+  //   lat: position.coords.latitude,
+  //   lng: position.coords.longitude,
+  //   val: 5,
+  // });
+  // $.post({
+  //   url: '/set',
+  //   data: location,
+  //   success: () => {
+  //     successCounter += 1;
+  //     $('#successCounter').text(successCounter);
+  //   },
+  //   fail: (message) => {
+  //     console.error(message);
+  //     log(message, 'danger');
+  //     errorCounter += 1;
+  //     $('#errorCounter').text(errorCounter);
+  //   },
+  //   contentType: "application/json",
+  //   dataType: 'json',
+  // });
 }
 
 var options = {
@@ -112,17 +186,62 @@ function toggle() {
   }
 }
 
-function start(){
-  if(handle === null){
-    log('Asking for permission', 'warning');
+function start() {
+  if(handle === null) {
+    log('Locating...', 'info');
     handle = navigator.geolocation.watchPosition(showPosition, (positionError) => {
       log(positionError.message, 'danger');
     }, options);
   }
 }
 
-function stop(){
+function stop() {
   navigator.geolocation.clearWatch(handle);
   handle = null;
-  $('#toggleBtn').text('Start Logging');
+  $('#toggleBtn').text('Follow Position');
+}
+
+function snapshot() {
+  $('#snapshotBtn').prop("disabled", true);
+  $('#snapshotBtn').text('Measuring, please stay still...')
+  log('Locating...', 'info');
+  navigator.geolocation.getCurrentPosition(
+    (position) => {
+      var location = JSON.stringify({
+        lon: position.coords.longitude,
+        lat: position.coords.latitude,
+        acc: position.coords.accuracy,
+        t: position.timestamp
+      });
+      log(location, 'info');
+      log('sending position and gathering access points...', 'info');
+      $.post({
+        url: '/snapshot',
+        data: location,
+        success: () => {
+          successCounter += 1;
+          log('Saved snapshot', 'success');
+          $('#successCounter').text(successCounter);
+          $('#snapshotBtn').prop("disabled", false);
+          $('#snapshotBtn').text('Snapshot')
+        },
+        fail: (message) => {
+          console.error(message);
+          log(message, 'danger');
+          errorCounter += 1;
+          $('#errorCounter').text(errorCounter);
+          $('#snapshotBtn').prop("disabled", false);
+          $('#snapshotBtn').text('Snapshot')
+        },
+        contentType: "application/json",
+        dataType: 'json',
+      });
+    },
+    (positionError) => {
+      log(positionError.message, 'danger');
+      $('#snapshotBtn').prop("disabled", false);
+      $('#snapshotBtn').text('Snapshot')
+    },
+    options,
+  );
 }

templates/index.html

@@ -20,7 +20,10 @@
         <h1>
           WebGPS
           <button type="button" id="toggleBtn" class="btn btn-primary" onclick="toggle()">
-            Start Logging
+            Follow Position
+          </button>
+          <button type="button" id="snapshotBtn" class="btn btn-danger" onclick="snapshot()">
+            Snapshot
           </button>
         </h1>
       </div>