buffered MAC processing (experimental)

include/globals.h

@@ -18,10 +18,6 @@
 #include "mallocator.h"
 #include <bsec.h>
 
-// sniffing types
-#define MAC_SNIFF_WIFI 0
-#define MAC_SNIFF_BLE 1
-
 // bits in payloadmask for filtering payload data
 #define GPS_DATA (0x01)
 #define ALARM_DATA (0x02)
@@ -75,6 +71,9 @@ enum runmode_t {
   RUNMODE_UPDATE
 };
 
+// sniffing types
+enum snifftype_t { MAC_SNIFF_WIFI, MAC_SNIFF_BLE, MAC_SNIFF_BLE_ENS };
+
 // Struct holding devices's runtime configuration
 // using packed to avoid compiler padding, because struct will be memcpy'd to
 // byte array
@@ -114,6 +113,13 @@ typedef struct {
   uint8_t Message[PAYLOAD_BUFFER_SIZE];
 } MessageBuffer_t;
 
+// Struct for MAC processing queue
+typedef struct {
+  uint8_t mac[6];
+  int8_t rssi;
+  snifftype_t sniff_type;
+} MacBuffer_t;
+
 typedef struct {
   int32_t latitude;
   int32_t longitude;
@@ -151,7 +157,7 @@ extern bool volatile TimePulseTick; // 1sec pps flag set by GPS or RTC
 extern timesource_t timeSource;
 extern hw_timer_t *displayIRQ, *matrixDisplayIRQ, *ppsIRQ;
 extern SemaphoreHandle_t I2Caccess;
-extern TaskHandle_t irqHandlerTask, ClockTask;
+extern TaskHandle_t irqHandlerTask, ClockTask, macProcessTask;
 extern TimerHandle_t WifiChanTimer;
 extern Timezone myTZ;
 extern RTC_DATA_ATTR runmode_t RTC_runmode;

include/macsniff.h

@@ -14,13 +14,12 @@
 #include "corona.h"
 #endif
 
-#define MAC_SNIFF_WIFI 0
-#define MAC_SNIFF_BLE 1
-#define MAC_SNIFF_BLE_CWA 2
-
 uint16_t get_salt(void);
 uint64_t macConvert(uint8_t *paddr);
-uint16_t mac_add(uint8_t *paddr, int8_t rssi, bool sniff_type);
+esp_err_t macQueueInit(void);
+void mac_process(void *pvParameters);
+void IRAM_ATTR mac_add(uint8_t *paddr, int8_t rssi, snifftype_t sniff_type);
+uint16_t mac_analyze(uint8_t *paddr, int8_t rssi, snifftype_t sniff_type);
 void printKey(const char *name, const uint8_t *key, uint8_t len, bool lsb);
 
 #endif

src/blecsan.cpp

@@ -6,14 +6,10 @@
 #define BT_BD_ADDR_HEX(addr)                                                   \
   addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]
 
-// UUID of Exposure Notification Service (ENS)
-// see
-// https://blog.google/documents/70/Exposure_Notification_-_Bluetooth_Specification_v1.2.2.pdf
-static const char ensMagicBytes[] = "\x16\x6f\xfd";
-
 // local Tag for logging
 static const char TAG[] = "bluetooth";
 
+#ifdef VERBOSE
 const char *bt_addr_t_to_string(esp_ble_addr_type_t type) {
   switch (type) {
   case BLE_ADDR_TYPE_PUBLIC:
@@ -110,15 +106,24 @@ const char *btsig_gap_type(uint32_t gap_type) {
     return "Unknown type";
   }
 } // btsig_gap_type
+#endif
 
-// using IRAM_:ATTR here to speed up callback function
+// using IRAM_ATTR here to speed up callback function
 IRAM_ATTR void gap_callback_handler(esp_gap_ble_cb_event_t event,
                                     esp_ble_gap_cb_param_t *param) {
 
   esp_ble_gap_cb_param_t *p = (esp_ble_gap_cb_param_t *)param;
 
+#if (COUNT_ENS)
+  // UUID of Exposure Notification Service (ENS)
+  // https://blog.google/documents/70/Exposure_Notification_-_Bluetooth_Specification_v1.2.2.pdf
+  static const char ensMagicBytes[] = "\x16\x6f\xfd";
+#endif
+
+#ifdef VERBOSE
   ESP_LOGV(TAG, "BT payload rcvd -> type: 0x%.2x -> %s", *p->scan_rst.ble_adv,
            btsig_gap_type(*p->scan_rst.ble_adv));
+#endif
 
   switch (event) {
   case ESP_GAP_BLE_SCAN_PARAM_SET_COMPLETE_EVT:
@@ -138,11 +143,13 @@ IRAM_ATTR void gap_callback_handler(esp_gap_ble_cb_event_t event,
     if (p->scan_rst.search_evt ==
         ESP_GAP_SEARCH_INQ_RES_EVT) // Inquiry result for a peer device
     {                               // evaluate sniffed packet
+#ifdef VERBOSE
       ESP_LOGV(TAG, "Device address (bda): %02x:%02x:%02x:%02x:%02x:%02x",
                BT_BD_ADDR_HEX(p->scan_rst.bda));
       ESP_LOGV(TAG, "Addr_type           : %s",
                bt_addr_t_to_string(p->scan_rst.ble_addr_type));
       ESP_LOGV(TAG, "RSSI                : %d", p->scan_rst.rssi);
+#endif
 
       if ((cfg.rssilimit) &&
           (p->scan_rst.rssi < cfg.rssilimit)) { // rssi is negative value
@@ -154,26 +161,26 @@ IRAM_ATTR void gap_callback_handler(esp_gap_ble_cb_event_t event,
 #if (VENDORFILTER)
       if ((p->scan_rst.ble_addr_type == BLE_ADDR_TYPE_RANDOM) ||
           (p->scan_rst.ble_addr_type == BLE_ADDR_TYPE_RPA_RANDOM)) {
+#ifdef VERBOSE
         ESP_LOGV(TAG, "BT device filtered");
+#endif
         break;
       }
 #endif
 
-      // hash and add this device and show new count total if it was not
+      // add this device mac to processing queue
-      // previously added
-
-#if (COUNT_ENS)
-      uint16_t hashedmac =
-#endif
-
-          mac_add((uint8_t *)p->scan_rst.bda, p->scan_rst.rssi, MAC_SNIFF_BLE);
 
 #if (COUNT_ENS)
+      // check for ens signature
       if (cfg.enscount) {
-        // check for ens signature
+        if (strstr((const char *)p->scan_rst.ble_adv, ensMagicBytes) != NULL)
-        if (NULL != strstr((const char *)p->scan_rst.ble_adv, ensMagicBytes))
+          mac_add((uint8_t *)p->scan_rst.bda, p->scan_rst.rssi,
-          cwa_mac_add(hashedmac);
+                  MAC_SNIFF_BLE_ENS);
+        else
+          mac_add((uint8_t *)p->scan_rst.bda, p->scan_rst.rssi, MAC_SNIFF_BLE);
       }
+#else
+      mac_add((uint8_t *)p->scan_rst.bda, p->scan_rst.rssi, MAC_SNIFF_BLE);
 #endif
 
       /* to be improved in vendorfilter if:
@@ -186,8 +193,8 @@ IRAM_ATTR void gap_callback_handler(esp_gap_ble_cb_event_t event,
       // uint8_t *data = esp_ble_resolve_adv_data(p->scan_rst.ble_adv,
       ESP_BLE_AD_TYPE_NAME_CMPL, &len);
 
-      filter BLE devices using their advertisements to get filter alternative to
+      filter BLE devices using their advertisements to get filter alternative
-      vendor OUI if vendorfiltering is on, we ...
+      to vendor OUI if vendorfiltering is on, we ...
       - want to count: mobile phones and tablets
       - don't want to count: beacons, peripherals (earphones, headsets,
       printers), cars and machines see
@@ -211,13 +218,13 @@ IRAM_ATTR void gap_callback_handler(esp_gap_ble_cb_event_t event,
 
   default:
     break;
-  }
+  } // switch
 } // gap_callback_handler
 
 esp_err_t register_ble_callback(void) {
   ESP_LOGI(TAG, "Register GAP callback");
 
-  // This function is called to occur gap event, such as scan result.
+  // This function is called when gap event occurs, such as scan result.
   // register the scan callback function to the gap module
   ESP_ERROR_CHECK(esp_ble_gap_register_callback(&gap_callback_handler));
 

src/corona.cpp

@@ -23,18 +23,13 @@ static std::map<uint16_t, unsigned long> cwaSeenNotifiers;
 
 // Remove notifiers last seen over FORGET_AFTER_MINUTES ago.
 void cwa_clear() {
-  /*
+#ifdef VERBOSE
-
+  ESP_LOGV(TAG, "CWA: forget old notifier: %d", cwaSeenNotifiers.size());
-    #ifdef SOME_FORM_OF_DEBUG
+  for (auto const &notifier : cwaSeenNotifiers) {
-  ESP_LOGD(TAG, "CWA: forget old notifier: %d", cwaSeenNotifiers.size());
+    ESP_LOGD(TAG, "CWA forget <%04X>", notifier.first);
-    for (auto const &notifier : cwaSeenNotifiers) {
+    //    }
-      ESP_LOGD(TAG, "CWA forget <%X>", notifier.first);
+  }
-      //    }
+#endif
-    }
-  #endif
-
-  */
-
   // clear everything, otherwise we would count the same device again, as in the
   // next cycle it likely will advertise with a different hash-value
   cwaSeenNotifiers.clear();

src/cyclic.cpp

@@ -40,6 +40,9 @@ void doHousekeeping() {
   ESP_LOGD(TAG, "IRQhandler %d bytes left | Taskstate = %d",
            uxTaskGetStackHighWaterMark(irqHandlerTask),
            eTaskGetState(irqHandlerTask));
+  ESP_LOGD(TAG, "MACprocessor %d bytes left | Taskstate = %d",
+           uxTaskGetStackHighWaterMark(macProcessTask),
+           eTaskGetState(macProcessTask));
 #if (HAS_LORA)
   ESP_LOGD(TAG, "LMiCtask %d bytes left | Taskstate = %d",
            uxTaskGetStackHighWaterMark(lmicTask), eTaskGetState(lmicTask));

src/macsniff.cpp

@@ -10,6 +10,9 @@
 // Local logging tag
 static const char TAG[] = __FILE__;
 
+QueueHandle_t MacQueue;
+TaskHandle_t macProcessTask;
+
 uint16_t salt = 0;
 
 uint16_t get_salt(void) {
@@ -43,7 +46,60 @@ uint64_t macConvert(uint8_t *paddr) {
   return (__builtin_bswap64(*mac) >> 16);
 }
 
-uint16_t mac_add(uint8_t *paddr, int8_t rssi, bool sniff_type) {
+esp_err_t macQueueInit() {
+  _ASSERT(MAC_QUEUE_SIZE > 0);
+  MacQueue = xQueueCreate(MAC_QUEUE_SIZE, sizeof(MacBuffer_t));
+  if (MacQueue == 0) {
+    ESP_LOGE(TAG, "Could not create MAC processing queue. Aborting.");
+    return ESP_FAIL;
+  }
+  ESP_LOGI(TAG, "MAC processing queue created, size %d Bytes",
+           MAC_QUEUE_SIZE * sizeof(MacBuffer_t));
+
+  xTaskCreatePinnedToCore(mac_process,     // task function
+                          "mac_process",   // name of task
+                          2048,            // stack size of task
+                          (void *)1,       // parameter of the task
+                          1,               // priority of the task
+                          &macProcessTask, // task handle
+                          1);              // CPU core
+
+  return ESP_OK;
+}
+
+// sniffed MAC processing task
+void mac_process(void *pvParameters) {
+  _ASSERT((uint32_t)pvParameters == 1); // FreeRTOS check
+
+  MacBuffer_t MacBuffer;
+
+  while (1) {
+
+    // fetch next or wait for incoming MAC from sniffing queue
+    if (xQueueReceive(MacQueue, &MacBuffer, portMAX_DELAY) != pdTRUE) {
+      ESP_LOGE(TAG, "Premature return from xQueueReceive() with no data!");
+      continue;
+    } else
+      mac_analyze(MacBuffer.mac, MacBuffer.rssi, MacBuffer.sniff_type);
+  }
+  delay(2); // yield to CPU
+}
+
+// enqueue message in MAC processing queue
+void IRAM_ATTR mac_add(uint8_t *paddr, int8_t rssi, snifftype_t sniff_type) {
+
+  MacBuffer_t MacBuffer;
+
+  MacBuffer.rssi = rssi;
+  MacBuffer.sniff_type = sniff_type;
+  memcpy(MacBuffer.mac, paddr, 6);
+
+  if (xQueueSendToBackFromISR(MacQueue, (void *)&MacBuffer, (TickType_t)0) !=
+      pdTRUE)
+    ESP_LOGW(TAG, "Dense radio traffic, packet lost!");
+}
+
+uint16_t mac_analyze(uint8_t *paddr, int8_t rssi, snifftype_t sniff_type) {
 
   if (salt == 0) // ensure we have salt (appears after radio is turned on)
     return 0;
@@ -86,18 +142,27 @@ uint16_t mac_add(uint8_t *paddr, int8_t rssi, bool sniff_type) {
       // increment counter and one blink led
       if (sniff_type == MAC_SNIFF_WIFI) {
         macs_wifi++; // increment Wifi MACs counter
+
 #if (HAS_LED != NOT_A_PIN) || defined(HAS_RGB_LED)
         blink_LED(COLOR_GREEN, 50);
 #endif
       }
 #if (BLECOUNTER)
-      else if (sniff_type == MAC_SNIFF_BLE) {
+      else if ((sniff_type == MAC_SNIFF_BLE) ||
+               (sniff_type = MAC_SNIFF_BLE_ENS)) {
         macs_ble++; // increment BLE Macs counter
+
+#if (COUNT_ENS)
+        if (sniff_type == MAC_SNIFF_BLE_ENS)
+          cwa_mac_add(hashedmac);
+#endif
+
 #if (HAS_LED != NOT_A_PIN) || defined(HAS_RGB_LED)
         blink_LED(COLOR_MAGENTA, 50);
 #endif
       }
-#endif
+
+#endif // BLECOUNTER
 
       // in beacon monitor mode check if seen MAC is a known beacon
       if (cfg.monitormode) {
@@ -142,4 +207,4 @@ uint16_t mac_add(uint8_t *paddr, int8_t rssi, bool sniff_type) {
   // if a new and unique Wifi or BLE mac was counted, returs hash of this mac,
   // else 0
   return hashedmac;
-}
+}

src/main.cpp

@@ -35,9 +35,10 @@ IDLE          0     0     ESP32 arduino scheduler -> runs wifi sniffer
 lmictask      1     2     MCCI LMiC LORAWAN stack
 clockloop     1     4     generates realtime telegrams for external clock
 timesync_proc 1     3     processes realtime time sync requests
-irqhandler    1     1     cyclic tasks (i.e. displayrefresh) triggered by timers
+irqhandler    1     2     cyclic tasks (i.e. displayrefresh) triggered by timers
 gpsloop       1     1     reads data from GPS via serial or i2c
 lorasendtask  1     1     feeds data from lora sendqueue to lmcic
+macprocess    1     1     analyzes sniffed MACs
 IDLE          1     0     ESP32 arduino scheduler -> runs wifi channel rotator
 
 Low priority numbers denote low priority tasks.
@@ -285,6 +286,10 @@ void setup() {
     start_ota_update();
 #endif
 
+  // start mac processing task
+  ESP_LOGI(TAG, "Starting MAC processor...");
+  macQueueInit();
+
 // start BLE scan callback if BLE function is enabled in NVRAM configuration
 // or switch off bluetooth, if not compiled
 #if (BLECOUNTER)

src/sensor.cpp

@@ -57,7 +57,7 @@ uint8_t *sensor_read(uint8_t sensor) {
     // note: Sensor1 fields are used for ENS count, if ENS detection enabled
 #if (COUNT_ENS)
     if (cfg.enscount)
-      payload.addCount(cwa_report(), MAC_SNIFF_BLE_CWA);
+      payload.addCount(cwa_report(), MAC_SNIFF_BLE_ENS);
 #else
     buf[0] = length;
     buf[1] = 0x01;