Merge branch 'development' of https://github.com/cyberman54/ESP32-Paxcounter into development

LICENSE

@@ -217,6 +217,11 @@ Arduino-LMIC Library
 TTN OTAA Example
 https://github.com/matthijskooijman/arduino-lmic/blob/master/examples/ttn-otaa/
 
+and it's fork
+
+LoraWAN-in-C library, adapted to run under the Arduino environment
+https://github.com/jpmeijers/arduino-lmic
+
 under this Licence:
 
 "License

README.md

@@ -69,7 +69,9 @@ For the LoPy/LoPy4 the original Pycom firmware is not needed here, so there is n
 
 Note: If you use this software you do this at your own risk. That means that you alone - not the authors of this software - are responsible for the legal compliance of an application using this or build from this software and/or usage of a device created using this software. You should take special care and get prior legal advice if you plan metering passengers in public areas and/or publish data drawn from doing so.
 
-Disclosure: The Paxcounter code stores scanned MAC adresses in the device's RAM, and keeps it in RAM temporary for a configurable scan cycle time (default 240 seconds). After each scan cycle the collected MAC data is erased from RAM. MAC data never is transferred to the LoRaWAN network. No kind of tracking and no persistent storing of MAC data or timestamps on the device and no other kind of analytics than counting is implemented in this code. Wireless networks are not touched by this code, but MAC adresses from wireless devices as well within as not within wireless networks, regardless if encrypted or unencrypted, are made visible and scanned by this code. The same applies to Bluetooth MACs, if the bluetooth option in the code is enabled.
+# Privacy disclosure
+
+Paxcounter generates identifiers for sniffed MAC adresses and collects them temporary in the device's RAM for a configurable scan cycle time (default 240 seconds). After each scan cycle the collected identifiers are cleared. Identifiers are generated by salting and hashing MAC adresses. The random salt value changes after each scan cycle. Identifiers and MAC adresses are never transferred to the LoRaWAN network. No persistent storing of MAC adresses, identifiers or timestamps and no other kind of analytics than counting are implemented in this code. Wireless networks are not touched by this code, but MAC adresses from wireless devices as well within as not within wireless networks, regardless if encrypted or unencrypted, are sniffed and processed by this code. If the bluetooth option in the code is enabled, bluetooth MACs are scanned and processed by the included BLE stack, and counted by this code.
 
 # Payload format description
 

platformio.ini

@@ -10,9 +10,9 @@
 
 ; ---> SELECT TARGET PLATFORM HERE! <---
 [platformio]
-env_default = heltec_wifi_lora_32
+;env_default = heltec_wifi_lora_32
 ;env_default = ttgov1
-;env_default = ttgov2
+env_default = ttgov2
 ;env_default = lopy
 ;env_default = lopy4
 ;env_default = lolin32lite_lora
@@ -70,7 +70,9 @@ board = esp32dev
 framework = arduino
 monitor_baud = 115200
 upload_speed = 921600
-lib_deps = U8g2
+lib_deps = 
+    U8g2
+    ESP32 BLE Arduino@>=0.4.9
 build_flags = 
 ;set log level, we need build_flag for this, otherwise we can't use ESP_LOGx in arduino framework
     -DCORE_DEBUG_LEVEL=ARDUHAL_LOG_LEVEL_VERBOSE

src/blecount.cpp

@@ -27,6 +27,6 @@ void BLECount() {
     u8x8.clearLine(3);
     u8x8.setCursor(0,3);
     blenum=foundDevices.getCount();
-    u8x8.printf("BLE#: %4i",blenum);
+    u8x8.printf("BLE#: %-5i",blenum);
 }
 #endif

src/globals.h

@@ -12,7 +12,6 @@
 #include <hal/hal.h>
 
 // Struct holding devices's runtime configuration
-
 typedef struct {
   int8_t lorasf;                       // 7-12, lora spreadfactor
   int8_t txpower;                      // 2-15, lora tx power
@@ -33,9 +32,10 @@ extern configData_t cfg;
 extern uint8_t mydata[];
 extern uint64_t uptimecounter;
 extern osjob_t sendjob;
-extern int macnum, blenum, countermode, screensaver, adrmode, lorasf, txpower, rlim;
+extern uint16_t macnum, blenum, salt;
+extern int countermode, screensaver, adrmode, lorasf, txpower, rlim;
 extern bool joinstate;
-extern std::set<uint64_t, std::greater <uint64_t> > macs;
+extern std::set<uint16_t> macs; 
 
 #ifdef HAS_DISPLAY
     extern HAS_DISPLAY u8x8;

src/lorawan.cpp

@@ -79,10 +79,10 @@ void printKeys(void) {
 #endif // VERBOSE
 
 void do_send(osjob_t* j){
-    mydata[0] = (macnum & 0x0000ff00) >> 8;
+    mydata[0] = (macnum & 0xff00) >> 8;
-    mydata[1] = macnum  & 0x000000ff;
+    mydata[1] = macnum  & 0x00ff;
-    mydata[2] = (blenum & 0x0000ff00) >> 8;
+    mydata[2] = (blenum & 0xff00) >> 8;
-    mydata[3] = blenum  & 0x000000ff;
+    mydata[3] = blenum  & 0x00ff;
 
     // Check if there is not a current TX/RX job running
     if (LMIC.opmode & OP_TXRXPEND) {

src/main.cpp

@@ -25,9 +25,6 @@ Refer to LICENSE.txt file in repository for more details.
 #include "main.h"
 #include "globals.h"
 
-// std::set for unified array functions
-#include <set>
-
 // OLED driver
 #include <U8x8lib.h>
 
@@ -45,11 +42,11 @@ configData_t cfg; // struct holds current device configuration
 osjob_t sendjob, initjob; // LMIC
 
 // Initialize global variables
-int macnum = 0, blenum = 0;
+uint16_t macnum = 0, blenum = 0, salt;
 uint64_t uptimecounter = 0;
 bool joinstate = false;
 
-std::set<uint64_t, std::greater <uint64_t> > macs; // storage holds MAC frames
+std::set<uint16_t> macs; // associative container holds filtered MAC adresses
 
 // this variable will be changed in the ISR, and read in main loop
 static volatile bool ButtonTriggered = false;
@@ -70,16 +67,29 @@ void eraseConfig(void);
 void saveConfig(void);
 void loadConfig(void);
 
-
 /* begin LMIC specific parts ------------------------------------------------------------ */
 
+// LMIC enhanced Pin mapping 
+const lmic_pinmap lmic_pins = {
+    .mosi = PIN_SPI_MOSI,
+    .miso = PIN_SPI_MISO,
+    .sck = PIN_SPI_SCK,
+    .nss = PIN_SPI_SS,
+    .rxtx = LMIC_UNUSED_PIN,
+    .rst = RST,
+    .dio = {DIO0, DIO1, DIO2}
+};
+
 // defined in lorawan.cpp
 void gen_lora_deveui(uint8_t * pdeveui);
 void RevBytes(unsigned char* b, size_t c);
-
 #ifdef VERBOSE
     void printKeys(void);
-#endif // VERBOSE
+#endif
+
+// LMIC functions
+void onEvent(ev_t ev);
+void do_send(osjob_t* j);
 
 // LMIC callback functions
 void os_getDevKey (u1_t *buf) { 
@@ -102,21 +112,6 @@ void os_getDevEui (u1_t* buf) {
         gen_lora_deveui(buf); // generate DEVEUI from device's MAC
 }
 
-// LMIC enhanced Pin mapping 
-const lmic_pinmap lmic_pins = {
-    .mosi = PIN_SPI_MOSI,
-    .miso = PIN_SPI_MISO,
-    .sck = PIN_SPI_SCK,
-    .nss = PIN_SPI_SS,
-    .rxtx = LMIC_UNUSED_PIN,
-    .rst = RST,
-    .dio = {DIO0, DIO1, DIO2}
-};
-
-// LMIC functions
-void onEvent(ev_t ev);
-void do_send(osjob_t* j);
-
 // LoRaWAN Initjob
 static void lora_init (osjob_t* j) {
     // reset MAC state
@@ -208,20 +203,21 @@ void wifi_sniffer_loop(void * pvParameters) {
             
             // execute BLE count if BLE function is enabled
             #ifdef BLECOUNTER
-                if ( cfg.blescan )
+                if (cfg.blescan)
                     BLECount();
             #endif
             
             // Prepare and execute LoRaWAN data upload
             u8x8.setCursor(0,4);
-            u8x8.printf("MAC#: %4i", macnum);
+            u8x8.printf("MAC#: %-5i", macnum);
             do_send(&sendjob); // send payload
             vTaskDelay(500/portTICK_PERIOD_MS);
             yield();
 
             // clear counter if not in cumulative counter mode
-            if ( cfg.countermode != 1 ) {
+            if (cfg.countermode != 1) {
-                macs.erase(macs.begin(), macs.end()); // clear RAM
+                macs.clear(); // clear macs container
+                salt = random(65536); // get new 16bit random for salting hashes
                 macnum = 0;
                 u8x8.clearLine(0); u8x8.clearLine(1); // clear Display counter
             }      
@@ -384,11 +380,14 @@ void setup() {
     antenna_init();
 #endif
 
+    // initialize salt value using esp_random() called by random in arduino-esp32 core
+    salt = random(65536); // get new 16bit random for salting hashes
+
     // initialize display
     init_display(PROGNAME, PROGVERSION);     
     u8x8.setPowerSave(!cfg.screenon); // set display off if disabled
     u8x8.setCursor(0,5);
-    u8x8.printf(!cfg.rssilimit ? "RLIM:  off" : "RLIM: %4i", cfg.rssilimit);
+    u8x8.printf(!cfg.rssilimit ? "RLIM: off" : "RLIM: %4i", cfg.rssilimit);
     u8x8.drawString(0,6,"Join Wait       ");
     
     // output LoRaWAN keys to console

src/main.h

@@ -1,5 +1,5 @@
 // program version
-#define PROGVERSION                     "1.2.5" // use max 10 chars here!
+#define PROGVERSION                     "1.2.62" // use max 10 chars here!
 #define PROGNAME                        "PAXCNT"
 
 // Verbose enables serial output
@@ -16,8 +16,10 @@
 #define SEND_SECS                       120 // [seconds/2] -> 240 sec.
 
 // WiFi sniffer config
-#define	WIFI_CHANNEL_MAX                13
+#define WIFI_CHANNEL_MIN                1   // start channel number where scan begings
-#define	WIFI_CHANNEL_SWITCH_INTERVAL    50 // [seconds/100] -> 0,5 sec.
+#define	WIFI_CHANNEL_MAX                13  // total channel number to scan
+#define WIFI_MY_COUNTRY                 "EU"  // for Wifi RF settings
+#define	WIFI_CHANNEL_SWITCH_INTERVAL    50  // [seconds/100] -> 0,5 sec.     
 
 // Default LoRa Spreadfactor
 #define LORASFDEFAULT                   9 // 7 ... 12

src/rcommand.cpp

@@ -67,8 +67,9 @@ void set_reset(int val) {
             break;
         case 1: // reset MAC counter
             ESP_LOGI(TAG, "Remote command: reset MAC counter");
-            macs.erase(macs.begin(), macs.end()); // clear RAM
+            macs.clear(); // clear macs container
             macnum = 0;
+            salt = random(65536); // get new 16bit random for salting hashes
             u8x8.clearLine(0); u8x8.clearLine(1); // clear Display counter
             u8x8.clearLine(5);
             u8x8.setCursor(0, 5);

src/rokkithash.cpp

@@ -0,0 +1,83 @@
+/*
+ * RokkitHash - Arduino port for Paul Hsieh's "SuperFastHash"
+ *
+ * A very quick hash function, (c) Paul Hsieh
+ *
+ * See http://www.azillionmonkeys.com/qed/hash.html for more information
+ * about its inner workings
+ *
+ * - Initial Arduino version: 2014 Alex K
+ * - 8-bit improvements: robtillaart
+ * - Current maintainer: SukkoPera <software@sukkology.net>
+ *
+ * See http://forum.arduino.cc/index.php?topic=226686.0 for some talk
+ * about the various improvements.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <inttypes.h>
+
+uint32_t rokkit(const char * data, int len) {
+  uint32_t hash, tmp;
+  int rem;
+
+    if (len <= 0 || data == 0) return 0;
+    hash = len;
+    rem = len & 3;
+    len >>= 2;
+
+    /* Main loop */
+    while (len > 0) {
+        hash  += *((uint16_t*)data);
+        tmp    = (*((uint16_t*)(data+2)) << 11) ^ hash;
+        hash   = (hash << 16) ^ tmp;
+        data  += 2*2;
+        hash  += hash >> 11;
+		len--;
+    }
+
+    /* Handle end cases */
+    switch (rem) {
+        case 3: hash += *((uint16_t*)data);
+                hash ^= hash << 16;
+                hash ^= ((signed char)data[2]) << 18;
+                hash += hash >> 11;
+                break;
+        case 2: hash += *((uint16_t*)data);
+                hash ^= hash << 11;
+                hash += hash >> 17;
+                break;
+        case 1: hash += (signed char)*data;
+                hash ^= hash << 10;
+                hash += hash >> 1;
+    }
+
+    /* Force "avalanching" of final 127 bits */
+    hash ^= hash << 3;
+    hash += hash >> 5;
+    hash ^= hash << 4;
+    hash += hash >> 17;
+    hash ^= hash << 25;
+    hash += hash >> 6;
+
+    return hash;
+}

src/wifisniffer.cpp

@@ -14,7 +14,10 @@
 // Local logging tag
 static const char *TAG = "wifisniffer";
 
-static wifi_country_t wifi_country = {.cc="EU", .schan=1, .nchan=13, .policy=WIFI_COUNTRY_POLICY_AUTO};
+// function defined in rokkithash.cpp
+uint32_t rokkit(const char * , int );
+
+static wifi_country_t wifi_country = {.cc=WIFI_MY_COUNTRY, .schan=WIFI_CHANNEL_MIN, .nchan=WIFI_CHANNEL_MAX, .policy=WIFI_COUNTRY_POLICY_MANUAL};
 
 typedef struct {
 	unsigned frame_ctrl:16;
@@ -37,17 +40,15 @@ extern void wifi_sniffer_packet_handler(void *buff, wifi_promiscuous_pkt_type_t
 
 void wifi_sniffer_init(void) {
 		wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
-		cfg.nvs_enable = 0; // we don't want wifi settings from NVRAM
+		cfg.nvs_enable = 0; // we don't need any wifi settings from NVRAM
-    	ESP_ERROR_CHECK(esp_wifi_init(&cfg));
+		wifi_promiscuous_filter_t filter = {.filter_mask = WIFI_PROMIS_FILTER_MASK_MGMT}; // we need only MGMT frames
-    	ESP_ERROR_CHECK(esp_wifi_set_country(&wifi_country));
+    	ESP_ERROR_CHECK(esp_wifi_init(&cfg));						// configure Wifi with cfg
-		ESP_ERROR_CHECK(esp_wifi_set_storage(WIFI_STORAGE_RAM) );
+    	ESP_ERROR_CHECK(esp_wifi_set_country(&wifi_country));		// set locales for RF and channels
-		ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_NULL) );
+		ESP_ERROR_CHECK(esp_wifi_set_storage(WIFI_STORAGE_RAM));	// we don't need NVRAM
-    	//ESP_ERROR_CHECK(esp_wifi_start()); // not sure if we need this in this application?
+		ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_NULL));
-    	//ESP_ERROR_CHECK(esp_wifi_set_max_tx_power(-128)); // we don't need to TX, so we use lowest power level to save energy
+    	ESP_ERROR_CHECK(esp_wifi_set_promiscuous_filter(&filter));	// set MAC frame filter
-    	wifi_promiscuous_filter_t filter = {.filter_mask = WIFI_PROMIS_FILTER_MASK_MGMT}; // we need only MGMT frames
-    	ESP_ERROR_CHECK(esp_wifi_set_promiscuous_filter(&filter)); // set MAC frame filter
     	ESP_ERROR_CHECK(esp_wifi_set_promiscuous_rx_cb(&wifi_sniffer_packet_handler));
-    	ESP_ERROR_CHECK(esp_wifi_set_promiscuous(true));
+    	ESP_ERROR_CHECK(esp_wifi_set_promiscuous(true));			// now switch on monitor mode
 }
 
 void wifi_sniffer_set_channel(uint8_t channel) {
@@ -58,36 +59,40 @@ void wifi_sniffer_packet_handler(void* buff, wifi_promiscuous_pkt_type_t type) {
 	const wifi_promiscuous_pkt_t *ppkt = (wifi_promiscuous_pkt_t *)buff;
 	const wifi_ieee80211_packet_t *ipkt = (wifi_ieee80211_packet_t *)ppkt->payload;
 	const wifi_ieee80211_mac_hdr_t *hdr = &ipkt->hdr;
-	char counter [10];
+	char counter [6]; // uint16_t -> 2 byte -> 5 decimals + '0' terminator -> 6 chars
+	char macbuf [21]; // uint64_t -> 8 byte -> 20 decimals + '0' terminator -> 21 chars
+	uint64_t addr2int;
+	uint32_t vendor2int;
+	uint16_t hashedmac;
+	std::pair<std::set<uint16_t>::iterator, bool> newmac;
 
 	if (( cfg.rssilimit == 0 ) || (ppkt->rx_ctrl.rssi > cfg.rssilimit )) { // rssi is negative value
-	    uint64_t addr2int = ( (uint64_t)hdr->addr2[0] ) | ( (uint64_t)hdr->addr2[1] << 8 ) | ( (uint64_t)hdr->addr2[2] << 16 ) | \
+	    addr2int = ( (uint64_t)hdr->addr2[0] ) | ( (uint64_t)hdr->addr2[1] << 8 ) | ( (uint64_t)hdr->addr2[2] << 16 ) | \
 			( (uint64_t)hdr->addr2[3] << 24 ) | ( (uint64_t)hdr->addr2[4] << 32 ) | ( (uint64_t)hdr->addr2[5] << 40 );
 
-#ifdef VENDORFILTER
+#ifdef VENDORFILTER // uses vendor array with prefiltered OUIs (no local nd no group MACs, bits 0+1 in 1st byte of OUI)
-		uint32_t vendor2int = ( (uint32_t)hdr->addr2[2] ) | ( (uint32_t)hdr->addr2[1] << 8 ) | ( (uint32_t)hdr->addr2[0] << 16 );
+		vendor2int = ( (uint32_t)hdr->addr2[2] ) | ( (uint32_t)hdr->addr2[1] << 8 ) | ( (uint32_t)hdr->addr2[0] << 16 );
-
 		if ( std::find(vendors.begin(), vendors.end(), vendor2int) != vendors.end() ) {
 #endif
-		    macs.insert(addr2int);
+			// salt and hash MAC, and if new unique one, store identifier in container and increment counter on display
+			// https://en.wikipedia.org/wiki/MAC_Address_Anonymization
 			
-			// INFO: RSSI when adding MAC
+			addr2int |= (uint64_t) salt << 48;	// prepend 12 bit salt to 48 bit MAC
-			ESP_LOGI(TAG, "WiFi RSSI: %02d", ppkt->rx_ctrl.rssi);
+			itoa(addr2int, macbuf, 10);			// convert 64 bit MAC to base 10 decimal string
-
+			hashedmac = rokkit(macbuf, 5);		// hash MAC, use 5 chars to fit hash in uint16_t container
-			// if new unique MAC logged increment counter on display
+			newmac = macs.insert(hashedmac);	// store hashed MAC if new unique
-			if ( macs.size() > macnum ) {
+			if (newmac.second) {				// first time seen MAC
-				macnum = macs.size();
+				macnum++;						// increment MAC counter
-				itoa(macnum, counter, 10);
+				itoa(macnum, counter, 10);		// base 10 decimal counter value
 				u8x8.draw2x2String(0, 0, counter);
-				ESP_LOGI(TAG, "MAC counter: %4i", macnum);
+				ESP_LOGI(TAG, "#%05i: RSSI %04d -> Salt %04x -> Hash %04x", macnum, ppkt->rx_ctrl.rssi, salt, hashedmac);
 			}
 
-			
 #ifdef VENDORFILTER
 		}
 #endif
-	} else {
+	} else
-		ESP_LOGI(TAG, "Ignoring RSSI %02d (limit: %i)", ppkt->rx_ctrl.rssi, cfg.rssilimit );
+		ESP_LOGI(TAG, "RSSI %04d -> ignoring (limit: %i)", ppkt->rx_ctrl.rssi, cfg.rssilimit);
-	}
+
 	yield();
 }