Merge pull request #514 from cyberman54/development

Add SDcard support | bugfix TTN decoder | bugfix IRQ handler

README.md

@@ -20,7 +20,7 @@ Paxcounter is a proof-of-concept device for metering passenger flows in realtime
 
 Intention of this project is to do this without intrusion in privacy: You don't need to track people owned devices, if you just want to count them. Therefore, Paxcounter does not persistenly store MAC adresses and does no kind of fingerprinting the scanned devices.
 
-Data is transferred to a server via a LoRaWAN network, and/or a wired SPI slave interface.
+Data is transferred to a server via a LoRaWAN network, and/or a wired SPI slave interface. It can also be stored on a local SD-card.
 
 You can build this project battery powered and reach a full day uptime with a single 18650 Li-Ion cell.
 
@@ -33,7 +33,7 @@ This can all be done with a single small and cheap ESP32 board for less than $20
 *LoRa & SPI*:
 
 - Heltec: LoRa-32 v1 and v2
-- TTGO: T1, T2, T3, T-Beam, T-Fox
+- TTGO: T1*, T2*, T3*, T-Beam, T-Fox (* supports microSD-card)
 - Pycom: LoPy, LoPy4, FiPy
 - Radioshuttle.de: [ECO Power Board](https://www.radioshuttle.de/esp32-eco-power/esp32-eco-power-board/)
 - WeMos: LoLin32 + [LoraNode32 shield](https://github.com/hallard/LoLin32-Lora), 
@@ -60,6 +60,7 @@ Depending on board hardware following features are supported:
 - IF482 (serial) and DCF77 (gpio) time telegram generator
 - Switch external power / battery
 - LED Matrix display (similar to [this 64x16 model](https://www.instructables.com/id/64x16-RED-LED-Marquee/), can be ordered on [Aliexpress](https://www.aliexpress.com/item/P3-75-dot-matrix-led-module-3-75mm-high-clear-top1-for-text-display-304-60mm/32616683948.html))
+- SD-card (see section SD-card here)
 
 Target platform must be selected in [platformio.ini](https://github.com/cyberman54/ESP32-Paxcounter/blob/master/platformio.ini).<br>
 Hardware dependent settings (pinout etc.) are stored in board files in /hal directory. If you want to use a ESP32 board which is not yet supported, use hal file generic.h and tailor pin mappings to your needs. Pull requests for new boards welcome.<br>
@@ -196,6 +197,43 @@ Follow all steps so far for preparing the device, use the packed payload format.
 There in the sensor configuration select "TheThingsNetwork" and set Decoding Profil to "LoRa serialization", enter your TTN Application and Device Id. Decoding option has to be
 	[{"decoder":"latLng"},{"decoder":"uint16","sensor_id":"yoursensorid"}] 
 
+# SD-card
+Data can be stored on an SD-card if one is availabe. Simply choose the file in src/hal and add the following lines to your hal-file:
+
+    #define HAS_SDCARD 1     // this board has an SD-card-reader/writer
+    // Pins for SD-card
+    #define SDCARD_CS   (13) // fill in the correct numbers for your board
+    #define SDCARD_MOSI (15)
+    #define SDCARD_MISO (2)
+    #define SDCARD_SCLK (14)
+
+Please choose the correct number for the connection of the reader/writer.
+
+This is an example of a board with SD-card: https://www.aliexpress.com/item/32990008126.html
+In this case you take the file src/hal/ttgov21new.h and add the lines given above (numbers given are for this board).
+
+Another approach would be this tiny board: https://www.aliexpress.com/item/32424558182.html (needs 5V).
+In this case you choose the correct file for your ESP32-board in the src/hal-directory and add the lines given above to the correct h-file. Please correct the numbers given in the example to the numbers used corresponding to your wiring.
+
+Some hints:
+These cheap devices often handle SD-cards up to 32GB, not bigger ones. They can handle files in the old DOS-way, to say the filenames are in the 8.3-format. And they often cannot handle subdirectories.
+
+The software included here writes data in a file named PAXCOUNT.xx, where xx can range from 00 to 99. The software starts with 00, checks to see if such a file already exists and if yes it will continue with the next number (up to 99 - in this case it will return no sd-card). So an existing file will not be overwritten.
+
+The data is written to the card and after 3 write-operations the data is flushed to the disk. So maybe the last 3 minutes of data get lost when you disconnect the PAXCOUNTER from power.
+
+And finally: this is the data written to the disk:
+
+    date, time, wifi, bluet
+    00.00.1970,00:01:09,2,0
+    00.00.1970,00:02:09,1,0
+    00.00.1970,00:03:09,2,0
+
+Format of the data is CSV, which can easily imported into LibreOffice, Excel, .....
+
+If you want to change this please look into src/sdcard.cpp and include/sdcard.h.
+
+
 # Payload format
 
 You can select different payload formats in [paxcounter.conf](src/paxcounter.conf#L12):
@@ -507,3 +545,4 @@ Thanks to
 - [terrillmoore](https://github.com/mcci-catena) for maintaining the LMIC for arduino LoRaWAN stack
 - [sbamueller](https://github.com/sbamueller) for writing the tutorial in Make Magazine
 - [Stefan](https://github.com/nerdyscout) for paxcounter opensensebox integration
+- [August Quint](https://github.com/AugustQu) for adding SD card data logger support

include/sdcard.h

@@ -0,0 +1,16 @@
+#ifndef _SDCARD_H
+#define _SDCARD_H
+
+#include <globals.h>
+#include <stdio.h>
+
+#include <SPI.h>
+#include <mySD.h>
+
+#define SDCARD_FILE_NAME       "paxcount.%02d"
+#define SDCARD_FILE_HEADER     "date, time, wifi, bluet"
+
+bool sdcardInit( void );
+void sdcardWriteData( uint16_t, uint16_t);
+
+#endif

include/senddata.h

@@ -12,6 +12,10 @@
 #include "display.h"
 #endif
 
+#ifdef HAS_SDCARD
+#include "sdcard.h"
+#endif
+
 extern Ticker sendcycler;
 
 void SendPayload(uint8_t port, sendprio_t prio);

platformio.ini

@@ -7,7 +7,7 @@
 
 ; ---> SELECT THE TARGET PLATFORM HERE! <---
 [board]
-halfile = generic.h
+;halfile = generic.h
 ;halfile = ebox.h
 ;halfile = eboxtube.h
 ;halfile = ecopower.h
@@ -19,7 +19,7 @@ halfile = generic.h
 ;halfile = ttgov21new.h
 ;halfile = ttgofox.h
 ;halfile = ttgobeam.h
-;halfile = ttgobeam10.h
+halfile = ttgobeam10.h
 ;halfile = fipy.h
 ;halfile = lopy.h
 ;halfile = lopy4.h
@@ -43,7 +43,7 @@ description = Paxcounter is a device for metering passenger flows in realtime. I
 
 [common]
 ; for release_version use max. 10 chars total, use any decimal format like "a.b.c"
-release_version = 1.9.84
+release_version = 1.9.85
 ; DEBUG LEVEL: For production run set to 0, otherwise device will leak RAM while running!
 ; 0=None, 1=Error, 2=Warn, 3=Info, 4=Debug, 5=Verbose
 debug_level = 3
@@ -56,28 +56,28 @@ monitor_speed = 115200
 upload_speed = 115200
 lib_deps_lora =
     ;MCCI LoRaWAN LMIC library@>=3.1.0 ; MCCI LMIC by Terrill Moore
-    https://github.com/mcci-catena/arduino-lmic.git#8a62487
+    https://github.com/mcci-catena/arduino-lmic.git
 lib_deps_display =
-    ;ss_oled@>=3.1.1 ; simple and small OLED lib by Larry Bank
+    ss_oled@>=3.2.0 ; simple and small OLED lib by Larry Bank
-    https://github.com/bitbank2/ss_oled.git#246d963
+    BitBang_I2C@>=1.3.0
-    BitBang_I2C@>=1.2.0
     QRCode@>=0.0.1
 lib_deps_matrix_display =
-    https://github.com/Seeed-Studio/Ultrathin_LED_Matrix.git
+    Ultrathin_LED_Matrix
 lib_deps_rgbled =
     SmartLeds@>=1.1.6
 lib_deps_gps =
     1655@>=1.0.2 ; #1655 TinyGPSPlus by Mikal Hart
 lib_deps_sensors =
-    Adafruit Unified Sensor@>=1.0.3
+    Adafruit Unified Sensor@>=1.1.1
-    Adafruit BME280 Library@>=1.0.10
+    Adafruit BME280 Library@>=2.0.0
     Adafruit BMP085 Library@>=1.0.1
 lib_deps_basic =
     ArduinoJson@^5.13.1
-    76@>=1.2.2 ; #76 Timezone by Jack Christensen
+    76@>=1.2.3 ; #76 Timezone by Jack Christensen
     274@>=2.3.3 ; #274 RTC by Michael Miller
     SimpleButton
     AXP202X_Library@>=1.1.0 ; AXP202 PMU lib by Lewis He
+    esp32-micro-sdcard
 lib_deps_all =
     ${common.lib_deps_basic}
     ${common.lib_deps_lora}

src/TTN/packed_decoder.js

@@ -4,6 +4,8 @@
 function Decoder(bytes, port) {
 
     var decoded = {};
+    decoded.wifi = 0;
+    decoded.ble = 0;
 
     if (bytes.length === 0) {
         return {};
@@ -147,7 +149,7 @@ var int8 = function (bytes) {
     if (bytes.length !== int8.BYTES) {
         throw new Error('int8 must have exactly 1 byte');
     }
-	var value = +(bytesToInt(bytes));
+    var value = +(bytesToInt(bytes));
     if (value > 127) {
         value -= 256;
     }
@@ -159,7 +161,7 @@ var int16 = function (bytes) {
     if (bytes.length !== int16.BYTES) {
         throw new Error('int16 must have exactly 2 bytes');
     }
-	var value = +(bytesToInt(bytes));
+    var value = +(bytesToInt(bytes));
     if (value > 32767) {
         value -= 65536;
     }
@@ -171,7 +173,7 @@ var int32 = function (bytes) {
     if (bytes.length !== int32.BYTES) {
         throw new Error('int32 must have exactly 4 bytes');
     }
-	var value = +(bytesToInt(bytes));
+    var value = +(bytesToInt(bytes));
     if (value > 2147483647) {
         value -= 4294967296;
     }

src/TTN/plain_decoder.js

@@ -3,6 +3,8 @@
 
 function Decoder(bytes, port) {
   var decoded = {};
+  decoded.wifi = 0;
+  decoded.ble = 0;
 
   if (port === 1) {
     var i = 0;

src/hal/generic.h

@@ -18,6 +18,14 @@
 #define SPI_SCLK GPIO_NUM_18
 #define SPI_CS   GPIO_NUM_5
 
+// enable only if you want to store a local paxcount table on the device
+#define HAS_SDCARD  1      // this board has an SD-card-reader/writer
+// Pins for SD-card
+#define SDCARD_CS    (13)
+#define SDCARD_MOSI  (15)
+#define SDCARD_MISO  (2)
+#define SDCARD_SCLK  (14)
+
 // enable only if device has these sensors, otherwise comment these lines
 // tutorial to connect BME sensor see here:
 // https://sbamueller.wordpress.com/2019/02/26/paxcounter-mit-umweltsensor/

src/hal/ttgov1.h

@@ -18,6 +18,14 @@
 #define LED_ACTIVE_LOW 1  // Onboard LED is active when pin is LOW
 #define HAS_BUTTON KEY_BUILTIN
 
+// enable only if you want to store a local paxcount table on the device
+#define HAS_SDCARD  1      // this board has an SD-card-reader/writer
+// Pins for SD-card
+#define SDCARD_CS    (13)
+#define SDCARD_MOSI  (15)
+#define SDCARD_MISO  (2)
+#define SDCARD_SCLK  (14)
+
 // Pins for I2C interface of OLED Display
 #define MY_OLED_SDA (4)
 #define MY_OLED_SCL (15)

src/hal/ttgov2.h

@@ -19,6 +19,14 @@
 // disable brownout detection (needed on TTGOv2 for battery powered operation)
 #define DISABLE_BROWNOUT 1 // comment out if you want to keep brownout feature
 
+// enable only if you want to store a local paxcount table on the device
+#define HAS_SDCARD  1      // this board has an SD-card-reader/writer
+// Pins for SD-card
+#define SDCARD_CS    (13)
+#define SDCARD_MOSI  (15)
+#define SDCARD_MISO  (2)
+#define SDCARD_SCLK  (14)
+
 // Pins for I2C interface of OLED Display
 #define MY_OLED_SDA (21)
 #define MY_OLED_SCL (22)

src/hal/ttgov21new.h

@@ -15,6 +15,14 @@
 #define HAS_LORA 1       // comment out if device shall not send data via LoRa
 #define CFG_sx1276_radio 1 // HPD13A LoRa SoC
 
+// enable only if you want to store a local paxcount table on the device
+#define HAS_SDCARD  1      // this board has an SD-card-reader/writer
+// Pins for SD-card
+#define SDCARD_CS    (13)
+#define SDCARD_MOSI  (15)
+#define SDCARD_MISO  (2)
+#define SDCARD_SCLK  (14)
+
 // enable only if device has these sensors, otherwise comment these lines
 // BME280 sensor on I2C bus
 //#define HAS_BME 1 // Enable BME sensors in general

src/hal/ttgov21old.h

@@ -19,6 +19,14 @@
 #define HAS_LED NOT_A_PIN // no usable LED on board
 #define DISABLE_BROWNOUT 1 // comment out if you want to keep brownout feature
 
+// enable only if you want to store a local paxcount table on the device
+#define HAS_SDCARD  1      // this board has an SD-card-reader/writer
+// Pins for SD-card
+#define SDCARD_CS    (13)
+#define SDCARD_MOSI  (15)
+#define SDCARD_MISO  (2)
+#define SDCARD_SCLK  (14)
+
 #define HAS_DISPLAY 1
 #define DISPLAY_FLIP  1 // rotated display
 //#define BAT_MEASURE_ADC ADC1_GPIO35_CHANNEL // battery probe GPIO pin -> ADC1_CHANNEL_7

src/irqhandler.cpp

@@ -8,8 +8,7 @@ void irqHandler(void *pvParameters) {
 
   configASSERT(((uint32_t)pvParameters) == 1); // FreeRTOS check
 
-  static uint32_t InterruptStatus = 0x00;
+  uint32_t InterruptStatus;
-  static bool mask_irq = false;
 
   // task remains in blocked state until it is notified by an irq
   for (;;) {
@@ -19,22 +18,16 @@ void irqHandler(void *pvParameters) {
                     portMAX_DELAY);   // wait forever
 
     if (InterruptStatus & UNMASK_IRQ) // interrupt handler to be enabled?
-      mask_irq = false;
+      InterruptStatus &= ~MASK_IRQ;   // then clear irq mask flag
-
+    // else suppress processing if interrupt handler is disabled
-      // suppress processing if interrupt handler is disabled
+    // or time critical lmic jobs are pending in next 100ms
-      // or time critical lmic jobs are pending in next 100ms
+    else if ((InterruptStatus & MASK_IRQ)
 #if (HAS_LORA)
-    else if (mask_irq || os_queryTimeCriticalJobs(ms2osticks(100)))
+             || os_queryTimeCriticalJobs(ms2osticks(100))
-#else
-    else if (mask_irq)
 #endif
+    )
       continue;
 
-    else if (InterruptStatus & MASK_IRQ) { // interrupt handler to be disabled?
-      mask_irq = true;
-      continue;
-    }
-
 // button pressed?
 #ifdef HAS_BUTTON
     if (InterruptStatus & BUTTON_IRQ) {
@@ -59,21 +52,6 @@ void irqHandler(void *pvParameters) {
     }
 #endif
 
-// BME sensor data to be read?
-#if (HAS_BME)
-    if (InterruptStatus & BME_IRQ) {
-      bme_storedata(&bme_status);
-      InterruptStatus &= ~BME_IRQ;
-    }
-
-#endif
-
-    // are cyclic tasks due?
-    if (InterruptStatus & CYCLIC_IRQ) {
-      doHousekeeping();
-      InterruptStatus &= ~CYCLIC_IRQ;
-    }
-
 #if (TIME_SYNC_INTERVAL)
     // is time to be synced?
     if (InterruptStatus & TIMESYNC_IRQ) {
@@ -83,6 +61,20 @@ void irqHandler(void *pvParameters) {
     }
 #endif
 
+// BME sensor data to be read?
+#if (HAS_BME)
+    if (InterruptStatus & BME_IRQ) {
+      bme_storedata(&bme_status);
+      InterruptStatus &= ~BME_IRQ;
+    }
+#endif
+
+    // are cyclic tasks due?
+    if (InterruptStatus & CYCLIC_IRQ) {
+      doHousekeeping();
+      InterruptStatus &= ~CYCLIC_IRQ;
+    }
+
 // do we have a power event?
 #if (HAS_PMU)
     if (InterruptStatus & PMU_IRQ) {
@@ -96,8 +88,8 @@ void irqHandler(void *pvParameters) {
       sendData();
       InterruptStatus &= ~SENDCYCLE_IRQ;
     }
-  }
+  } // for
-}
+} // irqHandler()
 
 // esp32 hardware timer triggered interrupt service routines
 // they notify the irq handler task

src/main.cpp

@@ -323,6 +323,11 @@ void setup() {
   assert(spi_init() == ESP_OK);
 #endif
 
+#ifdef HAS_SDCARD
+  if (sdcardInit())
+    strcat_P(features, " SD");
+#endif
+
 #if (VENDORFILTER)
   strcat_P(features, " FILTER");
 #endif

src/sdcard.cpp

@@ -0,0 +1,71 @@
+// routines for writing data to an SD-card, if present
+
+#if (HAS_SDCARD)
+
+// Local logging tag
+static const char TAG[] = __FILE__;
+
+#include "sdcard.h"
+
+static bool useSDCard;
+
+static void createFile(void);
+
+File fileSDCard;
+
+bool sdcardInit() {
+  ESP_LOGD(TAG, "looking for SD-card...");
+  useSDCard = SD.begin(SDCARD_CS, SDCARD_MOSI, SDCARD_MISO, SDCARD_SCLK);
+  if (useSDCard)
+    createFile();
+  return useSDCard;
+}
+
+void sdcardWriteData(uint16_t noWifi, uint16_t noBle) {
+  static int counterWrites = 0;
+  char tempBuffer[12 + 1];
+  time_t t = now();
+
+  if (!useSDCard)
+    return;
+
+  ESP_LOGD(TAG, "writing to SD-card");
+  sprintf(tempBuffer, "%02d.%02d.%4d,", day(t), month(t), year(t));
+  fileSDCard.print(tempBuffer);
+  sprintf(tempBuffer, "%02d:%02d:%02d,", hour(t), minute(t), second(t));
+  fileSDCard.print(tempBuffer);
+  sprintf(tempBuffer, "%d,%d", noWifi, noBle);
+  fileSDCard.println(tempBuffer);
+
+  if (++counterWrites > 2) {
+    // force writing to SD-card
+    ESP_LOGD(TAG, "flushing data to card");
+    fileSDCard.flush();
+    counterWrites = 0;
+  }
+}
+
+void createFile(void) {
+  char bufferFilename[8 + 1 + 3 + 1];
+
+  useSDCard = false;
+
+  for (int i = 0; i < 100; i++) {
+    sprintf(bufferFilename, SDCARD_FILE_NAME, i);
+    ESP_LOGD(TAG, "SD: looking for file <%s>", bufferFilename);
+    bool fileExists = SD.exists(bufferFilename);
+    if (!fileExists) {
+      ESP_LOGD(TAG, "SD: file does not exist: opening");
+      fileSDCard = SD.open(bufferFilename, FILE_WRITE);
+      if (fileSDCard) {
+        ESP_LOGD(TAG, "SD: name opended: <%s>", bufferFilename);
+        fileSDCard.println(SDCARD_FILE_HEADER);
+        useSDCard = true;
+        break;
+      }
+    }
+  }
+  return;
+}
+
+#endif // (HAS_SDCARD)

src/senddata.cpp

@@ -51,6 +51,11 @@ void SendPayload(uint8_t port, sendprio_t prio) {
   spi_enqueuedata(&SendBuffer);
 #endif
 
+// write data to sdcard, if present
+#ifdef HAS_SDCARD
+  sdcardWriteData(macs_wifi, macs_ble);
+#endif
+
 } // SendPayload
 
 // interrupt triggered function to prepare payload to send