Merge pull request #564 from cyberman54/development

v1.9.93

include/globals.h

@@ -127,7 +127,6 @@ extern SemaphoreHandle_t I2Caccess;
 extern TaskHandle_t irqHandlerTask, ClockTask;
 extern TimerHandle_t WifiChanTimer;
 extern Timezone myTZ;
-extern time_t userUTCTime;
 extern RTC_DATA_ATTR runmode_t RTC_runmode;
 
 // application includes

include/lorawan.h

@@ -5,9 +5,6 @@
 #include "rcommand.h"
 #include "timekeeper.h"
 #include <driver/rtc_io.h>
-#if (TIME_SYNC_LORASERVER)
-#include "timesync.h"
-#endif
 
 // LMIC-Arduino LoRaWAN Stack
 #include <lmic.h>
@@ -54,9 +51,4 @@ const char *getSfName(rps_t rps);
 const char *getBwName(rps_t rps);
 const char *getCrName(rps_t rps);
 
-#if (TIME_SYNC_LORAWAN)
-void user_request_network_time_callback(void *pVoidUserUTCTime,
-                                               int flagSuccess);
-#endif
-
 #endif

include/timekeeper.h

@@ -5,6 +5,7 @@
 #include "rtctime.h"
 #include "TimeLib.h"
 #include "irqhandler.h"
+#include "timesync.h"
 
 #if (HAS_GPS)
 #include "gpsread.h"

include/timesync.h

@@ -1,30 +1,31 @@
 #ifndef _TIMESYNC_H
 #define _TIMESYNC_H
 
-#include <chrono>
 #include "globals.h"
 #include "irqhandler.h"
 #include "timekeeper.h"
 
-//#define TIME_SYNC_TRIGGER 100 // threshold for time sync [milliseconds]
 #define TIME_SYNC_FRAME_LENGTH 0x07 // timeserver answer frame length [bytes]
-#define TIME_SYNC_FIXUP 16 // empirical calibration to fixup processing time [milliseconds]
+#define TIME_SYNC_FIXUP 16 // compensation for processing time [milliseconds]
 #define TIMEREQUEST_MAX_SEQNO 0xfe // threshold for wrap around seqno
 #define TIMEREQUEST_FINISH                                                     \
   (TIMEREQUEST_MAX_SEQNO + 1) // marker for end of timesync handshake
+#define GPS_UTC_DIFF 315964800
 
 enum timesync_t {
   timesync_tx,
   timesync_rx,
   gwtime_sec,
   gwtime_msec,
+  gwtime_tzsec,
   no_of_timestamps
 };
 
 void timesync_init(void);
 void send_timesync_req(void);
 int recv_timesync_ans(const uint8_t buf[], uint8_t buf_len);
-void process_timesync_req(void *taskparameter);
 void store_timestamp(uint32_t timestamp, timesync_t timestamp_type);
+void IRAM_ATTR process_timesync_req(void *taskparameter);
+void IRAM_ATTR process_timesync_req(void *pVoidUserUTCTime, int flagSuccess);
 
 #endif

platformio.ini

@@ -45,7 +45,7 @@ description = Paxcounter is a device for metering passenger flows in realtime. I
 
 [common]
 ; for release_version use max. 10 chars total, use any decimal format like "a.b.c"
-release_version = 1.9.92
+release_version = 1.9.93
 ; DEBUG LEVEL: For production run set to 0, otherwise device will leak RAM while running!
 ; 0=None, 1=Error, 2=Warn, 3=Info, 4=Debug, 5=Verbose
 debug_level = 3

src/Timeserver/timeserver.java

@@ -0,0 +1,88 @@
+/* LoRaWAN Timeserver
+
+construct 7 byte timesync_answer from gateway timestamp and node's time_sync_req
+
+byte    meaning
+1       sequence number (taken from node's time_sync_req)
+2       timezone in 15 minutes steps
+3..6    current second (from epoch time 1970)
+7       1/250ths fractions of current second
+
+*/
+
+function timecompare(a, b) {
+  
+  const timeA = a.time;
+  const timeB = b.time;
+
+  let comparison = 0;
+  if (timeA > timeB) {
+    comparison = 1;
+  } else if (timeA < timeB) {
+    comparison = -1;
+  }
+  return comparison;
+}
+
+let confidence = 2000; // max millisecond diff gateway time to server time
+
+// guess if we have received a valid time_sync_req command
+if (msg.payload.payload_raw.length != 1)
+  return;
+
+var deviceMsg = { payload: msg.payload.dev_id };
+var seqNo = msg.payload.payload_raw[0];
+var seqNoMsg = { payload: seqNo };
+var gateway_list = msg.payload.metadata.gateways;
+
+// filter all gateway timestamps that have milliseconds part (which we assume have a ".")
+var gateways = gateway_list.filter(function (element) {
+  return (element.time.includes("."));
+});
+
+var gateway_time = gateways.map(gw => {
+    return {
+      time: new Date(gw.time),
+      eui: gw.gtw_id,
+      }
+  });
+var server_time = new Date(msg.payload.metadata.time);
+
+// validate all gateway timestamps against lorawan server_time (which is assumed to be recent)
+var gw_timestamps = gateway_time.filter(function (element) {
+  return ((element.time > (server_time - confidence) && element.time <= server_time));
+});
+
+// if no timestamp left, we have no valid one and exit
+if (gw_timestamps.length === 0) {
+    var notavailMsg = { payload: "n/a" };
+    var notimeMsg = { payload: 0xff };    
+    var buf2 = Buffer.alloc(1);
+    msg.payload = new Buffer(buf2.fill(0xff));
+    msg.port = 9; // Paxcounter TIMEPORT
+    return [notavailMsg, notavailMsg, deviceMsg, seqNoMsg, msg];}
+
+// sort time array in ascending order to find most recent timestamp for time answer
+gw_timestamps.sort(timecompare);
+
+var timestamp = gw_timestamps[0].time;
+var eui = gw_timestamps[0].eui;
+var offset = server_time - timestamp;
+
+var seconds = Math.floor(timestamp/1000);
+var fractions = (timestamp % 1000) / 4;
+
+let buf = new ArrayBuffer(7);
+new DataView(buf).setUint8(0, seqNo);
+// Timezone (in 15min steps)
+var timezone = 8; // CET = UTC+2h
+new DataView(buf).setUint8(1, timezone);
+new DataView(buf).setUint32(2, seconds);
+new DataView(buf).setUint8(6, fractions);
+
+msg.payload = new Buffer(new Uint8Array(buf));
+msg.port = 9; // Paxcounter TIMEPORT
+var euiMsg = { payload: eui };
+var offsetMsg = { payload: offset };
+
+return [euiMsg, offsetMsg, deviceMsg, seqNoMsg, msg];

src/lmic_config.h

@@ -21,7 +21,7 @@
 #define LMIC_USE_INTERRUPTS 1
 
 // time sync via LoRaWAN network, note: not supported by TTNv2
-//#define LMIC_ENABLE_DeviceTimeReq 1
+#define LMIC_ENABLE_DeviceTimeReq 1
 
 // use callback event handlers, not onEvent() reference
 #define LMIC_ENABLE_onEvent 0

src/lorawan.cpp

@@ -381,56 +381,6 @@ void lora_enqueuedata(MessageBuffer_t *message) {
 
 void lora_queuereset(void) { xQueueReset(LoraSendQueue); }
 
-#if (TIME_SYNC_LORAWAN)
-void IRAM_ATTR user_request_network_time_callback(void *pVoidUserUTCTime,
-                                                  int flagSuccess) {
-  // Explicit conversion from void* to uint32_t* to avoid compiler errors
-  time_t *pUserUTCTime = (time_t *)pVoidUserUTCTime;
-
-  // A struct that will be populated by LMIC_getNetworkTimeReference.
-  // It contains the following fields:
-  //  - tLocal: the value returned by os_GetTime() when the time
-  //            request was sent to the gateway, and
-  //  - tNetwork: the seconds between the GPS epoch and the time
-  //              the gateway received the time request
-  lmic_time_reference_t lmicTimeReference;
-
-  if (flagSuccess != 1) {
-    ESP_LOGW(TAG, "LoRaWAN network did not answer time request");
-    return;
-  }
-
-  // Populate lmic_time_reference
-  flagSuccess = LMIC_getNetworkTimeReference(&lmicTimeReference);
-  if (flagSuccess != 1) {
-    ESP_LOGW(TAG, "LoRaWAN time request failed");
-    return;
-  }
-
-  // mask application irq to ensure accurate timing
-  mask_user_IRQ();
-
-  // Update userUTCTime, considering the difference between the GPS and UTC
-  // time, and the leap seconds until year 2019
-  *pUserUTCTime = lmicTimeReference.tNetwork + 315964800;
-  // Current time, in ticks
-  ostime_t ticksNow = os_getTime();
-  // Time when the request was sent, in ticks
-  ostime_t ticksRequestSent = lmicTimeReference.tLocal;
-  // Add the delay between the instant the time was transmitted and
-  // the current time
-  time_t requestDelaySec = osticks2ms(ticksNow - ticksRequestSent) / 1000;
-
-  // Update system time with time read from the network
-  setMyTime(*pUserUTCTime + requestDelaySec, 0, _lora);
-
-finish:
-  // end of time critical section: release app irq lock
-  unmask_user_IRQ();
-
-} // user_request_network_time_callback
-#endif // TIME_SYNC_LORAWAN
-
 // LMIC lorawan stack task
 void lmictask(void *pvParameters) {
   configASSERT(((uint32_t)pvParameters) == 1);

src/main.cpp

@@ -87,7 +87,6 @@ hw_timer_t *ppsIRQ = NULL, *displayIRQ = NULL, *matrixDisplayIRQ = NULL;
 TaskHandle_t irqHandlerTask = NULL, ClockTask = NULL;
 SemaphoreHandle_t I2Caccess;
 bool volatile TimePulseTick = false;
-time_t userUTCTime = 0;
 timesource_t timeSource = _unsynced;
 
 // container holding unique MAC address hashes with Memory Alloctor using PSRAM,

src/paxcounter.conf

@@ -72,15 +72,15 @@
 #define OTA_MIN_BATT                    3600    // minimum battery level for OTA [millivolt]
 #define RESPONSE_TIMEOUT_MS             60000   // firmware binary server connection timeout [milliseconds]
 
-// settings for syncing time of node with external time source
+// settings for syncing time of node with a time source (network / gps / rtc / timeserver)
+#define TIME_SYNC_LORAWAN               1       // set to 1 to use LORA network as time source, 0 means off [default = 1]
 #define TIME_SYNC_INTERVAL              60      // sync time attempt each .. minutes from time source (GPS/LORA/RTC) [default = 60], 0 means off
 #define TIME_SYNC_INTERVAL_RETRY        10      // retry time sync after lost sync each .. minutes [default = 10], 0 means off
 #define TIME_SYNC_COMPILEDATE           0       // set to 1 to use compile date to initialize RTC after power outage [default = 0]
-#define TIME_SYNC_LORAWAN               0       // set to 1 to use LORA network as time source, 0 means off [default = 0]
-#define TIME_SYNC_LORASERVER            0       // set to 1 to use LORA timeserver as time source, 0 means off [default = 0]
 
-// settings for syncing time with timeserver applications
+// specific settings for syncing time of node with a timeserver
-#define TIME_SYNC_SAMPLES               1       // number of time requests for averaging
+#define TIME_SYNC_LORASERVER            0       // set to 1 to use LORA timeserver as time source, 0 means off [default = 0]
+#define TIME_SYNC_SAMPLES               1       // number of time requests for averaging, max. 255
 #define TIME_SYNC_CYCLE                 60      // delay between two time samples [seconds]
 #define TIME_SYNC_TIMEOUT               300     // timeout waiting for timeserver answer [seconds]
 

src/timekeeper.cpp

@@ -37,12 +37,9 @@ void calibrateTime(void) {
   }
 #endif
 
-// kick off asychronous Lora timeserver timesync if we have
+// kick off asychronous lora timesync if we have
-#if (HAS_LORA) && (TIME_SYNC_LORASERVER)
+#if (HAS_LORA) && (TIME_SYNC_LORASERVER) || (TIME_SYNC_LORAWAN)
   send_timesync_req();
-// kick off asychronous lora network sync if we have
-#elif (HAS_LORA) && (TIME_SYNC_LORAWAN)
-  LMIC_requestNetworkTime(user_request_network_time_callback, &userUTCTime);
 #endif
 
 // no time from GPS -> fallback to RTC time while trying lora sync
@@ -233,7 +230,7 @@ void clock_init(void) {
   pinMode(HAS_DCF77, OUTPUT);
 #endif
 
-  userUTCTime = now();
+  time_t userUTCTime = now();
 
   xTaskCreatePinnedToCore(clock_loop,           // task function
                           "clockloop",          // name of task

src/timesync.cpp

@@ -1,31 +1,37 @@
 /*
 
-///--> IMPORTANT LICENSE NOTE for this file <--///
+///--> IMPORTANT LICENSE NOTE for timesync option 1 in this file <--///
 
 PLEASE NOTE: There is a patent filed for the time sync algorithm used in the
 code of this file. The shown implementation example is covered by the
 repository's licencse, but you may not be eligible to deploy the applied
 algorithm in applications without granted license by the patent holder.
 
+You may use timesync option 2 if you do not want or cannot accept this.
+
 */
 
-#if (TIME_SYNC_LORASERVER) && (HAS_LORA)
-
 #include "timesync.h"
 
+#if (TIME_SYNC_LORASERVER) && (TIME_SYNC_LORAWAN) && (HAS_LORA)
+#error Duplicate timesync method selected. You must select either LORASERVER or LORAWAN timesync.
+#endif
+
 // Local logging tag
 static const char TAG[] = __FILE__;
 
-TaskHandle_t timeSyncReqTask = NULL;
+// timesync option 1: use external timeserver (for LoRAWAN < 1.0.3)
 
+#if (TIME_SYNC_LORASERVER) && (HAS_LORA)
+
+static TaskHandle_t timeSyncReqTask = NULL;
+static bool timeSyncPending = false;
 static uint8_t time_sync_seqNo = (uint8_t)random(TIMEREQUEST_MAX_SEQNO);
 static uint8_t sample_idx = 0;
-static bool timeSyncPending = false;
 static uint32_t timesync_timestamp[TIME_SYNC_SAMPLES][no_of_timestamps] = {0};
 
 // send time request message
-void send_timesync_req() {
+void send_timesync_req(void) {
-
   // if a timesync handshake is pending then exit
   if (timeSyncPending)
     return;
@@ -37,7 +43,7 @@ void send_timesync_req() {
 }
 
 // task for sending time sync requests
-void process_timesync_req(void *taskparameter) {
+void IRAM_ATTR process_timesync_req(void *taskparameter) {
 
   uint32_t rcv_seq_no = TIMEREQUEST_FINISH, time_offset_ms;
 
@@ -78,8 +84,6 @@ void process_timesync_req(void *taskparameter) {
         }
       }
 
-      ESP_LOGD(TAG, "sample_idx = %d", sample_idx);
-
       // calculate time diff from collected timestamps
       time_offset_ms += timesync_timestamp[sample_idx][timesync_rx] -
                         timesync_timestamp[sample_idx][timesync_tx];
@@ -137,9 +141,8 @@ void process_timesync_req(void *taskparameter) {
 // called from lorawan.cpp
 void store_timestamp(uint32_t timestamp, timesync_t timestamp_type) {
 
-  ESP_LOGD(TAG, "[%0.3f] seq#%d[%d]: timestamp(t%d)=%d",
+  ESP_LOGD(TAG, "[%0.3f] seq#%d[%d]: timestamp(t%d)=%d", millis() / 1000.0,
-           millis() / 1000.0, time_sync_seqNo, sample_idx, timestamp_type,
+           time_sync_seqNo, sample_idx, timestamp_type, timestamp);
-           timestamp);
 
   timesync_timestamp[sample_idx][timestamp_type] = timestamp;
 }
@@ -178,22 +181,24 @@ int recv_timesync_ans(const uint8_t buf[], const uint8_t buf_len) {
 
   else { // we received a probably valid time frame
 
-    // pointers to 4 bytes containing UTC seconds since unix epoch, msb
+    // pointers to 4 bytes msb order
     uint32_t timestamp_sec, *timestamp_ptr;
 
-    // extract 1 byte timezone from payload (one step being 15min * 60s = 900s)
+    // extract 1 byte containing timezone offset
-    // uint32_t timezone_sec = buf[0] * 900; // for future use
+    // one step being 15min * 60sec = 900sec
+    uint32_t timestamp_tzsec = buf[0] * 900; // timezone offset in secs
     buf++;
 
-    // extract 4 bytes timestamp from payload
+    // extract 4 bytes containing gateway time in UTC seconds since unix
-    // and convert it to uint32_t, octet order is big endian
+    // epoch and convert it to uint32_t, octet order is big endian
     timestamp_ptr = (uint32_t *)buf;
-    // swap byte order from msb to lsb, note: this is platform dependent
+    // swap byte order from msb to lsb, note: this is a platform dependent hack
     timestamp_sec = __builtin_bswap32(*timestamp_ptr);
     buf += 4;
-    // extract 1 byte fractional seconds in 2^-8 second steps
+
-    // (= 1/250th sec), we convert this to ms
+    // extract 1 byte containing fractional seconds in 2^-8 second steps
-    uint16_t timestamp_msec = 4 * buf[0];
+    // one step being 1/250th sec * 1000 = 4msec
+    uint16_t timestamp_msec = buf[0] * 4;
     // calculate absolute time received from gateway
     time_t t = timestamp_sec + timestamp_msec / 1000;
 
@@ -205,6 +210,7 @@ int recv_timesync_ans(const uint8_t buf[], const uint8_t buf_len) {
       // store time received from gateway
       store_timestamp(timestamp_sec, gwtime_sec);
       store_timestamp(timestamp_msec, gwtime_msec);
+      store_timestamp(timestamp_tzsec, gwtime_tzsec);
 
       // inform processing task
       xTaskNotify(timeSyncReqTask, seq_no, eSetBits);
@@ -230,3 +236,56 @@ void timesync_init() {
 }
 
 #endif
+
+// timesync option 2: use LoRAWAN network time (requires LoRAWAN >= 1.0.3)
+
+#if (TIME_SYNC_LORAWAN) && (HAS_LORA)
+
+static time_t networkUTCTime;
+
+// send time request message
+void send_timesync_req(void) {
+  LMIC_requestNetworkTime(process_timesync_req, &networkUTCTime);
+}
+
+void IRAM_ATTR process_timesync_req(void *pVoidUserUTCTime, int flagSuccess) {
+  // Explicit conversion from void* to uint32_t* to avoid compiler errors
+  time_t *pUserUTCTime = (time_t *)pVoidUserUTCTime;
+
+  // A struct that will be populated by LMIC_getNetworkTimeReference.
+  // It contains the following fields:
+  //  - tLocal: the value returned by os_GetTime() when the time
+  //            request was sent to the gateway, and
+  //  - tNetwork: the seconds between the GPS epoch and the time
+  //              the gateway received the time request
+  lmic_time_reference_t lmicTimeReference;
+
+  if (flagSuccess != 1) {
+    ESP_LOGW(TAG, "LoRaWAN network did not answer time request");
+    return;
+  }
+
+  // Populate lmic_time_reference
+  flagSuccess = LMIC_getNetworkTimeReference(&lmicTimeReference);
+  if (flagSuccess != 1) {
+    ESP_LOGW(TAG, "LoRaWAN time request failed");
+    return;
+  }
+
+  // mask application irq to ensure accurate timing
+  mask_user_IRQ();
+
+  // Update networkUTCTime, considering the difference between GPS and UTC time
+  *pUserUTCTime = lmicTimeReference.tNetwork + GPS_UTC_DIFF;
+  // Add delay between the instant the time was transmitted and the current time
+  uint16_t requestDelaymSec =
+      osticks2ms(os_getTime() - lmicTimeReference.tLocal);
+
+  // Update system time with time read from the network
+  setMyTime(*pUserUTCTime, requestDelaymSec, _lora);
+
+  // end of time critical section: release app irq lock
+  unmask_user_IRQ();
+
+} // user_request_network_time_callback
+#endif // TIME_SYNC_LORAWAN