timesync (experimental)

include/payload.h

@@ -20,7 +20,8 @@
 
 #endif
 
-// MyDevices CayenneLPP 2.0 types for Packed Sensor Payload, not using channels, but different FPorts
+// MyDevices CayenneLPP 2.0 types for Packed Sensor Payload, not using channels,
+// but different FPorts
 #define LPP_GPS 136          // 3 byte lon/lat 0.0001 °, 3 bytes alt 0.01m
 #define LPP_TEMPERATURE 103  // 2 bytes, 0.1°C signed MSB
 #define LPP_DIGITAL_INPUT 0  // 1 byte
@@ -40,11 +41,13 @@ public:
   void reset(void);
   uint8_t getSize(void);
   uint8_t *getBuffer(void);
+  void addByte(uint8_t value);
+  void addWord(uint16_t value);
   void addCount(uint16_t value, uint8_t sniffytpe);
   void addConfig(configData_t value);
   void addStatus(uint16_t voltage, uint64_t uptime, float cputemp, uint32_t mem,
                  uint8_t reset1, uint8_t reset2);
-  void add2Bytes(int8_t rssi, uint8_t message);
+  void addAlarm(int8_t rssi, uint8_t message);
   void addVoltage(uint16_t value);
   void addGPS(gpsStatus_t value);
   void addBME(bmeStatus_t value);

include/timesync.h

@@ -7,19 +7,27 @@
 
 #define TIME_SYNC_SAMPLES 3   // number of time requests for averaging
 #define TIME_SYNC_CYCLE 20    // seconds between two time requests
-#define TIME_SYNC_TIMEOUT 30  // timeout seconds waiting for timeserver answer
+#define TIME_SYNC_TIMEOUT 180 // timeout seconds waiting for timeserver answer
-#define TIME_SYNC_THRESHOLD 1 // time deviation threshold triggering time sync
+#define TIME_SYNC_THRESHOLD                                                    \
-#define TIME_SYNC_START_OPCODE 0x90 // force time sync on node
+  1.0f // time deviation threshold triggering time sync
-#define TIME_SYNC_REQ_OPCODE 0x92   // node requests time at server
+#define TIME_SYNC_START_OPCODE 0x90 // start time sync on node
-#define TIME_SYNC_ANS_OPCODE 0x93   // server answers time to node
+#define TIME_SYNC_STOP_OPCODE 0x91  // stop time sync on node
+#define TIME_SYNC_REQ_OPCODE 0x92   // node request at timeserver
+#define TIME_SYNC_ANS_OPCODE 0x93   // timeserver answer to node
 
-extern uint32_t time_sync_messages[], time_sync_answers[];
+typedef struct {
-extern uint8_t volatile time_sync_seqNo;
+  uint32_t seconds;
+  uint8_t fractions; // 1/250ths second = 4 milliseconds resolution
+} time_sync_message_t;
+
+extern time_sync_message_t time_sync_messages[], time_sync_answers[];
+extern uint8_t time_sync_seqNo;
 
 void send_Servertime_req(void);
 void recv_Servertime_ans(uint8_t val[]);
 void process_Servertime_sync_req(void *taskparameter);
 void process_Servertime_sync_ans(void *taskparameter);
 void force_Servertime_sync(uint8_t val[]);
+void store_time_sync_req(time_t secs, uint32_t micros);
 
 #endif

platformio.ini

@@ -6,7 +6,7 @@
 
 ; ---> SELECT TARGET PLATFORM HERE! <---
 [platformio]
-env_default = generic
+;env_default = generic
 ;env_default = ebox
 ;env_default = eboxtube
 ;env_default = heltec
@@ -15,7 +15,7 @@ env_default = generic
 ;env_default = ttgov2
 ;env_default = ttgov21old
 ;env_default = ttgov21new
-;env_default = ttgobeam
+env_default = ttgobeam
 ;env_default = ttgofox
 ;env_default = lopy
 ;env_default = lopy4
@@ -30,10 +30,10 @@ description = Paxcounter is a proof-of-concept ESP32 device for metering passeng
 
 [common]
 ; for release_version use max. 10 chars total, use any decimal format like "a.b.c"
-release_version = 1.7.361
+release_version = 1.7.37
 ; DEBUG LEVEL: For production run set to 0, otherwise device will leak RAM while running!
 ; 0=None, 1=Error, 2=Warn, 3=Info, 4=Debug, 5=Verbose
-debug_level = 3
+debug_level = 4
 ; UPLOAD MODE: select esptool to flash via USB/UART, select custom to upload to cloud for OTA
 upload_protocol = esptool
 ;upload_protocol = custom

src/irqhandler.cpp

@@ -35,8 +35,11 @@ void irqHandler(void *pvParameters) {
 
 #if (TIME_SYNC_INTERVAL)
     // is time to be synced?
-    if (InterruptStatus & TIMESYNC_IRQ)
+    if (InterruptStatus & TIMESYNC_IRQ) {
-      setTime(timeProvider());
+      time_t t = timeProvider();
+      if (timeIsValid(t))
+        setTime(t);
+    }
 #endif
 
     // is time to send the payload?

src/lorawan.cpp

@@ -175,6 +175,8 @@ void showLoraKeys(void) {
 
 void onEvent(ev_t ev) {
   char buff[24] = "";
+  uint32_t now_micros;
+
   switch (ev) {
 
   case EV_SCAN_TIMEOUT:
@@ -227,11 +229,10 @@ void onEvent(ev_t ev) {
 
 #if (TIME_SYNC_TIMESERVER)
     // if last packet sent was a timesync request was sent, store TX timestamp
-    if (LMIC.pendTxPort == TIMEPORT) {
+    if ((LMIC.pendTxPort == TIMEPORT) &&
-      time_sync_messages[time_sync_seqNo] = osticks2ms(LMIC.txend);
+        (LMIC.pendTxData[0] == TIME_SYNC_REQ_OPCODE))
-      ESP_LOGD(TAG, "Timeserver request #%d was sent at %d",
+      store_time_sync_req(now(now_micros), now_micros);
-               time_sync_seqNo, time_sync_messages[time_sync_seqNo]);
+      // maybe using more precise osticks2ms(LMIC.txend) here?
-    }
 #endif
 
     strcpy_P(buff, (LMIC.txrxFlags & TXRX_ACK) ? PSTR("RECEIVED_ACK")
@@ -247,7 +248,6 @@ void onEvent(ev_t ev) {
       if ((LMIC.txrxFlags & TXRX_PORT) &&
           (LMIC.frame[LMIC.dataBeg - 1] == RCMDPORT))
         rcommand(LMIC.frame + LMIC.dataBeg, LMIC.dataLen);
-
     }
     break;
 
@@ -462,8 +462,7 @@ void user_request_network_time_callback(void *pVoidUserUTCTime,
   }
 
   // Update userUTCTime, considering the difference between the GPS and UTC
-  // time, and the leap seconds
+  // time, and the leap seconds until year 2019
-  // !!! DANGER !!! This code will expire in next year with leap second
   *pUserUTCTime = lmicTimeReference.tNetwork + 315964800;
   // Current time, in ticks
   ostime_t ticksNow = os_getTime();

src/macsniff.cpp

@@ -106,7 +106,7 @@ bool mac_add(uint8_t *paddr, int8_t rssi, bool sniff_type) {
           blink_LED(COLOR_WHITE, 2000);
 #endif
           payload.reset();
-          payload.add2Bytes(rssi, beaconID);
+          payload.addAlarm(rssi, beaconID);
           SendPayload(BEACONPORT, prio_high);
         }
       };

src/paxcounter.conf

@@ -68,7 +68,7 @@
 // settings for syncing time of node with external time source
 #define TIME_SYNC_INTERVAL              60      // sync time attempt each .. minutes from time source (GPS/LORA/RTC) [default = 60], 0 means off
 #define TIME_SYNC_LORAWAN               0       // set to 1 to use LORA network as time source, 0 means off [default = 0]
-#define TIME_SYNC_TIMESERVER            0       // set to 1 to use LORA timeserver as time source, 0 means off [default = 0]
+#define TIME_SYNC_TIMESERVER            1       // set to 1 to use LORA timeserver as time source, 0 means off [default = 0]
 
 // time zone, see https://github.com/JChristensen/Timezone/blob/master/examples/WorldClock/WorldClock.ino
 #define DAYLIGHT_TIME                   {"CEST", Last, Sun, Mar, 2, 120}     // Central European Summer Time

src/payload.cpp

@@ -18,12 +18,19 @@ uint8_t *PayloadConvert::getBuffer(void) { return buffer; }
 
 #if PAYLOAD_ENCODER == 1
 
+void PayloadConvert::addByte(uint8_t value) { buffer[cursor++] = (value); }
+
+void PayloadConvert::addWord(uint16_t value) {
+  buffer[cursor++] = lowByte(value);
+  buffer[cursor++] = highByte(value);
+}
+
 void PayloadConvert::addCount(uint16_t value, uint8_t snifftype) {
   buffer[cursor++] = highByte(value);
   buffer[cursor++] = lowByte(value);
 }
 
-void PayloadConvert::add2Bytes(int8_t rssi, uint8_t msg) {
+void PayloadConvert::addAlarm(int8_t rssi, uint8_t msg) {
   buffer[cursor++] = rssi;
   buffer[cursor++] = msg;
 }
@@ -141,11 +148,15 @@ void PayloadConvert::addTime(time_t value) {
 
 #elif PAYLOAD_ENCODER == 2
 
+void PayloadConvert::addByte(uint8_t value) { writeUint8(value); }
+
+void PayloadConvert::addWord(uint16_t value) { writeUint16(value); }
+
 void PayloadConvert::addCount(uint16_t value, uint8_t snifftype) {
   writeUint16(value);
 }
 
-void PayloadConvert::add2Bytes(int8_t rssi, uint8_t msg) {
+void PayloadConvert::addAlarm(int8_t rssi, uint8_t msg) {
   writeUint8(rssi);
   writeUint8(msg);
 }
@@ -299,6 +310,16 @@ void PayloadConvert::writeBitmap(bool a, bool b, bool c, bool d, bool e, bool f,
 
 #elif (PAYLOAD_ENCODER == 3 || PAYLOAD_ENCODER == 4)
 
+void PayloadConvert::addByte(uint8_t value) { 
+  /* 
+  not implemented
+  */ }
+
+void PayloadConvert::addWord(uint16_t value) { 
+  /*
+  not implemented
+  */ }
+
 void PayloadConvert::addCount(uint16_t value, uint8_t snifftype) {
   switch (snifftype) {
   case MAC_SNIFF_WIFI:
@@ -322,7 +343,7 @@ void PayloadConvert::addCount(uint16_t value, uint8_t snifftype) {
   }
 }
 
-void PayloadConvert::add2Bytes(int8_t rssi, uint8_t msg) {
+void PayloadConvert::addAlarm(int8_t rssi, uint8_t msg) {
 #if (PAYLOAD_ENCODER == 3)
   buffer[cursor++] = LPP_ALARM_CHANNEL;
 #endif

src/rcommand.cpp

@@ -308,7 +308,7 @@ cmd_t table[] = {{0x01, set_rssi, 1, true},
                  {0x86, get_time, 0, false}
 #if(TIME_SYNC_TIMESERVER)
                  ,
-                 {TIME_SYNC_ANS_OPCODE, recv_Servertime_ans, 5, false},
+                 {TIME_SYNC_ANS_OPCODE, recv_Servertime_ans, 6, false},
                  {TIME_SYNC_START_OPCODE, force_Servertime_sync, 0, false}
 #endif
 };

src/timesync.cpp

@@ -18,10 +18,11 @@ patent holder.
 static const char TAG[] = __FILE__;
 
 TaskHandle_t timeSyncReqTask, timeSyncAnsTask;
-uint32_t time_sync_messages[TIME_SYNC_SAMPLES +
+time_sync_message_t
-                            1] = {0},
+    time_sync_messages[TIME_SYNC_SAMPLES + 1] = {0},
-                            time_sync_answers[TIME_SYNC_SAMPLES + 1] = {0};
+                                           time_sync_answers[TIME_SYNC_SAMPLES +
-uint8_t volatile time_sync_seqNo = 0; // used in lorawan.cpp to store timestamp
+                                                             1] = {0};
+uint8_t time_sync_seqNo = 0; // used in lorawan.cpp to store timestamp
 
 // send time request message
 void send_Servertime_req() {
@@ -35,9 +36,12 @@ void send_Servertime_req() {
 
     // clear timestamp array
     for (uint8_t i = 0; i <= TIME_SYNC_SAMPLES + 1; i++) {
-      time_sync_messages[i] = time_sync_answers[i] = 0;
+      time_sync_messages[i].seconds = time_sync_answers[i].seconds = 0;
+      time_sync_messages[i].fractions = time_sync_answers[i].fractions = 0;
     }
 
+    time_sync_seqNo = 0;
+
     // create temporary task for processing sync answers if not already active
     if (!timeSyncAnsTask)
       xTaskCreatePinnedToCore(process_Servertime_sync_ans, // task function
@@ -68,14 +72,16 @@ void recv_Servertime_ans(uint8_t val[]) {
     return;
 
   uint8_t seq_no = val[0];
-  uint32_t timestamp = 0;
+  uint32_t timestamp_sec = 0, timestamp_ms = 0;
 
-  for (int i = 1; i <= 4; i++)
+  for (int i = 1; i <= 4; i++) {
-    timestamp = (timestamp << 8) | val[i];
+    timestamp_sec = (timestamp_sec << 8) | val[i];
-  time_sync_answers[seq_no] = timestamp;
+    time_sync_answers[seq_no].seconds = timestamp_sec;
+  }
+  time_sync_answers[seq_no].fractions = val[5];
 
-  ESP_LOGD(TAG, "Timeserver timestamp #%d received: time=%d", seq_no,
+  ESP_LOGD(TAG, "Timeserver timestamp #%d received: time=%d.%d", seq_no,
-           timestamp);
+           timestamp_sec, timestamp_ms);
 
   // inform processing task
   if (timeSyncAnsTask)
@@ -88,59 +94,95 @@ void process_Servertime_sync_req(void *taskparameter) {
   for (uint8_t i = 1; i <= TIME_SYNC_SAMPLES; i++) {
     time_sync_seqNo++;
     payload.reset();
-    payload.add2Bytes(TIME_SYNC_REQ_OPCODE, i);
+    payload.addWord(TIME_SYNC_REQ_OPCODE | time_sync_seqNo << 8);
     SendPayload(TIMEPORT, prio_high);
-    ESP_LOGD(TAG, "Timeserver request #%d sent", i);
+    ESP_LOGD(TAG, "Timeserver request #%d sent", time_sync_seqNo);
     // Wait for the next cycle
     vTaskDelay(pdMS_TO_TICKS(TIME_SYNC_CYCLE * 1000));
   }
+  payload.reset();
+  payload.addByte(TIME_SYNC_STOP_OPCODE);
+  SendPayload(TIMEPORT, prio_high); // necessary to receive last timestamp
   timeSyncReqTask = NULL;
   vTaskDelete(NULL); // end task
 }
 
 // task for processing a timesync handshake
 void process_Servertime_sync_ans(void *taskparameter) {
-  uint32_t seq_no = 0;
+
-  uint32_t NetworkTime = 0;
+  uint32_t seq_no = 0, time_diff_sec = 0, time_diff_ms = 0;
-  int32_t time_diff = 0;
+  time_t time_to_set = 0;
-  uint8_t ans_counter = TIME_SYNC_SAMPLES;
+  float time_offset = 0.0f;
 
   // collect incoming timestamp samples notified by rcommand
   for (uint8_t i = 1; i <= TIME_SYNC_SAMPLES; i++) {
-    if (xTaskNotifyWait(0x00, ULONG_MAX, &seq_no,
+    if ((xTaskNotifyWait(0x00, ULONG_MAX, &seq_no,
                          (TIME_SYNC_CYCLE + TIME_SYNC_TIMEOUT) * 1000 /
-                            portTICK_PERIOD_MS) == pdFALSE)
+                             portTICK_PERIOD_MS) == pdFALSE) ||
-      continue; // no answer received before timeout
+        (seq_no != i)) {
+      ESP_LOGW(TAG, "Timesync handshake timeout");
+      goto finish; // no valid sequence received before timeout
+    } else         // calculate time diff from set of collected timestamps
+    {
+      time_diff_sec += time_sync_messages[seq_no].seconds -
+                       time_sync_answers[seq_no].seconds;
+      time_diff_ms += 4 * (time_sync_messages[seq_no].fractions -
+                           time_sync_answers[seq_no].fractions);
+    }
+  } // for
 
-    time_diff += time_sync_messages[seq_no] - time_sync_answers[seq_no];
+  time_offset = (time_diff_sec + time_diff_ms / 1000.0) / TIME_SYNC_SAMPLES;
-    ans_counter--;
+
+  ESP_LOGD(TAG, "Timesync finished, time offset=%.4f seconds", time_offset);
+
+  // check time diff and if necessary set time
+  if (time_offset >= TIME_SYNC_THRESHOLD) {
+
+    // wait until top of second
+    if (time_diff_ms > 0) {
+      vTaskDelay(1000 - time_diff_ms); // clock is fast
+      time_diff_sec--;
+    } else if (time_diff_ms < 0) { // clock is stale
+      vTaskDelay(1000 + time_diff_ms);
+      time_diff_sec++;
     }
 
-  if (ans_counter) {
+    time_to_set = time_t(now() - time_diff_sec);
-    ESP_LOGW(TAG, "Timesync handshake timeout");
+
-  } else {
+    ESP_LOGD(TAG, "Time to set = %d", time_to_set);
-    // calculate time diff from set of collected timestamps
+
-    if (time_diff / TIME_SYNC_SAMPLES) {
-      NetworkTime = now() + time_diff;
-      ESP_LOGI(TAG, "Timesync finished, time offset=%d seconds", time_diff);
     // Update system time with time read from the network
-      if (timeIsValid(NetworkTime)) {
+    if (timeIsValid(time_to_set)) {
-        setTime(NetworkTime);
+      setTime(time_to_set);
+      SyncToPPS();
       timeSource = _lora;
       timesyncer.attach(TIME_SYNC_INTERVAL * 60,
                         timeSync); // set to regular repeat
-        ESP_LOGI(TAG, "Recent time received from timeserver");
+      ESP_LOGI(TAG, "Timesync finished, time was adjusted");
     } else
       ESP_LOGW(TAG, "Invalid time received from timeserver");
   } else
     ESP_LOGI(TAG, "Timesync finished, time is up to date");
-  } // if (ans_counter)
+
+finish:
 
   time_sync_seqNo = 0;
   timeSyncAnsTask = NULL;
   vTaskDelete(NULL); // end task
 }
 
+// called from lorawan.cpp when tine_sync_req was sent
+void store_time_sync_req(time_t secs, uint32_t micros) {
+
+  time_sync_messages[time_sync_seqNo].seconds = secs;
+  time_sync_messages[time_sync_seqNo].fractions =
+      micros / 250; // 4ms resolution
+
+  ESP_LOGD(TAG, "Timeserver request #%d was sent at %d.%d", time_sync_seqNo,
+           time_sync_messages[time_sync_seqNo].seconds,
+           time_sync_messages[time_sync_seqNo].fractions);
+}
+
 void force_Servertime_sync(uint8_t val[]) {
   ESP_LOGI(TAG, "Timesync requested by timeserver");
   timeSync();