rework SD card logging

include/sdcard.h

@@ -6,11 +6,15 @@
 #include "globals.h"
 #include <stdio.h>
 #include <SPI.h>
+#include "esp_vfs_fat.h"
+#include "sdmmc_cmd.h"
+
+#define MOUNT_POINT "/sdcard"
 
 #if HAS_SDCARD == 1
-#include <SD.h>
+#include "driver/sdspi_host.h"
 #elif HAS_SDCARD == 2
-#include <SD_MMC.h>
+#include "driver/sdmmc_host.h"
 #else
 #error HAS_SDCARD unknown card reader value, must be either 1 or 2
 #endif
@@ -35,25 +39,6 @@
 #define SDCARD_SCLK SCK
 #endif
 
-// Default config for SDMMC_HOST_DEFAULT (4-bit bus width, slot 1)
-// https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/peripherals/sdmmc_host.html
-
-#ifndef SDCARD_DATA0
-#define SDCARD_DATA0 2
-#endif
-
-#ifndef SDCARD_DATA1
-#define SDCARD_DATA1 4
-#endif
-
-#ifndef SDCARD_DATA2
-#define SDCARD_DATA2 12
-#endif
-
-#ifndef SDCARD_DATA3
-#define SDCARD_DATA3 13
-#endif
-
 #define SDCARD_FILE_NAME clientId
 #define SDCARD_FILE_HEADER "timestamp,wifi,ble"
 
@@ -62,6 +47,7 @@
 #endif
 
 bool sdcard_init(bool create = true);
+void sdcard_flush(void);
 void sdcard_close(void);
 void sdcardWriteData(uint16_t, uint16_t, uint16_t = 0);
 

src/cyclic.cpp

@@ -126,6 +126,10 @@ void doHousekeeping() {
   }
 #endif
 
+#if (HAS_SDCARD)
+  sdcard_flush();
+#endif
+
 } // doHousekeeping()
 
 uint32_t getFreeRAM() {

src/sdcard.cpp

@@ -1,64 +1,249 @@
+#ifdef HAS_SDCARD
+
 // routines for writing data to an SD-card, if present
 // use FAT32 formatted card
 // check whether your card reader supports SPI oder SDMMC and select appropriate
-// SD low level driver in board hal file
+// SD interface in board hal file
 
 // Local logging tag
 static const char TAG[] = __FILE__;
 
 #include "sdcard.h"
 
-#ifdef HAS_SDCARD
+sdmmc_card_t *card;
 
-static bool useSDCard;
+const char mount_point[] = MOUNT_POINT;
-static void openFile(void);
+static bool useSDCard = false;
 
-File fileSDCard;
+// This file stream will be used for payload logging
+static FILE *data_file;
+// This file stream will be used for system logging
 
-#if HAS_SDCARD == 1
+#ifdef SD_LOGGING
-SPIClass sd_spi;
+static FILE *log_file;
+
+// Save UART stdout stream
+static FILE *uart_stdout = stdout;
+
+// This function will be called by the ESP log library every time ESP_LOG needs
+// to be performed.
+//      @important Do NOT use the ESP_LOG* macro's in this function ELSE
+//      recursive loop and stack overflow! So use printf() instead for debug
+//      messages.
+// CURRENTLY NOT WORKING DUE TO AN ISSUE IN ARDUINO-ESP32
+int print_to_sd_card(const char *fmt, va_list args) {
+  static bool static_fatal_error = false;
+  static const uint32_t WRITE_CACHE_CYCLE = 5;
+  static uint32_t counter_write = 0;
+  int iresult;
+
+  // #1 Write to file
+  if (log_file == NULL) {
+    printf("%s() ABORT. file handle log_file is NULL\n", __FUNCTION__);
+    return -1;
+  }
+  if (static_fatal_error == false) {
+    iresult = vfprintf(log_file, fmt, args);
+    if (iresult < 0) {
+      printf("%s() ABORT. failed vfprintf() -> logging disabled \n",
+             __FUNCTION__);
+      // MARK FATAL
+      static_fatal_error = true;
+      return iresult;
+    }
+
+    // #2 Smart commit after x writes
+    counter_write++;
+    if (counter_write % WRITE_CACHE_CYCLE == 0) {
+      printf("%s() fsync'ing log file (WRITE_CACHE_CYCLE=%u)\n",
+             WRITE_CACHE_CYCLE);
+      fsync(fileno(log_file));
+    }
+  }
+
+  // #3 ALWAYS Write to stdout!
+  return vprintf(fmt, args);
+}
 #endif
 
+bool openFile(FILE **fd, const char *filename) {
+
+  char _filename[50];
+  sprintf(_filename, "%s%s", MOUNT_POINT, filename);
+
+  if ((*fd = fopen(_filename, "a")) == NULL) {
+    ESP_LOGE(TAG, "file <%s> open error", _filename);
+    return false;
+  } else {
+    ESP_LOGI(TAG, "file <%s> opened", _filename);
+    return true;
+  }
+} // openfile
+
 bool sdcard_init(bool create) {
 
   // for usage of SD drivers on ESP32 platform see
   // https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/peripherals/sdspi_host.html
   // https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/peripherals/sdmmc_host.html
 
+  // Options for mounting the filesystem.
+  // If format_if_mount_failed is set to true, SD card will be partitioned and
+  // formatted in case when mounting fails.
+  esp_vfs_fat_mount_config_t mount_config = {.format_if_mount_failed = false,
+                                             .max_files = 5};
+
   ESP_LOGI(TAG, "looking for SD-card...");
-#if HAS_SDCARD == 1 // use SD SPI host driver
+
-  digitalWrite(SDCARD_CS, HIGH);
+#if (HAS_SDCARD == 1) // use SD interface in SPI host mode
-  sd_spi.begin(SDCARD_SCLK, SDCARD_MISO, SDCARD_MOSI, SDCARD_CS);
+
-  digitalWrite(SDCARD_CS, LOW);
+  sdspi_host_t host = SDSPI_HOST_DEFAULT();
-  useSDCard = SD.begin(SDCARD_CS, sd_spi);
+
-#elif HAS_SDCARD == 2 // use SD MMC host driver
+  // This initializes the slot without card detect (CD) and write protect (WP)
-  // enable internal pullups of sd-data lines
+  // signals. Modify slot_config.gpio_cd and slot_config.gpio_wp if you want
-  gpio_set_pull_mode(gpio_num_t(SDCARD_DATA0), GPIO_PULLUP_ONLY);
+  // to use these signals (if your board has them)
-  gpio_set_pull_mode(gpio_num_t(SDCARD_DATA1), GPIO_PULLUP_ONLY);
+  sdspi_slot_config_t slot_config = SDSPI_SLOT_CONFIG_DEFAULT();
-  gpio_set_pull_mode(gpio_num_t(SDCARD_DATA2), GPIO_PULLUP_ONLY);
+
-  gpio_set_pull_mode(gpio_num_t(SDCARD_DATA3), GPIO_PULLUP_ONLY);
+  spi_bus_config_t bus_cfg = {
-  useSDCard = SD_MMC.begin();
+      .mosi_io_num = SDCARD_MOSI,
+      .miso_io_num = SDCARD_MISO,
+      .sclk_io_num = SDCARD_SCLK,
+      .quadwp_io_num = -1,
+      .quadhd_io_num = -1,
+      .max_transfer_sz = 4000,
+  };
+
+  ret = spi_bus_initialize(host.slot, &bus_cfg, SPI_DMA_CHAN);
+  if (ret != ESP_OK) {
+    ESP_LOGE(TAG, "failed to initialize SPI bus");
+    return;
+  }
+
+  // This initializes the slot without card detect (CD) and write protect (WP)
+  // signals. Modify slot_config.gpio_cd and slot_config.gpio_wp if your board
+  // has these signals.
+  sdspi_device_config_t slot_config = SDSPI_DEVICE_CONFIG_DEFAULT();
+  slot_config.gpio_cs = SDCARD_CS;
+  slot_config.host_id = host.slot;
+
+  // Use settings defined above to initialize SD card and mount FAT filesystem.
+  esp_err_t ret = esp_vfs_fat_sdspi_mount(mount_point, &host, &slot_config,
+                                          &mount_config, &card);
+
+#elif (HAS_SDCARD == 2) // use SD interface in MMC host mode
+
+  sdmmc_host_t host = SDMMC_HOST_DEFAULT();
+
+  // This initializes the slot without card detect (CD) and write protect (WP)
+  // signals. Modify slot_config.gpio_cd and slot_config.gpio_wp if your board
+  // has these signals.
+  // Default config for SDMMC_HOST_DEFAULT (4-bit bus width, slot 1)
+  // https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/peripherals/sdmmc_host.html
+  sdmmc_slot_config_t slot_config = SDMMC_SLOT_CONFIG_DEFAULT();
+
+  // To use 4-line SD mode, change this to 4:
+  slot_config.width = 1;
+
+  // Enable internal pullups on enabled pins. The internal pullups
+  // are insufficient however, please make sure 10k external pullups are
+  // connected on the bus. This is for debug / example purpose only.
+  slot_config.flags |= SDMMC_SLOT_FLAG_INTERNAL_PULLUP;
+
+  // Use settings defined above to initialize SD card and mount FAT filesystem.
+  esp_err_t ret = esp_vfs_fat_sdmmc_mount(mount_point, &host, &slot_config,
+                                          &mount_config, &card);
+
 #endif
 
-  if (useSDCard) {
+  // mount error handling
-    ESP_LOGI(TAG, "SD-card found");
+  if (ret != ESP_OK) {
-    openFile();
+    if (ret == ESP_FAIL) {
-    return true;
+      ESP_LOGE(TAG, "failed to mount filesystem");
-  } else {
+    } else {
-    ESP_LOGI(TAG, "SD-card not found");
+      ESP_LOGE(TAG, "SD-card not found (%d)", ret);
+    }
     return false;
   }
+
+  // SD card is now initialized
+  useSDCard = true;
+  ESP_LOGI(TAG, "filesystem mounted");
+  sdmmc_card_print_info(stdout, card);
+
+  // open files for data and, optional, system logging
+  char bufferFilename[50];
+
+  snprintf(bufferFilename, sizeof(bufferFilename), "/%s.csv", SDCARD_FILE_NAME);
+
+  if (openFile(&data_file, bufferFilename)) {
+
+    fpos_t position;
+    fgetpos(data_file, &position);
+
+    // empty file? then we write a header line
+    if (position = 0) {
+      fprintf(data_file, "%s", SDCARD_FILE_HEADER);
+#if (defined BAT_MEASURE_ADC || defined HAS_PMU)
+      fprintf(data_file, "%s", SDCARD_FILE_HEADER_VOLTAGE);
+#endif
+#if (HAS_SDS011)
+      fprintf(data_file, "%s", SDCARD_FILE_HEADER_SDS011);
+#endif
+      fprintf(data_file, "\n");
+    }
+
+  } else {
+    useSDCard = false;
+  }
+
+#ifdef SD_LOGGING
+  snprintf(bufferFilename, sizeof(bufferFilename), "/%s.log", SDCARD_FILE_NAME);
+
+  if (openFile(&log_file, bufferFilename)) {
+    ESP_LOGI(TAG, "redirecting serial output to SD-card");
+    esp_log_set_vprintf(&print_to_sd_card);
+    // Change stdout for THIS TASK ONLY
+    // stdout = log_file;
+    // Change stdout for all new tasks which will be created
+    //_GLOBAL_REENT->_stdout = log_file;
+  } else {
+    useSDCard = false;
+  }
+#endif
+
+  return useSDCard;
+
+} // sdcard_init
+
+void sdcard_flush(void) {
+  if (data_file)
+    fsync(fileno(data_file));
+#ifdef SD_LOGGING
+  if (log_file)
+    fsync(fileno(log_file));
+#endif
 }
 
 void sdcard_close(void) {
   ESP_LOGI(TAG, "closing SD-card");
-  fileSDCard.flush();
+  sdcard_flush();
-  fileSDCard.close();
+#ifdef SD_LOGGING
+  // Reset logging output back to normal
+  ESP_LOGI(TAG, "redirect console back to serial output");
+  // stdout = uart_stdout;
+  //_GLOBAL_REENT->_stdout = uart_stdout;
+  esp_log_set_vprintf(&vprintf);
+#endif
+  fcloseall();
+  esp_vfs_fat_sdcard_unmount(mount_point, card);
+  ESP_LOGI(TAG, "SD-card unmounted");
 }
 
 void sdcardWriteData(uint16_t noWifi, uint16_t noBle,
                      __attribute__((unused)) uint16_t voltage) {
-  static int counterWrites = 0;
+
+  if (!useSDCard)
+    return;
+
   char tempBuffer[20 + 1];
   time_t t = time(NULL);
   struct tm tt;
@@ -68,89 +253,17 @@ void sdcardWriteData(uint16_t noWifi, uint16_t noBle,
   sdsStatus_t sds;
 #endif
 
-  if (!useSDCard)
+  ESP_LOGI(TAG, "writing data to SD-card");
-    return;
+  fprintf(data_file, "%FT%TZ", &tt);
-
+  fprintf(data_file, ",%d,%d", noWifi, noBle);
-  ESP_LOGI(TAG, "SD: writing data");
-  strftime(tempBuffer, sizeof(tempBuffer), "%FT%TZ", &tt);
-  fileSDCard.print(tempBuffer);
-  snprintf(tempBuffer, sizeof(tempBuffer), ",%d,%d", noWifi, noBle);
-  fileSDCard.print(tempBuffer);
 #if (defined BAT_MEASURE_ADC || defined HAS_PMU)
-  snprintf(tempBuffer, sizeof(tempBuffer), ",%d", voltage);
+  fprintf(data_file, ",%d", voltage);
-  fileSDCard.print(tempBuffer);
 #endif
 #if (HAS_SDS011)
   sds011_store(&sds);
-  snprintf(tempBuffer, sizeof(tempBuffer), ",%5.1f,%4.1f", sds.pm10 / 10,
+  fprintf(data_file, ",%5.1f,%4.1f", sds.pm10 / 10, sds.pm25 / 10);
-           sds.pm25 / 10);
-  fileSDCard.print(tempBuffer);
 #endif
-  fileSDCard.println();
+  fprintf(data_file, "\n");
-
-  if (++counterWrites > 2) {
-    // force writing to SD-card
-    ESP_LOGI(TAG, "SD: flushing data");
-    fileSDCard.flush();
-    counterWrites = 0;
-  }
-}
-
-void openFile(void) {
-  char bufferFilename[30];
-
-  useSDCard = false;
-
-  snprintf(bufferFilename, sizeof(bufferFilename), "/%s.csv", SDCARD_FILE_NAME);
-  ESP_LOGI(TAG, "SD: looking for file <%s>", bufferFilename);
-
-#if HAS_SDCARD == 1
-  bool fileExists = SD.exists(bufferFilename);
-#elif HAS_SDCARD == 2
-  bool fileExists = SD_MMC.exists(bufferFilename);
-#endif
-
-  // file not exists, create it
-  if (!fileExists) {
-    ESP_LOGD(TAG, "SD: file not found, creating...");
-
-#if HAS_SDCARD == 1
-    fileSDCard = SD.open(bufferFilename, FILE_WRITE);
-#elif HAS_SDCARD == 2
-    fileSDCard = SD_MMC.open(bufferFilename, FILE_WRITE);
-#endif
-
-    if (fileSDCard) {
-      ESP_LOGD(TAG, "SD: name opened: <%s>", bufferFilename);
-      fileSDCard.print(SDCARD_FILE_HEADER);
-#if (defined BAT_MEASURE_ADC || defined HAS_PMU)
-      fileSDCard.print(SDCARD_FILE_HEADER_VOLTAGE); // for battery level data
-#endif
-#if (HAS_SDS011)
-      fileSDCard.print(SDCARD_FILE_HEADER_SDS011);
-#endif
-      fileSDCard.println();
-      useSDCard = true;
-    }
-  }
-
-  // file exists, append data
-  else {
-    ESP_LOGD(TAG, "SD: file found, opening...");
-
-#if HAS_SDCARD == 1
-    fileSDCard = SD.open(bufferFilename, FILE_APPEND);
-#elif HAS_SDCARD == 2
-    fileSDCard = SD_MMC.open(bufferFilename, FILE_APPEND);
-#endif
-
-    if (fileSDCard) {
-      ESP_LOGD(TAG, "SD: name opened: <%s>", bufferFilename);
-      useSDCard = true;
-    }
-  }
-
-  return;
 }
 
 #endif // (HAS_SDCARD)