From 42aa481c70cfbd4ed6eb16e724cef118a83cf5e3 Mon Sep 17 00:00:00 2001 From: cyberman54 Date: Sat, 26 Feb 2022 17:30:23 +0100 Subject: [PATCH] rework SD card logging --- include/sdcard.h | 28 +--- src/cyclic.cpp | 4 + src/sdcard.cpp | 325 +++++++++++++++++++++++++++++++---------------- 3 files changed, 230 insertions(+), 127 deletions(-) diff --git a/include/sdcard.h b/include/sdcard.h index 1c9eb3d7..ba59ec1c 100644 --- a/include/sdcard.h +++ b/include/sdcard.h @@ -6,11 +6,15 @@ #include "globals.h" #include #include +#include "esp_vfs_fat.h" +#include "sdmmc_cmd.h" + +#define MOUNT_POINT "/sdcard" #if HAS_SDCARD == 1 -#include +#include "driver/sdspi_host.h" #elif HAS_SDCARD == 2 -#include +#include "driver/sdmmc_host.h" #else #error HAS_SDCARD unknown card reader value, must be either 1 or 2 #endif @@ -35,25 +39,6 @@ #define SDCARD_SCLK SCK #endif -// Default config for SDMMC_HOST_DEFAULT (4-bit bus width, slot 1) -// https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/peripherals/sdmmc_host.html - -#ifndef SDCARD_DATA0 -#define SDCARD_DATA0 2 -#endif - -#ifndef SDCARD_DATA1 -#define SDCARD_DATA1 4 -#endif - -#ifndef SDCARD_DATA2 -#define SDCARD_DATA2 12 -#endif - -#ifndef SDCARD_DATA3 -#define SDCARD_DATA3 13 -#endif - #define SDCARD_FILE_NAME clientId #define SDCARD_FILE_HEADER "timestamp,wifi,ble" @@ -62,6 +47,7 @@ #endif bool sdcard_init(bool create = true); +void sdcard_flush(void); void sdcard_close(void); void sdcardWriteData(uint16_t, uint16_t, uint16_t = 0); diff --git a/src/cyclic.cpp b/src/cyclic.cpp index 385f9afe..27697efc 100644 --- a/src/cyclic.cpp +++ b/src/cyclic.cpp @@ -126,6 +126,10 @@ void doHousekeeping() { } #endif +#if (HAS_SDCARD) + sdcard_flush(); +#endif + } // doHousekeeping() uint32_t getFreeRAM() { diff --git a/src/sdcard.cpp b/src/sdcard.cpp index c8696bb1..51b4d9a0 100644 --- a/src/sdcard.cpp +++ b/src/sdcard.cpp @@ -1,64 +1,249 @@ +#ifdef HAS_SDCARD + // routines for writing data to an SD-card, if present // use FAT32 formatted card // check whether your card reader supports SPI oder SDMMC and select appropriate -// SD low level driver in board hal file +// SD interface in board hal file // Local logging tag static const char TAG[] = __FILE__; #include "sdcard.h" -#ifdef HAS_SDCARD +sdmmc_card_t *card; -static bool useSDCard; -static void openFile(void); +const char mount_point[] = MOUNT_POINT; +static bool useSDCard = false; -File fileSDCard; +// This file stream will be used for payload logging +static FILE *data_file; +// This file stream will be used for system logging -#if HAS_SDCARD == 1 -SPIClass sd_spi; +#ifdef SD_LOGGING +static FILE *log_file; + +// Save UART stdout stream +static FILE *uart_stdout = stdout; + +// This function will be called by the ESP log library every time ESP_LOG needs +// to be performed. +// @important Do NOT use the ESP_LOG* macro's in this function ELSE +// recursive loop and stack overflow! So use printf() instead for debug +// messages. +// CURRENTLY NOT WORKING DUE TO AN ISSUE IN ARDUINO-ESP32 +int print_to_sd_card(const char *fmt, va_list args) { + static bool static_fatal_error = false; + static const uint32_t WRITE_CACHE_CYCLE = 5; + static uint32_t counter_write = 0; + int iresult; + + // #1 Write to file + if (log_file == NULL) { + printf("%s() ABORT. file handle log_file is NULL\n", __FUNCTION__); + return -1; + } + if (static_fatal_error == false) { + iresult = vfprintf(log_file, fmt, args); + if (iresult < 0) { + printf("%s() ABORT. failed vfprintf() -> logging disabled \n", + __FUNCTION__); + // MARK FATAL + static_fatal_error = true; + return iresult; + } + + // #2 Smart commit after x writes + counter_write++; + if (counter_write % WRITE_CACHE_CYCLE == 0) { + printf("%s() fsync'ing log file (WRITE_CACHE_CYCLE=%u)\n", + WRITE_CACHE_CYCLE); + fsync(fileno(log_file)); + } + } + + // #3 ALWAYS Write to stdout! + return vprintf(fmt, args); +} #endif +bool openFile(FILE **fd, const char *filename) { + + char _filename[50]; + sprintf(_filename, "%s%s", MOUNT_POINT, filename); + + if ((*fd = fopen(_filename, "a")) == NULL) { + ESP_LOGE(TAG, "file <%s> open error", _filename); + return false; + } else { + ESP_LOGI(TAG, "file <%s> opened", _filename); + return true; + } +} // openfile + bool sdcard_init(bool create) { // for usage of SD drivers on ESP32 platform see // https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/peripherals/sdspi_host.html // https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/peripherals/sdmmc_host.html + // Options for mounting the filesystem. + // If format_if_mount_failed is set to true, SD card will be partitioned and + // formatted in case when mounting fails. + esp_vfs_fat_mount_config_t mount_config = {.format_if_mount_failed = false, + .max_files = 5}; + ESP_LOGI(TAG, "looking for SD-card..."); -#if HAS_SDCARD == 1 // use SD SPI host driver - digitalWrite(SDCARD_CS, HIGH); - sd_spi.begin(SDCARD_SCLK, SDCARD_MISO, SDCARD_MOSI, SDCARD_CS); - digitalWrite(SDCARD_CS, LOW); - useSDCard = SD.begin(SDCARD_CS, sd_spi); -#elif HAS_SDCARD == 2 // use SD MMC host driver - // enable internal pullups of sd-data lines - gpio_set_pull_mode(gpio_num_t(SDCARD_DATA0), GPIO_PULLUP_ONLY); - gpio_set_pull_mode(gpio_num_t(SDCARD_DATA1), GPIO_PULLUP_ONLY); - gpio_set_pull_mode(gpio_num_t(SDCARD_DATA2), GPIO_PULLUP_ONLY); - gpio_set_pull_mode(gpio_num_t(SDCARD_DATA3), GPIO_PULLUP_ONLY); - useSDCard = SD_MMC.begin(); + +#if (HAS_SDCARD == 1) // use SD interface in SPI host mode + + sdspi_host_t host = SDSPI_HOST_DEFAULT(); + + // This initializes the slot without card detect (CD) and write protect (WP) + // signals. Modify slot_config.gpio_cd and slot_config.gpio_wp if you want + // to use these signals (if your board has them) + sdspi_slot_config_t slot_config = SDSPI_SLOT_CONFIG_DEFAULT(); + + spi_bus_config_t bus_cfg = { + .mosi_io_num = SDCARD_MOSI, + .miso_io_num = SDCARD_MISO, + .sclk_io_num = SDCARD_SCLK, + .quadwp_io_num = -1, + .quadhd_io_num = -1, + .max_transfer_sz = 4000, + }; + + ret = spi_bus_initialize(host.slot, &bus_cfg, SPI_DMA_CHAN); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "failed to initialize SPI bus"); + return; + } + + // This initializes the slot without card detect (CD) and write protect (WP) + // signals. Modify slot_config.gpio_cd and slot_config.gpio_wp if your board + // has these signals. + sdspi_device_config_t slot_config = SDSPI_DEVICE_CONFIG_DEFAULT(); + slot_config.gpio_cs = SDCARD_CS; + slot_config.host_id = host.slot; + + // Use settings defined above to initialize SD card and mount FAT filesystem. + esp_err_t ret = esp_vfs_fat_sdspi_mount(mount_point, &host, &slot_config, + &mount_config, &card); + +#elif (HAS_SDCARD == 2) // use SD interface in MMC host mode + + sdmmc_host_t host = SDMMC_HOST_DEFAULT(); + + // This initializes the slot without card detect (CD) and write protect (WP) + // signals. Modify slot_config.gpio_cd and slot_config.gpio_wp if your board + // has these signals. + // Default config for SDMMC_HOST_DEFAULT (4-bit bus width, slot 1) + // https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/peripherals/sdmmc_host.html + sdmmc_slot_config_t slot_config = SDMMC_SLOT_CONFIG_DEFAULT(); + + // To use 4-line SD mode, change this to 4: + slot_config.width = 1; + + // Enable internal pullups on enabled pins. The internal pullups + // are insufficient however, please make sure 10k external pullups are + // connected on the bus. This is for debug / example purpose only. + slot_config.flags |= SDMMC_SLOT_FLAG_INTERNAL_PULLUP; + + // Use settings defined above to initialize SD card and mount FAT filesystem. + esp_err_t ret = esp_vfs_fat_sdmmc_mount(mount_point, &host, &slot_config, + &mount_config, &card); + #endif - if (useSDCard) { - ESP_LOGI(TAG, "SD-card found"); - openFile(); - return true; - } else { - ESP_LOGI(TAG, "SD-card not found"); + // mount error handling + if (ret != ESP_OK) { + if (ret == ESP_FAIL) { + ESP_LOGE(TAG, "failed to mount filesystem"); + } else { + ESP_LOGE(TAG, "SD-card not found (%d)", ret); + } return false; } + + // SD card is now initialized + useSDCard = true; + ESP_LOGI(TAG, "filesystem mounted"); + sdmmc_card_print_info(stdout, card); + + // open files for data and, optional, system logging + char bufferFilename[50]; + + snprintf(bufferFilename, sizeof(bufferFilename), "/%s.csv", SDCARD_FILE_NAME); + + if (openFile(&data_file, bufferFilename)) { + + fpos_t position; + fgetpos(data_file, &position); + + // empty file? then we write a header line + if (position = 0) { + fprintf(data_file, "%s", SDCARD_FILE_HEADER); +#if (defined BAT_MEASURE_ADC || defined HAS_PMU) + fprintf(data_file, "%s", SDCARD_FILE_HEADER_VOLTAGE); +#endif +#if (HAS_SDS011) + fprintf(data_file, "%s", SDCARD_FILE_HEADER_SDS011); +#endif + fprintf(data_file, "\n"); + } + + } else { + useSDCard = false; + } + +#ifdef SD_LOGGING + snprintf(bufferFilename, sizeof(bufferFilename), "/%s.log", SDCARD_FILE_NAME); + + if (openFile(&log_file, bufferFilename)) { + ESP_LOGI(TAG, "redirecting serial output to SD-card"); + esp_log_set_vprintf(&print_to_sd_card); + // Change stdout for THIS TASK ONLY + // stdout = log_file; + // Change stdout for all new tasks which will be created + //_GLOBAL_REENT->_stdout = log_file; + } else { + useSDCard = false; + } +#endif + + return useSDCard; + +} // sdcard_init + +void sdcard_flush(void) { + if (data_file) + fsync(fileno(data_file)); +#ifdef SD_LOGGING + if (log_file) + fsync(fileno(log_file)); +#endif } void sdcard_close(void) { ESP_LOGI(TAG, "closing SD-card"); - fileSDCard.flush(); - fileSDCard.close(); + sdcard_flush(); +#ifdef SD_LOGGING + // Reset logging output back to normal + ESP_LOGI(TAG, "redirect console back to serial output"); + // stdout = uart_stdout; + //_GLOBAL_REENT->_stdout = uart_stdout; + esp_log_set_vprintf(&vprintf); +#endif + fcloseall(); + esp_vfs_fat_sdcard_unmount(mount_point, card); + ESP_LOGI(TAG, "SD-card unmounted"); } void sdcardWriteData(uint16_t noWifi, uint16_t noBle, __attribute__((unused)) uint16_t voltage) { - static int counterWrites = 0; + + if (!useSDCard) + return; + char tempBuffer[20 + 1]; time_t t = time(NULL); struct tm tt; @@ -68,89 +253,17 @@ void sdcardWriteData(uint16_t noWifi, uint16_t noBle, sdsStatus_t sds; #endif - if (!useSDCard) - return; - - ESP_LOGI(TAG, "SD: writing data"); - strftime(tempBuffer, sizeof(tempBuffer), "%FT%TZ", &tt); - fileSDCard.print(tempBuffer); - snprintf(tempBuffer, sizeof(tempBuffer), ",%d,%d", noWifi, noBle); - fileSDCard.print(tempBuffer); + ESP_LOGI(TAG, "writing data to SD-card"); + fprintf(data_file, "%FT%TZ", &tt); + fprintf(data_file, ",%d,%d", noWifi, noBle); #if (defined BAT_MEASURE_ADC || defined HAS_PMU) - snprintf(tempBuffer, sizeof(tempBuffer), ",%d", voltage); - fileSDCard.print(tempBuffer); + fprintf(data_file, ",%d", voltage); #endif #if (HAS_SDS011) sds011_store(&sds); - snprintf(tempBuffer, sizeof(tempBuffer), ",%5.1f,%4.1f", sds.pm10 / 10, - sds.pm25 / 10); - fileSDCard.print(tempBuffer); + fprintf(data_file, ",%5.1f,%4.1f", sds.pm10 / 10, sds.pm25 / 10); #endif - fileSDCard.println(); - - if (++counterWrites > 2) { - // force writing to SD-card - ESP_LOGI(TAG, "SD: flushing data"); - fileSDCard.flush(); - counterWrites = 0; - } -} - -void openFile(void) { - char bufferFilename[30]; - - useSDCard = false; - - snprintf(bufferFilename, sizeof(bufferFilename), "/%s.csv", SDCARD_FILE_NAME); - ESP_LOGI(TAG, "SD: looking for file <%s>", bufferFilename); - -#if HAS_SDCARD == 1 - bool fileExists = SD.exists(bufferFilename); -#elif HAS_SDCARD == 2 - bool fileExists = SD_MMC.exists(bufferFilename); -#endif - - // file not exists, create it - if (!fileExists) { - ESP_LOGD(TAG, "SD: file not found, creating..."); - -#if HAS_SDCARD == 1 - fileSDCard = SD.open(bufferFilename, FILE_WRITE); -#elif HAS_SDCARD == 2 - fileSDCard = SD_MMC.open(bufferFilename, FILE_WRITE); -#endif - - if (fileSDCard) { - ESP_LOGD(TAG, "SD: name opened: <%s>", bufferFilename); - fileSDCard.print(SDCARD_FILE_HEADER); -#if (defined BAT_MEASURE_ADC || defined HAS_PMU) - fileSDCard.print(SDCARD_FILE_HEADER_VOLTAGE); // for battery level data -#endif -#if (HAS_SDS011) - fileSDCard.print(SDCARD_FILE_HEADER_SDS011); -#endif - fileSDCard.println(); - useSDCard = true; - } - } - - // file exists, append data - else { - ESP_LOGD(TAG, "SD: file found, opening..."); - -#if HAS_SDCARD == 1 - fileSDCard = SD.open(bufferFilename, FILE_APPEND); -#elif HAS_SDCARD == 2 - fileSDCard = SD_MMC.open(bufferFilename, FILE_APPEND); -#endif - - if (fileSDCard) { - ESP_LOGD(TAG, "SD: name opened: <%s>", bufferFilename); - useSDCard = true; - } - } - - return; + fprintf(data_file, "\n"); } #endif // (HAS_SDCARD)