From f165c628ae4ec80c41d521644a55d1f0e8ad745d Mon Sep 17 00:00:00 2001 From: Norbert Walter Date: Fri, 13 Mar 2026 20:28:08 +0000 Subject: [PATCH 1/3] Fix NetworkClient.cpp for more robust data transmission and decompression --- lib/obp60task/NetworkClient.cpp | 221 +++++++++++++++++++++++++++++-- lib/obp60task/NetworkClient.h | 12 ++ lib/obp60task/PageNavigation.cpp | 11 +- 3 files changed, 226 insertions(+), 18 deletions(-) diff --git a/lib/obp60task/NetworkClient.cpp b/lib/obp60task/NetworkClient.cpp index 96a649d..d4a1605 100644 --- a/lib/obp60task/NetworkClient.cpp +++ b/lib/obp60task/NetworkClient.cpp @@ -7,12 +7,29 @@ extern "C" { #include "puff.h" } +static uint32_t crc32_update(uint32_t crc, const uint8_t* data, size_t len) { + crc = ~crc; + for (size_t i = 0; i < len; ++i) { + crc ^= data[i]; + for (int bit = 0; bit < 8; ++bit) { + uint32_t mask = -(int32_t)(crc & 1U); + crc = (crc >> 1) ^ (0xEDB88320U & mask); + } + } + return ~crc; +} + // Constructor NetworkClient::NetworkClient(size_t reserveSize) : _doc(reserveSize), _valid(false), _jsonRaw(nullptr), - _jsonRawLen(0) + _jsonRawLen(0), + _imageWidth(0), + _imageHeight(0), + _numberPixels(0), + _pictureBase64(nullptr), + _pictureBase64Len(0) { } @@ -24,6 +41,100 @@ NetworkClient::~NetworkClient() { } } +bool NetworkClient::findJsonIntField(const char* json, size_t len, const char* key, int& outValue) { + if (json == nullptr || key == nullptr || len == 0) { + return false; + } + + char pattern[64]; + int plen = snprintf(pattern, sizeof(pattern), "\"%s\"", key); + if (plen <= 0 || (size_t)plen >= sizeof(pattern)) { + return false; + } + + const char* keyPos = strstr(json, pattern); + if (keyPos == nullptr) { + return false; + } + + const char* end = json + len; + const char* colon = strchr(keyPos + plen, ':'); + if (colon == nullptr || colon >= end) { + return false; + } + + const char* p = colon + 1; + while (p < end && (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n')) { + ++p; + } + if (p >= end) { + return false; + } + + char* parseEnd = nullptr; + long value = strtol(p, &parseEnd, 10); + if (parseEnd == p) { + return false; + } + outValue = (int)value; + return true; +} + +bool NetworkClient::extractJsonStringInPlace(char* json, size_t len, const char* key, char*& outValue, size_t& outLen) { + outValue = nullptr; + outLen = 0; + + if (json == nullptr || key == nullptr || len == 0) { + return false; + } + + char pattern[64]; + int plen = snprintf(pattern, sizeof(pattern), "\"%s\"", key); + if (plen <= 0 || (size_t)plen >= sizeof(pattern)) { + return false; + } + + char* keyPos = strstr(json, pattern); + if (keyPos == nullptr) { + return false; + } + + char* end = json + len; + char* colon = strchr(keyPos + plen, ':'); + if (colon == nullptr || colon >= end) { + return false; + } + + char* p = colon + 1; + while (p < end && (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n')) { + ++p; + } + if (p >= end || *p != '"') { + return false; + } + + char* valueStart = p + 1; + char* cur = valueStart; + while (cur < end) { + if (*cur == '\\') { + ++cur; + if (cur < end) { + ++cur; + } + continue; + } + if (*cur == '"') { + *cur = '\0'; + outValue = valueStart; + outLen = (size_t)(cur - valueStart); + return true; + } + ++cur; + } + + return false; +} + // Skip GZIP Header an goto DEFLATE content int NetworkClient::skipGzipHeader(const uint8_t* data, size_t len) { if (len < 10) return -1; @@ -207,8 +318,16 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou return false; } + if (total > 0 && (int)len != total) { + Serial.printf("Plain response incomplete: got=%d expected=%d\n", (int)len, total); + if (stream) stream->stop(); + http.end(); + free(buffer); + return false; + } + // Return plain body to caller - outData = (uint8_t*)malloc(len); + outData = (uint8_t*)malloc(len + 1); if (!outData) { Serial.println("Malloc failed outData (plain)."); // --- Added: Force-close connection only if aborted to avoid TCP RST storms --- @@ -218,6 +337,7 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou return false; } memcpy(outData, buffer, len); + outData[len] = 0; outLen = len; http.end(); @@ -284,6 +404,13 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou // NEW: only attempt gzip parse/decompress after we have a complete body (when Content-Length is known) // This avoids wasting heap with repeated malloc/free and reduces fragmentation over long runtimes. + if (!aborting) { + if (total > 0 && (int)len != total) { + Serial.printf("GZIP response incomplete: got=%d expected=%d\n", (int)len, total); + aborting = true; + } + } + if (!aborting) { if (len < 20) { aborting = true; @@ -308,7 +435,7 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou } aborting = true; } else { - uint8_t* test = (uint8_t*)malloc((size_t)outNeeded); + uint8_t* test = (uint8_t*)malloc((size_t)outNeeded + 1); if (!test) { Serial.println("Malloc failed test buffer, aborting."); aborting = true; @@ -318,10 +445,36 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou int res = puff(test, &testLen, buffer + headerOffset, &srcLen); if (res == 0) { - if (DEBUGING) {Serial.printf("Decompress OK! Size: %lu bytes\n", testLen);} - outData = test; - outLen = (size_t)testLen; - complete = true; + uint32_t trailerCrc = + (uint32_t)buffer[len - 8] | + ((uint32_t)buffer[len - 7] << 8) | + ((uint32_t)buffer[len - 6] << 16) | + ((uint32_t)buffer[len - 5] << 24); + uint32_t trailerIsize = + (uint32_t)buffer[len - 4] | + ((uint32_t)buffer[len - 3] << 8) | + ((uint32_t)buffer[len - 2] << 16) | + ((uint32_t)buffer[len - 1] << 24); + uint32_t calcCrc = crc32_update(0, test, (size_t)testLen); + uint32_t calcIsize = (uint32_t)testLen; + + if (calcCrc != trailerCrc || calcIsize != trailerIsize) { + Serial.printf( + "GZIP CRC/ISIZE mismatch crc=%08lx/%08lx isize=%lu/%lu\n", + (unsigned long)calcCrc, + (unsigned long)trailerCrc, + (unsigned long)calcIsize, + (unsigned long)trailerIsize + ); + free(test); + aborting = true; + } else { + test[testLen] = 0; + if (DEBUGING) {Serial.printf("Decompress OK! Size: %lu bytes\n", testLen);} + outData = test; + outLen = (size_t)testLen; + complete = true; + } } else { if (DEBUGING) { Serial.printf("Decompress failed: res=%d out=%lu src=%lu\n", res, testLen, srcLen); @@ -355,6 +508,11 @@ bool NetworkClient::fetchAndDecompressJson(const String& url) { _valid = false; _doc.clear(); + _imageWidth = 0; + _imageHeight = 0; + _numberPixels = 0; + _pictureBase64 = nullptr; + _pictureBase64Len = 0; if (_jsonRaw != nullptr) { free(_jsonRaw); @@ -370,11 +528,24 @@ bool NetworkClient::fetchAndDecompressJson(const String& url) { return false; } - // Parse in zero-copy mode and keep the backing buffer alive in the class. - DeserializationError err = deserializeJson(_doc, reinterpret_cast(raw), rawLen); + char* json = reinterpret_cast(raw); + bool ok = true; + ok = findJsonIntField(json, rawLen, "number_pixels", _numberPixels) && ok; + ok = findJsonIntField(json, rawLen, "width", _imageWidth) && ok; + ok = findJsonIntField(json, rawLen, "height", _imageHeight) && ok; + ok = extractJsonStringInPlace(json, rawLen, "picture_base64", _pictureBase64, _pictureBase64Len) && ok; - if (err) { - Serial.printf("JSON ERROR: %s\n", err.c_str()); + if (!ok) { + Serial.println("JSON field extraction failed."); + free(raw); + return false; + } + + if (_imageWidth <= 0 || _imageHeight <= 0 || _pictureBase64Len == 0) { + Serial.printf("JSON invalid geometry/data w=%d h=%d b64=%u\n", + _imageWidth, + _imageHeight, + (unsigned int)_pictureBase64Len); free(raw); return false; } @@ -382,7 +553,13 @@ bool NetworkClient::fetchAndDecompressJson(const String& url) { _jsonRaw = raw; _jsonRawLen = rawLen; - if (DEBUGING) {Serial.println("JSON OK!");} + if (DEBUGING) { + Serial.printf("JSON fields OK: num=%d w=%d h=%d b64=%u\n", + _numberPixels, + _imageWidth, + _imageHeight, + (unsigned int)_pictureBase64Len); + } _valid = true; return true; } @@ -391,6 +568,26 @@ JsonDocument& NetworkClient::json() { return _doc; } +int NetworkClient::imageWidth() const { + return _imageWidth; +} + +int NetworkClient::imageHeight() const { + return _imageHeight; +} + +int NetworkClient::numberPixels() const { + return _numberPixels; +} + +const char* NetworkClient::pictureBase64() const { + return _pictureBase64; +} + +size_t NetworkClient::pictureBase64Len() const { + return _pictureBase64Len; +} + bool NetworkClient::isValid() const { return _valid; } diff --git a/lib/obp60task/NetworkClient.h b/lib/obp60task/NetworkClient.h index 80e0703..a04bf1f 100644 --- a/lib/obp60task/NetworkClient.h +++ b/lib/obp60task/NetworkClient.h @@ -16,6 +16,11 @@ public: bool fetchAndDecompressJson(const String& url); JsonDocument& json(); + int imageWidth() const; + int imageHeight() const; + int numberPixels() const; + const char* pictureBase64() const; + size_t pictureBase64Len() const; bool isValid() const; private: @@ -23,8 +28,15 @@ private: bool _valid; uint8_t* _jsonRaw; size_t _jsonRawLen; + int _imageWidth; + int _imageHeight; + int _numberPixels; + char* _pictureBase64; + size_t _pictureBase64Len; int skipGzipHeader(const uint8_t* data, size_t len); bool httpGetGzip(const String& url, uint8_t*& outData, size_t& outLen); + static bool findJsonIntField(const char* json, size_t len, const char* key, int& outValue); + static bool extractJsonStringInPlace(char* json, size_t len, const char* key, char*& outValue, size_t& outLen); }; diff --git a/lib/obp60task/PageNavigation.cpp b/lib/obp60task/PageNavigation.cpp index 44045fd..31153e7 100644 --- a/lib/obp60task/PageNavigation.cpp +++ b/lib/obp60task/PageNavigation.cpp @@ -456,10 +456,9 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map failCount = 0; nextAllowedMs = now + 1000; // keep 1 Hz on success - auto& json = net.json(); // Extract JSON content - int numPix = json["number_pixels"] | 0; // Read number of pixels - imgWidth = json["width"] | 0; // Read width of image - imgHeight = json["height"] | 0; // Read height og image + int numPix = net.numberPixels(); // Read number of pixels + imgWidth = net.imageWidth(); // Read width of image + imgHeight = net.imageHeight(); // Read height of image size_t requiredBytesMono = 0; size_t requiredBytesRgb565 = 0; if (imgWidth > 0 && imgHeight > 0){ @@ -471,12 +470,12 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map return PAGE_UPDATE; } - const char* b64src = json["picture_base64"].as(); // Read picture as Base64 content + const char* b64src = net.pictureBase64(); // Read picture as Base64 content if (b64src == nullptr){ LOG_DEBUG(GwLog::ERROR,"Error PageNavigation: picture_base64 missing"); return PAGE_UPDATE; } - size_t b64len = strlen(b64src); // Calculate length of Base64 content + size_t b64len = net.pictureBase64Len(); // Calculate length of Base64 content // Copy Base64 content in PSRAM char* b64 = (char*) heap_caps_malloc(b64len + 1, MALLOC_CAP_SPIRAM); // Allcate PSRAM for Base64 content if (!b64) { From a7cdfe700bf17d1d26fc40ecca3bd7a415a31b57 Mon Sep 17 00:00:00 2001 From: Norbert Walter Date: Fri, 13 Mar 2026 20:57:51 +0000 Subject: [PATCH 2/3] Typo --- lib/obp60task/PageNavigation.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/obp60task/PageNavigation.cpp b/lib/obp60task/PageNavigation.cpp index 31153e7..8be5e7e 100644 --- a/lib/obp60task/PageNavigation.cpp +++ b/lib/obp60task/PageNavigation.cpp @@ -475,7 +475,7 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map LOG_DEBUG(GwLog::ERROR,"Error PageNavigation: picture_base64 missing"); return PAGE_UPDATE; } - size_t b64len = net.pictureBase64Len(); // Calculate length of Base64 content + size_t b64len = net.pictureBase64Len(); // Calculate length of Base64 content // Copy Base64 content in PSRAM char* b64 = (char*) heap_caps_malloc(b64len + 1, MALLOC_CAP_SPIRAM); // Allcate PSRAM for Base64 content if (!b64) { @@ -485,7 +485,6 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map memcpy(b64, b64src, b64len + 1); // Copy Base64 content in PSRAM // Set image buffer in PSRAM - //size_t imgSize = getdisplay().width() * getdisplay().height(); size_t imgSize = (numPix > 0) ? (size_t)numPix : requiredBytesMono; // Calculate image size if (imgSize < requiredBytesMono){ imgSize = requiredBytesMono; @@ -538,7 +537,7 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map } #endif - // Copy actual navigation man to ackup map + // Copy actual navigation map to backup map imageBackupWidth = imgWidth; imageBackupHeight = imgHeight; imageBackupSize = imgSize; From d13c4af9cbe48112b44183f116acfdb26f9fc3c9 Mon Sep 17 00:00:00 2001 From: norbert-walter Date: Fri, 13 Mar 2026 21:58:29 +0100 Subject: [PATCH 3/3] Changes --- lib/obp60task/debugging.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/obp60task/debugging.txt b/lib/obp60task/debugging.txt index c0a22e0..73e4f3e 100644 --- a/lib/obp60task/debugging.txt +++ b/lib/obp60task/debugging.txt @@ -3,4 +3,7 @@ Debugging tool log.txt = text file with error messages from terminal console -tools/decoder.py -p ESP32S3 -t ~/.platformio/packages/toolchain-xtensa-esp32s3/ -e .pio/build/obp60_s3/firmware.elf log.txt \ No newline at end of file +cd /home/norbert/esp32-nmea2000-obp60 +tools/decoder.py -p ESP32S3 -t ~/.platformio/packages/toolchain-xtensa-esp32s3/ -e .pio/build/obp60_s3/firmware.elf log.txt + +tools/decoder.py -p ESP32S3 -t ~/.platformio/packages/toolchain-xtensa-esp32s3/ -e .pio/build/obp70_s3/firmware.elf /home/norbert/Dokumente/Multifunktionsdisplay_OBP60/Crashes/20260313/log.txt \ No newline at end of file