diff --git a/lib/obp60task/Changes_to_original.txt b/lib/obp60task/Changes_to_original.txt new file mode 100644 index 0000000..052d644 --- /dev/null +++ b/lib/obp60task/Changes_to_original.txt @@ -0,0 +1,12 @@ +Changes to original project (wellenvogel) + +* esp32-nmea2000-obp60/gwwifi/GwWifi.CPPDEFINES + - any fixes for reconnect handling +* esp32-nmea2000-obp60/platformio.ini + - change to newer versions for AsyncTCP and AsyncWebServer (better handling for bad WiFi connections) + + From AsyncTCP-esphome @ 2.0.1 + ottowinter/ESPAsyncWebServer-esphome@2.0.1 + + To AsyncTCP-esphome @ 2.1.1 + ottowinter/ESPAsyncWebServer-esphome@3.4.0 \ No newline at end of file diff --git a/lib/obp60task/NetworkClient.cpp b/lib/obp60task/NetworkClient.cpp index c6a7962..b8ebaee 100644 --- a/lib/obp60task/NetworkClient.cpp +++ b/lib/obp60task/NetworkClient.cpp @@ -51,14 +51,16 @@ int NetworkClient::skipGzipHeader(const uint8_t* data, size_t len) { // HTTP GET + GZIP Decompression (reading in chunks) bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& outLen) { - const size_t capacity = READLIMIT; // Read limit for data (can be adjusted in NetworkClient.h) + const size_t capacity = READLIMIT; // Read limit for data (can be adjusted in NetworkClient.h) uint8_t* buffer = (uint8_t*)malloc(capacity); + // If not with WiFi connectetd then return without any activities if (!gwWifi.clientConnected()) { if (DEBUGING) {Serial.println("No WiFi connection");} return false; } + // If frame buffer not correct allocated then return without any activities if (!buffer) { if (DEBUGING) {Serial.println("Malloc failed buffer");} return false; @@ -71,20 +73,50 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou http.setTimeout(TCPREADTIMEOUT); // Read timeout in ms (can be adjusted in NetworkClient.h) http.begin(url); + + // NEW: force server to close the connection after the response (prevents "stuck" keep-alive reads) + http.addHeader("Connection", "close"); + + // NEW: request gzip, but we will only decompress if the server actually answers with gzip http.addHeader("Accept-Encoding", "gzip"); + // NEW: register headers BEFORE GET() (more reliable with Arduino HTTPClient) + if (DEBUGING) { + // We need follow key words + const char* keys[] = { + "Content-Encoding", + "Transfer-Encoding", + "Content-Length" + }; + // Read header + http.collectHeaders(keys, 3); + } + int code = http.GET(); if (code != HTTP_CODE_OK) { - Serial.printf("HTTP ERROR: %d\n", code); + Serial.printf("HTTP Client ERROR: %d (%s)\n", code, http.errorToString(code).c_str()); // Hard reset HTTP + socket WiFiClient* tmp = http.getStreamPtr(); if (tmp) tmp->stop(); // Force close TCP socket + http.end(); - free(buffer); return false; } + else{ + if (DEBUGING) { + String ce = http.header("Content-Encoding"); + String te = http.header("Transfer-Encoding"); + String cl = http.header("Content-Length"); + + // Print header informations + Serial.printf("Content-Encoding=%s Transfer-Encoding=%s Content-Length=%s\n", + ce.c_str(), + te.c_str(), + cl.c_str()); + } + } WiFiClient* stream = http.getStreamPtr(); @@ -93,55 +125,189 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou const uint32_t READ_TIMEOUT = READDATATIMEOUT; // Timeout for reading data (can be adjusted in NetworkClient.h) bool complete = false; + bool aborting = false; // NEW: remember if we must force-close socket - while (http.connected() && !complete) { + // NEW: detect if server really sent gzip + String ce = http.header("Content-Encoding"); + bool isGzip = ce.equalsIgnoreCase("gzip"); - size_t avail = stream->available(); + // NEW: read expected body size if provided by server (prevents waiting forever for missing bytes) + int total = http.getSize(); // returns Content-Length, or -1 if unknown/chunked - if (avail == 0) { - if (millis() - lastData > READ_TIMEOUT) { - Serial.println("TIMEOUT waiting for data!"); - break; - } - delay(1); - continue; - } - - if (len + avail > capacity) - avail = capacity - len; - - int read = stream->readBytes(buffer + len, avail); - len += read; - lastData = millis(); - - if (DEBUGING) {Serial.printf("Read chunk: %d (total: %d)\n", read, (int)len);} - - if (len < 20) continue; // Not enough data for header - - int headerOffset = skipGzipHeader(buffer, len); - if (headerOffset < 0) continue; - - unsigned long testLen = len * 8; // Dynamic expansion - uint8_t* test = (uint8_t*)malloc(testLen); - - if (!test) continue; - - unsigned long srcLen = len - headerOffset; - - int res = puff(test, &testLen, buffer + headerOffset, &srcLen); - if (res == 0) { - if (DEBUGING) {Serial.printf("Decompress OK! Size: %lu bytes\n", testLen);} - outData = test; - outLen = testLen; - complete = true; - break; - } - - free(test); + // NEW: fail fast if server claims something larger than our buffer + if (total > 0 && (size_t)total > capacity) { + Serial.println("Response exceeds READLIMIT."); + aborting = true; } - // --- Added: Force-close connection in all cases to avoid stuck TCP sockets --- - if (stream) stream->stop(); + // NEW: if not gzip, we will not try to decompress (prevents false "Decompress OK" / random success) + // You can either handle plain JSON here or just fail-fast. + if (!isGzip && !aborting) { + if (DEBUGING) { + Serial.println("Server response is NOT gzip (Content-Encoding != gzip)."); + Serial.println("Either disable Accept-Encoding: gzip or add plain-body handling here."); + } + + // --- Plain-body handling (recommended): read full body into outData as-is --- + // NEW: try to read Content-Length bytes if available (more robust) + if (total > 0 && (size_t)total > capacity) { + Serial.println("Plain response exceeds READLIMIT."); + aborting = true; + } else { + // Read until we have all bytes (Content-Length) or until connection closes + buffer drains + while ((http.connected() || (stream && stream->available())) && !aborting) { + size_t avail = stream ? stream->available() : 0; + if (avail == 0) { + if (millis() - lastData > READ_TIMEOUT) { + Serial.println("TIMEOUT waiting for data (plain)!"); + aborting = true; + break; + } + delay(1); + continue; + } + + if (len >= capacity) { + Serial.println("READLIMIT reached, aborting (plain)."); + aborting = true; + break; + } + + if (len + avail > capacity) + avail = capacity - len; + + int read = stream->readBytes(buffer + len, avail); + if (read > 0) { + len += (size_t)read; + lastData = millis(); + } + + // NEW: stop reading as soon as we have the full response + if (total > 0 && (int)len >= total) { + break; // we got full body + } + } + } + + if (aborting) { + // --- Added: Force-close connection only if aborted to avoid TCP RST storms --- + if (stream) stream->stop(); // Force close TCP socket + http.end(); + free(buffer); + return false; + } + + // Return plain body to caller + outData = (uint8_t*)malloc(len); + if (!outData) { + Serial.println("Malloc failed outData (plain)."); + // --- Added: Force-close connection only if aborted to avoid TCP RST storms --- + if (stream) stream->stop(); // Force close TCP socket + http.end(); + free(buffer); + return false; + } + memcpy(outData, buffer, len); + outLen = len; + + http.end(); + free(buffer); + return true; + } + + // --- GZIP path (only if Content-Encoding is gzip) --- + if (!aborting) { + + // NEW: read exactly Content-Length bytes when available (prevents partial-body timeout loops) + while ((http.connected() || (stream && stream->available())) && !complete && !aborting) { + + size_t avail = stream ? stream->available() : 0; + + if (avail == 0) { + // NEW: if Content-Length is known and we already read it all, stop immediately + if (total > 0 && (int)len >= total) { + break; + } + + if (millis() - lastData > READ_TIMEOUT) { + Serial.println("TIMEOUT waiting for data!"); + aborting = true; // NEW: mark abnormal exit + break; + } + delay(1); + continue; + } + + // NEW: safety check if buffer limit is reached + if (len >= capacity) { + Serial.println("READLIMIT reached, aborting."); + aborting = true; + break; + } + + // NEW: if Content-Length is known, do not read beyond it + if (total > 0) { + size_t remaining = (size_t)total - len; + if (avail > remaining) avail = remaining; + } + + if (len + avail > capacity) + avail = capacity - len; + + int read = stream->readBytes(buffer + len, avail); + if (read <= 0) { + // NEW: avoid tight loop if read returns zero + delay(1); + continue; + } + + len += (size_t)read; + lastData = millis(); + + if (DEBUGING) {Serial.printf("Read chunk: %d (total: %d)\n", read, (int)len);} + + // NEW: if Content-Length is known and fully received, we can stop reading + if (total > 0 && (int)len >= total) { + break; + } + } + + // NEW: only attempt gzip parse/decompress after we have a complete body (when Content-Length is known) + // This avoids wasting heap with repeated malloc/free and reduces fragmentation over long runtimes. + if (!aborting) { + if (len < 20) { + aborting = true; + } else { + int headerOffset = skipGzipHeader(buffer, len); + if (headerOffset < 0) { + aborting = true; + } else { + unsigned long testLen = len * 8; // Dynamic expansion + uint8_t* test = (uint8_t*)malloc(testLen); + + if (!test) { + Serial.println("Malloc failed test buffer, aborting."); + aborting = true; + } else { + unsigned long srcLen = (unsigned long)(len - (size_t)headerOffset); + int res = puff(test, &testLen, buffer + headerOffset, &srcLen); + + if (res == 0) { + if (DEBUGING) {Serial.printf("Decompress OK! Size: %lu bytes\n", testLen);} + outData = test; + outLen = (size_t)testLen; + complete = true; + } else { + free(test); + } + } + } + } + } + } + + // --- Added: Force-close connection only if aborted to avoid TCP RST storms --- + if (aborting && stream) stream->stop(); // NEW: stop() only on abnormal termination http.end(); free(buffer); diff --git a/lib/obp60task/NetworkClient.h b/lib/obp60task/NetworkClient.h index 03e7f83..5d3f448 100644 --- a/lib/obp60task/NetworkClient.h +++ b/lib/obp60task/NetworkClient.h @@ -3,7 +3,7 @@ #include #include -#define DEBUGING false // Debug flag for NetworkClient for more live information +#define DEBUGING true // Debug flag for NetworkClient for more live information #define READLIMIT 200000 // HTTP read limit in byte for gzip content (can be adjusted) #define CONNECTIONTIMEOUT 3000 // Timeout in ms for HTTP connection #define TCPREADTIMEOUT 2000 // Timeout in ms for read HTTP client stack diff --git a/lib/obp60task/PageNavigation.cpp b/lib/obp60task/PageNavigation.cpp index bd4adf1..5652cad 100644 --- a/lib/obp60task/PageNavigation.cpp +++ b/lib/obp60task/PageNavigation.cpp @@ -374,8 +374,21 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map getdisplay().setPartialWindow(0, 0, getdisplay().width(), getdisplay().height()); // Set partial update getdisplay().setTextColor(commonData->fgcolor); + // NEW: simple exponential backoff for 1 Hz polling (prevents connection-refused storms) + static uint32_t nextAllowedMs = 0; + static uint8_t failCount = 0; + + uint32_t now = millis(); + + // NEW: if we are in backoff window, skip network call and use backup immediately + bool allowFetch = ((int32_t)(now - nextAllowedMs) >= 0); + // If a network connection to URL then load the navigation map - if (net.fetchAndDecompressJson(url)) { + if (allowFetch && net.fetchAndDecompressJson(url)) { + + // NEW: reset backoff on success + failCount = 0; + nextAllowedMs = now + 1000; // keep 1 Hz on success auto& json = net.json(); // Extract JSON content int numPix = json["number_pixels"] | 0; // Read number of pixels @@ -397,7 +410,7 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map size_t imgSize = numPix; // Calculate image size uint8_t* imageData = (uint8_t*) heap_caps_malloc(imgSize, MALLOC_CAP_SPIRAM); // Allocate PSRAM for image if (!imageData) { - LOG_DEBUG(GwLog::ERROR,"Error PageNavigation: PPSRAM alloc image buffer failed"); + LOG_DEBUG(GwLog::ERROR,"Error PageNavigation: PSRAM alloc image buffer failed"); free(b64); return PAGE_UPDATE; } @@ -426,12 +439,25 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map } // If no network connection then use backup navigation map else{ + + // NEW: update backoff only if we actually attempted a fetch (not when skipping due to backoff) + if (allowFetch) { + // NEW: exponential backoff: 1s,2s,4s,8s,16s,30s (capped) + if (failCount < 6) failCount++; + uint32_t backoffMs = 1000u << failCount; + if (backoffMs > 30000u) backoffMs = 30000u; + nextAllowedMs = now + backoffMs; + } else { + // NEW: we are currently backing off; do not increase failCount further + // nextAllowedMs stays unchanged + } + // Show backup image (backup navigation map) if (hasImageBackup) { getdisplay().drawBitmap(0, 25, imageBackupData, imageBackupWidth, imageBackupHeight, commonData->fgcolor); } - // Show info: Connection lost when 5 page refreshes has a connection lost to the map server + // Show connection lost info when 5 page refreshes has a connection lost to the map server // Short connection losts are uncritical if(lostCounter >= 5){ getdisplay().setFont(&Ubuntu_Bold12pt8b); @@ -444,7 +470,6 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map lostCounter++; // Increment lost counter } - // ############### Draw Values ################ getdisplay().setFont(&Ubuntu_Bold12pt8b); diff --git a/platformio.ini b/platformio.ini index 341083c..2ecdd94 100644 --- a/platformio.ini +++ b/platformio.ini @@ -21,8 +21,8 @@ lib_deps = ttlappalainen_NMEA2000=https://github.com/wellenvogel/NMEA2000.git#20251126 ttlappalainen/NMEA0183 @ 1.10.1 ArduinoJson @ 6.18.5 - AsyncTCP-esphome @ 2.0.1 - ottowinter/ESPAsyncWebServer-esphome@2.0.1 + AsyncTCP-esphome @ 2.1.1 + ottowinter/ESPAsyncWebServer-esphome@3.4.0 FS Preferences ESPmDNS @@ -34,8 +34,8 @@ lib_deps= ttlappalainen_NMEA2000=symlink://../NMEA2000 ttlappalainen/NMEA0183 @ 1.10.1 ArduinoJson @ 6.18.5 - AsyncTCP-esphome @ 2.0.1 - ottowinter/ESPAsyncWebServer-esphome@2.0.1 + AsyncTCP-esphome @ 2.1.1 + ottowinter/ESPAsyncWebServer-esphome@3.4.0 FS Preferences ESPmDNS