Merge branch 'autopilot3'

lib/obp60task/Changes_to_original.txt

@@ -0,0 +1,12 @@
+Changes to original project (wellenvogel)
+
+* esp32-nmea2000-obp60/gwwifi/GwWifi.CPPDEFINES
+    - any fixes for reconnect handling
+* esp32-nmea2000-obp60/platformio.ini
+    - change to newer versions for AsyncTCP and AsyncWebServer (better handling for bad WiFi connections)
+
+    From AsyncTCP-esphome @ 2.0.1
+         ottowinter/ESPAsyncWebServer-esphome@2.0.1
+
+    To AsyncTCP-esphome @ 2.1.1
+         ottowinter/ESPAsyncWebServer-esphome@3.4.0

lib/obp60task/NetworkClient.cpp

@@ -51,14 +51,16 @@ int NetworkClient::skipGzipHeader(const uint8_t* data, size_t len) {
 // HTTP GET + GZIP Decompression (reading in chunks)
 bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& outLen) {
 
-    const size_t capacity = READLIMIT;   // Read limit for data (can be adjusted in NetworkClient.h)
+    const size_t capacity = READLIMIT;          // Read limit for data (can be adjusted in NetworkClient.h)
     uint8_t* buffer = (uint8_t*)malloc(capacity);
 
+    // If not with WiFi connectetd then return without any activities
     if (!gwWifi.clientConnected()) {
         if (DEBUGING) {Serial.println("No WiFi connection");}
         return false;
     }
     
+    // If frame buffer not correct allocated then return without any activities
     if (!buffer) {
         if (DEBUGING) {Serial.println("Malloc failed buffer");}
         return false;
@@ -71,20 +73,50 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou
     http.setTimeout(TCPREADTIMEOUT);            // Read timeout in ms (can be adjusted in NetworkClient.h)
 
     http.begin(url);
+
+    // NEW: force server to close the connection after the response (prevents "stuck" keep-alive reads)
+    http.addHeader("Connection", "close");
+
+    // NEW: request gzip, but we will only decompress if the server actually answers with gzip
     http.addHeader("Accept-Encoding", "gzip");
 
+    // NEW: register headers BEFORE GET() (more reliable with Arduino HTTPClient)
+    if (DEBUGING) {
+        // We need follow key words
+        const char* keys[] = {
+            "Content-Encoding",
+            "Transfer-Encoding",
+            "Content-Length"
+        };
+        // Read header
+        http.collectHeaders(keys, 3);
+    }
+
     int code = http.GET();
     if (code != HTTP_CODE_OK) {
-        Serial.printf("HTTP ERROR: %d\n", code);
+        Serial.printf("HTTP Client ERROR: %d (%s)\n", code, http.errorToString(code).c_str());
 
         // Hard reset HTTP + socket
         WiFiClient* tmp = http.getStreamPtr();
         if (tmp) tmp->stop();   // Force close TCP socket
+        
         http.end();
-
         free(buffer);
         return false;
     }
+    else{       
+        if (DEBUGING) {
+            String ce = http.header("Content-Encoding");
+            String te = http.header("Transfer-Encoding");
+            String cl = http.header("Content-Length");
+
+            // Print header informations
+            Serial.printf("Content-Encoding=%s Transfer-Encoding=%s Content-Length=%s\n",
+                        ce.c_str(),
+                        te.c_str(),
+                        cl.c_str());
+        }                
+    }
 
     WiFiClient* stream = http.getStreamPtr();
 
@@ -93,55 +125,189 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou
     const uint32_t READ_TIMEOUT = READDATATIMEOUT;   // Timeout for reading data (can be adjusted in NetworkClient.h)
 
     bool complete = false;
+    bool aborting = false;   // NEW: remember if we must force-close socket
 
-    while (http.connected() && !complete) {
+    // NEW: detect if server really sent gzip
+    String ce = http.header("Content-Encoding");
+    bool isGzip = ce.equalsIgnoreCase("gzip");
 
-        size_t avail = stream->available();
+    // NEW: read expected body size if provided by server (prevents waiting forever for missing bytes)
+    int total = http.getSize();  // returns Content-Length, or -1 if unknown/chunked
 
-        if (avail == 0) {
+    // NEW: fail fast if server claims something larger than our buffer
-            if (millis() - lastData > READ_TIMEOUT) {
+    if (total > 0 && (size_t)total > capacity) {
-                Serial.println("TIMEOUT waiting for data!");
+        Serial.println("Response exceeds READLIMIT.");
-                break;
+        aborting = true;
-            }
-            delay(1);
-            continue;
-        }
-
-        if (len + avail > capacity)
-            avail = capacity - len;
-
-        int read = stream->readBytes(buffer + len, avail);
-        len += read;
-        lastData = millis();
-
-        if (DEBUGING) {Serial.printf("Read chunk: %d (total: %d)\n", read, (int)len);}
-
-        if (len < 20) continue;  // Not enough data for header
-
-        int headerOffset = skipGzipHeader(buffer, len);
-        if (headerOffset < 0) continue;
-
-        unsigned long testLen = len * 8; // Dynamic expansion
-        uint8_t* test = (uint8_t*)malloc(testLen);
-
-        if (!test) continue;
-
-        unsigned long srcLen = len - headerOffset;
-
-        int res = puff(test, &testLen, buffer + headerOffset, &srcLen);
-        if (res == 0) {
-            if (DEBUGING) {Serial.printf("Decompress OK! Size: %lu bytes\n", testLen);}
-            outData = test;
-            outLen = testLen;
-            complete = true;
-            break;
-        }
-
-        free(test);
     }
 
-    // --- Added: Force-close connection in all cases to avoid stuck TCP sockets ---
+    // NEW: if not gzip, we will not try to decompress (prevents false "Decompress OK" / random success)
-    if (stream) stream->stop();
+    // You can either handle plain JSON here or just fail-fast.
+    if (!isGzip && !aborting) {
+        if (DEBUGING) {
+            Serial.println("Server response is NOT gzip (Content-Encoding != gzip).");
+            Serial.println("Either disable Accept-Encoding: gzip or add plain-body handling here.");
+        }
+
+        // --- Plain-body handling (recommended): read full body into outData as-is ---
+        // NEW: try to read Content-Length bytes if available (more robust)
+        if (total > 0 && (size_t)total > capacity) {
+            Serial.println("Plain response exceeds READLIMIT.");
+            aborting = true;
+        } else {
+            // Read until we have all bytes (Content-Length) or until connection closes + buffer drains
+            while ((http.connected() || (stream && stream->available())) && !aborting) {
+                size_t avail = stream ? stream->available() : 0;
+                if (avail == 0) {
+                    if (millis() - lastData > READ_TIMEOUT) {
+                        Serial.println("TIMEOUT waiting for data (plain)!");
+                        aborting = true;
+                        break;
+                    }
+                    delay(1);
+                    continue;
+                }
+
+                if (len >= capacity) {
+                    Serial.println("READLIMIT reached, aborting (plain).");
+                    aborting = true;
+                    break;
+                }
+
+                if (len + avail > capacity)
+                    avail = capacity - len;
+
+                int read = stream->readBytes(buffer + len, avail);
+                if (read > 0) {
+                    len += (size_t)read;
+                    lastData = millis();
+                }
+
+                // NEW: stop reading as soon as we have the full response
+                if (total > 0 && (int)len >= total) {
+                    break; // we got full body
+                }
+            }
+        }
+
+        if (aborting) {
+            // --- Added: Force-close connection only if aborted to avoid TCP RST storms ---
+            if (stream) stream->stop();   // Force close TCP socket
+            http.end();
+            free(buffer);
+            return false;
+        }
+
+        // Return plain body to caller
+        outData = (uint8_t*)malloc(len);
+        if (!outData) {
+            Serial.println("Malloc failed outData (plain).");
+            // --- Added: Force-close connection only if aborted to avoid TCP RST storms ---
+            if (stream) stream->stop();   // Force close TCP socket
+            http.end();
+            free(buffer);
+            return false;
+        }
+        memcpy(outData, buffer, len);
+        outLen = len;
+
+        http.end();
+        free(buffer);
+        return true;
+    }
+
+    // --- GZIP path (only if Content-Encoding is gzip) ---
+    if (!aborting) {
+
+        // NEW: read exactly Content-Length bytes when available (prevents partial-body timeout loops)
+        while ((http.connected() || (stream && stream->available())) && !complete && !aborting) {
+
+            size_t avail = stream ? stream->available() : 0;
+
+            if (avail == 0) {
+                // NEW: if Content-Length is known and we already read it all, stop immediately
+                if (total > 0 && (int)len >= total) {
+                    break;
+                }
+
+                if (millis() - lastData > READ_TIMEOUT) {
+                    Serial.println("TIMEOUT waiting for data!");
+                    aborting = true;   // NEW: mark abnormal exit
+                    break;
+                }
+                delay(1);
+                continue;
+            }
+
+            // NEW: safety check if buffer limit is reached
+            if (len >= capacity) {
+                Serial.println("READLIMIT reached, aborting.");
+                aborting = true;
+                break;
+            }
+
+            // NEW: if Content-Length is known, do not read beyond it
+            if (total > 0) {
+                size_t remaining = (size_t)total - len;
+                if (avail > remaining) avail = remaining;
+            }
+
+            if (len + avail > capacity)
+                avail = capacity - len;
+
+            int read = stream->readBytes(buffer + len, avail);
+            if (read <= 0) {
+                // NEW: avoid tight loop if read returns zero
+                delay(1);
+                continue;
+            }
+
+            len += (size_t)read;
+            lastData = millis();
+
+            if (DEBUGING) {Serial.printf("Read chunk: %d (total: %d)\n", read, (int)len);}
+
+            // NEW: if Content-Length is known and fully received, we can stop reading
+            if (total > 0 && (int)len >= total) {
+                break;
+            }
+        }
+
+        // NEW: only attempt gzip parse/decompress after we have a complete body (when Content-Length is known)
+        // This avoids wasting heap with repeated malloc/free and reduces fragmentation over long runtimes.
+        if (!aborting) {
+            if (len < 20) {
+                aborting = true;
+            } else {
+                int headerOffset = skipGzipHeader(buffer, len);
+                if (headerOffset < 0) {
+                    aborting = true;
+                } else {
+                    unsigned long testLen = len * 8; // Dynamic expansion
+                    uint8_t* test = (uint8_t*)malloc(testLen);
+
+                    if (!test) {
+                        Serial.println("Malloc failed test buffer, aborting.");
+                        aborting = true;
+                    } else {
+                        unsigned long srcLen = (unsigned long)(len - (size_t)headerOffset);
+                        int res = puff(test, &testLen, buffer + headerOffset, &srcLen);
+
+                        if (res == 0) {
+                            if (DEBUGING) {Serial.printf("Decompress OK! Size: %lu bytes\n", testLen);}
+                            outData = test;
+                            outLen = (size_t)testLen;
+                            complete = true;
+                        } else {
+                            free(test);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    // --- Added: Force-close connection only if aborted to avoid TCP RST storms ---
+    if (aborting && stream) stream->stop();   // NEW: stop() only on abnormal termination
 
     http.end();
     free(buffer);

lib/obp60task/NetworkClient.h

@@ -3,7 +3,7 @@
 #include <WiFi.h>
 #include <HTTPClient.h>
 
-#define DEBUGING false             // Debug flag for NetworkClient for more live information
+#define DEBUGING true             // Debug flag for NetworkClient for more live information
 #define READLIMIT 200000        // HTTP read limit in byte for gzip content (can be adjusted)
 #define CONNECTIONTIMEOUT 3000  // Timeout in ms for HTTP connection 
 #define TCPREADTIMEOUT 2000     // Timeout in ms for read HTTP client stack

lib/obp60task/PageNavigation.cpp

@@ -374,8 +374,21 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map
         getdisplay().setPartialWindow(0, 0, getdisplay().width(), getdisplay().height()); // Set partial update
         getdisplay().setTextColor(commonData->fgcolor);
 
+        // NEW: simple exponential backoff for 1 Hz polling (prevents connection-refused storms)
+        static uint32_t nextAllowedMs = 0;
+        static uint8_t  failCount = 0;
+
+        uint32_t now = millis();
+
+        // NEW: if we are in backoff window, skip network call and use backup immediately
+        bool allowFetch = ((int32_t)(now - nextAllowedMs) >= 0);
+        
         // If a network connection to URL then load the navigation map
-        if (net.fetchAndDecompressJson(url)) {
+        if (allowFetch && net.fetchAndDecompressJson(url)) {
+
+            // NEW: reset backoff on success
+            failCount = 0;
+            nextAllowedMs = now + 1000; // keep 1 Hz on success
 
             auto& json = net.json();                // Extract JSON content
             int numPix = json["number_pixels"] | 0; // Read number of pixels
@@ -397,7 +410,7 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map
             size_t imgSize = numPix;    // Calculate image size
             uint8_t* imageData = (uint8_t*) heap_caps_malloc(imgSize, MALLOC_CAP_SPIRAM);           // Allocate PSRAM for image
             if (!imageData) {
-                LOG_DEBUG(GwLog::ERROR,"Error PageNavigation: PPSRAM alloc image buffer failed");
+                LOG_DEBUG(GwLog::ERROR,"Error PageNavigation: PSRAM alloc image buffer failed");
                 free(b64);
                 return PAGE_UPDATE;
             }
@@ -426,12 +439,25 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map
         }
         // If no network connection then use backup navigation map
         else{
+
+            // NEW: update backoff only if we actually attempted a fetch (not when skipping due to backoff)
+            if (allowFetch) {
+                // NEW: exponential backoff: 1s,2s,4s,8s,16s,30s (capped)
+                if (failCount < 6) failCount++;
+                uint32_t backoffMs = 1000u << failCount;
+                if (backoffMs > 30000u) backoffMs = 30000u;
+                nextAllowedMs = now + backoffMs;
+            } else {
+                // NEW: we are currently backing off; do not increase failCount further
+                // nextAllowedMs stays unchanged
+            }
+
             // Show backup image (backup navigation map)
             if (hasImageBackup) {
                 getdisplay().drawBitmap(0, 25, imageBackupData, imageBackupWidth, imageBackupHeight, commonData->fgcolor);
             }    
 
-            // Show info: Connection lost when 5 page refreshes has a connection lost to the map server
+            // Show connection lost info when 5 page refreshes has a connection lost to the map server
             // Short connection losts are uncritical
             if(lostCounter >= 5){
                 getdisplay().setFont(&Ubuntu_Bold12pt8b);
@@ -444,7 +470,6 @@ bool showValues = false; // Show values HDT, SOG, DBT in navigation map
             lostCounter++; // Increment lost counter
         }
 
-
         // ############### Draw Values ################
         getdisplay().setFont(&Ubuntu_Bold12pt8b);
 

platformio.ini

@@ -21,8 +21,8 @@ lib_deps =
     ttlappalainen_NMEA2000=https://github.com/wellenvogel/NMEA2000.git#20251126
 	ttlappalainen/NMEA0183 @ 1.10.1
 	ArduinoJson @ 6.18.5
-	AsyncTCP-esphome @ 2.0.1
+	AsyncTCP-esphome @ 2.1.1
-	ottowinter/ESPAsyncWebServer-esphome@2.0.1
+	ottowinter/ESPAsyncWebServer-esphome@3.4.0
 	FS
 	Preferences
 	ESPmDNS
@@ -34,8 +34,8 @@ lib_deps=
 	ttlappalainen_NMEA2000=symlink://../NMEA2000
 	ttlappalainen/NMEA0183 @ 1.10.1
 	ArduinoJson @ 6.18.5
-	AsyncTCP-esphome @ 2.0.1
+	AsyncTCP-esphome @ 2.1.1
-	ottowinter/ESPAsyncWebServer-esphome@2.0.1
+	ottowinter/ESPAsyncWebServer-esphome@3.4.0
 	FS
 	Preferences
 	ESPmDNS