diff --git a/lib/obp60task/NetworkClient.cpp b/lib/obp60task/NetworkClient.cpp index c6a7962..4d091f5 100644 --- a/lib/obp60task/NetworkClient.cpp +++ b/lib/obp60task/NetworkClient.cpp @@ -51,14 +51,16 @@ int NetworkClient::skipGzipHeader(const uint8_t* data, size_t len) { // HTTP GET + GZIP Decompression (reading in chunks) bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& outLen) { - const size_t capacity = READLIMIT; // Read limit for data (can be adjusted in NetworkClient.h) + const size_t capacity = READLIMIT; // Read limit for data (can be adjusted in NetworkClient.h) uint8_t* buffer = (uint8_t*)malloc(capacity); + // If not with WiFi connectetd then return without any activities if (!gwWifi.clientConnected()) { if (DEBUGING) {Serial.println("No WiFi connection");} return false; } + // If frame buffer not correct allocated then return without any activities if (!buffer) { if (DEBUGING) {Serial.println("Malloc failed buffer");} return false; @@ -71,20 +73,51 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou http.setTimeout(TCPREADTIMEOUT); // Read timeout in ms (can be adjusted in NetworkClient.h) http.begin(url); + + // NEW: force server to close the connection after the response (prevents "stuck" keep-alive reads) + http.addHeader("Connection", "close"); + + // NEW: request gzip, but we will only decompress if the server actually answers with gzip http.addHeader("Accept-Encoding", "gzip"); + // NEW: register headers BEFORE GET() (more reliable with Arduino HTTPClient) + if (DEBUGING) { + // We need follow key words + const char* keys[] = { + "Content-Encoding", + "Transfer-Encoding", + "Content-Length" + }; + // Read header + http.collectHeaders(keys, 3); + } + int code = http.GET(); if (code != HTTP_CODE_OK) { - Serial.printf("HTTP ERROR: %d\n", code); + Serial.printf("HTTP Client ERROR: %d (%s)\n", code, http.errorToString(code).c_str()); // Hard reset HTTP + socket WiFiClient* tmp = http.getStreamPtr(); if (tmp) tmp->stop(); // Force close TCP socket + http.end(); - free(buffer); return false; } + else{ + + if (DEBUGING) { + String ce = http.header("Content-Encoding"); + String te = http.header("Transfer-Encoding"); + String cl = http.header("Content-Length"); + + // Print header informations + Serial.printf("Content-Encoding=%s Transfer-Encoding=%s Content-Length=%s\n", + ce.c_str(), + te.c_str(), + cl.c_str()); + } + } WiFiClient* stream = http.getStreamPtr(); @@ -93,25 +126,120 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou const uint32_t READ_TIMEOUT = READDATATIMEOUT; // Timeout for reading data (can be adjusted in NetworkClient.h) bool complete = false; + bool aborting = false; // NEW: remember if we must force-close socket - while (http.connected() && !complete) { + // NEW: detect if server really sent gzip + String ce = http.header("Content-Encoding"); + bool isGzip = ce.equalsIgnoreCase("gzip"); - size_t avail = stream->available(); + // NEW: if not gzip, we will not try to decompress (prevents false "Decompress OK" / random success) + // You can either handle plain JSON here or just fail-fast. + if (!isGzip) { + if (DEBUGING) { + Serial.println("Server response is NOT gzip (Content-Encoding != gzip)."); + Serial.println("Either disable Accept-Encoding: gzip or add plain-body handling here."); + } + + // --- Plain-body handling (recommended): read full body into outData as-is --- + // NEW: try to read Content-Length bytes if available (more robust) + int total = http.getSize(); // returns Content-Length, or -1 if unknown/chunked + if (total > 0 && (size_t)total > capacity) { + Serial.println("Plain response exceeds READLIMIT."); + aborting = true; + } else { + // Read until we have all bytes (Content-Length) or until connection closes + buffer drains + while ((http.connected() || (stream && stream->available())) && !aborting) { + size_t avail = stream ? stream->available() : 0; + if (avail == 0) { + if (millis() - lastData > READ_TIMEOUT) { + Serial.println("TIMEOUT waiting for data (plain)!"); + aborting = true; + break; + } + delay(1); + continue; + } + + if (len >= capacity) { + Serial.println("READLIMIT reached, aborting (plain)."); + aborting = true; + break; + } + + if (len + avail > capacity) + avail = capacity - len; + + int read = stream->readBytes(buffer + len, avail); + if (read > 0) { + len += (size_t)read; + lastData = millis(); + } + + if (total > 0 && (int)len >= total) { + break; // we got full body + } + } + } + + if (aborting) { + // --- Added: Force-close connection only if aborted to avoid TCP RST storms --- + if (stream) stream->stop(); // Force close TCP socket + http.end(); + free(buffer); + return false; + } + + // Return plain body to caller + outData = (uint8_t*)malloc(len); + if (!outData) { + Serial.println("Malloc failed outData (plain)."); + // --- Added: Force-close connection only if aborted to avoid TCP RST storms --- + if (stream) stream->stop(); // Force close TCP socket + http.end(); + free(buffer); + return false; + } + memcpy(outData, buffer, len); + outLen = len; + + http.end(); + free(buffer); + return true; + } + + // --- GZIP path (only if Content-Encoding is gzip) --- + while ((http.connected() || (stream && stream->available())) && !complete) { + + size_t avail = stream ? stream->available() : 0; if (avail == 0) { if (millis() - lastData > READ_TIMEOUT) { Serial.println("TIMEOUT waiting for data!"); + aborting = true; // NEW: mark abnormal exit break; } delay(1); continue; } + // NEW: safety check if buffer limit is reached + if (len >= capacity) { + Serial.println("READLIMIT reached, aborting."); + aborting = true; + break; + } + if (len + avail > capacity) avail = capacity - len; int read = stream->readBytes(buffer + len, avail); - len += read; + if (read <= 0) { + // NEW: avoid tight loop if read returns zero + delay(1); + continue; + } + + len += (size_t)read; lastData = millis(); if (DEBUGING) {Serial.printf("Read chunk: %d (total: %d)\n", read, (int)len);} @@ -124,7 +252,12 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou unsigned long testLen = len * 8; // Dynamic expansion uint8_t* test = (uint8_t*)malloc(testLen); - if (!test) continue; + if (!test) { + // NEW: abort if allocation fails to prevent endless retry loop + Serial.println("Malloc failed test buffer, aborting."); + aborting = true; + break; + } unsigned long srcLen = len - headerOffset; @@ -132,7 +265,7 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou if (res == 0) { if (DEBUGING) {Serial.printf("Decompress OK! Size: %lu bytes\n", testLen);} outData = test; - outLen = testLen; + outLen = (size_t)testLen; complete = true; break; } @@ -140,8 +273,8 @@ bool NetworkClient::httpGetGzip(const String& url, uint8_t*& outData, size_t& ou free(test); } - // --- Added: Force-close connection in all cases to avoid stuck TCP sockets --- - if (stream) stream->stop(); + // --- Added: Force-close connection only if aborted to avoid TCP RST storms --- + if (aborting && stream) stream->stop(); // NEW: stop() only on abnormal termination http.end(); free(buffer); diff --git a/lib/obp60task/NetworkClient.h b/lib/obp60task/NetworkClient.h index 03e7f83..5d3f448 100644 --- a/lib/obp60task/NetworkClient.h +++ b/lib/obp60task/NetworkClient.h @@ -3,7 +3,7 @@ #include #include -#define DEBUGING false // Debug flag for NetworkClient for more live information +#define DEBUGING true // Debug flag for NetworkClient for more live information #define READLIMIT 200000 // HTTP read limit in byte for gzip content (can be adjusted) #define CONNECTIONTIMEOUT 3000 // Timeout in ms for HTTP connection #define TCPREADTIMEOUT 2000 // Timeout in ms for read HTTP client stack diff --git a/platformio.ini b/platformio.ini index 341083c..c32a227 100644 --- a/platformio.ini +++ b/platformio.ini @@ -21,8 +21,10 @@ lib_deps = ttlappalainen_NMEA2000=https://github.com/wellenvogel/NMEA2000.git#20251126 ttlappalainen/NMEA0183 @ 1.10.1 ArduinoJson @ 6.18.5 - AsyncTCP-esphome @ 2.0.1 - ottowinter/ESPAsyncWebServer-esphome@2.0.1 +; AsyncTCP-esphome @ 2.0.1 +; ottowinter/ESPAsyncWebServer-esphome@2.0.1 + AsyncTCP-esphome @ 2.1.1 + ottowinter/ESPAsyncWebServer-esphome@3.4.0 FS Preferences ESPmDNS @@ -34,8 +36,10 @@ lib_deps= ttlappalainen_NMEA2000=symlink://../NMEA2000 ttlappalainen/NMEA0183 @ 1.10.1 ArduinoJson @ 6.18.5 - AsyncTCP-esphome @ 2.0.1 - ottowinter/ESPAsyncWebServer-esphome@2.0.1 +; AsyncTCP-esphome @ 2.0.1 +; ottowinter/ESPAsyncWebServer-esphome@2.0.1 + AsyncTCP-esphome @ 2.1.1 + ottowinter/ESPAsyncWebServer-esphome@3.4.0 FS Preferences ESPmDNS