Prepare web server with embedded files

.gitignore

@@ -0,0 +1,2 @@
+*~
+.pio

README

@@ -0,0 +1,148 @@
+OBP Keypad 
+==========
+
+- Stromversorgung über M12-Anschluß über NMEA2000
+  Eingangsbereich 6~21V
+- Ein- und Ausschalten durch langen Tastendruck auf DST/ ONOFF
+- Konfiguration über Web-Interface
+- PWR leuchtet grün wen NMEA2000-Verbindung etabliert
+- PWR leuchtet rot wenn nur Stromversorgung aktiv ist
+
+Optionen
+- I²C Temp/Hum-Sensor
+- Seatalk1 Anschluß für Fernbedienung Raymarine Pinnenpilot
+- EPaper-Display 2.9" zur Anzeige der Tastenbelegung
+
+Damit die LEDs nicht stören, kann umgeschaltet werden zwischen
+permanentem Leuchten und nur kurzem Aufblinken bei Betätigung.
+
+
+Bohrung Taster: 12mm
+Taster Außenmaß: 17.5mm
+
+Verbindungskabel CPU-Platine 
+  JST 2.54 XH 6 Pin Steckverbinder -> LED
+
+Anschlußmöglichkeiten
+
+  für Stromversorgung +12V und NMEA2000
+    4pin Terminalblock steck-/schraubbar +12V, DNG, CAN-L, CAN-H
+
+  für I²C-Module
+    2x 4pin Buchsenleiste weibl. 
+    1x QWIIC-Buchse (JST_SH_BM04B-SRSS-TB_04x1.00mm)
+
+  für mechanische Taster
+    1x JST 2.54 XH 7 Pin Steckverbinder -> Tasten
+    Masseverbindung über einzelnes getrenntes Kabel
+
+  für LEDs
+
+
+Bemerkungen
+-----------
+
+Bei den aktuell verwendeten Tasten sind die Anschlußdrähte extrem
+filigran. Leichtes Brechen und schlechte Verarbeitung.
+
+Beschaltung MCU Nano
+--------------------
+
+Wiki: https://www.waveshare.com/wiki/ESP32-S3-Nano
+
+Der Nano hat 30 Pins.
+
+Stromversorgung über VIN. Lt. Spezifikation können dort 6 bis 21V
+anliegen. Vmtl. ist ein Betrieb mit 5V auch möglich.
+Den 3.3V-Pin nicht benutzen. Dieser ist als Ausgang gedacht! 
+
+Das Mapping von Nano-Pin zu GPIO muß noch überprüft werden.
+Der nano kann in zwei verschiedenen Mapping-Modi betrieben
+werden!
+
+Die Pins für i²C (A4, A5)und SPI (D11, D12, D13) sind absichtlich
+nicht belegt um frei für Erweiterungen zu sein. An SPI kann
+ggf. ein Epaper angeschlossen werden.
+
+
+ Key   Color   Pin      Remarks 
+----- ------- -------- --------------------
+ 1      B       D2      GPIO5
+ 2      B       D3      GPIO6
+ 3      B       D4      GPIO7
+ 4      B       D5      GPIO8
+ 5      B       D6      GPIO9
+ 6      Y       D7      GPIO10 Illumination
+ DST    Y       D8      GPIO17 Destination, On/Off
+
+ LED    Pin
+------ ----------
+ A      A0  GPIO1 
+ B      A1  GPIO2
+ C      A2  GPIO3
+(D      A3 reserved for future)
+ RGBA   A6  GPIO4
+ RGBB   A7  GPIO13
+ RGBC   B1  GPIO14
+
+ CAN    Pin
+------ ----------
+ RX     D9
+ TX     D10
+
+
+Beschaltung MCU Pico !!! Nicht fertig / ungültig !!!
+--------------------
+
+Stromversorgung über VSYS mit 5V.
+
+
+ Key   Color   Pin      Remarks 
+----- ------- -------- --------------------
+ 1      B       GP11
+ 2      B       GP12
+ 3      B       GP13
+ 4      B       GP14
+ 5      B       GP15
+ 6      Y       GP16     Illumination
+ DST    Y       GP17     Destination, On/Off
+
+ LED    Pin
+------ ----------
+ A      GP1
+ B      GP2
+ C      GP4
+ RGBA   GP5
+ RGBB   GP6
+ RGBC   GP7
+
+ CAN    Pin
+------ ----------
+ RX     GP9
+ TX     GP10
+
+Bauteilliste
+------------
+
+  1x ESP32-S3 Nano oder ESP32-S3 Pico
+  5x Taster schwarz
+  2x Taster gelb
+  1x M12 Einbaubuchse
+  1x Spannungswandler 12V -> 3.3V
+  1x RGB LED (gemeinsame Anode)
+  3x LED grün
+  1x SN65HVD230 CAN Transceiver
+  1x Gehäuse 150x60x40
+  4x Befestigungsschraube M4
+  1x Kabelsatz
+
+
+
+Konfiguration
+-------------
+
+- Instanz-Nummer, es können mehrere Keypads im System sein
+- Namen des gekoppelten Geräts, an dieses werden die Tasten gesendet
+- Tastencodes Tasten 1 bis 6
+- Tastennamen
+- Web-AP

boards/esp32_s3_nano.json

@@ -0,0 +1,55 @@
+{
+  "build": {
+    "arduino":{
+      "ldscript": "esp32s3_out.ld",
+      "partitions": "default_16MB.csv",
+      "memory_type": "qio_opi"
+    },
+    "core": "esp32",
+    "extra_flags": [
+      "-DBOARD_HAS_PSRAM",
+      "-DARDUINO_ESP32S3_DEV",
+      "-DARDUINO_USB_MODE=1",
+      "-DARDUINO_USB_CDC_ON_BOOT=1",
+      "-DARDUINO_RUNNING_CORE=1",
+      "-DARDUINO_EVENT_RUNNING_CORE=1"
+    ],
+    "f_cpu": "240000000L",
+    "f_flash": "80000000L",
+    "flash_mode": "qio",
+    "hwids": [
+      [
+        "0x303A",
+        "0x1001"
+      ]
+    ],
+    "mcu": "esp32s3",
+    "variant": "esp32s3"
+  },
+  "connectivity": [
+    "bluetooth",
+    "wifi"
+  ],
+  "debug": {
+    "default_tool": "esp-builtin",
+    "onboard_tools": [
+      "esp-builtin"
+    ],
+    "openocd_target": "esp32s3.cfg"
+  },
+  "frameworks": [
+    "arduino",
+    "espidf"
+  ],
+  "name": "OBPkb61 ESP32-S3-N16R8 16 MB QD, 8 MB PSRAM)",
+  "upload": {
+    "flash_size": "16MB",
+    "maximum_ram_size": 327680,
+    "maximum_size": 16777216,
+    "wait_for_upload_port": true,
+    "require_upload_port": true,
+    "speed": 460800
+  },
+  "url": "https://computerclub.hoogi.de/obpkeypad/",
+  "vendor": "Open Boat Projects"
+}

extra_pre.py

@@ -0,0 +1,164 @@
+print("running extra script...")
+
+import os
+import sys
+import inspect
+import gzip
+import shutil
+
+Import("env")
+
+GEN_DIR = 'lib/generated'
+OWN_FILE = "extra_script.py"
+
+INDEXJS = "index.js"
+INDEXCSS = "index.css"
+
+# embedded files definition is generated inside here
+EMBEDDED_INCLUDE="embeddedfiles.h"
+
+def basePath():
+    #see: https://stackoverflow.com/questions/16771894/python-nameerror-global-name-file-is-not-defined
+    return os.path.dirname(inspect.getfile(lambda: None))
+
+def is_current(infile, outfile):
+    if os.path.exists(outfile):
+        otime = os.path.getmtime(outfile)
+        itime = os.path.getmtime(infile)
+        if (otime >= itime):
+            own = os.path.join(basePath(), OWN_FILE)
+            if os.path.exists(own):
+                owntime = os.path.getmtime(own)
+                if owntime > otime:
+                    return False
+            print(f"{outfile} is newer than {infile}, no need to recreate")
+            return True
+    return False
+
+def compress_file(infile, outfile):
+    if is_current(infile, outfile):
+        return
+    print(f"compressing {infile}")
+    with open(infile, 'rb') as f_in:
+        with gzip.open(outfile, 'wb') as f_out:
+            shutil.copyfileobj(f_in, f_out)
+
+def write_file_if_changed(filename, data):
+    """
+    Create or update file if it not contains data as content
+    """
+    if os.path.exists(filename):
+        changetype = 'updating'
+        with open(filename, 'r') as fh:
+            if fh.read() == data:
+                print(f"{filename} is up to date")
+                return False
+    else:
+        changetype = 'generating'
+    print(f"#{changetype} {filename}")
+    with open(filename, 'w') as fh:
+        fh.write(data)
+    return True
+
+def get_embedded_files(env):
+    filelist = []
+    efiles = env.GetProjectOption("board_build.embed_files")
+    for f in efiles.split("\n"):
+        if f == '':
+            continue
+        filelist.append(f)
+    return filelist  
+
+def get_content_type(fn):
+    if (fn.endswith('.gz')):
+        fn = fn[0:-3]
+    if (fn.endswith('html')):
+        return "text/html"
+    if (fn.endswith('json')):
+        return "application/json"
+    if (fn.endswith('js')):
+        return "text/javascript"
+    if (fn.endswith('css')):
+        return "text/css"
+    if (fn.endswith('png')):
+        return "image/png"
+    if (fn.endswith('jpg') or fn.endswith('jpeg')):
+        return "image/jpeg"
+    return "application/octet-stream"
+
+def join_files(target, filename, dirlist):
+    """
+    Join files named filename within different directories
+    into one single gzip archive (located in generated dir)
+    """
+    print("joinfiles: {} into {} from dirs [{}]".format(filename, target, ','.join(dirlist)))
+    flist = []
+    for dir in dirlist:
+        fn = os.path.join(dir, filename)
+        if os.path.exists(fn):
+            flist.append(fn)
+    current = False
+    if os.path.exists(target):
+        current = True
+        for f in flist:
+            if not is_current(f, target):
+                current = False
+                break
+    if current:
+        print(f"{target} is up to date")
+        return
+    print(f"creating {target}")
+
+    # add multiple files
+    with gzip.open(target, "wb") as oh:
+        for fn in flist:
+            print("adding {} to {}".format(fn, target))
+            with open(fn, "rb") as rh:
+                shutil.copyfileobj(rh, oh)
+
+def prebuild(env):
+    print("#prebuild running")
+
+    # directory for dynamically generated files
+    gendir = os.path.join(basePath(), GEN_DIR)
+    if not os.path.exists(gendir):
+        os.makedirs(gendir)
+    if not os.path.isdir(gendir):
+        print("unable to create directory {}".format(gendir))
+        sys.exit(1)
+
+    # join static web server files
+
+    # only useful for different task dirs with custom configs
+    # join_files(os.path.join(gendir, INDEXJS+".gz"), INDEXJS, ["web"])
+    # join_files(os.path.join(gendir, INDEXCSS+".gz"), INDEXCSS, ["web"])
+
+    # platformio defined embedded files as list
+    pio_embedded = get_embedded_files(env)
+
+    filedefs = []
+    for ef in pio_embedded:
+        print(f"#checking embedded file {ef}")
+        # files are defined with relative path to platformio.ini
+        # the name can be with extension gz or without
+        (dn, fn) = os.path.split(ef)
+        pureName = fn
+        if pureName.endswith('.gz'):
+            pureName = pureName[0:-3]
+        usname = ef.replace('/','_').replace('.','_') # sanitize filenames
+        filedefs.append((pureName, usname, get_content_type(pureName)))
+        infile = os.path.join(basePath(), "web", pureName)
+        if os.path.exists(infile):
+            compress_file(infile, ef)
+        else:
+            print("#WARNING: infile {infile} for {ef} not found")
+
+    # generate c-header file for embedding zip files
+    content = ""
+    for entry in filedefs:
+        content += 'EMBED_GZ_FILE("{}", {}, "{}");\n'.format(*entry)
+    write_file_if_changed(os.path.join(gendir, EMBEDDED_INCLUDE), content)  
+
+print("#prescript...")
+prebuild(env)
+

include/hardware.h

@@ -1,34 +0,0 @@
-// General hardware definitions
-#pragma once
-
-// Keys
-#define KEY_1 1         // Key or touchpad
-#define KEY_2 2         // Key or touchpad
-
-// I2S audio output -> PCM5102 DAC
-#define I2S_DOUT 4      // data out
-#define I2S_BCLK 5      // bit clock	
-#define I2S_LRC 6       // left right channel select
-#define I2S_XSMT 7      // PCM5102 soft mute
-
-// SPI for SD-Card; VSPI pins for higher performance
-#define SD_CS 10
-#define SD_MOSI 11
-#define SD_MISO 13
-#define SD_SCK 12
-
-// I2C for control interface
-#define I2C_SDA 8
-#define I2C_CLK 9
-
-// CAN bus for NMEA2000 connection
-#define CAN_RX 47
-#define CAN_TX 48
-
-// RS485 for NMEA0183 connection / UART1
-#define SER_RX 18
-#define SER_TX 17
-
-// I2C Addresses
-// Address of DAC module
-// Address of switchbank

include/main.h

@@ -1,4 +1,4 @@
-#pragma
+#pragma once
 
 // WIFI AP
 #define WIFI_CHANNEL 9

platformio.ini

@@ -15,6 +15,13 @@ lib_deps =
     robtillaart/SHT31@^0.5.2
 #   adafruit/Adafruit NeoPixel
 
+# only these files will be emedded into firmware
+board_build.embed_files = 
+	lib/generated/index.html.gz
+	lib/generated/index.js.gz
+	lib/generated/index.css.gz
+	lib/generated/config.json.gz
+
 extra_scripts =
     pre:extra_pre.py
     post:extra_post.py
@@ -27,8 +34,9 @@ build_flags =
 
 [env:esp32-s3-nano]
 build_type = release # debug | release
-#board = esp32-s3-devkitc-1
+# board = esp32-s3-devkitc-1
-board = arduino_nano_esp32
+board = esp32_s3_nano
+#board = arduino_nano_esp32 # ATTENTION! Pin numbering scheme changes
 board_upload.flash_size = 16MB
 board_build.partitions = default.csv
 upload_port = /dev/ttyACM0

src/main.cpp

@@ -10,25 +10,47 @@
 #include <N2kMessages.h>
 #include "main.h"
 #include "Nmea2kTwai.h"
+#include <map>
 
 Preferences preferences; // persistent storage for configuration
 
+class EmbeddedFile;
+static std::map<String, EmbeddedFile*> embeddedFiles;
+class EmbeddedFile {
+  public:
+    const uint8_t *start;
+    int len;
+    String contentType;
+    EmbeddedFile(String name, String contentType, const uint8_t *start, int len) {
+      this->start = start;
+      this->len = len;
+      this->contentType = contentType;
+      embeddedFiles[name] = this;
+    }
+} ;
+
+#define EMBED_GZ_FILE(fileName, binName, contentType) \
+  extern const uint8_t  binName##_File[] asm("_binary_" #binName "_start"); \
+  extern const uint8_t  binName##_FileLen[] asm("_binary_" #binName "_size"); \
+  const EmbeddedFile binName##_Config(fileName,contentType,(const uint8_t*)binName##_File,(int)binName##_FileLen);
+#include "embeddedfiles.h"
+
+void send_embedded_file(String name, AsyncWebServerRequest *request)
+{
+    std::map<String, EmbeddedFile*>::iterator it = embeddedFiles.find(name);
+    if (it != embeddedFiles.end()) {
+        EmbeddedFile* found = it->second;
+        AsyncWebServerResponse *response = request->beginResponse(200, found->contentType, found->start, found->len);
+        response->addHeader(F("Content-Encoding"), F("gzip"));
+        request->send(response);
+    } else {
+        request->send(404, "text/plain", "Not found");
+    }
+}
+
 const char* wifi_ssid = "OBPKP61";
 const char* wifi_pass = "keypad61";
 AsyncWebServer server(80);
-const char index_html[] PROGMEM = R"rawliteral(
-<!DOCTYPE HTML><html>
-<head>
-  <title>ESP Web Server</title>
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-  <link rel="icon" href="data:,">
-</head>
-<body>
-  <h2>ESP Web Server</h2>
-  <p>Work in progress</p>
-</body>
-</html>
-)rawliteral";
 
 unsigned long lastPrint = 0;
 unsigned long counter = 0;
@@ -110,20 +132,29 @@ void setup() {
     WiFi.persistent(false);
     WiFi.mode(WIFI_MODE_AP);
 
-    IPAddress ap_addr(192, 168, 15, 1);
-    IPAddress ap_subnet(255, 255, 255, 0);
-    IPAddress ap_gateway(ap_addr);
-
     int channel = WIFI_CHANNEL;
     bool hidden = false;
     WiFi.softAP(wifi_ssid, wifi_pass, channel, hidden, WIFI_MAX_STA);
 
+    IPAddress ap_addr(192, 168, 15, 1);
+    IPAddress ap_subnet(255, 255, 255, 0);
+    IPAddress ap_gateway(ap_addr);
+    WiFi.softAPConfig(ap_addr, ap_gateway, ap_subnet);
+ 
     // Route for root / web page
     server.on("/", HTTP_GET, [](AsyncWebServerRequest *request) {
-            request->send(200, "text/html", index_html, processor);
+        send_embedded_file("index.html", request);
+    });
+    // Route for all other defined pages
+    for (auto it = embeddedFiles.begin(); it != embeddedFiles.end(); it++) {
+        String uri = String("/") + it->first;
+        server.on(uri.c_str(), HTTP_GET, [it](AsyncWebServerRequest *request) {
+            send_embedded_file(it->first, request);
         });
+    }
     server.begin();
 
+
     NMEA2000.SetProductInformation("00000001",          // Manufacturer's Model serial code
                                  74,                    // Manufacturer's product code
                                  "OBPkeypad6/1",        // Manufacturer's Model ID
@@ -284,7 +315,6 @@ void loop() {
         delay(200);
     }
 
-
     // ---- PRINT NUMBER EVERY SECOND ----
     if (millis() - lastPrint >= 1000) {
         lastPrint = millis();

web/config.json

@@ -0,0 +1,20 @@
+[
+{
+    "name": "systemName",
+    "label": "system name",
+    "type": "string",
+    "default": "OBPkeypad61",
+    "check": "checkSystemName",
+    "description": "system name, used for the access point and for services",
+    "category": "system"
+},
+{
+    "name": "apPassword",
+    "type": "password",
+    "default": "keypad61",
+    "check": "checkApPass",
+    "description": "set the password for the Wifi access point",
+    "category": "system",
+    "capabilities":{"apPwChange":["true"]}
+}
+]

web/index.html

@@ -0,0 +1,17 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+<title>OBPkeypad 6/1</title>
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<link rel="icon" href="data:,">
+<script type="text/javascript" src="index.js"></script>
+<link rel="stylesheet" href="index.css">
+</head>
+<body>
+<div class="main">
+<h1>OBPkeypad 6/1</h1>
+<p>Work in progress</p>
+<p><a href="https://computerclub.hoogi.de/obpkeypad/">Furter Information</a></p>
+</div>
+</body>
+</html>