Improve build and integrate firmware versioning

extra_post.py

@@ -0,0 +1,102 @@
+Import("env", "projenv")
+import os
+import glob
+
+print("##post script running")
+
+def getString(buffer, offset, length):
+    return buffer[offset:offset+length].rstrip(b'\0').decode('utf-8')
+
+def getFirmwareInfo(fwfile):
+    """
+    ESP-IDF firmware descriptor
+
+    typedef struct {
+        uint32_t magic_word;
+        uint32_t secure_version;
+        uint32_t reserv1;
+        uint32_t reserv2;
+        char version[32];
+        char project_name[32];
+        char time[16];
+        char date[16];
+        char idf_ver[32];
+        uint8_t app_elf_sha256[32];
+    } esp_app_desc_t
+
+    """
+    print(f"checking firmware {fwfile}")
+    with open(fwfile, "rb") as fh:
+        MAGIC = b"\x32\x54\xcd\xab" # little endian
+        buffer = fh.read(1024)
+        idx = buffer.find(MAGIC, 288) # find MAGIC
+        if idx < 0:
+            raise Exception("no app descriptor found")
+        else:
+            print("found app descriptor at {}".format(idx))
+        version = getString(buffer, idx + 16, 32)
+        name = getString(buffer, idx + 48, 32)
+        fwtime = getString(buffer, idx + 80, 16)
+        fwdate = getString(buffer, idx + 96, 16)
+        idf_ver = getString(buffer, idx + 112, 16)
+    print(f"  name: '{name}'")
+    print(f"  version: '{version}'")
+    print(f"  time: '{fwtime}'")
+    print(f"  date: '{fwdate}'")
+    print(f"  idf: '{idf_ver}'")
+    return (name, version)
+
+def postbuild(source, target, env):
+    print("##postbuild")
+    print("source={}".format(source[0]))
+    print("target={}".format(target[0]))
+    firmware = env.subst("$BUILD_DIR/${PROGNAME}.bin")
+    (fwname, fwversion) = getFirmwareInfo(firmware)
+
+    esptool = env.get('UPLOADER')
+    uploaderflags = env.subst("${UPLOADERFLAGS}")
+    base = env.subst("$PIOENV")
+    appoffset = env.subst("$ESP32_APP_OFFSET")
+    python = env.subst("$PYTHONEXE")
+
+    print("base=%s,esptool=%s,appoffset=%s,uploaderflags=%s"%(base,esptool,appoffset,uploaderflags))
+
+    uploadparts = uploaderflags.split(" ")
+
+    chip = "esp32"
+    if len(uploadparts) < 6:
+        print("uploaderflags does not have enough parameter")
+        return
+    for i in range(0, len(uploadparts)):
+        if uploadparts[i] == "--chip":
+            if i < (len(uploadparts) - 1):
+                chip = uploadparts[i+1]
+    uploadfiles = uploadparts[-6:]
+    for i in range(1, len(uploadfiles), 2):
+        if not os.path.isfile(uploadfiles[i]):
+            print("file %s for combine not found"%uploadfiles[i])
+            return
+    offset = uploadfiles[0]
+
+    outdir = env.subst("$BUILD_DIR")
+    for f in glob.glob(os.path.join(outdir, base + "*.bin")):
+        print("removing old file {}".format(f))
+        os.unlink(f)
+
+    outfile = os.path.join(outdir, "{}-all.bin".format((base)))
+    cmd = [python, esptool, "--chip", chip, "merge_bin", "--target-offset", offset, "-o", outfile]
+    cmd += uploadfiles
+    cmd += [appoffset, firmware]
+    print("running {}".format(' '.join(cmd)))
+    env.Execute(' '.join(cmd), "#merging bin files")
+
+    # Create symlinks to better identify firmware
+    fw_update = os.path.join(outdir, f"{base}-{fwversion}-update.bin")
+    os.symlink(firmware, fw_update)
+    fw_full = os.path.join(outdir, f"{base}-{fwversion}-all.bin")
+    os.symlink(outfile, fw_full)
+
+env.AddPostAction(
+    "$BUILD_DIR/${PROGNAME}.bin",
+    postbuild
+)

extra_pre.py

@@ -5,11 +5,12 @@ import sys
 import inspect
 import gzip
 import shutil
+from datetime import datetime
 
 Import("env")
 
 GEN_DIR = 'lib/generated'
-OWN_FILE = "extra_script.py"
+OWN_FILE = "extra_pre.py"
 
 INDEXJS = "index.js"
 INDEXCSS = "index.css"
@@ -118,6 +119,7 @@ def join_files(target, filename, dirlist):
 
 def prebuild(env):
     print("#prebuild running")
+    #versfmt = env.GetProjectOption('versionformat')
 
     # directory for dynamically generated files
     gendir = os.path.join(basePath(), GEN_DIR)
@@ -159,6 +161,29 @@ def prebuild(env):
         content += 'EMBED_GZ_FILE("{}", {}, "{}");\n'.format(*entry)
     write_file_if_changed(os.path.join(gendir, EMBEDDED_INCLUDE), content)  
 
-print("#prescript...")
+    # prepare version information
+    versfmt = env.GetProjectOption('custom_versionformat')
+    if versfmt == 'DEVELOP':
+        print("Develop version style")
+        version = "dev"+datetime.now().strftime("%Y%m%d")
+    else:
+        print("Production version style")
+        version = env.GetProjectOption('custom_version')
+
+    now = datetime.utcnow()
+    env.Append(
+        CPPDEFINES=[
+            ('FWVERSION', version),
+            ("FWBUILDDATE", f"{now:%Y-%m-%d}"),
+            ("FWBUILDTIME", f"{now:%H:%M:%S}"),
+        ]
+    )
+
+print("##prescript...")
 prebuild(env)
 
+ # We are using our custom ESP-IDF firmware descriptor
+env.Append(
+    LINKFLAGS = [ "-u", "custom_app_desc" ]
+)
+

include/main.h

@@ -1,5 +1,25 @@
 #pragma once
 
+#define STRINGIFY_IMPL(x) #x
+#define STRINGIFY(x) STRINGIFY_IMPL(x)
+
+#ifndef FWVERSION
+#define VERSION "*undef*"
+#else
+#define VERSION STRINGIFY(FWVERSION)
+#endif
+
+#ifndef BUILD_DATE
+#define BUILD_DATE STRINGIFY(FWBUILDDATE)
+#endif
+
+#ifndef BUILD_TIME
+#define BUILD_TIME STRINGIFY(FWBUILDTIME)
+#endif
+
+#define FIRMWARE_TYPE STRINGIFY(PIO_ENV_BUILD)
+#define IDF_VERSION STRINGIFY(ESP_IDF_VERSION_MAJOR) "." STRINGIFY(ESP_IDF_VERSION_MINOR) "." STRINGIFY(ESP_IDF_VERSION_PATCH)
+
 // WIFI AP
 #define WIFI_CHANNEL 9
 #define WIFI_MAX_STA 2

platformio.ini

@@ -1,6 +1,6 @@
 [platformio]
 default_envs=
-    esp32-s3-nano
+    obpkp61
 
 [env]
 platform = espressif32
@@ -41,7 +41,10 @@ build_flags =
 build_unflags =
     -std=gnu++11
 
-[env:esp32-s3-nano]
+custom_version = 0.2.0
+custom_versionformat = DEVELOP # Version format switch: e.g.DEVELOP=dev<yyyymmdd>|RELEASE=1.0.0
+
+[env:obpkp61]
 build_type = release # debug | release
 board = esp32_s3_nano
 #board = arduino_nano_esp32 # ATTENTION! Pin numbering scheme changes

src/main.cpp

@@ -1,4 +1,5 @@
 #include <Arduino.h>
+#include "main.h"
 #include <Preferences.h>
 #include <ArduinoJson.h>
 #include <WiFi.h>
@@ -10,7 +11,6 @@
 #include <NMEA2000.h>
 #include <N2kMsg.h>
 #include <N2kMessages.h>
-#include "main.h"
 #include "Nmea2kTwai.h"
 #include "N2kDeviceList.h"
 #include <map>
@@ -18,6 +18,21 @@
 #include "mbedtls/md.h" // for SHA256
 #include <esp32/clk.h> // for cpu frequency
 #include "driver/rtc_io.h" // for wakeup from deep sleep
+#include "esp_app_format.h" // for custom fw descriptor
+
+// ESP-IDF firmware descriptor
+__attribute__((section(".rodata_custom_desc"))) esp_app_desc_t custom_app_desc = {
+    ESP_APP_DESC_MAGIC_WORD,
+    1, // secure version
+    {0, 0}, // reserved
+    VERSION,
+    FIRMWARE_TYPE,
+    BUILD_DATE,
+    BUILD_TIME,
+    IDF_VERSION,
+    {},
+    {}
+};
 
 String get_sha256(String payload) {
     byte shaResult[32];
@@ -442,7 +457,7 @@ void setup() {
         StaticJsonDocument<512> doc;
         doc["systemName"] = "Keypad1";
         doc["logLevel"] = 0;
-        doc["version"] = "0.0";
+        doc["version"] = VERSION;
         doc["fwtype"] = "unknown";
         doc["salt"] = "secret";
         doc["AdminPassword"] = "********";

web/config.json

@@ -123,9 +123,19 @@
         "C",
         "F"
     ],
-    "description": "Temperature format: Kelvin, Celsius or Fahrenheit [K|C|F].",
+    "description": "Temperature format: Kelvin, Celsius or Fahrenheit\npossible values [K|C|F].",
     "category": "Units"
 },
+{
+    "name": "switchBank",
+    "label": "Switch bank #",
+    "type": "number",
+    "default": 0,
+    "min": 0,
+    "max": 252,
+    "description": "The number uf switch bank the keys belong to\nRange [0 .. 252] default 0",
+    "category": "Keys"
+},
 {
     "name": "key1",
     "label": "Key 1 code",