allow usercode to define config and set capabilities

extra_script.py

@@ -91,31 +91,57 @@ def writeFileIfChanged(fileName,data):
     with open(fileName,"w") as oh:
         oh.write(data)
 
-def generateCfg(ch,oh,inFile=''):
+def mergeConfig(base,other):
+    for bdir in other:
+        cname=os.path.join(bdir,"config.json")
+        if os.path.exists(cname):
+            print("merge config %s"%cname)
+            with open(cname,'rb') as ah:
+                merge=json.load(ah)
+                base=base+merge
+    return base
+
+def generateMergedConfig(inFile,outFile,addDirs=[]):
+    if not os.path.exists(inFile):
+        raise Exception("unable to read cfg file %s"%inFile)
+    data=""
+    with open(inFile,'rb') as ch:    
         config=json.load(ch)
-    oh.write("//generated from %s\n"%inFile)
+        config=mergeConfig(config,addDirs)
-    oh.write('#include "GwConfigItem.h"\n')
+        data=json.dumps(config,indent=2)
+        writeFileIfChanged(outFile,data)
+
+def generateCfg(inFile,outFile,addDirs=[]):
+    if not os.path.exists(inFile):
+        raise Exception("unable to read cfg file %s"%inFile)
+    data=""
+    with open(inFile,'rb') as ch:    
+        config=json.load(ch)
+        config=mergeConfig(config,addDirs)      
+        data+="//generated from %s\n"%inFile
+        data+='#include "GwConfigItem.h"\n'
         l=len(config)
-    oh.write('class GwConfigDefinitions{\n')
+        data+='class GwConfigDefinitions{\n'
-    oh.write('  public:\n')
+        data+='  public:\n'
-    oh.write('  int getNumConfig() const{return %d;}\n'%(l))
+        data+='  int getNumConfig() const{return %d;}\n'%(l)
         for item in config:
             n=item.get('name')
             if n is None:
                 continue
             if len(n) > 15:
                 raise Exception("%s: config names must be max 15 caracters"%n)
-        oh.write('  const String %s=F("%s");\n'%(n,n))
+            data+='  const String %s=F("%s");\n'%(n,n)
-    oh.write('  protected:\n')    
+        data+='  protected:\n'    
-    oh.write('  GwConfigItem *configs[%d]={\n'%(l))
+        data+='  GwConfigItem *configs[%d]={\n'%(l)
         first=True
         for item in config:
             if not first:
-            oh.write(',\n')
+                data+=',\n'
             first=False    
-        oh.write("    new GwConfigItem(%s,\"%s\")"%(item.get('name'),item.get('default')))
+            data+="    new GwConfigItem(%s,\"%s\")"%(item.get('name'),item.get('default'))
-    oh.write('};\n')  
+        data+='};\n'  
-    oh.write('};\n')
+        data+='};\n'
+    writeFileIfChanged(outFile,data)    
                     
 
 def generateXdrMappings(fp,oh,inFile=''):
@@ -157,10 +183,17 @@ def generateXdrMappings(fp,oh,inFile=''):
     oh.write("\n")
     oh.write("};\n")
 
-def genereateUserTasks(outfile):
+userTaskDirs=[]
-    includes=[]
+
+def getUserTaskDirs():
+    rt=[]
     taskdirs=glob.glob(os.path.join('lib','*task*'))
     for task in taskdirs:
+        rt.append(task)
+    return rt
+def genereateUserTasks(outfile):
+    includes=[]
+    for task in userTaskDirs:
         #print("##taskdir=%s"%task)
         base=os.path.basename(task)
         includeNames=[base.lower()+".h",'gw'+base.lower()+'.h']
@@ -201,9 +234,11 @@ def getContentType(fn):
     return "application/octet-stream"
 
 def prebuild(env):
+    global userTaskDirs
     print("#prebuild running")
     if not checkDir():
         sys.exit(1)
+    userTaskDirs=getUserTaskDirs()
     embedded=getEmbeddedFiles(env)
     filedefs=[]
     for ef in embedded:
@@ -223,7 +258,10 @@ def prebuild(env):
             print("#WARNING: infile %s for %s not found"%(inFile,ef))
     generateEmbedded(filedefs,os.path.join(outPath(),EMBEDDED_INCLUDE))
     genereateUserTasks(os.path.join(outPath(), TASK_INCLUDE))
-    generateFile(os.path.join(basePath(),CFG_FILE),os.path.join(outPath(),CFG_INCLUDE),generateCfg)
+    mergedConfig=os.path.join(outPath(),os.path.basename(CFG_FILE))
+    generateMergedConfig(os.path.join(basePath(),CFG_FILE),mergedConfig,userTaskDirs)
+    compressFile(mergedConfig,mergedConfig+".gz")
+    generateCfg(mergedConfig,os.path.join(outPath(),CFG_INCLUDE))
     generateFile(os.path.join(basePath(),XDR_FILE),os.path.join(outPath(),XDR_INCLUDE),generateXdrMappings)
     version="dev"+datetime.now().strftime("%Y%m%d")
     env.Append(CPPDEFINES=[('GWDEVVERSION',version)])

lib/api/GwApi.h

@@ -19,4 +19,7 @@ class GwApi{
 #ifndef DECLARE_USERTASK
 #define DECLARE_USERTASK(task)
 #endif
+#ifndef DECLARE_CAPABILITY
+#define DECLARE_CAPABILITY(name,value)
+#endif
 #endif

lib/exampletask/GwExampleTask.cpp

@@ -35,10 +35,15 @@ class GetBoatDataRequest: public GwMessage{
 void exampleTask(void *param){
     GwApi *api=(GwApi*)param;
     GwLog *logger=api->getLogger();
+    //get some configuration data
+    bool exampleSwitch=api->getConfig()->getConfigItem(
+        api->getConfig()->exampleConfig,
+        true)->asBoolean();
     //------
     //initialization goes here
     //------
     bool hasPosition=false;
+    LOG_DEBUG(GwLog::DEBUG,"example switch ist %s",exampleSwitch?"true":"false");
     while(true){
         delay(1000);
         /*
@@ -65,14 +70,14 @@ void exampleTask(void *param){
         }
         if (r->latitude == INVALID_COORD || r->longitude == INVALID_COORD){
             if (hasPosition){
-                logger->logDebug(GwLog::ERROR,"position lost...");
+                if (exampleSwitch) logger->logDebug(GwLog::ERROR,"position lost...");
                 hasPosition=false;
             }
         }
         else{
             //do something with the data we have from boatData
             if (! hasPosition){
-                logger->logDebug(GwLog::LOG,"postion now available lat=%f, lon=%f",
+                if (exampleSwitch) logger->logDebug(GwLog::LOG,"postion now available lat=%f, lon=%f",
                     r->latitude,r->longitude);
                     hasPosition=true;
             }

lib/exampletask/GwExampleTask.h

@@ -7,5 +7,9 @@
 void exampleTask(void *param);
 //make the task known to the core
 DECLARE_USERTASK(exampleTask);
+//we declare a capability that we can
+//use in config.json to only show some
+//elements when this capability is set correctly
+DECLARE_CAPABILITY(testboard,true);
 #endif
 #endif

lib/exampletask/Readme.md

@@ -1,8 +1,9 @@
 Extending the Core
 ==================
 This directory contains an example on how you can extend the base functionality of the gateway.
-Basically you can define own boards here and can add one or more tasks that will be started by the core.
+Maybe you have another interesting hardware or need some additional functions but would like to use the base functionality of the gateway.
-You can also add additional libraries that will be used for your task.
+You can define own hardware configurations (environments) here and can add one or more tasks that will be started by the core.
+You can also add additional libraries that will be used to build your task.
 In this example we define an addtional board (environment) with the name "testboard".
 When building for this board we add the -DTEST_BOARD to the compilation - see [platformio.ini](platformio.ini).
 The additional task that we defined will only be compiled and started for this environment (see the #ifdef TEST_BOARD in the code).
@@ -10,22 +11,41 @@ You can add your own directory below "lib". The name of the directory must conta
 
 Files
 -----
-   * [platformio.ini](platformio.ini)
+   * [platformio.ini](platformio.ini)<br>
-    extend the base configuration - we add a dummy library here and define our buil environment (board)
+    This file is completely optional.
-   * [GwExampleTask.h](GwExampleTask.h) the name of this include must match the name of the directory (ignoring case) with a "gw" in front. This file includes our special hardware definitions and registers our task at the core (DECLARE_USERTASK in the code).
+    You only need this if you want to
+    extend the base configuration - we add a dummy library here and define one additional build environment (board)
+   * [GwExampleTask.h](GwExampleTask.h) the name of this include must match the name of the directory (ignoring case) with a "gw" in front. This file includes our special hardware definitions and registers our task at the core (DECLARE_USERTASK in the code). Optionally it can define some capabilities (using DECLARE_CAPABILITY) that can be used in the config UI (see below).
+   Avoid including headers from other libraries in this file as this could interfere with the main code. Just only include them in your .cpp files (or in other headers).
    * [GwExampleTaks.cpp](GwExampleTask.cpp) includes the implementation of our task. This tasks runs in an own thread - see the comments in the code.
+   We can have as many cpp (and header files) as we need to structure our code.  
    * [GwExampleHardware.h](GwExampleHardware.h) includes our pin definitions for the board.
+   * [config.json](config.json)<br>
+     This file allows to add some config definitions that are needed for our task. For the possible options have a look at the global [config.json](../../web/config.json). Be careful not to overwrite config defitions from the global file. A good practice wood be to prefix the names of definitions with parts of the library name. Always put them in a separate category so that they do not interfere with the system ones.
+     The defined config items can later be accessed in the code (see the example in [GwExampleTask.cpp](GwExampleTask.cpp)).
 
  Hints
  -----
- Just be careful not to interfere with names from the core - so it is a good practice to prefix your files and class like in the example.
+ Just be careful not to interfere with C symbols from the core - so it is a good practice to prefix your files and class like in the example.
 
  Developing
  ----------
  To develop I recommend forking the gateway repository and adding your own directory below lib (with the string task in it's name).
  As your code goes into a separate directory it should be very easy to fetch upstream changes without the need to adapt your code.
+ Typically after forking the repo on github (https://github.com/wellenvogel/esp32-nmea2000) and initially cloning it you will add my repository as an "upstream repo":
+ ```
+ git remote add upstream https://github.com/wellenvogel/esp32-nmea2000.git
+ ```
+ To merge in a new version use:
+ ```
+ git fetch upstream
+ git merge upstream/master
+ ```
+ Refer to https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork
+ 
+ By following the hints in this doc the merge should always succeed without conflicts.
 
  Future Plans
  ------------
- If there will be a need we can extend this extension API by means of adding config items and specific java script code and css for the UI.
+ If there will be a need we can extend this extension API by means of adding  specific java script code and css for the UI.
  

lib/exampletask/config.json

@@ -0,0 +1,13 @@
+[
+    {
+        "name": "exampleConfig",
+        "label": "logging on",
+        "type": "boolean",
+        "default": "false",
+        "description": "switch on logging of position acquired/failed",
+        "category": "example",
+        "capabilities": {
+            "testboard":"true"
+        }
+    }
+]

lib/usercode/GwUserCode.cpp

@@ -1,6 +1,7 @@
 #include "GwUserCode.h"
 #include <Arduino.h>
 #include <vector>
+#include <map>
 //user task handling
 class UserTask{
     public:
@@ -11,7 +12,9 @@ class UserTask{
             this->task=task;
         }
 };
+
 std::vector<UserTask> userTasks;
+GwUserCode::Capabilities userCapabilities;
 
 void registerUserTask(TaskFunction_t task,String name){
   userTasks.push_back(UserTask(name,task));
@@ -23,7 +26,14 @@ class GwUserTask{
             registerUserTask(task,name);
         }
 };
+class GwUserCapability{
+    public:
+        GwUserCapability(String name,String value){
+            userCapabilities[name]=value;  
+        }
+};
 #define DECLARE_USERTASK(task) GwUserTask __##task##__(task,#task);
+#define DECLARE_CAPABILITY(name,value) GwUserCapability __CAP##name__(#name,#value); 
 #include "GwUserTasks.h"
 #include "GwApi.h"
 class TaskApi : public GwApi
@@ -87,3 +97,7 @@ void GwUserCode::startAddonTask(String name, TaskFunction_t task, int id){
     LOG_DEBUG(GwLog::LOG,"starting addon task %s with id %d",name.c_str(),id);
     startAddOnTask(api,task,id);
 }
+
+GwUserCode::Capabilities * GwUserCode::getCapabilities(){
+    return &userCapabilities;
+}

lib/usercode/GwUserCode.h

@@ -1,14 +1,17 @@
 #ifndef _GWUSERCODE_H
 #define _GWUSERCODE_H
 #include <Arduino.h>
+#include <map>
 class GwLog;
 class GwApi;
 class GwUserCode{
     GwLog *logger;
     GwApi *api;
     public:
+        typedef std::map<String,String> Capabilities;
         GwUserCode(GwApi *api);
         void startUserTasks(int baseId);
         void startAddonTask(String name,TaskFunction_t task, int id);
+        Capabilities *getCapabilities();
 };
 #endif

src/main.cpp

@@ -265,6 +265,7 @@ bool delayedRestart(){
   },"reset",1000,&logger,0,NULL) == pdPASS;
 }
 
+GwUserCode userCodeHandler(new ApiImpl(200));
 
 #define JSON_OK "{\"status\":\"OK\"}"
 
@@ -329,7 +330,12 @@ class CapabilitiesRequest : public GwRequestMessage{
     CapabilitiesRequest() : GwRequestMessage(F("application/json"),F("capabilities")){};
   protected:
     virtual void processRequest(){
-      DynamicJsonDocument json(JSON_OBJECT_SIZE(6));
+      int numCapabilities=userCodeHandler.getCapabilities()->size();
+      DynamicJsonDocument json(JSON_OBJECT_SIZE(numCapabilities*3+6));
+      for (auto it=userCodeHandler.getCapabilities()->begin();
+        it != userCodeHandler.getCapabilities()->end();it++){
+          json[it->first]=it->second;
+        }
       #ifdef GWSERIAL_MODE
       String serial(F(GWSERIAL_MODE));
       #else
@@ -692,11 +698,10 @@ void setup() {
   NMEA2000.Open();
   logger.logDebug(GwLog::LOG,"starting addon tasks");
   logger.flush();
-  GwUserCode userHandler(new ApiImpl(200));
+  userCodeHandler.startAddonTask(F("handleButtons"),handleButtons,100);
-  userHandler.startAddonTask(F("handleButtons"),handleButtons,100);
   setLedMode(LED_GREEN);
-  userHandler.startAddonTask(F("handleLeds"),handleLeds,101);
+  userCodeHandler.startAddonTask(F("handleLeds"),handleLeds,101);
-  userHandler.startUserTasks(200);
+  userCodeHandler.startUserTasks(200);
   
   logger.logDebug(GwLog::LOG,"setup done");
 }  

tools/testServer.py

@@ -49,26 +49,31 @@ class RequestHandler(http.server.SimpleHTTPRequestHandler):
             path = urllib.parse.unquote(path)
         path = posixpath.normpath(path)
         words = path.split('/')
-        words = filter(None, words)
+        words = list(filter(None, words))
-        path = self.server.baseDir
+        for baseDir in [
+            os.path.join(self.server.baseDir,'lib','generated'),
+            os.path.join(self.server.baseDir,'web')]:
+            rpath = baseDir
             for word in words:
                 if os.path.dirname(word) or word in (os.curdir, os.pardir):
                     # Ignore components that are not a simple file/directory name
                     continue
-            path = os.path.join(path, word)
+                rpath = os.path.join(rpath, word)
             if trailing_slash:
-            path += '/'
+                rpath += '/'
-        return path
+            if os.path.exists(rpath):
+                return rpath
                 
-def run(baseDir,port,apiUrl,server_class=http.server.HTTPServer, handler_class=RequestHandler):
+def run(port,apiUrl,server_class=http.server.HTTPServer, handler_class=RequestHandler):
+    basedir=os.path.join(os.path.dirname(__file__),'..')
     server_address = ('', port)
     httpd = server_class(server_address, handler_class)
     httpd.proxyUrl=apiUrl
-    httpd.baseDir=baseDir
+    httpd.baseDir=basedir
     httpd.serve_forever()
 
 if __name__ == '__main__':
-    if len(sys.argv) != 4:
+    if len(sys.argv) != 3:
-        print("usage: %s basedir port apiurl"%sys.argv[0])
+        print("usage: %s port apiurl"%sys.argv[0])
         sys.exit(1)
-    run(sys.argv[1],int(sys.argv[2]),sys.argv[3])    
+    run(int(sys.argv[1]),sys.argv[2])    

web/index.js

@@ -829,10 +829,12 @@ function toggleClass(el,id,classList){
 function createConfigDefinitions(parent, capabilities, defs,includeXdr) {
     let category;
     let categoryEl;
+    let categoryFrame;
     let frame = parent.querySelector('.configFormRows');
     if (!frame) throw Error("no config form");
     frame.innerHTML = '';
     configDefinitions = defs;
+    let currentCategoryPopulated=true;
     defs.forEach(function (item) {
         if (!item.type) return;
         if (item.category.match(/^xdr/)){
@@ -842,7 +844,11 @@ function createConfigDefinitions(parent, capabilities, defs,includeXdr) {
             if(includeXdr) return;
         } 
         if (item.category != category || !categoryEl) {
-            let categoryFrame = addEl('div', 'category', frame);
+            if (categoryFrame && ! currentCategoryPopulated){
+                categoryFrame.remove();
+            }
+            currentCategoryPopulated=false;
+            categoryFrame = addEl('div', 'category', frame);
             categoryFrame.setAttribute('data-category',item.category)
             let categoryTitle = addEl('div', 'title', categoryFrame);
             let categoryButton = addEl('span', 'icon icon-more', categoryTitle);
@@ -862,17 +868,20 @@ function createConfigDefinitions(parent, capabilities, defs,includeXdr) {
             })
             category = item.category;
         }
+        let showItem=true;
         if (item.capabilities !== undefined) {
             for (let capability in item.capabilities) {
                 let values = item.capabilities[capability];
-                if (!capabilities[capability]) return;
                 let found = false;
+                if (! (values instanceof Array)) values=[values];
                 values.forEach(function (v) {
                     if (capabilities[capability] == v) found = true;
                 });
-                if (!found) return;
+                if (!found) showItem=false;
             }
         }
+        if (showItem) {
+            currentCategoryPopulated=true;
             let row = addEl('div', 'row', categoryEl);
             let label = item.label || item.name;
             addEl('span', 'label', row, label);
@@ -882,14 +891,14 @@ function createConfigDefinitions(parent, capabilities, defs,includeXdr) {
             valueEl.setAttribute('data-default', item.default);
             valueEl.addEventListener('change', function (ev) {
                 let el = ev.target;
-            checkChange(el, row,item.name);
+                checkChange(el, row, item.name);
             })
-        let condition=getConditions(item.name);
+            let condition = getConditions(item.name);
-        if (condition){
+            if (condition) {
-            condition.forEach(function(cel){
+                condition.forEach(function (cel) {
-                for (let c in cel){
+                    for (let c in cel) {
-                    if (!conditionRelations[c]){
+                        if (!conditionRelations[c]) {
-                        conditionRelations[c]=[];
+                            conditionRelations[c] = [];
                         }
                         conditionRelations[c].push(valueEl);
                     }
@@ -906,16 +915,20 @@ function createConfigDefinitions(parent, capabilities, defs,includeXdr) {
             })
             bt = addEl('button', 'infoButton', btContainer, '?');
             bt.addEventListener('click', function (ev) {
-            if (item.description){
+                if (item.description) {
                     showOverlay(item.description);
                 }
-            else{
+                else {
-                if (item.category.match(/^xdr/)){
+                    if (item.category.match(/^xdr/)) {
                         showXdrHelp();
                     }
                 }
             });
-    })
+        }
+    });
+    if (categoryFrame && ! currentCategoryPopulated){
+        categoryFrame.remove();
+    }
 }
 function loadConfigDefinitions() {
     getJson("api/capabilities")