generate embedded files automatically

extra_script.py

@@ -10,10 +10,18 @@ Import("env")
 GEN_DIR='generated'
 CFG_FILE='web/config.json'
 XDR_FILE='web/xdrconfig.json'
-FILES=['web/index.html',CFG_FILE,XDR_FILE,'web/index.js','web/index.css']
 CFG_INCLUDE='GwConfigDefinitions.h'
 XDR_INCLUDE='GwXdrTypeMappings.h'
+EMBEDDED_INCLUDE="GwEmbeddedFiles.h"
 
+def getEmbeddedFiles(env):
+    rt=[]
+    efiles=env.GetProjectOption("board_build.embed_files")
+    for f in efiles.split("\n"):
+        if f == '':
+            continue
+        rt.append(f)
+    return rt  
 
 def basePath():
     #see: https://stackoverflow.com/questions/16771894/python-nameerror-global-name-file-is-not-defined
@@ -38,29 +46,45 @@ def isCurrent(infile,outfile):
             print("%s is newer then %s, no need to recreate"%(outfile,infile))
             return True
     return False        
-def compressFile(inFile):
+def compressFile(inFile,outfile):
-    outfile=os.path.basename(inFile)+".gz"
-    inFile=os.path.join(basePath(),inFile)
-    outfile=os.path.join(outPath(),outfile)
     if isCurrent(inFile,outfile):
         return
     with open(inFile, 'rb') as f_in:
         with gzip.open(outfile, 'wb') as f_out:
             shutil.copyfileobj(f_in, f_out)
 
-
+def generateFile(infile,outfile,callback,inMode='rb',outMode='w'):
-def generateCfg():
-    outfile=os.path.join(outPath(),CFG_INCLUDE)
-    infile=os.path.join(basePath(),CFG_FILE)
     if isCurrent(infile,outfile):
         return
-    print("creating %s"%CFG_INCLUDE)
+    print("creating %s"%outfile)
     oh=None
-    with open(CFG_FILE,'rb') as ch:
+    with open(infile,inMode) as ch:
-        config=json.load(ch)
-        try:
         with open(outfile,'w') as oh:
-                oh.write("//generated from %s\n"%CFG_FILE)
+            try:
+                callback(ch,oh,inFile=infile)
+                oh.close()
+            except Exception as e:
+                try:
+                    oh.close()
+                except:
+                    pass
+                os.unlink(outfile)
+                raise
+
+def writeFileIfChanged(fileName,data):
+    if os.path.exists(fileName):
+        with open(fileName,"r") as ih:
+            old=ih.read()
+            ih.close()
+            if old == data:
+                return
+    print("#generating %s"%fileName)
+    with open(fileName,"w") as oh:
+        oh.write(data)
+
+def generateCfg(ch,oh,inFile=''):
+    config=json.load(ch)      
+    oh.write("//generated from %s\n"%inFile)
     oh.write('#include "GwConfigItem.h"\n')
     l=len(config)
     oh.write('class GwConfigDefinitions{\n')
@@ -83,28 +107,10 @@ def generateCfg():
         oh.write("    new GwConfigItem(%s,\"%s\")"%(item.get('name'),item.get('default')))
     oh.write('};\n')  
     oh.write('};\n')
-                oh.close()  
-        except Exception as e:
-            if oh is not None:
-                try:
-                    oh.close()
-                except:
-                    pass
-            os.unlink(outfile)
-            raise        
                     
-def generateXdrMappings():
-    outfile=os.path.join(outPath(),XDR_INCLUDE)
-    infile=os.path.join(basePath(),XDR_FILE)
-    if isCurrent(infile,outfile):
-        return
-    print("creating %s"%XDR_INCLUDE)
-    oh=None
 
-    with open(infile,"rb") as fp:
+def generateXdrMappings(fp,oh,inFile=''):
     jdoc=json.load(fp)
-        try:
-            with open(outfile,"w") as oh:
     oh.write("static GwXDRTypeMapping* typeMappings[]={\n")
     first=True
     for cat in jdoc:
@@ -141,23 +147,49 @@ def generateXdrMappings():
             oh.write("   new GwXDRTypeMapping(%d,%d,%d) /*%s:%s*/"%(cid,tc,id,cat,l))
     oh.write("\n")
     oh.write("};\n")
-        except Exception as e:
-            if oh:
-                try:
-                    oh.close()
-                except:
-                    pass
-            os.unlink(outfile)
-            raise
 
+def generateEmbedded(elist,outFile):
+    content=""
+    for entry in elist:
+        content+="EMBED_GZ_FILE(\"%s\",%s,\"%s\");\n"%entry
+    writeFileIfChanged(outFile,content)    
 
+def getContentType(fn):
+    if (fn.endswith('.gz')):
+        fn=fn[0:-3]
+    if (fn.endswith('html')):
+        return "text/html"
+    if (fn.endswith('json')):
+        return "application/json"
+    if (fn.endswith('js')):
+        return "text/javascript"
+    return "application/octet-stream"
 
-if not checkDir():
+def prebuild(env):
+    print("#prebuild running")
+    if not checkDir():
         sys.exit(1)
-for f in FILES:
+    embedded=getEmbeddedFiles(env)
-    print("compressing %s"%f)
+    filedefs=[]
-    compressFile(f)
+    for ef in embedded:
-generateCfg()
+        print("#checking embedded file %s"%ef)
-generateXdrMappings()
+        (dn,fn)=os.path.split(ef)
-version="dev"+datetime.now().strftime("%Y%m%d")
+        pureName=fn
-env.Append(CPPDEFINES=[('GWDEVVERSION',version)])
+        if pureName.endswith('.gz'):
+            pureName=pureName[0:-3]
+        ct=getContentType(pureName)
+        usname=ef.replace('/','_').replace('.','_')
+        filedefs.append((pureName,usname,ct))
+        inFile=os.path.join(basePath(),"web",pureName)
+        if os.path.exists(inFile):
+            print("compressing %s"%inFile)
+            compressFile(inFile,ef)
+        else:
+            print("#WARNING: infile %s for %s not found"%(inFile,ef))
+    generateEmbedded(filedefs,os.path.join(outPath(),EMBEDDED_INCLUDE))
+    generateFile(os.path.join(basePath(),CFG_FILE),os.path.join(outPath(),CFG_INCLUDE),generateCfg)
+    generateFile(os.path.join(basePath(),XDR_FILE),os.path.join(outPath(),XDR_INCLUDE),generateXdrMappings)
+    version="dev"+datetime.now().strftime("%Y%m%d")
+    env.Append(CPPDEFINES=[('GWDEVVERSION',version)])
+
+prebuild(env)

lib/webserver/GwWebServer.cpp

@@ -16,15 +16,12 @@ class EmbeddedFile {
       embeddedFiles[name]=this;
     }
 } ;
-#define EMBED_GZ_FILE(fileName, fileExt, contentType) \
+#define EMBED_GZ_FILE(fileName, binName, contentType) \
-  extern const uint8_t  fileName##_##fileExt##_File[] asm("_binary_generated_" #fileName "_" #fileExt "_gz_start"); \
+  extern const uint8_t  binName##_File[] asm("_binary_" #binName "_start"); \
-  extern const uint8_t  fileName##_##fileExt##_FileLen[] asm("_binary_generated_" #fileName "_" #fileExt "_gz_size"); \
+  extern const uint8_t  binName##_FileLen[] asm("_binary_" #binName "_size"); \
-  const EmbeddedFile fileName##_##fileExt##_Config(#fileName "." #fileExt,contentType,(const uint8_t*)fileName##_##fileExt##_File,(int)fileName##_##fileExt##_FileLen);
+  const EmbeddedFile binName##_Config(fileName,contentType,(const uint8_t*)binName##_File,(int)binName##_FileLen);
 
-EMBED_GZ_FILE(index,html,"text/html")
+#include "GwEmbeddedFiles.h"
-EMBED_GZ_FILE(config,json,"application/json")
-EMBED_GZ_FILE(index,js,"text/javascript")
-EMBED_GZ_FILE(index,css,"text/css")
 
 void sendEmbeddedFile(String name,String contentType,AsyncWebServerRequest *request){
     std::map<String,EmbeddedFile*>::iterator it=embeddedFiles.find(name);

platformio.ini

@@ -23,6 +23,7 @@ board_build.embed_files =
 	generated/index.js.gz
 	generated/index.css.gz
 	generated/config.json.gz
+	generated/xdrconfig.json.gz
 board_build.partitions = partitions_custom.csv	
 extra_scripts = 
 	pre:extra_script.py