[SYS] Update arduino core to 3.1.1 + refactor ino to cpp (#2177)

* Update arduino core to 3.1.1

* Fix Blufi build

* Update arduinojson, fix build errors with idf

* Fix narrowing

* fix RF builds

* WIP-Convert ino files to cpp

* Fix pilight build

* Fix Somfy actuator build.

* Update esp32dev-rf partition

* Fix Weatherstation build

* Fix GFSunInverter build

* Fix esp32dev-ir build

* Fix ble-aws build

* Fix eth builds

* Fix m5Stack missing pins_arduino.h

* Fix build errors for M5 stack/tough, others are upstream issues.

* Fix RTL 433 build - remaining errors are from radolib

* Fix nodemcu build

* fix 2g builds

* Fix serial build

* Fix actuator on off builds

* Fix SSD1306 build - remaining errors are from radiolib

* Fix multiple definition of OTAserver_cert

* Fix nodemcu rf2 build

* Fix ADC builds

* Fix sensor builds

* Fix LORA builds

* Fix multi-receiver builds - remaining errors are in radiolib

* Fix fastled builds

* Fix theegns board builds

* Fix broker builds

* Update fastled - old version failed all-test build

* Fix RN8209 builds

* Fix max temp actuator builds

* Fix PWM builds

* Fix C37 sensor builds

* Fix HTU21 builds

* Fix INA266 builds

* Fix undefined variables in mqtt discovery

* Fix webui build

* Fix fastled invalid pin error

* Fix wifi manual setup builds

* Fix onewire/all-test build - bin too big error remaining

* Fix theengs plug build

* Fix rfbridge builds

* Fix blufi builds

* Fix undefined functions in serial

* Fix preprocessor definition checks

* Set IDF log level to erre

* Add delay in loop to prevent watchdog timeout

* Use xTaskCreateUniveral to support single core processors

* Remove old HTTPUpdate files - upsteam fixed.

* Cleanup and move common declarations to header file

* Use custom partiton table to fix builds where bin is too large

* Update M5StickC - fixs esp32-m5stick-c-ble build

* Revert to Arduino core 2.x for builds with incompatible libs

* Remove "Z" from file names and rename omg_common to TheengsCommon

* Fix gateway name when using MAC with new Arduino core

* Update IDF config to reduce loop task stack use

* Update esp-nimble-cpp version, corrects BLE uppercase ID's

* Update wifi manager to fix watchdog timeout error

Author: h2zero

.gitignore

@@ -15,4 +15,5 @@ dependencies.lock
 sdkconfig.*
 !sdkconfig.defaults
 __pycache__
-*.pem
+*.pem
+managed_components

environments.ini

@@ -188,7 +188,7 @@ custom_hardware = Theengs Plug
 platform = ${com.esp32_platform}
 board = esp32dev
 extra_scripts = ${com-esp32.extra_scripts}
-board_build.partitions = min_spiffs.csv
+board_build.partitions = partitions/no_coredump.csv
 lib_deps =
   ${com-esp32.lib_deps}
   ${libraries.ble}
@@ -270,6 +270,7 @@ build_flags =
 [env:esp32dev-rf]
 platform = ${com.esp32_platform}
 board = esp32dev
+board_build.partitions = min_spiffs.csv
 lib_deps =
   ${com-esp32.lib_deps}
   ${libraries.rc-switch}
@@ -311,6 +312,7 @@ custom_description = Gateway using ESPilight library only, need CC1101
 [env:esp32dev-somfy-cc1101]
 platform = ${com.esp32_platform}
 board = esp32dev
+board_build.partitions = min_spiffs.csv
 lib_deps =
   ${com-esp32.lib_deps}
   ${libraries.somfy_remote}
@@ -342,6 +344,7 @@ custom_description = Gateway using Somfy Remote and ESPilight library, need CC11
 [env:esp32dev-weatherstation]
 platform = ${com.esp32_platform}
 board = esp32dev
+board_build.partitions = min_spiffs.csv
 lib_deps =
   ${com-esp32.lib_deps}
   ${libraries.rfWeatherStation}
@@ -354,6 +357,7 @@ custom_description = Gateway to retrieve weather station data Ventus W174/W132 (
 [env:esp32dev-gf-sun-inverter]
 platform = ${com.esp32_platform}
 board = esp32dev
+board_build.partitions = min_spiffs.csv
 lib_deps =
   ${com-esp32.lib_deps}
   ${libraries.emodbus}
@@ -598,10 +602,10 @@ custom_description = BLE gateway using wifi
 custom_hardware = OLIMEX ESP32 Gateway
 
 [env:esp32-m5stick-ble]
-platform = ${com.esp32_platform}
+platform = [email protected]
 board = m5stack-core-esp32
 extra_scripts = ${com-esp32.extra_scripts}
-board_build.partitions = min_spiffs.csv
+board_build.partitions = partitions/no_coredump.csv
 lib_deps =
   ${com-esp32.lib_deps}
   ${libraries.ble}
@@ -622,7 +626,7 @@ custom_description = Expandable module with BLE gateway, display, and little IR
 custom_hardware = M5Stick Grey
 
 [env:esp32-m5stack-ble]
-platform = ${com.esp32_platform}
+platform = [email protected]
 board = m5stack-core-esp32
 extra_scripts = ${com-esp32.extra_scripts}
 board_build.partitions = min_spiffs.csv
@@ -647,7 +651,7 @@ custom_description = Expandable module with BLE gateway and display
 custom_hardware = M5Stack Core
 
 [env:esp32-m5tough-ble]
-platform = ${com.esp32_platform}
+platform = [email protected]
 board = m5stack-core-esp32
 extra_scripts = ${com-esp32.extra_scripts}
 board_build.partitions = min_spiffs.csv
@@ -666,7 +670,7 @@ custom_description = Expandable module with BLE gateway and display, suitable fo
 custom_hardware = M5Tough rugged
 
 [env:esp32-m5stick-c-ble]
-platform = ${com.esp32_platform}
+platform = [email protected]
 board = m5stick-c
 extra_scripts = ${com-esp32.extra_scripts}
 board_build.partitions = min_spiffs.csv
@@ -697,10 +701,10 @@ custom_description = Expandable module with BLE gateway, display, and little IR
 custom_hardware = M5Stick C
 
 [env:esp32-m5stick-cp-ble]
-platform = ${com.esp32_platform}
+platform = [email protected]
 board = pico32
 extra_scripts = ${com-esp32.extra_scripts}
-board_build.partitions = min_spiffs.csv
+board_build.partitions = partitions/no_coredump.csv
 lib_deps =
   ${com-esp32.lib_deps}
   ${libraries.ble}
@@ -731,7 +735,7 @@ custom_hardware = M5Stick CP
 platform = ${com.esp32_platform}
 board = pico32
 extra_scripts = ${com-esp32.extra_scripts}
-board_build.partitions = min_spiffs.csv
+board_build.partitions = partitions/no_coredump.csv
 lib_deps =
   ${com-esp32.lib_deps}
   ${libraries.ble}
@@ -762,7 +766,7 @@ custom_hardware = M5 ATOM Matrix
 platform = ${com.esp32_platform}
 board = pico32
 extra_scripts = ${com-esp32.extra_scripts}
-board_build.partitions = min_spiffs.csv
+board_build.partitions = partitions/no_coredump.csv
 lib_deps =
   ${com-esp32.lib_deps}
   ${libraries.ble}
@@ -785,7 +789,7 @@ custom_description = Compact enclosure ESP32 with BLE gateway
 custom_hardware = M5 ATOM Lite
 
 [env:esp32doitv1-aithinker-r01-sx1278]
-platform = ${com.esp32_platform}
+platform = [email protected]
 board = esp32doit-devkit-v1
 board_build.partitions = min_spiffs.csv
 lib_deps =
@@ -864,7 +868,7 @@ upload_flags =
   --port=8266
 
 [env:esp32dev-rtl_433]
-platform = ${com.esp32_platform}
+platform = [email protected]
 board = esp32dev
 board_build.partitions = min_spiffs.csv
 lib_deps =
@@ -904,7 +908,7 @@ build_flags =
 custom_description = Gateway using RTL_433_ESP library, need CC1101
 
 [env:heltec-rtl_433]
-platform = ${com.esp32_platform}
+platform = [email protected]
 board = heltec_wifi_lora_32_V2
 ; ~/.platformio/packages/framework-arduinoespressif32/variants/.../pins_arduino.h
 board_build.partitions = min_spiffs.csv
@@ -933,7 +937,7 @@ custom_description = Gateway using RTL_433_ESP and RadioLib
 custom_hardware = ESP32 HELTEC LORA32 V2
 
 [env:heltec-rtl_433-fsk]
-platform = ${com.esp32_platform}
+platform = [email protected]
 board = heltec_wifi_lora_32_V2
 ; ~/.platformio/packages/framework-arduinoespressif32/variants/.../pins_arduino.h
 board_build.partitions = min_spiffs.csv
@@ -991,7 +995,7 @@ custom_description = Heltec BLE gateway with adaptive scanning activated, automa
 custom_hardware = ESP32 HELTEC LORA32 V2
 
 [env:lilygo-rtl_433]
-platform = ${com.esp32_platform}
+platform = [email protected]
 board = ttgo-lora32-v21
 ; ~/.platformio/packages/framework-arduinoespressif32/variants/.../pins_arduino.h
 board_build.partitions = min_spiffs.csv
@@ -1020,7 +1024,7 @@ custom_description = For ESP32, Gateway using RTL_433_ESP and RadioLib
 custom_hardware = ESP32 LILYGO LoRa32 V2.1
 
 [env:lilygo-rtl_433-fsk]
-platform = ${com.esp32_platform}
+platform = [email protected]
 board = ttgo-lora32-v21
 ; ~/.platformio/packages/framework-arduinoespressif32/variants/.../pins_arduino.h
 board_build.partitions = min_spiffs.csv
@@ -1078,7 +1082,7 @@ custom_description = LilyGo BLE gateway with adaptive scanning activated, automa
 custom_hardware = ESP32 LILYGO LoRa32 V2.1
 
 [env:esp32dev-multi_receiver]
-platform = ${com.esp32_platform}
+platform = [email protected]
 board = esp32dev
 board_build.partitions = min_spiffs.csv
 lib_deps =
@@ -1896,11 +1900,16 @@ framework = arduino, espidf
 board = esp32dev
 extra_scripts = ${com-esp32.extra_scripts}
 board_build.partitions = partitions/min_spiffs.csv
+board_build.embed_txtfiles =
+    managed_components/espressif__esp_insights/server_certs/https_server.crt
+    managed_components/espressif__esp_rainmaker/server_certs/rmaker_mqtt_server.crt
+    managed_components/espressif__esp_rainmaker/server_certs/rmaker_claim_service_server.crt
+    managed_components/espressif__esp_rainmaker/server_certs/rmaker_ota_server.crt
 lib_deps =
   ${com-esp32.lib_deps}
   ${libraries.decoder}
-  h2zero/esp-nimble-cpp@2.0.1
-  h2zero/NimBLEOta@0.1.0
+  h2zero/esp-nimble-cpp@2.1.1
+  h2zero/NimBLEOta@0.2.0
 build_flags =
   ${com-esp32.build_flags}
   '-DZgatewayBT="BT"'
@@ -1909,5 +1918,4 @@ build_flags =
   '-DGateway_Name="OMG_ESP32_BLE"'
   '-DUSE_BLUFI=1'
   '-DenableMultiGTWSync=false'
-  '-DCONFIG_NIMBLE_CPP_FREERTOS_TASK_BLOCK_BIT=31'
 custom_description = Regular BLE gateway based on esp-idf with adaptive scanning activated, automatically adapts the scan parameters depending on your devices

main/TheengsCommon.h

@@ -0,0 +1,100 @@
+#pragma once
+
+#include "User_config.h"
+
+#define ARDUINOJSON_USE_LONG_LONG     1
+#define ARDUINOJSON_ENABLE_STD_STRING 1
+#include <ArduinoJson.h>
+#include <ArduinoLog.h>
+
+#if defined(ESP32)
+#  include <Preferences.h>
+extern Preferences preferences;
+#endif
+
+// Flags definition for white list, black list, discovery management
+#define device_flags_init     0 << 0
+#define device_flags_isDisc   1 << 0
+#define device_flags_isWhiteL 1 << 1
+#define device_flags_isBlackL 1 << 2
+#define device_flags_connect  1 << 3
+#define isWhite(device)       device->isWhtL
+#define isBlack(device)       device->isBlkL
+#define isDiscovered(device)  device->isDisc
+
+enum GatewayState {
+  WAITING_ONBOARDING,
+  ONBOARDING,
+  OFFLINE,
+  NTWK_CONNECTED,
+  BROKER_CONNECTED,
+  PROCESSING,
+  NTWK_DISCONNECTED,
+  BROKER_DISCONNECTED,
+  LOCAL_OTA_IN_PROGRESS,
+  REMOTE_OTA_IN_PROGRESS,
+  SLEEPING,
+  ERROR
+};
+
+enum PowerMode {
+  DEACTIVATED = -1,
+  ALWAYS_ON,
+  INTERVAL,
+  ACTION
+};
+
+struct SYSConfig_s {
+  bool mqtt; // if true the gateway will publish the received data on the MQTT broker
+  bool serial; // if true the gateway will publish the received data on the SERIAL
+  bool blufi; // if true the gateway will be accesible with blufi
+  bool offline;
+  bool discovery; // HA discovery convention
+#ifdef LED_ADDRESSABLE
+  int rgbbrightness; // brightness of the RGB LED
+#endif
+  enum PowerMode powerMode;
+};
+
+extern GatewayState gatewayState;
+extern SYSConfig_s SYSConfig;
+extern bool ready_to_sleep;
+extern char mqtt_topic[];
+extern char gateway_name[];
+extern unsigned long lastDiscovery; // Time of the last discovery to trigger automaticaly to off after DiscoveryAutoOffTimer
+
+extern bool enqueueJsonObject(const StaticJsonDocument<JSON_MSG_BUFFER>& jsonDoc, int timeout);
+extern bool enqueueJsonObject(const StaticJsonDocument<JSON_MSG_BUFFER>& jsonDoc);
+extern void buildTopicFromId(JsonObject& Jsondata, const char* origin);
+extern bool pubMQTT(const char* topic, const char* payload);
+extern bool pubMQTT(const char* topic, const char* payload, bool retainFlag);
+extern bool pubMQTT(String topic, const char* payload);
+extern bool pubMQTT(const char* topic, unsigned long payload);
+extern bool pubMQTT(const char* topic, unsigned long long payload);
+extern bool pubMQTT(const char* topic, String payload);
+extern bool pubMQTT(String topic, String payload);
+extern bool pubMQTT(String topic, int payload);
+extern bool pubMQTT(String topic, unsigned long long payload);
+extern bool pubMQTT(String topic, float payload);
+extern bool pubMQTT(const char* topic, float payload);
+extern bool pubMQTT(const char* topic, int payload);
+extern bool pubMQTT(const char* topic, unsigned int payload);
+extern bool pubMQTT(const char* topic, long payload);
+extern bool pubMQTT(const char* topic, double payload);
+extern bool pubMQTT(String topic, unsigned long payload);
+extern unsigned long uptime();
+extern bool cmpToMainTopic(const char*, const char*);
+extern bool pub(const char*, const char*, bool);
+extern bool pub(const char*, const char*);
+
+template <typename T>
+void Config_update(JsonObject& data, const char* key, T& var) {
+  if (data.containsKey(key)) {
+    if (var != data[key].as<T>()) {
+      var = data[key].as<T>();
+      Log.notice(F("Config %s changed to: %T" CR), key, data[key].as<T>());
+    } else {
+      Log.notice(F("Config %s unchanged, currently: %T" CR), key, data[key].as<T>());
+    }
+  }
+}