Merge pull request #63 from technyon/add-mqtt-logging

Add mqtt logging

CMakeLists.txt

@@ -33,7 +33,7 @@ include_directories(${PROJECT_NAME}
         lib/pubsubclient/src
         lib/WebServer/src
         lib/Ethernet/src
-#        include
+        lib/MqttLogger/src
 )
 
 file(GLOB SRCFILES
@@ -54,6 +54,7 @@ file(GLOB SRCFILES
         PresenceDetection.cpp
         PreferencesKeys.h
         Gpio.cpp
+        Logger.cpp
         Version.h
 #        include/RTOS.h
         lib/WiFiManager/WiFiManager.cpp
@@ -72,6 +73,7 @@ file(GLOB SRCFILES
         lib/BleScanner/src/BleInterfaces.h
         lib/BleScanner/src/BleScanner.cpp
         lib/pubsubclient/src/PubSubClient.cpp
+        lib/MqttLogger/src/MqttLogger.cpp
 )
 
 file(GLOB_RECURSE SRCFILESREC

Gpio.cpp

@@ -2,6 +2,7 @@
 #include "Gpio.h"
 #include "Arduino.h"
 #include "Pins.h"
+#include "Logger.h"
 
 Gpio* Gpio::_inst = nullptr;
 NukiWrapper* Gpio::_nuki = nullptr;
@@ -26,7 +27,7 @@ void Gpio::isrLock()
     if(millis() < _lockedTs) return;
     _nuki->lock();
     _lockedTs = millis() + _debounceTime;
-    Serial.println(F("Lock via GPIO"));;
+    Log->println(F("Lock via GPIO"));;
 }
 
 void Gpio::isrUnlock()
@@ -34,7 +35,7 @@ void Gpio::isrUnlock()
     if(millis() < _lockedTs) return;
     _nuki->unlock();
     _lockedTs = millis() + _debounceTime;
-    Serial.println(F("Unlock via GPIO"));;
+    Log->println(F("Unlock via GPIO"));;
 }
 
 void Gpio::isrUnlatch()
@@ -42,5 +43,5 @@ void Gpio::isrUnlatch()
     if(millis() < _lockedTs) return;
     _nuki->unlatch();
     _lockedTs = millis() + _debounceTime;
-    Serial.println(F("Unlatch via GPIO"));;
+    Log->println(F("Unlatch via GPIO"));;
 }

Logger.cpp

@@ -0,0 +1,3 @@
+#include "Logger.h"
+
+Print* Log = nullptr;

Logger.h

@@ -0,0 +1,7 @@
+#ifndef MQTT_LOGGER_GLOBAL
+#define MQTT_LOGGER_GLOBAL
+
+#include "MqttLogger.h"
+extern Print* Log;
+
+#endif

MqttTopics.h

@@ -42,4 +42,5 @@
 #define mqtt_topic_reset "/maintenance/reset"
 #define mqtt_topic_uptime "/maintenance/uptime"
 #define mqtt_topic_wifi_rssi "/maintenance/wifiRssi"
+#define mqtt_topic_log "/maintenance/log"
 #define mqtt_topic_freeheap "/maintenance/freeHeap"

Network.cpp

@@ -3,6 +3,7 @@
 #include "MqttTopics.h"
 #include "networkDevices/W5500Device.h"
 #include "networkDevices/WifiDevice.h"
+#include "Logger.h"
 
 
 Network* Network::_inst = nullptr;
@@ -29,15 +30,15 @@ void Network::setupDevice(const NetworkDeviceType hardware)
     switch(hardware)
     {
         case NetworkDeviceType::W5500:
-            Serial.println(F("Network device: W5500"));
+            Log->println(F("Network device: W5500"));
             _device = new W5500Device(_hostname, _preferences);
             break;
         case NetworkDeviceType::WiFi:
-            Serial.println(F("Network device: Builtin WiFi"));
+            Log->println(F("Network device: Builtin WiFi"));
             _device = new WifiDevice(_hostname, _preferences);
             break;
         default:
-            Serial.println(F("Unknown network device type, defaulting to WiFi"));
+            Log->println(F("Unknown network device type, defaulting to WiFi"));
             _device = new WifiDevice(_hostname, _preferences);
             break;
     }
@@ -55,8 +56,8 @@ void Network::initialize()
 
     _device->initialize();
 
-    Serial.print(F("Host name: "));
+    Log->print(F("Host name: "));
-    Serial.println(_hostname);
+    Log->println(_hostname);
 
     String brokerAddr = _preferences->getString(preference_mqtt_broker);
     strcpy(_mqttBrokerAddr, brokerAddr.c_str());
@@ -88,10 +89,10 @@ void Network::initialize()
         }
     }
 
-    Serial.print(F("MQTT Broker: "));
+    Log->print(F("MQTT Broker: "));
-    Serial.print(_mqttBrokerAddr);
+    Log->print(_mqttBrokerAddr);
-    Serial.print(F(":"));
+    Log->print(F(":"));
-    Serial.println(port);
+    Log->println(port);
 
     _device->mqttClient()->setServer(_mqttBrokerAddr, port);
     _device->mqttClient()->setCallback(Network::onMqttDataReceivedCallback);
@@ -117,16 +118,16 @@ int Network::update()
             ESP.restart();
         }
 
-        Serial.println(F("Network not connected. Trying reconnect."));
+        Log->println(F("Network not connected. Trying reconnect."));
         bool success = _device->reconnect();
-        Serial.println(success ? F("Reconnect successful") : F("Reconnect failed"));
+        Log->println(success ? F("Reconnect successful") : F("Reconnect failed"));
     }
 
     if(!_device->isConnected())
     {
         if(_networkTimeout > 0 && (ts - _lastConnectedTs > _networkTimeout * 1000))
         {
-            Serial.println("Network timeout has been reached, restarting ...");
+            Log->println("Network timeout has been reached, restarting ...");
             delay(200);
             ESP.restart();
         }
@@ -149,8 +150,8 @@ int Network::update()
         bool success = publishString(_mqttPresencePrefix, mqtt_topic_presence, _presenceCsv);
         if(!success)
         {
-            Serial.println(F("Failed to publish presence CSV data."));
+            Log->println(F("Failed to publish presence CSV data."));
-            Serial.println(_presenceCsv);
+            Log->println(_presenceCsv);
         }
         _presenceCsv = nullptr;
     }
@@ -184,23 +185,23 @@ bool Network::reconnect()
 
     while (!_device->mqttClient()->connected() && millis() > _nextReconnect)
     {
-        Serial.println(F("Attempting MQTT connection"));
+        Log->println(F("Attempting MQTT connection"));
         bool success = false;
 
         if(strlen(_mqttUser) == 0)
         {
-            Serial.println(F("MQTT: Connecting without credentials"));
+            Log->println(F("MQTT: Connecting without credentials"));
             success = _device->mqttClient()->connect(_preferences->getString(preference_hostname).c_str());
         }
         else
         {
-            Serial.print(F("MQTT: Connecting with user: ")); Serial.println(_mqttUser);
+            Log->print(F("MQTT: Connecting with user: ")); Log->println(_mqttUser);
             success = _device->mqttClient()->connect(_preferences->getString(preference_hostname).c_str(), _mqttUser, _mqttPass);
         }
 
         if (success)
         {
-            Serial.println(F("MQTT connected"));
+            Log->println(F("MQTT connected"));
             _mqttConnected = true;
             delay(100);
             for(const String& topic : _subscribedTopics)
@@ -218,8 +219,8 @@ bool Network::reconnect()
         }
         else
         {
-            Serial.print(F("MQTT connect failed, rc="));
+            Log->print(F("MQTT connect failed, rc="));
-            Serial.println(_device->mqttClient()->state());
+            Log->println(_device->mqttClient()->state());
             _device->printError();
             _device->mqttClient()->disconnect();
             _mqttConnected = false;

NetworkOpener.cpp

@@ -4,6 +4,7 @@
 #include "MqttTopics.h"
 #include "PreferencesKeys.h"
 #include "Pins.h"
+#include "Logger.h"
 
 NetworkOpener::NetworkOpener(Network* network, Preferences* preferences)
         : _preferences(preferences),
@@ -70,8 +71,8 @@ void NetworkOpener::onMqttDataReceived(char *&topic, byte *&payload, unsigned in
     {
         if(strcmp(value, "") == 0 || strcmp(value, "--") == 0 || strcmp(value, "ack") == 0 || strcmp(value, "unknown_action") == 0) return;
 
-        Serial.print(F("Opener lock action received: "));
+        Log->print(F("Opener lock action received: "));
-        Serial.println(value);
+        Log->println(value);
         bool success = false;
         if(_lockActionReceivedCallback != NULL)
         {

NukiWrapper.cpp

@@ -2,6 +2,7 @@
 #include <RTOS.h>
 #include "PreferencesKeys.h"
 #include "MqttTopics.h"
+#include "Logger.h"
 #include <NukiLockUtils.h>
 
 NukiWrapper* nukiInst;
@@ -70,12 +71,12 @@ void NukiWrapper::initialize()
 
     _nukiLock.setEventHandler(this);
 
-    Serial.print(F("Lock state interval: "));
+    Log->print(F("Lock state interval: "));
-    Serial.print(_intervalLockstate);
+    Log->print(_intervalLockstate);
-    Serial.print(F(" | Battery interval: "));
+    Log->print(F(" | Battery interval: "));
-    Serial.print(_intervalBattery);
+    Log->print(_intervalBattery);
-    Serial.print(F(" | Publish auth data: "));
+    Log->print(F(" | Publish auth data: "));
-    Serial.println(_publishAuthData ? "yes" : "no");
+    Log->println(_publishAuthData ? "yes" : "no");
 
     if(!_publishAuthData)
     {
@@ -91,14 +92,14 @@ void NukiWrapper::update()
 
     if (!_paired)
     {
-        Serial.println(F("Nuki start pairing"));
+        Log->println(F("Nuki start pairing"));
 
         Nuki::AuthorizationIdType idType = _preferences->getBool(preference_register_as_app) ?
                                            Nuki::AuthorizationIdType::App :
                                            Nuki::AuthorizationIdType::Bridge;
 
         if (_nukiLock.pairNuki(idType) == Nuki::PairingResult::Success) {
-            Serial.println(F("Nuki paired"));
+            Log->println(F("Nuki paired"));
             _paired = true;
             setupHASS();
         }
@@ -111,9 +112,9 @@ void NukiWrapper::update()
 
     if(_restartBeaconTimeout > 0 && (millis() - _nukiLock.getLastReceivedBeaconTs() > _restartBeaconTimeout * 1000))
     {
-        Serial.print("No BLE beacon received from the lock for ");
+        Log->print("No BLE beacon received from the lock for ");
-        Serial.print((millis() - _nukiLock.getLastReceivedBeaconTs()) / 1000);
+        Log->print((millis() - _nukiLock.getLastReceivedBeaconTs()) / 1000);
-        Serial.println(" seconds, restarting device.");
+        Log->println(" seconds, restarting device.");
         delay(200);
         ESP.restart();
     }
@@ -165,8 +166,8 @@ void NukiWrapper::update()
 
         _network->publishCommandResult(resultStr);
 
-        Serial.print(F("Lock action result: "));
+        Log->print(F("Lock action result: "));
-        Serial.println(resultStr);
+        Log->println(resultStr);
 
         _nextLockAction = (NukiLock::LockAction)0xff;
         if(_intervalLockstate > 10)
@@ -219,8 +220,8 @@ void NukiWrapper::updateKeyTurnerState()
     {
         char lockStateStr[20];
         lockstateToString(_keyTurnerState.lockState, lockStateStr);
-        Serial.print(F("Nuki lock state: "));
+        Log->print(F("Nuki lock state: "));
-        Serial.println(lockStateStr);
+        Log->println(lockStateStr);
     }
 
     if(_publishAuthData)
@@ -430,7 +431,7 @@ void NukiWrapper::onKeypadCommandReceived(const char *command, const uint &id, c
         memcpy(&entry.name, name.c_str(), nameLen > 20 ? 20 : nameLen);
         entry.code = codeInt;
         result = _nukiLock.addKeypadEntry(entry);
-        Serial.print("Add keypad code: "); Serial.println((int)result);
+        Log->print("Add keypad code: "); Log->println((int)result);
         updateKeypad();
     }
     else if(strcmp(command, "delete") == 0)
@@ -441,7 +442,7 @@ void NukiWrapper::onKeypadCommandReceived(const char *command, const uint &id, c
             return;
         }
         result = _nukiLock.deleteKeypadEntry(id);
-        Serial.print("Delete keypad code: "); Serial.println((int)result);
+        Log->print("Delete keypad code: "); Log->println((int)result);
         updateKeypad();
     }
     else if(strcmp(command, "update") == 0)
@@ -475,7 +476,7 @@ void NukiWrapper::onKeypadCommandReceived(const char *command, const uint &id, c
         entry.code = codeInt;
         entry.enabled = enabled == 0 ? 0 : 1;
         result = _nukiLock.updateKeypadEntry(entry);
-        Serial.print("Update keypad code: "); Serial.println((int)result);
+        Log->print("Update keypad code: "); Log->println((int)result);
         updateKeypad();
     }
     else if(command == "--")
@@ -522,18 +523,18 @@ void NukiWrapper::notify(Nuki::EventType eventType)
 
 void NukiWrapper::readConfig()
 {
-    Serial.print(F("Reading config. Result: "));
+    Log->print(F("Reading config. Result: "));
     Nuki::CmdResult result = _nukiLock.requestConfig(&_nukiConfig);
     _nukiConfigValid = result == Nuki::CmdResult::Success;
-    Serial.println(result);
+    Log->println(result);
 }
 
 void NukiWrapper::readAdvancedConfig()
 {
-    Serial.print(F("Reading advanced config. Result: "));
+    Log->print(F("Reading advanced config. Result: "));
     Nuki::CmdResult result = _nukiLock.requestAdvancedConfig(&_nukiAdvancedConfig);
     _nukiAdvancedConfigValid = result == Nuki::CmdResult::Success;
-    Serial.println(result);
+    Log->println(result);
 }
 
 void NukiWrapper::setupHASS()
@@ -552,7 +553,7 @@ void NukiWrapper::setupHASS()
     }
     else
     {
-        Serial.println(F("Unable to setup HASS. Invalid config received."));
+        Log->println(F("Unable to setup HASS. Invalid config received."));
     }
 }
 
@@ -571,6 +572,6 @@ void NukiWrapper::disableHASS()
     }
     else
     {
-        Serial.println(F("Unable to disable HASS. Invalid config received."));
+        Log->println(F("Unable to disable HASS. Invalid config received."));
     }
 }

PreferencesKeys.h

@@ -6,6 +6,7 @@
 #define preference_mqtt_broker_port "mqttport"
 #define preference_mqtt_user "mqttuser"
 #define preference_mqtt_password "mqttpass"
+#define preference_mqtt_log_enabled "mqttlog"
 #define preference_lock_enabled "lockena"
 #define preference_mqtt_lock_path "mqttpath"
 #define preference_opener_enabled "openerena"

WebCfgServer.cpp

@@ -3,6 +3,7 @@
 #include "PreferencesKeys.h"
 #include "Version.h"
 #include "hardware/WifiEthServer.h"
+#include "Logger.h"
 #include <esp_task_wdt.h>
 
 WebCfgServer::WebCfgServer(NukiWrapper* nuki, NukiOpenerWrapper* nukiOpener, Network* network, EthServer* ethServer, Preferences* preferences, bool allowRestartToPortal)
@@ -130,7 +131,7 @@ void WebCfgServer::initialize()
             String response = "";
             buildConfirmHtml(response, message);
             _server.send(200, "text/html", response);
-            Serial.println(F("Restarting"));
+            Log->println(F("Restarting"));
 
             waitAndProcess(true, 1000);
             ESP.restart();
@@ -276,6 +277,11 @@ bool WebCfgServer::processArgs(String& message)
             _preferences->putInt(preference_restart_timer, value.toInt());
             configChanged = true;
         }
+        else if(key == "MQTTLOG")
+        {
+            _preferences->putBool(preference_mqtt_log_enabled, (value == "1"));
+            configChanged = true;
+        }
         else if(key == "LSTINT")
         {
             _preferences->putInt(preference_query_interval_lockstate, value.toInt());
@@ -413,7 +419,7 @@ void WebCfgServer::update()
 {
     if(_otaStartTs > 0 && (millis() - _otaStartTs) > 120000)
     {
-        Serial.println(F("OTA time out, restarting"));
+        Log->println(F("OTA time out, restarting"));
         delay(200);
         ESP.restart();
     }
@@ -585,6 +591,7 @@ void WebCfgServer::buildMqttConfigHtml(String &response)
     printInputField(response, "NETTIMEOUT", "Network Timeout until restart (seconds; -1 to disable)", _preferences->getInt(preference_network_timeout), 5);
     printCheckBox(response, "RSTDISC", "Restart on disconnect", _preferences->getBool(preference_restart_on_disconnect));
     printInputField(response, "RSTTMR", "Restart timer (minutes; -1 to disable)", _preferences->getInt(preference_restart_timer), 10);
+    printCheckBox(response, "MQTTLOG", "Enable MQTT logging", _preferences->getBool(preference_mqtt_log_enabled));
     response.concat("</table>");
     response.concat("* If no encryption is configured for the MQTT broker, leave empty. Only supported for WiFi connections.<br>");
 
@@ -849,7 +856,7 @@ void WebCfgServer::handleOtaUpload()
 
     if(upload.filename == "")
     {
-        Serial.println("Invalid file for OTA upload");
+        Log->println("Invalid file for OTA upload");
         return;
     }
 
@@ -861,7 +868,7 @@ void WebCfgServer::handleOtaUpload()
         _otaStartTs = millis();
         esp_task_wdt_init(30, false);
         _network->disableAutoRestarts();
-        Serial.print("handleFileUpload Name: "); Serial.println(filename);
+        Log->print("handleFileUpload Name: "); Serial.println(filename);
     } else if (upload.status == UPLOAD_FILE_WRITE) {
         _transferredSize = _transferredSize + upload.currentSize;
         Serial.println(_transferredSize);

lib/MqttLogger/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2008-2020 Nicholas O'Leary
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

lib/MqttLogger/README.md

@@ -0,0 +1,71 @@
+# Remote logging to a MQTT server with the same print() interface as Serial
+
+This library provides an object that can be used just like `Serial` for printing logs, 
+however the text sent with `print()` etc. is published on a MQTT broker instead of 
+printing over the Serial console. This comes in handy when working with devices like the 
+ESP8266/ESP32 that are connected over WiFi. I use it for debugging my robots
+that are based on ESP32.
+
+The library uses [PubSubClient](https://github.com/knolleary/pubsubclient) for sending
+the MQTT messages.
+
+When no MQTT connection is available, the `MqttLogger` object behaves just like 
+`Serial`, i.e. your `print()` text is shown on the `Serial` console. The logger offers
+the following modes that can be passed as the third argument to the constructor
+when instantiating the object:
+
+* `MqttLoggerMode::MqttAndSerialFallback` - this is the default mode. `print()` will
+  publish to the MQTT server, and only when no MQTT connection is available `Serial` 
+  will be used. If you `print()` messages before the MQTT connection is established,
+  these messages will be sent to the `Serial` console. 
+* `MqttLoggerMode::MqttOnly` - no output on `Serial`. Beware: when no connection is 
+  available, no output is produced
+* `MqttLoggerMode::SerialOnly` - no messages are sent to the MQTT server. With this
+  configuration `MqttLogger` can be used as a substitute for logging with `Serial`. 
+* `MqttLoggerMode::MqttAndSerial` - messages are sent both to the MQTT server and to
+  the `Serial` console. 
+
+## Examples
+
+See directory `examples`. Currently there is only one example in directory `esp32`.
+
+In this directory, rename the file `wifi_secrets.h.txt` to `wifi_secrets.h` 
+and edit the file. Enter your WiFi ssid and password, the example uses this
+include file to set up your WiFi connection.
+
+You'll need a MQTT broker to publish your messages to, I use [Mosquitto](https://mosquitto.org/) 
+installed locally on my laptop. You can also use a free public service like 
+`test.mosquitto.org` or `broker.hivemq.com`, but this makes logging slower 
+(the messages have to be sent to and then downloaded from the online service). Also,
+make sure no private information is logged!
+
+The broker url is defined by the constant `mqtt_server` in the example, use
+`localhost` if you have a local install as recommended.
+
+For checking the mqtt logs events you'll use a MQTT client. The Mosquitto client 
+can be invoked in a terminal like
+
+    mosquitto_sub -h localhost -t mqttlogger/log
+
+but any other mqtt client will do (on Android try MQTT Dash, hivemq has a online
+version at (http://www.hivemq.com/demos/websocket-client/).
+
+## Compatible Hardware
+
+All devices that work with the PubSubClient should work with this libary, including:
+
+ - Arduino Ethernet
+ - Arduino Ethernet Shield
+ - Arduino YUN – use the included `YunClient` in place of `EthernetClient`, and
+   be sure to do a `Bridge.begin()` first
+ - Arduino WiFi Shield - if you want to send packets > 90 bytes with this shield,
+   enable the `MQTT_MAX_TRANSFER_SIZE` define in `PubSubClient.h`.
+ - Sparkfun WiFly Shield – [library](https://github.com/dpslwk/WiFly)
+ - TI CC3000 WiFi - [library](https://github.com/sparkfun/SFE_CC3000_Library)
+ - Intel Galileo/Edison
+ - ESP8266
+ - ESP32
+
+## License
+
+This code is released under the MIT License.

lib/MqttLogger/examples/esp32/esp32.ino

@@ -0,0 +1,94 @@
+#include <WiFi.h>
+#include <PubSubClient.h>
+#include <MqttLogger.h>
+#include "wifi_secrets.h"
+
+const char *ssid = WIFI_SSID;
+const char *password = WIFI_PASSWORD;
+const char *mqtt_server =  "broker.hivemq.com";
+
+WiFiClient espClient;
+PubSubClient client(espClient);
+
+// default mode is MqttLoggerMode::MqttAndSerialFallback
+MqttLogger mqttLogger(client,"mqttlogger/log");
+// other available modes:
+// MqttLogger mqttLogger(client,"mqttlogger/log",MqttLoggerMode::MqttAndSerial);
+// MqttLogger mqttLogger(client,"mqttlogger/log",MqttLoggerMode::MqttOnly);
+// MqttLogger mqttLogger(client,"mqttlogger/log",MqttLoggerMode::SerialOnly);
+
+// connect to wifi network
+void wifiConnect()
+{
+    mqttLogger.print("Connecting to WiFi: ");
+    mqttLogger.println(ssid);
+    WiFi.begin(ssid, password);
+    while (WiFi.status() != WL_CONNECTED)
+    {
+        delay(500);
+        mqttLogger.print(".");
+    }
+    mqttLogger.print("WiFi connected: ");
+    mqttLogger.println(WiFi.localIP());
+}
+
+// establish mqtt client connection
+void reconnect()
+{
+    // Loop until we're reconnected
+    while (!client.connected())
+    {
+        mqttLogger.print("Attempting MQTT connection...");
+        // Attempt to connect
+        if (client.connect("ESP32Logger"))
+        {
+            // as we have a connection here, this will be the first message published to the mqtt server
+            mqttLogger.println("connected.");  
+
+        }
+        else
+        {
+            mqttLogger.print("failed, rc=");
+            mqttLogger.print(client.state());
+            mqttLogger.println(" try again in 5 seconds");
+            // Wait 5 seconds before retrying
+            delay(5000);
+        }
+    }
+}
+
+// arduino setup
+void setup()
+{
+    Serial.begin(115200);
+    // MqttLogger uses mqtt when available and Serial as a fallback. Before any connection is established, 
+    // MqttLogger works just like Serial 
+    mqttLogger.println("Starting setup..");
+ 
+    // connect to wifi
+    wifiConnect();
+ 
+    // mqtt client
+    client.setServer(mqtt_server, 1883);
+    //client.setCallback(mqttIncomingCallback);
+    
+} 
+
+void loop()
+{
+    // here the mqtt connection is established
+    if (!client.connected())
+    {
+        reconnect();
+    }
+    // with a connection, subsequent print() publish on the mqtt broker, but in a buffered fashion
+    
+    mqttLogger.print(millis()/1000);
+    delay(1000);
+    mqttLogger.print(" seconds");
+    delay(1000);
+    mqttLogger.print(" online");
+    delay(1000);
+    // this finally flushes the buffer and published on the mqtt broker
+    mqttLogger.println();
+}

lib/MqttLogger/examples/esp32/wifi_secrets.h.txt

@@ -0,0 +1,3 @@
+// rename this file to wifi_secrets.h and provide the wifi secrets below
+#define WIFI_SSID "your wifi ssid here"
+#define WIFI_PASSWORD "your wifi password here"

lib/MqttLogger/library.json

@@ -0,0 +1,18 @@
+{
+    "name": "MqttLogger",
+    "keywords": "serial, mqtt, logging",
+    "description": "Remote logging on a mqtt broker with the same interface as Serial.print().",
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/androbi-com/MqttLogger.git"
+    },
+    "version": "0.2.3",
+    "exclude": "examples/*/wifi_secrets.h",
+    "examples": "examples/*/*.ino",
+    "frameworks": "arduino",
+    "platforms": [
+        "atmelavr",
+        "espressif8266",
+        "espressif32"
+    ]
+}

lib/MqttLogger/library.properties

@@ -0,0 +1,11 @@
+name=MqttLogger
+version=0.2.3
+author=androbi <info@androbi.com>
+maintainer=androbi <info@androbi.co>
+sentence=Remote logging on a mqtt broker with the same interface as Serial.print()
+paragraph=This library is a substitute for Serial as a logging/debug tool when your device has an internet connection (ESP32 etc.) and is not connected over the serial port. The text written by the print() commands is published to a given topic on a MQTT broker. By subscribing to the same topic you can display the log messages remotely. When no MQTT connection is available, Serial is used as a fallback.
+category=Communication
+url=https://github.com/androbi-com/MqttLogger
+architectures=*
+includes=MqttLogger.h
+depends=PubSubClient

lib/MqttLogger/src/MqttLogger.cpp

@@ -0,0 +1,116 @@
+#include "MqttLogger.h"
+#include "Arduino.h"
+
+MqttLogger::MqttLogger(MqttLoggerMode mode)
+{
+    this->setMode(mode);
+    this->setBufferSize(MQTT_MAX_PACKET_SIZE);
+}
+
+MqttLogger::MqttLogger(PubSubClient& client, const char* topic, MqttLoggerMode mode)
+{
+    this->setClient(client);
+    this->setTopic(topic);
+    this->setMode(mode);
+    this->setBufferSize(MQTT_MAX_PACKET_SIZE);
+}
+
+MqttLogger::~MqttLogger()
+{
+}
+
+void MqttLogger::setClient(PubSubClient& client)
+{
+    this->client = &client;
+}
+
+void MqttLogger::setTopic(const char* topic)
+{
+    this->topic = topic;
+}
+
+void MqttLogger::setMode(MqttLoggerMode mode)
+{
+    this->mode = mode;
+}
+
+uint16_t MqttLogger::getBufferSize()
+{
+    return this->bufferSize;
+}
+
+// allocate or reallocate local buffer, reset end to start of buffer
+boolean MqttLogger::setBufferSize(uint16_t size)
+{
+    if (size == 0)
+    {
+        return false;
+    }
+    if (this->bufferSize == 0)
+    {
+        this->buffer = (uint8_t *)malloc(size);
+        this->bufferEnd = this->buffer;
+    }
+    else
+    {
+        uint8_t *newBuffer = (uint8_t *)realloc(this->buffer, size);
+        if (newBuffer != NULL)
+        {
+            this->buffer = newBuffer;
+            this->bufferEnd = this->buffer;
+        }
+        else
+        {
+            return false;
+        }
+    }
+    this->bufferSize = size;
+    return (this->buffer != NULL);
+}
+
+// send & reset current buffer
+void MqttLogger::sendBuffer() 
+{
+    if (this->bufferCnt > 0)
+    {
+        bool doSerial = this->mode==MqttLoggerMode::SerialOnly || this->mode==MqttLoggerMode::MqttAndSerial;
+        if (this->mode!=MqttLoggerMode::SerialOnly && this->client != NULL && this->client->connected()) 
+        {
+            this->client->publish(this->topic, (byte *)this->buffer, this->bufferCnt, 1);
+        } else if (this->mode == MqttLoggerMode::MqttAndSerialFallback)
+        {
+            doSerial = true;
+        }
+        if (doSerial) 
+        {
+            Serial.write(this->buffer, this->bufferCnt);
+            Serial.println();
+        }
+        this->bufferCnt=0;
+    }
+    this->bufferEnd=this->buffer;
+}
+
+// implement Print::write(uint8_t c): store into a buffer until \n or buffer full
+size_t MqttLogger::write(uint8_t character)
+{
+    if (character == '\n') // when newline is printed we send the buffer
+    {
+        this->sendBuffer();
+    }
+    else
+    {
+        if (this->bufferCnt < this->bufferSize) // add char to end of buffer
+        {
+            *(this->bufferEnd++) = character;
+            this->bufferCnt++;
+        }
+        else // buffer is full, first send&reset buffer and then add char to buffer
+        {
+            this->sendBuffer();
+            *(this->bufferEnd++) = character;
+            this->bufferCnt++;
+        }
+    }
+    return 1;
+}

lib/MqttLogger/src/MqttLogger.h

@@ -0,0 +1,51 @@
+/*
+  MqttLogger - offer print() interface like Serial but by publishing to a given mqtt topic. 
+               Uses Serial as a fallback when no mqtt connection is available.
+
+  Claus Denk
+  https://androbi.com
+*/
+
+#ifndef MqttLogger_h
+#define MqttLogger_h
+
+#include <Arduino.h>
+#include <Print.h>
+#include <PubSubClient.h>
+
+enum MqttLoggerMode {
+    MqttAndSerialFallback = 0,
+    SerialOnly = 1,
+    MqttOnly = 2,
+    MqttAndSerial = 3,
+};
+
+class MqttLogger : public Print
+{
+private:
+    const char* topic;
+    uint8_t* buffer;
+    uint8_t* bufferEnd;
+    uint16_t bufferCnt = 0, bufferSize = 0;
+    PubSubClient* client;
+    MqttLoggerMode mode;
+    void sendBuffer();
+
+public:
+    MqttLogger(MqttLoggerMode mode=MqttLoggerMode::MqttAndSerialFallback);
+    MqttLogger(PubSubClient& client, const char* topic, MqttLoggerMode mode=MqttLoggerMode::MqttAndSerialFallback);
+    ~MqttLogger();
+
+    void setClient(PubSubClient& client);
+    void setTopic(const char* topic);
+    void setMode(MqttLoggerMode mode);
+    void setRetained(boolean retained);
+    
+    virtual size_t write(uint8_t);
+    using Print::write;
+    
+    uint16_t getBufferSize();
+    boolean setBufferSize(uint16_t size);
+};
+
+#endif

main.cpp

@@ -10,6 +10,7 @@
 #include "hardware/WifiEthServer.h"
 #include "NukiOpenerWrapper.h"
 #include "Gpio.h"
+#include "Logger.h"
 
 Network* network = nullptr;
 NetworkLock* networkLock = nullptr;
@@ -93,7 +94,7 @@ void checkMillisTask(void *pvParameters)
         // millis() is about to overflow. Restart device to prevent problems with overflow
         if(millis() > restartTs)
         {
-            Serial.println(F("Restart timer expired, restarting device."));
+            Log->println(F("Restart timer expired, restarting device."));
             delay(200);
             ESP.restart();
         }
@@ -161,6 +162,7 @@ void setup()
     pinMode(NETWORK_SELECT, INPUT_PULLUP);
 
     Serial.begin(115200);
+    Log = &Serial;
 
     initPreferences();
 
@@ -194,7 +196,7 @@ void setup()
     bleScanner->setScanDuration(10);
 
     lockEnabled = preferences->getBool(preference_lock_enabled);
-    Serial.println(lockEnabled ? F("NUKI Lock enabled") : F("NUKI Lock disabled"));
+    Log->println(lockEnabled ? F("NUKI Lock enabled") : F("NUKI Lock disabled"));
     if(lockEnabled)
     {
         nuki = new NukiWrapper("NukiHub", deviceId, bleScanner, networkLock, preferences);
@@ -207,7 +209,7 @@ void setup()
     }
 
     openerEnabled = preferences->getBool(preference_opener_enabled);
-    Serial.println(openerEnabled ? F("NUKI Opener enabled") : F("NUKI Opener disabled"));
+    Log->println(openerEnabled ? F("NUKI Opener enabled") : F("NUKI Opener disabled"));
     if(openerEnabled)
     {
         nukiOpener = new NukiOpenerWrapper("NukiHub", deviceId, bleScanner, networkOpener, preferences);

networkDevices/W5500Device.cpp

@@ -3,6 +3,8 @@
 #include "W5500Device.h"
 #include "../Pins.h"
 #include "../PreferencesKeys.h"
+#include "../Logger.h"
+#include "../MqttTopics.h"
 
 W5500Device::W5500Device(const String &hostname, Preferences* preferences)
 : NetworkDevice(hostname),
@@ -10,20 +12,20 @@ W5500Device::W5500Device(const String &hostname, Preferences* preferences)
 {
     initializeMacAddress(_mac);
 
-    Serial.print("MAC Adress: ");
+    Log->print("MAC Adress: ");
     for(int i=0; i < 6; i++)
     {
         if(_mac[i] < 10)
         {
-            Serial.print(F("0"));
+            Log->print(F("0"));
         }
-        Serial.print(_mac[i], 16);
+        Log->print(_mac[i], 16);
         if(i < 5)
         {
-            Serial.print(F(":"));
+            Log->print(F(":"));
         }
     }
-    Serial.println();
+    Log->println();
 }
 
 W5500Device::~W5500Device()
@@ -40,39 +42,49 @@ void W5500Device::initialize()
     _mqttClient = new PubSubClient(*_ethClient);
     _mqttClient->setBufferSize(_mqttMaxBufferSize);
 
+
+    if(_preferences->getBool(preference_mqtt_log_enabled))
+    {
+        _path = new char[200];
+        memset(_path, 0, sizeof(_path));
+
+        String pathStr = _preferences->getString(preference_mqtt_lock_path);
+        pathStr.concat(mqtt_topic_log);
+        strcpy(_path, pathStr.c_str());
+        Log = new MqttLogger(*_mqttClient, _path);
+    }
+
     reconnect();
 }
 
 
 bool W5500Device::reconnect()
 {
-
-
     _hasDHCPAddress = false;
 
     // start the Ethernet connection:
-    Serial.println(F("Initialize Ethernet with DHCP:"));
+    Log->println(F("Initialize Ethernet with DHCP:"));
 
     int dhcpRetryCnt = 0;
 
     while(dhcpRetryCnt < 3)
     {
-        Serial.print(F("DHCP connect try #"));
+        Log->print(F("DHCP connect try #"));
-        Serial.print(dhcpRetryCnt);
+        Log->print(dhcpRetryCnt);
-        Serial.println();
+        Log->println();
         dhcpRetryCnt++;
 
         if (Ethernet.begin(_mac, 1000, 1000) == 0)
         {
-            Serial.println(F("Failed to configure Ethernet using DHCP"));
+            Log->println(F("Failed to configure Ethernet using DHCP"));
             // Check for Ethernet hardware present
             if (Ethernet.hardwareStatus() == EthernetNoHardware)
             {
-                Serial.println(F("Ethernet module not found"));
+                Log->println(F("Ethernet module not found"));
             }
             if (Ethernet.linkStatus() == LinkOFF)
             {
-                Serial.println(F("Ethernet cable is not connected."));
+                Log->println(F("Ethernet cable is not connected."));
             }
 
             IPAddress ip;
@@ -91,8 +103,8 @@ bool W5500Device::reconnect()
         {
             _hasDHCPAddress = true;
             dhcpRetryCnt = 1000;
-            Serial.print(F("  DHCP assigned IP "));
+            Log->print(F("  DHCP assigned IP "));
-            Serial.println(Ethernet.localIP());
+            Log->println(Ethernet.localIP());
         }
     }
 
@@ -102,12 +114,12 @@ bool W5500Device::reconnect()
 
 void W5500Device::reconfigure()
 {
-    Serial.println(F("Reconfigure W5500 not implemented."));
+    Log->println(F("Reconfigure W5500 not implemented."));
 }
 
 void W5500Device::resetDevice()
 {
-    Serial.println(F("Resetting network hardware."));
+    Log->println(F("Resetting network hardware."));
     pinMode(ETHERNET_RESET_PIN, OUTPUT);
     digitalWrite(ETHERNET_RESET_PIN, HIGH);
     delay(250);
@@ -120,8 +132,8 @@ void W5500Device::resetDevice()
 
 void W5500Device::printError()
 {
-    Serial.print(F("Free Heap: "));
+    Log->print(F("Free Heap: "));
-    Serial.println(ESP.getFreeHeap());
+    Log->println(ESP.getFreeHeap());
 }
 
 PubSubClient *W5500Device::mqttClient()

networkDevices/W5500Device.h

@@ -33,6 +33,7 @@ private:
 
     int _maintainResult = 0;
     bool _hasDHCPAddress = false;
+    char* _path;
 
     byte _mac[6];
 };

networkDevices/WifiDevice.cpp

@@ -1,6 +1,8 @@
 #include <WiFi.h>
 #include "WifiDevice.h"
 #include "../PreferencesKeys.h"
+#include "../Logger.h"
+#include "../MqttTopics.h"
 
 RTC_NOINIT_ATTR char WiFiDevice_reconfdetect[17];
 
@@ -19,25 +21,36 @@ WifiDevice::WifiDevice(const String& hostname, Preferences* _preferences)
 
     if(caLength > 1) // length is 1 when empty
     {
-        Serial.println(F("MQTT over TLS."));
+        Log->println(F("MQTT over TLS."));
-        Serial.println(_ca);
+        Log->println(_ca);
         _wifiClientSecure = new WiFiClientSecure();
         _wifiClientSecure->setCACert(_ca);
         if(crtLength > 1 && keyLength > 1) // length is 1 when empty
         {
-            Serial.println(F("MQTT with client certificate."));
+            Log->println(F("MQTT with client certificate."));
-            Serial.println(_cert);
+            Log->println(_cert);
-            Serial.println(_key);
+            Log->println(_key);
             _wifiClientSecure->setCertificate(_cert);
             _wifiClientSecure->setPrivateKey(_key);
         }
         _mqttClient = new PubSubClient(*_wifiClientSecure);
     } else
     {
-        Serial.println(F("MQTT without TLS."));
+        Log->println(F("MQTT without TLS."));
         _wifiClient = new WiFiClient();
         _mqttClient = new PubSubClient(*_wifiClient);
     }
+
+    if(_preferences->getBool(preference_mqtt_log_enabled))
+    {
+        _path = new char[200];
+        memset(_path, 0, sizeof(_path));
+
+        String pathStr = _preferences->getString(preference_mqtt_lock_path);
+        pathStr.concat(mqtt_topic_log);
+        strcpy(_path, pathStr.c_str());
+        Log = new MqttLogger(*_mqttClient, _path);
+    }
 }
 
 PubSubClient *WifiDevice::mqttClient()
@@ -60,7 +73,7 @@ void WifiDevice::initialize()
 
     if(_startAp)
     {
-        Serial.println(F("Opening WiFi configuration portal."));
+        Log->println(F("Opening WiFi configuration portal."));
         res = _wm.startConfigPortal();
     }
     else
@@ -69,13 +82,13 @@ void WifiDevice::initialize()
     }
 
     if(!res) {
-        Serial.println(F("Failed to connect. Wait for ESP restart."));
+        Log->println(F("Failed to connect. Wait for ESP restart."));
         delay(1000);
         ESP.restart();
     }
     else {
-        Serial.print(F("WiFi connected: "));
+        Log->print(F("WiFi connected: "));
-        Serial.println(WiFi.localIP().toString());
+        Log->println(WiFi.localIP().toString());
     }
 
     if(_restartOnDisconnect)
@@ -101,10 +114,10 @@ void WifiDevice::printError()
     {
         char lastError[100];
         _wifiClientSecure->lastError(lastError,100);
-        Serial.println(lastError);
+        Log->println(lastError);
     }
-    Serial.print(F("Free Heap: "));
+    Log->print(F("Free Heap: "));
-    Serial.println(ESP.getFreeHeap());
+    Log->println(ESP.getFreeHeap());
 }
 
 bool WifiDevice::isConnected()

networkDevices/WifiDevice.h

@@ -34,6 +34,7 @@ private:
 //    SpiffsCookie _cookie;
     bool _restartOnDisconnect = false;
     bool _startAp = false;
+    char* _path;
 
     char _ca[TLS_CA_MAX_SIZE];
     char _cert[TLS_CERT_MAX_SIZE];