Automated updating to beta and nightly versions (#425)

* Github actions * Update firmware update and import/export * Auto commit fixes
2024-07-18 13:27:57 +02:00
parent eff4e57488
commit f4038f0429
5 changed files with 296 additions and 15 deletions
--- a/.github/workflows/beta.yml
+++ b/.github/workflows/beta.yml
@@ -0,0 +1,121 @@
+name: NukiHub Beta
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    name: Build ${{ matrix.board }} (${{ matrix.build }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        board: [esp32dev, esp32-s3, esp32-c3, esp32-c6, esp32solo1]
+        build: [release]
+    env:
+      BOARD: ${{ matrix.board }}
+      VARIANT: ${{ matrix.name || matrix.board }}
+      BUILD: ${{ matrix.build }}
+    steps:
+      - name: Fix variant name
+        run: |
+          # remove dash character
+          export VARIANT=${VARIANT//-/}
+
+          if [ "$VARIANT" = "esp32dev" ]; then
+            VARIANT="esp32"
+          fi
+
+          echo "VARIANT=${VARIANT}" | tee -a ${GITHUB_ENV}
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.platformio/.cache
+            ~/.platformio/packages
+          key: ${{ runner.os }}-pio-${{ matrix.board }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - name: Install dependencies
+        run: make deps
+      - name: Add version info
+        env:
+          Version: ${{ github.run_id }}.${{ github.run_number }}.${{ github.run_attempt }}
+        run: |
+          sed -i "s/unknownbuildnr/$Version (${BUILD})/g" src/Config.h
+      - name: Build ${{ matrix.build }} PlatformIO Project ${{ matrix.board }}
+        run: |
+          if [ "$BUILD" = "debug" ]; then
+            BOARD="${BOARD}_dbg"
+          fi
+          echo "::group::Building with PlatformIO"
+            if [ "$BUILD" = "release" ]; then
+              make updater_${BOARD}
+            fi
+            make $BOARD
+          echo "::endgroup::"
+      - name: Add flash script
+        env:
+          DOC: resources/how-to-flash.txt
+          FILES: ${{ format('{0}/{1}', env.BUILD, env.VARIANT) }}
+        run: |
+          BOARD=`echo $BOARD | tr '[:lower:]' '[:upper:]'`
+
+          # fix for docs
+          if [ "$BOARD" = "ESP32DEV" ]; then
+            BOARD="ESP32"
+          fi
+
+          # look for documentation on flash and copy the command
+          command=`sed -n '/^Howto flash (esptool)$/,$p' ${DOC} | sed -n '/^## '"${BOARD}"'$/,\${ n; n; p; }' | head -n1`
+
+          if [ -z "$command" ]; then
+            echo "::error::Command not found in document ${DOC} for board ${BOARD}"
+            exit 1
+          fi
+
+          echo -n "Command: "
+          echo "$command" | tee ${FILES}/flash.sh
+          chmod a+x ${FILES}/flash.sh
+
+      - name: Upload Artifact ${{ matrix.board }}-${{ matrix.build }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ format('{0}-{1}-assets', env.VARIANT, matrix.build) }}
+          path: ${{ matrix.build }}/${{ env.VARIANT }}
+  ota-beta:
+    name: Create beta from latest master
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Download release assets
+        uses: actions/download-artifact@v4
+        with:
+          path: release
+          pattern: '*-release-assets'
+      - name: Copy binaries to ota/beta
+        run: |
+          mkdir -p ota/beta/
+          cp -vf release/*/nuki_hub_*.bin ota/beta/
+          rm -rf release
+      - name: Commit binaries to master
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "Update beta binaries"
+          file_pattern: 'ota/beta/*.bin'
+          branch: master
+          skip_dirty_check: true
+          skip_fetch: true
+          skip_checkout: true
+          disable_globbing: true
+          add_options: '-f'
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -6,7 +6,6 @@ on:
  pull_request:
    branches: ["*"]
    tags: ["*"]
-
  workflow_dispatch:

 permissions:
@@ -96,7 +95,6 @@ jobs:
        with:
          name: ${{ format('{0}-{1}-assets', env.VARIANT, matrix.build) }}
          path: ${{ matrix.build }}/${{ env.VARIANT }}
-
  release:
    name: Release new version
    needs: build
@@ -168,16 +166,22 @@ jobs:
          name: "Nuki Hub ${{ steps.get_version.outputs.VERSION }}"
          artifacts: ${{ steps.zip.outputs.artifacts }}
          artifactContentType: application/zip
-      - name: Copy binaries to ota and webflash
+      - name: Copy binaries to ota and webflash and remove beta
        run: |
          cp -vf release/*/nuki_hub_*.bin ota/
          cp -vf release/*/webflash_nuki_hub_*.bin webflash/
          rm -rf release
          rm -rf debug
          rm -rf NukiHub-*.zip
+          git rm -r --cached ota/beta/*.bin
      - name: Commit binaries to master
        uses: stefanzweifel/git-auto-commit-action@v5
        with:
          commit_message: "Update binaries for version ${{ steps.get_version.outputs.VERSION }}"
-          file_pattern: 'ota/*.bin webflash/*.bin'
+          file_pattern: 'ota/*.bin ota/beta/*.bin ota/beta/debug/*.bin webflash/*.bin'
          branch: master
+          skip_dirty_check: true    
+          skip_fetch: true    
+          skip_checkout: true
+          disable_globbing: true
+          add_options: '-f'
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -0,0 +1,127 @@
+name: NukiHub Nightly
+on:
+  workflow_dispatch:
+  schedule:
+  - cron: "30 0 * * *"
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    name: Build ${{ matrix.board }} (${{ matrix.build }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        board: [esp32dev, esp32-s3, esp32-c3, esp32-c6, esp32solo1]
+        build: [release]
+    env:
+      BOARD: ${{ matrix.board }}
+      VARIANT: ${{ matrix.name || matrix.board }}
+      BUILD: ${{ matrix.build }}
+    steps:
+      - name: Fix variant name
+        run: |
+          # remove dash character
+          export VARIANT=${VARIANT//-/}
+
+          if [ "$VARIANT" = "esp32dev" ]; then
+            VARIANT="esp32"
+          fi
+
+          echo "VARIANT=${VARIANT}" | tee -a ${GITHUB_ENV}
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: recursive
+      - name: Get new commits
+        run: echo "NEW_COMMIT_COUNT=$(git log --oneline --since '24 hours ago' | wc -l)" >> $GITHUB_ENV
+      - uses: actions/cache@v4
+        if: ${{ env.NEW_COMMIT_COUNT > 1 }}
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.platformio/.cache
+            ~/.platformio/packages
+          key: ${{ runner.os }}-pio-${{ matrix.board }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - name: Install dependencies
+        run: make deps
+      - name: Add version info
+        env:
+          Version: ${{ github.run_id }}.${{ github.run_number }}.${{ github.run_attempt }}
+        run: |
+          sed -i "s/unknownbuildnr/$Version (${BUILD})/g" src/Config.h
+      - name: Build ${{ matrix.build }} PlatformIO Project ${{ matrix.board }}
+        run: |
+          if [ "$BUILD" = "debug" ]; then
+            BOARD="${BOARD}_dbg"
+          fi
+          echo "::group::Building with PlatformIO"
+            if [ "$BUILD" = "release" ]; then
+              make updater_${BOARD}
+            fi
+            make $BOARD
+          echo "::endgroup::"
+      - name: Add flash script
+        env:
+          DOC: resources/how-to-flash.txt
+          FILES: ${{ format('{0}/{1}', env.BUILD, env.VARIANT) }}
+        run: |
+          BOARD=`echo $BOARD | tr '[:lower:]' '[:upper:]'`
+
+          # fix for docs
+          if [ "$BOARD" = "ESP32DEV" ]; then
+            BOARD="ESP32"
+          fi
+
+          # look for documentation on flash and copy the command
+          command=`sed -n '/^Howto flash (esptool)$/,$p' ${DOC} | sed -n '/^## '"${BOARD}"'$/,\${ n; n; p; }' | head -n1`
+
+          if [ -z "$command" ]; then
+            echo "::error::Command not found in document ${DOC} for board ${BOARD}"
+            exit 1
+          fi
+
+          echo -n "Command: "
+          echo "$command" | tee ${FILES}/flash.sh
+          chmod a+x ${FILES}/flash.sh
+
+      - name: Upload Artifact ${{ matrix.board }}-${{ matrix.build }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ format('{0}-{1}-assets', env.VARIANT, matrix.build) }}
+          path: ${{ matrix.build }}/${{ env.VARIANT }}
+  ota-nightly:
+    name: Create nightly from latest master
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Download release assets
+        uses: actions/download-artifact@v4
+        with:
+          path: release
+          pattern: '*-release-assets'
+      - name: Copy binaries to ota/master
+        run: |
+          mkdir -p ota/master/
+          cp -vf release/*/nuki_hub_*.bin ota/master/
+          rm -rf release
+      - name: Commit binaries to master
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "Update master binaries"
+          file_pattern: 'ota/master/*.bin'
+          branch: master
+          skip_dirty_check: true
+          skip_fetch: true
+          skip_checkout: true
+          disable_globbing: true
+          add_options: '-f'
--- a/src/Config.h
+++ b/src/Config.h
@@ -2,7 +2,7 @@

 #include "sdkconfig.h"

-#define NUKI_HUB_VERSION "8.35"
+#define NUKI_HUB_VERSION "9.00"
 #define NUKI_HUB_BUILD "unknownbuildnr"

 #define GITHUB_LATEST_RELEASE_URL (char*)"https://github.com/technyon/nuki_hub/releases/latest"
@@ -13,27 +13,37 @@
 #define GITHUB_LATEST_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/nuki_hub_updater_esp32c3.bin"
 #define GITHUB_BETA_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/beta/nuki_hub_esp32c3.bin"
 #define GITHUB_BETA_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/beta/nuki_hub_updater_esp32c3.bin"
+#define GITHUB_MASTER_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/master/nuki_hub_esp32c3.bin"
+#define GITHUB_MASTER_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/master/nuki_hub_updater_esp32c3.bin"
 #elif defined(CONFIG_IDF_TARGET_ESP32S3)
 #define GITHUB_LATEST_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/nuki_hub_esp32s3.bin"
 #define GITHUB_LATEST_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/nuki_hub_updater_esp32s3.bin"
 #define GITHUB_BETA_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/beta/nuki_hub_esp32s3.bin"
 #define GITHUB_BETA_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/beta/nuki_hub_updater_esp32s3.bin"
+#define GITHUB_MASTER_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/master/nuki_hub_esp32s3.bin"
+#define GITHUB_MASTER_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/master/nuki_hub_updater_esp32s3.bin"
 #elif defined(CONFIG_IDF_TARGET_ESP32C6)
 #define GITHUB_LATEST_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/nuki_hub_esp32c6.bin"
 #define GITHUB_LATEST_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/nuki_hub_updater_esp32c6.bin"
 #define GITHUB_BETA_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/beta/nuki_hub_esp32c6.bin"
 #define GITHUB_BETA_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/beta/nuki_hub_updater_esp32c6.bin"
+#define GITHUB_MASTER_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/master/nuki_hub_esp32c6.bin"
+#define GITHUB_MASTER_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/master/nuki_hub_updater_esp32c6.bin"
 #else
 #if defined(FRAMEWORK_ARDUINO_SOLO1)
 #define GITHUB_LATEST_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/nuki_hub_esp32solo1.bin"
 #define GITHUB_LATEST_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/nuki_hub_updater_esp32solo1.bin"
 #define GITHUB_BETA_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/beta/nuki_hub_esp32solo1.bin"
 #define GITHUB_BETA_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/beta/nuki_hub_updater_esp32solo1.bin"
+#define GITHUB_MASTER_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/master/nuki_hub_esp32solo1.bin"
+#define GITHUB_MASTER_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/master/nuki_hub_updater_esp32solo1.bin"
 #else
 #define GITHUB_LATEST_RELEASE_BINARY_URL "https://github.com/technyon/nuki_hub/raw/master/ota/nuki_hub_esp32.bin"
 #define GITHUB_LATEST_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/nuki_hub_updater_esp32.bin"
 #define GITHUB_BETA_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/beta/nuki_hub_esp32.bin"
 #define GITHUB_BETA_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/beta/nuki_hub_updater_esp32.bin"
+#define GITHUB_MASTER_RELEASE_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/master/nuki_hub_esp32.bin"
+#define GITHUB_MASTER_UPDATER_BINARY_URL (char*)"https://github.com/technyon/nuki_hub/raw/master/ota/master/nuki_hub_updater_esp32.bin"
 #endif
 #endif

--- a/src/WebCfgServer.cpp
+++ b/src/WebCfgServer.cpp
@@ -329,6 +329,12 @@ void WebCfgServer::initialize()
            _preferences->putString(preference_ota_updater_url, GITHUB_BETA_UPDATER_BINARY_URL);
            _preferences->putString(preference_ota_main_url, GITHUB_BETA_RELEASE_BINARY_URL);
        }
+        else if(key == "master")
+        {
+            buildConfirmHtml(response, "Rebooting to update Nuki Hub and Nuki Hub updater<br/>Updating to latest development version", 2);
+            _preferences->putString(preference_ota_updater_url, GITHUB_MASTER_UPDATER_BINARY_URL);
+            _preferences->putString(preference_ota_main_url, GITHUB_MASTER_RELEASE_BINARY_URL);
+        }
        else
        {
            buildConfirmHtml(response, "Rebooting to update Nuki Hub and Nuki Hub updater<br/>Updating to latest RELEASE version", 2);
@@ -397,7 +403,7 @@ void WebCfgServer::buildOtaHtml(String &response, bool errored)

    if(_partitionType == 0)
    {
-        response.concat("<h4 class=\"warning\">You are currently running Nuki Hub with an outdated partition scheme. Because of this you cannot use OTA to update to 8.36 or higher. Please check GitHub for instructions on how to update to 8.36 and the new partition scheme</h4>");
+        response.concat("<h4 class=\"warning\">You are currently running Nuki Hub with an outdated partition scheme. Because of this you cannot use OTA to update to 9.00 or higher. Please check GitHub for instructions on how to update to 9.00 and the new partition scheme</h4>");
        response.concat("<button title=\"Open latest release on GitHub\" onclick=\" window.open('");
        response.concat(GITHUB_LATEST_RELEASE_URL);
        response.concat("', '_blank'); return false;\">Open latest release on GitHub</button>");
@@ -407,13 +413,18 @@ void WebCfgServer::buildOtaHtml(String &response, bool errored)
    response.concat("<div id=\"msgdiv\" style=\"visibility:hidden\">Initiating Over-the-air update. This will take about two minutes, please be patient.<br>You will be forwarded automatically when the update is complete.</div>");

    #if (ESP_IDF_VERSION > ESP_IDF_VERSION_VAL(5, 0, 0))
-    response.concat("<div id=\"autoupdform\"><h4>Auto update Nuki Hub</h4>");
+    response.concat("<div id=\"autoupdform\"><h4>Update Nuki Hub</h4>");
    response.concat("Click on the button to reboot and automatically update Nuki Hub and the Nuki Hub updater to the latest versions from GitHub");
-    response.concat("<form action=\"/autoupdate\" method=\"get\"><br><input type=\"submit\" value=\"Auto Update\" /></form><br><br></div>");
+    response.concat("<div style=\"clear: both\"></div>");
+    response.concat("<form onsubmit=\"return confirm('Do you really want to update to the latest release?');\" action=\"/autoupdate\" method=\"get\" style=\"float: left; margin-right: 10px\"><br><input type=\"submit\" style=\"background: green\" value=\"Update to latest release\"></form>");
+    response.concat("<form onsubmit=\"return confirm('Do you really want to update to the latest beta? This version could contain breaking bugs and necessitate downgrading to the latest release version using USB/Serial');\" action=\"/autoupdate?beta\" method=\"get\" style=\"float: left; margin-right: 10px\"><br><input type=\"submit\" style=\"color: black; background: yellow\"  value=\"Update to latest beta\"></form>");
+    response.concat("<form onsubmit=\"return confirm('Do you really want to update to the latest development version? This version could contain breaking bugs and necessitate downgrading to the latest release version using USB/Serial');\" action=\"/autoupdate?master\" method=\"get\" style=\"float: left; margin-right: 10px\"><br><input type=\"submit\" style=\"background: red\"  value=\"Update to latest development version\"></form>");
+    response.concat("<div style=\"clear: both\"></div><br><br></div>");
    #endif
    
    if(_partitionType == 1)
    {
+        response.concat("<h4><a onclick=\"hideshowmanual();\">Manually update Nuki Hub</a></h4><div id=\"manualupdate\" style=\"display: none\">");
        response.concat("<div id=\"rebootform\"><h4>Reboot to Nuki Hub Updater</h4>");
        response.concat("Click on the button to reboot to the Nuki Hub updater, where you can select the latest Nuki Hub binary to update");
        response.concat("<form action=\"/reboottoota\" method=\"get\"><br><input type=\"submit\" value=\"Reboot to Nuki Hub Updater\" /></form><br><br></div>");
@@ -423,6 +434,7 @@ void WebCfgServer::buildOtaHtml(String &response, bool errored)
    }
    else
    {
+        response.concat("<div id=\"manualupdate\">");
        response.concat("<div id=\"rebootform\"><h4>Reboot to Nuki Hub</h4>");
        response.concat("Click on the button to reboot to Nuki Hub");
        response.concat("<form action=\"/reboottoota\" method=\"get\"><br><input type=\"submit\" value=\"Reboot to Nuki Hub\" /></form><br><br></div>");
@@ -441,7 +453,7 @@ void WebCfgServer::buildOtaHtml(String &response, bool errored)
    response.concat("'); return false;\">Download latest binary from GitHub</button>");
    response.concat("<br><br><button title=\"Download latest updater binary from GitHub\" onclick=\" window.open('");
    response.concat(GITHUB_LATEST_UPDATER_BINARY_URL);
-    response.concat("'); return false;\">Download latest updater binary from GitHub</button></div>");
+    response.concat("'); return false;\">Download latest updater binary from GitHub</button></div></div>");
    response.concat("<script type=\"text/javascript\">");
    response.concat("window.addEventListener('load', function () {");
    response.concat("	var button = document.getElementById(\"submitbtn\");");
@@ -454,6 +466,14 @@ void WebCfgServer::buildOtaHtml(String &response, bool errored)
    response.concat("		document.getElementById('msgdiv').style.visibility = 'visible';");
    response.concat("	}");
    response.concat("});");
+    response.concat("function hideshowmanual() {");
+    response.concat("	var x = document.getElementById(\"manualupdate\");");
+    response.concat("	if (x.style.display === \"none\") {");
+    response.concat("	    x.style.display = \"block\";");
+    response.concat("	} else {");
+    response.concat("	    x.style.display = \"none\";");
+    response.concat("    }");
+    response.concat("}");
    response.concat("</script>");
    response.concat("</body></html>");
 }
@@ -1887,9 +1907,8 @@ void WebCfgServer::buildImportExportHtml(String &response)
    response.concat("<div id=\"gitdiv\">");
    response.concat("<h4>Export configuration</h4><br>");
    response.concat("<button title=\"Basic export\" onclick=\" window.open('/export', '_self'); return false;\">Basic export</button>");
-    response.concat("<br><br><button title=\"Export with redacted settings\" onclick=\" window.open('/export?redacted=1'); return false;\">Export with redacted settings</button>");if( _preferences->getBool(preference_show_secrets)) {
+    response.concat("<br><br><button title=\"Export with redacted settings\" onclick=\" window.open('/export?redacted=1'); return false;\">Export with redacted settings</button>");
    response.concat("<br><br><button title=\"Export with redacted settings and pairing data\" onclick=\" window.open('/export?redacted=1&pairing=1'); return false;\">Export with redacted settings and pairing data</button>");
-    }
    response.concat("</div><div id=\"msgdiv\" style=\"visibility:hidden\">Initiating config update. Please be patient.<br>You will be forwarded automatically when the import is complete.</div>");
    response.concat("<script type=\"text/javascript\">");
    response.concat("window.addEventListener('load', function () {");