diff --git a/.github/scripts/update_esptool.py b/.github/scripts/update_esptool.py new file mode 100644 index 00000000000..dd5de5526c3 --- /dev/null +++ b/.github/scripts/update_esptool.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 + +# This script is used to re-package the esptool if needed and update the JSON file +# for the Arduino ESP32 platform. +# +# The script has only been tested on macOS. +# +# For regular esptool releases, the generated packages already contain the correct permissions, +# extensions and are uploaded to the GitHub release assets. In this case, the script will only +# update the JSON file with the information from the GitHub release. +# +# The script can be used in two modes: +# 1. Local build: The build artifacts must be already downloaded and extracted in the base_folder. +# This is useful for esptool versions that are not yet released and that are grabbed from the +# GitHub build artifacts. +# 2. Release build: The script will get the release information from GitHub and update the JSON file. +# This is useful for esptool versions that are already released and that are uploaded to the +# GitHub release assets. +# +# For local build, the artifacts must be already downloaded and extracted in the base_folder +# set with the -l option. +# For example, a base folder "esptool" should contain the following folders extracted directly +# from the GitHub build artifacts: +# esptool/esptool-linux-aarch64 +# esptool/esptool-linux-amd64 +# esptool/esptool-linux-armv7 +# esptool/esptool-macos-amd64 +# esptool/esptool-macos-arm64 +# esptool/esptool-windows-amd64 + +import argparse +import json +import os +import shutil +import stat +import tarfile +import zipfile +import hashlib +import requests +from pathlib import Path + +def compute_sha256(filepath): + sha256 = hashlib.sha256() + with open(filepath, "rb") as f: + for block in iter(lambda: f.read(4096), b""): + sha256.update(block) + return f"SHA-256:{sha256.hexdigest()}" + +def get_file_size(filepath): + return os.path.getsize(filepath) + +def update_json_for_host(tmp_json_path, version, host, url, archiveFileName, checksum, size): + with open(tmp_json_path) as f: + data = json.load(f) + + for pkg in data.get("packages", []): + for tool in pkg.get("tools", []): + if tool.get("name") == "esptool_py": + tool["version"] = version + + if url is None: + # If the URL is not set, we need to find the old URL and update it + for system in tool.get("systems", []): + if system.get("host") == host: + url = system.get("url").replace(system.get("archiveFileName"), archiveFileName) + break + else: + print(f"No old URL found for host {host}. Using empty URL.") + url = "" + + # Preserve existing systems order and update or append the new system + systems = tool.get("systems", []) + system_updated = False + for i, system in enumerate(systems): + if system.get("host") == host: + systems[i] = { + "host": host, + "url": url, + "archiveFileName": archiveFileName, + "checksum": checksum, + "size": str(size), + } + system_updated = True + break + + if not system_updated: + systems.append({ + "host": host, + "url": url, + "archiveFileName": archiveFileName, + "checksum": checksum, + "size": str(size), + }) + tool["systems"] = systems + + with open(tmp_json_path, "w") as f: + json.dump(data, f, indent=2, sort_keys=False, ensure_ascii=False) + f.write("\n") + +def update_tools_dependencies(tmp_json_path, version): + with open(tmp_json_path) as f: + data = json.load(f) + + for pkg in data.get("packages", []): + for platform in pkg.get("platforms", []): + for dep in platform.get("toolsDependencies", []): + if dep.get("name") == "esptool_py": + dep["version"] = version + + with open(tmp_json_path, "w") as f: + json.dump(data, f, indent=2, sort_keys=False, ensure_ascii=False) + f.write("\n") + +def create_archives(version, base_folder): + archive_files = [] + + for dirpath in Path(base_folder).glob("esptool-*"): + if not dirpath.is_dir(): + continue + + base = dirpath.name[len("esptool-"):] + + if "windows" in dirpath.name: + zipfile_name = f"esptool-v{version}-{base}.zip" + print(f"Creating {zipfile_name} from {dirpath} ...") + with zipfile.ZipFile(zipfile_name, "w", zipfile.ZIP_DEFLATED) as zipf: + for root, _, files in os.walk(dirpath): + for file in files: + full_path = os.path.join(root, file) + zipf.write(full_path, os.path.relpath(full_path, start=dirpath)) + archive_files.append(zipfile_name) + else: + tarfile_name = f"esptool-v{version}-{base}.tar.gz" + print(f"Creating {tarfile_name} from {dirpath} ...") + for root, dirs, files in os.walk(dirpath): + for name in dirs + files: + os.chmod(os.path.join(root, name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + with tarfile.open(tarfile_name, "w:gz") as tar: + tar.add(dirpath, arcname=dirpath.name) + archive_files.append(tarfile_name) + + return archive_files + +def determine_hosts(archive_name): + if "linux-amd64" in archive_name: + return ["x86_64-pc-linux-gnu"] + elif "linux-armv7" in archive_name: + return ["arm-linux-gnueabihf"] + elif "linux-aarch64" in archive_name: + return ["aarch64-linux-gnu"] + elif "macos-amd64" in archive_name: + return ["x86_64-apple-darwin"] + elif "macos-arm64" in archive_name: + return ["arm64-apple-darwin"] + elif "windows-amd64" in archive_name: + return ["x86_64-mingw32", "i686-mingw32"] + else: + return [] + +def update_json_from_local_build(tmp_json_path, version, base_folder, archive_files): + for archive in archive_files: + print(f"Processing archive: {archive}") + hosts = determine_hosts(archive) + if not hosts: + print(f"Skipping unknown archive type: {archive}") + continue + + archive_path = Path(archive) + checksum = compute_sha256(archive_path) + size = get_file_size(archive_path) + + for host in hosts: + update_json_for_host(tmp_json_path, version, host, None, archive_path.name, checksum, size) + +def update_json_from_release(tmp_json_path, version, release_info): + assets = release_info.get("assets", []) + for asset in assets: + if (asset.get("name").endswith(".tar.gz") or asset.get("name").endswith(".zip")) and "esptool" in asset.get("name"): + asset_fname = asset.get("name") + print(f"Processing asset: {asset_fname}") + hosts = determine_hosts(asset_fname) + if not hosts: + print(f"Skipping unknown archive type: {asset_fname}") + continue + + asset_url = asset.get("browser_download_url") + asset_checksum = asset.get("digest") + asset_size = asset.get("size") + if asset_checksum is None: + asset_checksum = "" + print(f"Asset {asset_fname} has no checksum. Please set the checksum in the JSON file.") + + for host in hosts: + update_json_for_host(tmp_json_path, version, host, asset_url, asset_fname, asset_checksum, asset_size) + +def get_release_info(version): + url = f"https://api.github.com/repos/espressif/esptool/releases/tags/v{version}" + response = requests.get(url) + response.raise_for_status() + return response.json() + +def main(): + parser = argparse.ArgumentParser(description="Repack esptool and update JSON metadata.") + parser.add_argument("version", help="Version of the esptool (e.g. 5.0.dev1)") + parser.add_argument("-l", "--local", dest="base_folder", help="Enable local build mode and set the base folder with unpacked artifacts") + args = parser.parse_args() + + script_dir = Path(__file__).resolve().parent + json_path = (script_dir / "../../package/package_esp32_index.template.json").resolve() + tmp_json_path = Path(str(json_path) + ".tmp") + shutil.copy(json_path, tmp_json_path) + + local_build = args.base_folder is not None + + if local_build: + os.chdir(args.base_folder) + os.environ['COPYFILE_DISABLE'] = 'true' # this disables including resource forks in tar files on macOS + # Clear any existing archive files + for file in Path(args.base_folder).glob("esptool-*.*"): + file.unlink() + archive_files = create_archives(args.version, args.base_folder) + update_json_from_local_build(tmp_json_path, args.version, args.base_folder, archive_files) + else: + release_info = get_release_info(args.version) + update_json_from_release(tmp_json_path, args.version, release_info) + + print(f"Updating esptool version fields to {args.version}") + update_tools_dependencies(tmp_json_path, args.version) + + shutil.move(tmp_json_path, json_path) + print(f"Done. JSON updated at {json_path}") + +if __name__ == "__main__": + main() diff --git a/package/package_esp32_index.template.json b/package/package_esp32_index.template.json index eecc7c10788..3ae2ff09ee6 100644 --- a/package/package_esp32_index.template.json +++ b/package/package_esp32_index.template.json @@ -81,7 +81,7 @@ { "packager": "esp32", "name": "esptool_py", - "version": "4.9.dev3" + "version": "5.0.dev1" }, { "packager": "esp32", @@ -469,56 +469,56 @@ }, { "name": "esptool_py", - "version": "4.9.dev3", + "version": "5.0.dev1", "systems": [ { - "host": "x86_64-pc-linux-gnu", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-linux-amd64.tar.gz", - "archiveFileName": "esptool-v4.9.dev3-linux-amd64.tar.gz", - "checksum": "SHA-256:4ecaf51836cbf4ea3c19840018bfef3b0b8cd8fc3c95f6e1e043ca5bbeab9bf0", - "size": "64958202" + "host": "aarch64-linux-gnu", + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-linux-aarch64.tar.gz", + "archiveFileName": "esptool-v5.0.dev1-linux-aarch64.tar.gz", + "checksum": "SHA-256:bfafa7a7723ebbabfd8b6e3ca5ae00bfead0331de923754aeddb43b2c116a078", + "size": "58241736" }, { - "host": "arm-linux-gnueabihf", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-linux-armv7.tar.gz", - "archiveFileName": "esptool-v4.9.dev3-linux-armv7.tar.gz", - "checksum": "SHA-256:fff818573bce483ee793ac83c8211f6abf764aa3350f198228859f696a0a0b36", - "size": "31530030" + "host": "x86_64-pc-linux-gnu", + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-linux-amd64.tar.gz", + "archiveFileName": "esptool-v5.0.dev1-linux-amd64.tar.gz", + "checksum": "SHA-256:acd0486e96586b99d053a1479acbbbfcae8667227c831cdc53a171f9ccfa27ee", + "size": "100740042" }, { - "host": "aarch64-linux-gnu", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-linux-aarch64.tar.gz", - "archiveFileName": "esptool-v4.9.dev3-linux-aarch64.tar.gz", - "checksum": "SHA-256:5b274bdff2f62e6a07c3c1dfa51b1128924621f661747eca3dbe0f77972f2f06", - "size": "33663882" + "host": "arm-linux-gnueabihf", + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-linux-armv7.tar.gz", + "archiveFileName": "esptool-v5.0.dev1-linux-armv7.tar.gz", + "checksum": "SHA-256:ea77a38681506761bbb7b0b39c130811ed565667b67ebbdb4d6dcc6cb6e07368", + "size": "53451939" }, { "host": "x86_64-apple-darwin", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-macos-amd64.tar.gz", - "archiveFileName": "esptool-v4.9.dev3-macos-amd64.tar.gz", - "checksum": "SHA-256:c733c83b58fcf5f642fbb2fddb8ff24640c2c785126cba0821fb70c4a5ceea7a", - "size": "32767836" + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-macos-amd64.tar.gz", + "archiveFileName": "esptool-v5.0.dev1-macos-amd64.tar.gz", + "checksum": "SHA-256:900a8e90731208bee96647e0e207a43612b9452c2120c4fdc0ff4c6be226257b", + "size": "59631998" }, { "host": "arm64-apple-darwin", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-macos-arm64.tar.gz", - "archiveFileName": "esptool-v4.9.dev3-macos-arm64.tar.gz", - "checksum": "SHA-256:83c195a15981e6a5e7a130db2ccfb21e2d8093912e5b003681f9a5abadd71af7", - "size": "30121441" + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-macos-arm64.tar.gz", + "archiveFileName": "esptool-v5.0.dev1-macos-arm64.tar.gz", + "checksum": "SHA-256:3653f4de73cb4fc6a25351eaf663708e91c65ae3265d75bd54ca4315a4350bb4", + "size": "56349992" }, { - "host": "i686-mingw32", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-win64.zip", - "archiveFileName": "esptool-v4.9.dev3-win64.zip", - "checksum": "SHA-256:890051a4fdc684ff6f4af18d0bb27d274ca940ee0eef716a9455f8c64b25b215", - "size": "36072564" + "host": "x86_64-mingw32", + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-win64.zip", + "archiveFileName": "esptool-v5.0.dev1-win64.zip", + "checksum": "SHA-256:1e8fd89645daf94f2d4406ec73c9004e617ea921079515f9fd749205eece4d6d", + "size": "59102658" }, { - "host": "x86_64-mingw32", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-win64.zip", - "archiveFileName": "esptool-v4.9.dev3-win64.zip", - "checksum": "SHA-256:890051a4fdc684ff6f4af18d0bb27d274ca940ee0eef716a9455f8c64b25b215", - "size": "36072564" + "host": "i686-mingw32", + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-win64.zip", + "archiveFileName": "esptool-v5.0.dev1-win64.zip", + "checksum": "SHA-256:1e8fd89645daf94f2d4406ec73c9004e617ea921079515f9fd749205eece4d6d", + "size": "59102658" } ] }, diff --git a/platform.txt b/platform.txt index f41a8b4a764..8bfa69a79e0 100644 --- a/platform.txt +++ b/platform.txt @@ -120,7 +120,7 @@ recipe.hooks.prebuild.2.pattern.windows=cmd /c if not exist "{build.path}\partit recipe.hooks.prebuild.3.pattern.windows=cmd /c if not exist "{build.path}\partitions.csv" COPY "{runtime.platform.path}\tools\partitions\{build.partitions}.csv" "{build.path}\partitions.csv" # Check if custom bootloader exist: source > variant > build.boot -recipe.hooks.prebuild.4.pattern_args=--chip {build.mcu} elf2image --flash_mode {build.flash_mode} --flash_freq {build.img_freq} --flash_size {build.flash_size} -o +recipe.hooks.prebuild.4.pattern_args=--chip {build.mcu} elf2image --flash-mode {build.flash_mode} --flash-freq {build.img_freq} --flash-size {build.flash_size} -o recipe.hooks.prebuild.4.pattern=/usr/bin/env bash -c "[ -f "{build.source.path}"/bootloader.bin ] && cp -f "{build.source.path}"/bootloader.bin "{build.path}"/{build.project_name}.bootloader.bin || ( [ -f "{build.variant.path}"/{build.custom_bootloader}.bin ] && cp "{build.variant.path}"/{build.custom_bootloader}.bin "{build.path}"/{build.project_name}.bootloader.bin || "{tools.esptool_py.path}"/{tools.esptool_py.cmd} {recipe.hooks.prebuild.4.pattern_args} "{build.path}"/{build.project_name}.bootloader.bin "{compiler.sdk.path}"/bin/bootloader_{build.boot}_{build.boot_freq}.elf )" recipe.hooks.prebuild.4.pattern.windows=cmd /c IF EXIST "{build.source.path}\bootloader.bin" ( COPY /y "{build.source.path}\bootloader.bin" "{build.path}\{build.project_name}.bootloader.bin" ) ELSE ( IF EXIST "{build.variant.path}\{build.custom_bootloader}.bin" ( COPY "{build.variant.path}\{build.custom_bootloader}.bin" "{build.path}\{build.project_name}.bootloader.bin" ) ELSE ( "{tools.esptool_py.path}\{tools.esptool_py.cmd}" {recipe.hooks.prebuild.4.pattern_args} "{build.path}\{build.project_name}.bootloader.bin" "{compiler.sdk.path}\bin\bootloader_{build.boot}_{build.boot_freq}.elf" ) ) @@ -164,7 +164,7 @@ recipe.c.combine.pattern="{compiler.path}{compiler.c.elf.cmd}" {compiler.c.elf.f recipe.objcopy.partitions.bin.pattern={tools.gen_esp32part.cmd} -q "{build.path}/partitions.csv" "{build.path}/{build.project_name}.partitions.bin" ## Create bin -recipe.objcopy.bin.pattern_args=--chip {build.mcu} elf2image --flash_mode "{build.flash_mode}" --flash_freq "{build.img_freq}" --flash_size "{build.flash_size}" --elf-sha256-offset 0xb0 -o "{build.path}/{build.project_name}.bin" "{build.path}/{build.project_name}.elf" +recipe.objcopy.bin.pattern_args=--chip {build.mcu} elf2image --flash-mode "{build.flash_mode}" --flash-freq "{build.img_freq}" --flash-size "{build.flash_size}" --elf-sha256-offset 0xb0 -o "{build.path}/{build.project_name}.bin" "{build.path}/{build.project_name}.elf" recipe.objcopy.bin.pattern="{tools.esptool_py.path}/{tools.esptool_py.cmd}" {recipe.objcopy.bin.pattern_args} ## Create Insights Firmware Package @@ -177,7 +177,7 @@ recipe.hooks.objcopy.postobjcopy.2.pattern=/usr/bin/env bash -c "[ ! -d "{build. recipe.hooks.objcopy.postobjcopy.2.pattern.windows=cmd /c if exist "{build.path}\libraries\ESP_SR" if exist "{compiler.sdk.path}\esp_sr\srmodels.bin" COPY /y "{compiler.sdk.path}\esp_sr\srmodels.bin" "{build.path}\srmodels.bin" # Create merged binary -recipe.hooks.objcopy.postobjcopy.3.pattern_args=--chip {build.mcu} merge_bin -o "{build.path}/{build.project_name}.merged.bin" --fill-flash-size {build.flash_size} --flash_mode keep --flash_freq keep --flash_size keep {build.bootloader_addr} "{build.path}/{build.project_name}.bootloader.bin" 0x8000 "{build.path}/{build.project_name}.partitions.bin" 0xe000 "{runtime.platform.path}/tools/partitions/boot_app0.bin" 0x10000 "{build.path}/{build.project_name}.bin" +recipe.hooks.objcopy.postobjcopy.3.pattern_args=--chip {build.mcu} merge-bin -o "{build.path}/{build.project_name}.merged.bin" --pad-to-size {build.flash_size} --flash-mode keep --flash-freq keep --flash-size keep {build.bootloader_addr} "{build.path}/{build.project_name}.bootloader.bin" 0x8000 "{build.path}/{build.project_name}.partitions.bin" 0xe000 "{runtime.platform.path}/tools/partitions/boot_app0.bin" 0x10000 "{build.path}/{build.project_name}.bin" recipe.hooks.objcopy.postobjcopy.3.pattern="{tools.esptool_py.path}/{tools.esptool_py.cmd}" {recipe.hooks.objcopy.postobjcopy.3.pattern_args} ## Save bin @@ -294,14 +294,14 @@ debug.additional_config=debug_config.{build.mcu} tools.esptool_py.upload.protocol=serial tools.esptool_py.upload.params.verbose= tools.esptool_py.upload.params.quiet= -tools.esptool_py.upload.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default_reset --after hard_reset write_flash {upload.erase_cmd} -z --flash_mode keep --flash_freq keep --flash_size keep {build.bootloader_addr} "{build.path}/{build.project_name}.bootloader.bin" 0x8000 "{build.path}/{build.project_name}.partitions.bin" 0xe000 "{runtime.platform.path}/tools/partitions/boot_app0.bin" 0x10000 "{build.path}/{build.project_name}.bin" {upload.extra_flags} +tools.esptool_py.upload.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default-reset --after hard-reset write-flash {upload.erase_cmd} -z --flash-mode keep --flash-freq keep --flash-size keep {build.bootloader_addr} "{build.path}/{build.project_name}.bootloader.bin" 0x8000 "{build.path}/{build.project_name}.partitions.bin" 0xe000 "{runtime.platform.path}/tools/partitions/boot_app0.bin" 0x10000 "{build.path}/{build.project_name}.bin" {upload.extra_flags} tools.esptool_py.upload.pattern="{path}/{cmd}" {upload.pattern_args} ## Program Application ## ------------------- tools.esptool_py.program.params.verbose= tools.esptool_py.program.params.quiet= -tools.esptool_py.program.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default_reset --after hard_reset write_flash -z --flash_mode keep --flash_freq keep --flash_size keep 0x10000 "{build.path}/{build.project_name}.bin" +tools.esptool_py.program.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default-reset --after hard-reset write-flash -z --flash-mode keep --flash-freq keep --flash-size keep 0x10000 "{build.path}/{build.project_name}.bin" tools.esptool_py.program.pattern="{path}/{cmd}" {program.pattern_args} ## Erase Chip (before burning the bootloader) @@ -309,7 +309,7 @@ tools.esptool_py.program.pattern="{path}/{cmd}" {program.pattern_args} tools.esptool_py.erase.protocol=serial tools.esptool_py.erase.params.verbose= tools.esptool_py.erase.params.quiet= -tools.esptool_py.erase.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default_reset --after hard_reset erase_flash +tools.esptool_py.erase.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default-reset --after hard-reset erase-flash tools.esptool_py.erase.pattern="{path}/{cmd}" {erase.pattern_args} ## Burn Bootloader