From 68c660f1c5b7bf6318e429e825deec234c91c2cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Quentin?= Date: Thu, 13 Feb 2025 16:20:10 +0100 Subject: [PATCH] CI optimizations (#3129) * CI Improvements * Fix * Lint with stable * Fix * Xtensa * cleanup --- .github/actions/check-esp-hal/action.yml | 68 ------- .github/workflows/ci.yml | 26 +-- .github/workflows/ci_nightly.yml | 7 +- esp-ieee802154/Cargo.toml | 2 +- examples/Cargo.toml | 11 +- xtask/src/cargo.rs | 9 +- xtask/src/documentation.rs | 7 +- xtask/src/lib.rs | 7 + xtask/src/main.rs | 240 +++++++++++++++++++---- 9 files changed, 248 insertions(+), 129 deletions(-) delete mode 100644 .github/actions/check-esp-hal/action.yml diff --git a/.github/actions/check-esp-hal/action.yml b/.github/actions/check-esp-hal/action.yml deleted file mode 100644 index 4eac6e8e5..000000000 --- a/.github/actions/check-esp-hal/action.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Build and Check -description: Build and check the esp-hal and esp-lp-hal pacakges for a specified device -inputs: - device: - description: "Device SOC" - required: true - target: - description: "Target" - required: true - toolchain: - description: "Toolchain channel" - required: true -runs: - using: "composite" - steps: - - name: Set up cargo environment - shell: bash - run: | - # Convert the target triple from kebab-case to SCREAMING_SNAKE_CASE: - big_target=$(echo "${{ matrix.device.target }}" | tr [:lower:] [:upper:] | tr '-' '_') - # Set the *target specific* RUSTFLAGS for the current device: - echo "CARGO_TARGET_${big_target}_RUSTFLAGS=-Dwarnings" >> $GITHUB_ENV - # Linting toolchain (stable cant build documentation) - if [ "${{ inputs.toolchain }}" == "nightly" ]; then - echo "LINTING_TOOLCHAIN=+nightly" >> $GITHUB_ENV - else - echo "LINTING_TOOLCHAIN=+esp" >> $GITHUB_ENV - fi - # Clippy and docs checks - - name: Clippy - shell: bash - run: cargo $LINTING_TOOLCHAIN xtask lint-packages --chips ${{ inputs.device }} - - name: Check doc-tests - shell: bash - run: cargo $LINTING_TOOLCHAIN xtask run-doc-test esp-hal ${{ inputs.device }} - - name: Check documentation - shell: bash - run: cargo $LINTING_TOOLCHAIN xtask build-documentation --packages esp-hal --packages esp-wifi --packages esp-hal-embassy --chips ${{ inputs.device }} - # Build all supported examples for the low-power core first (if present): - - name: Build prerequisite examples (esp-lp-hal) - shell: bash - if: contains(fromJson('["esp32c6", "esp32s2", "esp32s3"]'), inputs.device) - run: cargo +${{ inputs.toolchain }} xtask build-examples esp-lp-hal ${{ inputs.device }} - - name: Check esp-lp-hal documentation - shell: bash - if: contains(fromJson('["esp32c6", "esp32s2", "esp32s3"]'), inputs.device) - run: cargo $LINTING_TOOLCHAIN xtask build-documentation --packages esp-lp-hal --chips ${{ inputs.device }} - # Make sure we're able to build the HAL without the default features - # enabled: - - name: Build (no features) - shell: bash - run: | - cargo xtask build-package \ - --no-default-features \ - --toolchain=${{ inputs.toolchain }} \ - --features=${{ inputs.device }} \ - --target=${{ inputs.target }} \ - esp-hal - - name: Build (examples) - env: - CI: 1 - shell: bash - run: cargo +${{ inputs.toolchain }} xtask build-examples esp-hal ${{ inputs.device }} --debug - - name: Build (qa-test) - env: - CI: 1 - shell: bash - run: cargo +${{ inputs.toolchain }} xtask build-examples qa-test ${{ inputs.device }} --debug diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b45325895..f0fd9140d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,8 @@ jobs: name: esp-hal (${{ matrix.device.soc }}) runs-on: ubuntu-latest env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + CI: 1 SSID: SSID PASSWORD: PASSWORD STATIC_IP: 1.1.1.1 @@ -54,14 +56,14 @@ jobs: matrix: device: [ # RISC-V devices: - { soc: "esp32c2", target: "riscv32imc-unknown-none-elf", toolchain: "stable" }, - { soc: "esp32c3", target: "riscv32imc-unknown-none-elf", toolchain: "stable" }, - { soc: "esp32c6", target: "riscv32imac-unknown-none-elf", toolchain: "stable" }, - { soc: "esp32h2", target: "riscv32imac-unknown-none-elf", toolchain: "stable" }, + { soc: "esp32c2", toolchain: "stable" }, + { soc: "esp32c3", toolchain: "stable" }, + { soc: "esp32c6", toolchain: "stable" }, + { soc: "esp32h2", toolchain: "stable" }, # Xtensa devices: - { soc: "esp32", target: "xtensa-esp32-none-elf", toolchain: "esp" }, - { soc: "esp32s2", target: "xtensa-esp32s2-none-elf", toolchain: "esp" }, - { soc: "esp32s3", target: "xtensa-esp32s3-none-elf", toolchain: "esp" }, + { soc: "esp32", toolchain: "esp" }, + { soc: "esp32s2", toolchain: "esp" }, + { soc: "esp32s3", toolchain: "esp" }, ] steps: - uses: actions/checkout@v4 @@ -79,13 +81,13 @@ jobs: components: rust-src - uses: Swatinem/rust-cache@v2 + with: + prefix-key: "ci-${{ matrix.device.soc }}" + cache-all-crates: true - name: Build and Check - uses: ./.github/actions/check-esp-hal - with: - device: ${{ matrix.device.soc }} - target: ${{ matrix.device.target }} - toolchain: ${{ matrix.device.toolchain }} + shell: bash + run: cargo +${{ matrix.device.toolchain }} xtask ci ${{ matrix.device.soc }} extras: runs-on: macos-m1-self-hosted diff --git a/.github/workflows/ci_nightly.yml b/.github/workflows/ci_nightly.yml index 628da18eb..9e54bf569 100644 --- a/.github/workflows/ci_nightly.yml +++ b/.github/workflows/ci_nightly.yml @@ -45,11 +45,8 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Build and Check - uses: ./.github/actions/check-esp-hal - with: - device: ${{ matrix.device.soc }} - target: ${{ matrix.device.target }} - toolchain: nightly + shell: bash + run: cargo +nightly xtask ci ${{ matrix.device.soc }} - if: failure() name: Create or Update GitHub Issue diff --git a/esp-ieee802154/Cargo.toml b/esp-ieee802154/Cargo.toml index fec421b3b..9c27285fc 100644 --- a/esp-ieee802154/Cargo.toml +++ b/esp-ieee802154/Cargo.toml @@ -22,7 +22,7 @@ byte = "0.2.7" critical-section = "1.2.0" document-features = "0.2.10" esp-hal = { version = "0.23.0", path = "../esp-hal" } -esp-wifi-sys = "0.7.0" +esp-wifi-sys = "0.7.1" heapless = "0.8.0" ieee802154 = "0.6.1" cfg-if = "1.0.0" diff --git a/examples/Cargo.toml b/examples/Cargo.toml index d956eb1d5..93403fa9e 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -9,7 +9,6 @@ publish = false aligned = { version = "0.4.2", optional = true } bleps = { git = "https://github.com/bjoernQ/bleps", package = "bleps", rev = "a5148d8ae679e021b78f53fd33afb8bb35d0b62e", features = [ "macros", "async"] } blocking-network-stack = { git = "https://github.com/bjoernQ/blocking-network-stack.git", rev = "b3ecefc222d8806edd221f266999ca339c52d34e" } -bt-hci = "0.2.0" cfg-if = "1.0.0" critical-section = "1.1.3" embassy-executor = { version = "0.7.0", features = ["task-arena-size-20480"] } @@ -18,7 +17,6 @@ embassy-net = { version = "0.6.0", features = [ "tcp", "udp", "dhcpv4", "medium- embassy-sync = "0.6.0" embassy-time = "0.4.0" embassy-usb = { version = "0.2.0", default-features = false } -embedded-can = "0.4.1" embedded-hal-async = "1.0.0" embedded-io = { version = "0.6.1", default-features = false } embedded-io-async = "0.6.1" @@ -37,17 +35,14 @@ ieee80211 = { version = "0.4.0", default-features = false } ieee802154 = "0.6.1" log = "0.4.22" nb = "1.1.0" -portable-atomic = { version = "1.9.0", default-features = false } sha2 = { version = "0.10.8", default-features = false } smoltcp = { version = "0.12.0", default-features = false, features = [ "medium-ethernet", "socket-raw"] } -embedded-time = "=0.12.1" static_cell = { version = "2.1.0", features = ["nightly"] } usb-device = "0.3.2" usbd-serial = "0.2.2" -edge-dhcp = { version = "0.5.0" } -edge-raw = { version = "0.5.0" } -edge-nal = { version = "0.5.0" } -edge-nal-embassy = { version = "0.5.0" } +edge-dhcp = { version = "0.5.0" } +edge-nal = { version = "0.5.0" } +edge-nal-embassy = { version = "0.5.0" } [features] esp32 = ["esp-hal/esp32", "esp-backtrace/esp32", "esp-hal-embassy?/esp32", "esp-println/esp32", "esp-storage?/esp32", "esp-wifi?/esp32"] diff --git a/xtask/src/cargo.rs b/xtask/src/cargo.rs index dac780331..e504c08e2 100644 --- a/xtask/src/cargo.rs +++ b/xtask/src/cargo.rs @@ -31,7 +31,7 @@ pub fn run(args: &[String], cwd: &Path) -> Result<()> { /// Execute cargo with the given arguments and from the specified directory. pub fn run_with_env(args: &[String], cwd: &Path, envs: I, capture: bool) -> Result where - I: IntoIterator, + I: IntoIterator + core::fmt::Debug, K: AsRef, V: AsRef, { @@ -45,6 +45,13 @@ where // using now or in future! let cwd = windows_safe_path(cwd); + log::debug!( + "Running `cargo {}` in {:?} - Environment {:?}", + args.join(" "), + cwd, + envs + ); + let output = Command::new(get_cargo()) .args(args) .current_dir(cwd) diff --git a/xtask/src/documentation.rs b/xtask/src/documentation.rs index e75458136..493625190 100644 --- a/xtask/src/documentation.rs +++ b/xtask/src/documentation.rs @@ -205,7 +205,12 @@ fn cargo_doc(workspace: &Path, package: Package, chip: Option) -> Result

bool { !matches!(self, Package::Examples | Package::HilTest | Package::QaTest) } + + /// Build on host + pub fn build_on_host(&self) -> bool { + use Package::*; + + matches!(self, EspBuild | EspConfig | EspMetadata) + } } #[derive(Debug, Clone, Copy, Display, ValueEnum)] diff --git a/xtask/src/main.rs b/xtask/src/main.rs index f24be0100..23e10801a 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -2,6 +2,7 @@ use std::{ fs, path::{Path, PathBuf}, process::Command, + time::Instant, }; use anyhow::{bail, ensure, Context as _, Result}; @@ -11,9 +12,7 @@ use strum::IntoEnumIterator; use xtask::{ cargo::{CargoAction, CargoArgsBuilder}, firmware::Metadata, - target_triple, - Package, - Version, + target_triple, Package, Version, }; // ---------------------------------------------------------------------------- @@ -51,6 +50,8 @@ enum Cli { RunTests(TestArgs), /// Run all ELFs in a folder. RunElfs(RunElfArgs), + /// Perform (parts of) the checks done in CI + Ci(CiArgs), } #[derive(Debug, Args)] @@ -185,6 +186,13 @@ struct RunElfArgs { path: PathBuf, } +#[derive(Debug, Args)] +struct CiArgs { + /// Chip to target. + #[arg(value_enum)] + chip: Chip, +} + // ---------------------------------------------------------------------------- // Application @@ -217,6 +225,7 @@ fn main() -> Result<()> { Cli::RunElfs(args) => run_elfs(args), Cli::RunExample(args) => examples(&workspace, args, CargoAction::Run), Cli::RunTests(args) => tests(&workspace, args, CargoAction::Run), + Cli::Ci(args) => run_ci_checks(&workspace, args), } } @@ -599,12 +608,12 @@ fn lint_packages(workspace: &Path, args: LintPackagesArgs) -> Result<()> { chip, &path, &[ - "-Zbuild-std=core", "--no-default-features", &format!("--target={}", chip.target()), &format!("--features={chip},defmt"), ], args.fix, + package.build_on_host(), )?; } @@ -627,12 +636,9 @@ fn lint_packages(workspace: &Path, args: LintPackagesArgs) -> Result<()> { lint_package( chip, &path, - &[ - "-Zbuild-std=core", - &format!("--target={}", chip.target()), - &features, - ], + &[&format!("--target={}", chip.target()), &features], args.fix, + package.build_on_host(), )?; } @@ -641,26 +647,23 @@ fn lint_packages(workspace: &Path, args: LintPackagesArgs) -> Result<()> { chip, &path, &[ - "-Zbuild-std=core", &format!("--target={}", chip.target()), &format!("--features={chip},executors,defmt,esp-hal/unstable"), ], args.fix, + package.build_on_host(), )?; } Package::EspIeee802154 => { if device.contains("ieee802154") { - let features = format!("--features={chip},sys-logs,esp-hal/unstable"); + let features = format!("--features={chip},esp-hal/unstable"); lint_package( chip, &path, - &[ - "-Zbuild-std=core", - &format!("--target={}", chip.target()), - &features, - ], + &[&format!("--target={}", chip.target()), &features], args.fix, + package.build_on_host(), )?; } } @@ -670,11 +673,11 @@ fn lint_packages(workspace: &Path, args: LintPackagesArgs) -> Result<()> { chip, &path, &[ - "-Zbuild-std=core", &format!("--target={}", chip.lp_target().unwrap()), &format!("--features={chip},embedded-io"), ], args.fix, + package.build_on_host(), )?; } } @@ -684,11 +687,11 @@ fn lint_packages(workspace: &Path, args: LintPackagesArgs) -> Result<()> { chip, &path, &[ - "-Zbuild-std=core", &format!("--target={}", chip.target()), &format!("--features={chip},defmt-espflash"), ], args.fix, + package.build_on_host(), )?; } @@ -697,8 +700,9 @@ fn lint_packages(workspace: &Path, args: LintPackagesArgs) -> Result<()> { lint_package( chip, &path, - &["-Zbuild-std=core", &format!("--target={}", chip.target())], + &[&format!("--target={}", chip.target())], args.fix, + package.build_on_host(), )?; } } @@ -708,18 +712,17 @@ fn lint_packages(workspace: &Path, args: LintPackagesArgs) -> Result<()> { chip, &path, &[ - "-Zbuild-std=core", &format!("--target={}", chip.target()), &format!("--features={chip},storage,nor-flash,low-level"), ], args.fix, + package.build_on_host(), )?; } Package::EspWifi => { - let mut features = format!( - "--features={chip},defmt,sys-logs,esp-hal/unstable,builtin-scheduler" - ); + let mut features = + format!("--features={chip},defmt,esp-hal/unstable,builtin-scheduler"); if device.contains("wifi") { features.push_str(",esp-now,sniffer") @@ -734,26 +737,38 @@ fn lint_packages(workspace: &Path, args: LintPackagesArgs) -> Result<()> { chip, &path, &[ - "-Zbuild-std=core,alloc", &format!("--target={}", chip.target()), "--no-default-features", &features, ], args.fix, + package.build_on_host(), )?; } + Package::XtensaLx => { + if matches!(device.arch(), Arch::Xtensa) { + lint_package( + chip, + &path, + &[&format!("--target={}", chip.target())], + args.fix, + package.build_on_host(), + )? + } + } + Package::XtensaLxRt => { if matches!(device.arch(), Arch::Xtensa) { lint_package( chip, &path, &[ - "-Zbuild-std=core", &format!("--target={}", chip.target()), &format!("--features={chip}"), ], args.fix, + package.build_on_host(), )? } } @@ -763,7 +778,7 @@ fn lint_packages(workspace: &Path, args: LintPackagesArgs) -> Result<()> { Package::Examples | Package::HilTest | Package::QaTest => {} // By default, no `clippy` arguments are required: - _ => lint_package(chip, &path, &[], args.fix)?, + _ => lint_package(chip, &path, &[], args.fix, package.build_on_host())?, } } } @@ -771,12 +786,24 @@ fn lint_packages(workspace: &Path, args: LintPackagesArgs) -> Result<()> { Ok(()) } -fn lint_package(chip: &Chip, path: &Path, args: &[&str], fix: bool) -> Result<()> { +fn lint_package( + chip: &Chip, + path: &Path, + args: &[&str], + fix: bool, + build_on_host: bool, +) -> Result<()> { log::info!("Linting package: {} ({})", path.display(), chip); let builder = CargoArgsBuilder::default().subcommand("clippy"); let mut builder = if chip.is_xtensa() { + let builder = if build_on_host { + builder + } else { + builder.arg("-Zbuild-std=core,alloc") + }; + // We only overwrite Xtensas so that externally set nightly/stable toolchains // are not overwritten. builder.toolchain("esp") @@ -788,15 +815,13 @@ fn lint_package(chip: &Chip, path: &Path, args: &[&str], fix: bool) -> Result<() builder = builder.arg(arg.to_string()); } - // build in release to reuse example artifacts - let cargo_args = builder.arg("--release"); - let cargo_args = if fix { - cargo_args.arg("--fix").arg("--lib").arg("--allow-dirty") + let builder = if fix { + builder.arg("--fix").arg("--lib").arg("--allow-dirty") } else { - cargo_args.arg("--").arg("-D").arg("warnings") + builder.arg("--").arg("-D").arg("warnings").arg("--no-deps") }; - let cargo_args = cargo_args.build(); + let cargo_args = builder.build(); xtask::cargo::run(&cargo_args, path) } @@ -913,3 +938,152 @@ fn run_doc_tests(workspace: &Path, args: ExampleArgs) -> Result<()> { Ok(()) } + +fn run_ci_checks(workspace: &Path, args: CiArgs) -> Result<()> { + let mut failure = false; + let started_at = Instant::now(); + + // Clippy and docs checks + + // Clippy + lint_packages( + workspace, + LintPackagesArgs { + packages: Package::iter().collect(), + chips: vec![args.chip], + fix: false, + }, + ) + .inspect_err(|_| failure = true) + .ok(); + + // Check doc-tests + run_doc_tests( + workspace, + ExampleArgs { + package: Package::EspHal, + chip: args.chip, + example: None, + debug: true, + }, + ) + .inspect_err(|_| failure = true) + .ok(); + + // Check documentation + build_documentation( + workspace, + BuildDocumentationArgs { + packages: vec![Package::EspHal, Package::EspWifi, Package::EspHalEmbassy], + chips: vec![args.chip], + base_url: None, + }, + ) + .inspect_err(|_| failure = true) + .ok(); + + // for chips with esp-lp-hal: Build all supported examples for the low-power + // core first + if args.chip.has_lp_core() { + // Build prerequisite examples (esp-lp-hal) + // `examples` copies the examples to a folder with the chip name as the last + // path element then we copy it to the place where the HP core example + // expects it + examples( + workspace, + ExampleArgs { + package: Package::EspLpHal, + chip: args.chip, + example: None, + debug: false, + }, + CargoAction::Build(PathBuf::from(format!( + "./esp-lp-hal/target/{}/release/examples", + args.chip.target() + ))), + ) + .inspect_err(|_| failure = true) + .and_then(|_| { + let from_dir = PathBuf::from(format!( + "./esp-lp-hal/target/{}/release/examples/{}", + args.chip.target(), + args.chip.to_string() + )); + let to_dir = PathBuf::from(format!( + "./esp-lp-hal/target/{}/release/examples", + args.chip.target() + )); + from_dir.read_dir()?.for_each(|entry| { + let entry = entry.unwrap(); + let path = entry.path(); + let to = to_dir.join(entry.file_name()); + fs::copy(path, to).expect("Failed to copy file"); + }); + Ok(()) + }) + .ok(); + + // Check documentation + build_documentation( + workspace, + BuildDocumentationArgs { + packages: vec![Package::EspLpHal], + chips: vec![args.chip], + base_url: None, + }, + ) + .inspect_err(|_| failure = true) + .ok(); + } + + // Make sure we're able to build the HAL without the default features enabled + build_package( + workspace, + BuildPackageArgs { + package: Package::EspHal, + target: Some(args.chip.target().to_string()), + features: vec![args.chip.to_string()], + toolchain: None, + no_default_features: true, + }, + ) + .inspect_err(|_| failure = true) + .ok(); + + // Build (examples) + examples( + workspace, + ExampleArgs { + package: Package::Examples, + chip: args.chip, + example: None, + debug: true, + }, + CargoAction::Build(PathBuf::from(format!("./examples/target/"))), + ) + .inspect_err(|_| failure = true) + .ok(); + + // Build (qa-test) + examples( + workspace, + ExampleArgs { + package: Package::QaTest, + chip: args.chip, + example: None, + debug: true, + }, + CargoAction::Build(PathBuf::from(format!("./qa-test/target/"))), + ) + .inspect_err(|_| failure = true) + .ok(); + + let completed_at = Instant::now(); + log::info!("CI checks completed in {:?}", completed_at - started_at); + + if failure { + bail!("CI checks failed"); + } + + Ok(()) +}