diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index 5fa205f..29bebc0 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -6,6 +6,7 @@ on: env: CARGO_TERM_COLOR: always + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: publish-release: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index afc670b..6a7361f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,14 +7,17 @@ on: - main paths-ignore: - "**/README.md" - - "**/cd.yml" + - "**/audit.yaml" + - "**/cd.yaml" pull_request: paths-ignore: - "**/README.md" - - "**/cd.yml" + - "**/audit.yaml" + - "**/cd.yaml" env: CARGO_TERM_COLOR: always + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: continuous-integration: diff --git a/Cargo.lock b/Cargo.lock index 18d43da..3d16534 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -534,6 +534,7 @@ dependencies = [ "openssl", "regex", "reqwest", + "retry", "serde", "serde_json", "strum", @@ -623,6 +624,12 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +[[package]] +name = "futures-io" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" + [[package]] name = "futures-sink" version = "0.3.25" @@ -642,9 +649,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-core", + "futures-io", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -1269,6 +1279,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "predicates" version = "2.1.5" @@ -1338,11 +1354,35 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] [[package]] name = "rayon" @@ -1468,6 +1508,15 @@ dependencies = [ "winreg", ] +[[package]] +name = "retry" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9166d72162de3575f950507683fac47e30f6f2c3836b71b7fbc61aa517c9c5f4" +dependencies = [ + "rand", +] + [[package]] name = "ring" version = "0.16.20" diff --git a/Cargo.toml b/Cargo.toml index 8c1f2b6..6359287 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ anyhow = "1.0.68" clap = { version = "4.0.32", features = ["derive"] } flate2 = "1.0.25" guess_host_triple = "0.1.3" -reqwest = "0.11.12" +reqwest = { version = "0.11.12", features = ["blocking"] } tar = "0.4.37" zip = "0.6.3" xz2 = "0.1.6" @@ -38,6 +38,8 @@ thiserror = "1.0.38" update-informer = "0.6.0" tokio = { version = "1.24.1", features = ["full"] } async-trait = "0.1.61" +retry = "2.0.0" + [target.'cfg(target_os = "linux")'.dependencies] openssl = { version = "0.10", features = ["vendored"] } diff --git a/src/error.rs b/src/error.rs index 3d4bf92..989bb67 100644 --- a/src/error.rs +++ b/src/error.rs @@ -38,6 +38,9 @@ pub enum Error { #[error("{} Unsuported file extension: '{0}'", emoji::ERROR)] UnsuportedFileExtension(String), // Toolchain - Rust + #[diagnostic(code(espup::toolchain::rust::failed_to_query_github))] + #[error("{} Failed To Query GitHub API.", emoji::ERROR)] + FailedGithubQuery, #[diagnostic(code(espup::toolchain::rust::failed_to_get_latest_version))] #[error("{} Failed To serialize Json from string.", emoji::ERROR)] FailedToSerializeJson, @@ -46,7 +49,7 @@ pub enum Error { XtensaToolchainAlreadyInstalled(String), #[diagnostic(code(espup::toolchain::rust::invalid_version))] #[error( - "{} Invalid toolchain version '{0}', must be in the form of '...'", + "{} Invalid toolchain version '{0}'. Verify that the format is correct: '...' or '..', and that the release exists in https://github.com/esp-rs/rust-build/releases", emoji::ERROR )] InvalidXtensaToolchanVersion(String), diff --git a/src/toolchain/mod.rs b/src/toolchain/mod.rs index 984ad1d..cf17688 100644 --- a/src/toolchain/mod.rs +++ b/src/toolchain/mod.rs @@ -1,9 +1,13 @@ use crate::{emoji, error::Error}; use async_trait::async_trait; use flate2::bufread::GzDecoder; -use log::info; +use log::{debug, info, warn}; use miette::Result; +use reqwest::blocking::Client; +use reqwest::header; +use retry::{delay::Fixed, retry}; use std::{ + env, fs::{create_dir_all, File}, io::Write, path::Path, @@ -95,55 +99,41 @@ pub async fn download_file( Ok(format!("{}/{}", output_directory, file_name)) } -#[cfg(test)] -mod tests { - use crate::toolchain::download_file; - use std::{fs::File, io::Write}; - - #[tokio::test] - async fn test_download_file() { - // Returns the correct file path when the file already exists - let temp_dir = tempfile::TempDir::new().unwrap(); - let file_name = "test.txt"; - let output_directory = temp_dir.path().to_str().unwrap(); - let file_path = format!("{}/{}", output_directory, file_name); - let mut file = File::create(file_path.clone()).unwrap(); - file.write_all(b"test content").unwrap(); - - let url = "https://example.com/test.txt"; - let result = download_file(url.to_string(), file_name, output_directory, false).await; - assert!(result.is_ok()); - let path = result.unwrap(); - assert_eq!(path, file_path); - - // Creates the output directory if it does not exist - let temp_dir = tempfile::TempDir::new().unwrap(); - let output_directory = temp_dir.path().join("test"); - let file_name = "test.txt"; - - let url = "https://example.com/test.txt"; - let result = download_file( - url.to_string(), - file_name, - output_directory.to_str().unwrap(), - false, - ) - .await; - assert!(result.is_ok()); - let path = result.unwrap(); - #[cfg(windows)] - let path = path.replace('/', "\\"); - assert_eq!(path, output_directory.join(file_name).to_str().unwrap()); - assert!(output_directory.exists()); - - // Downloads a ZIP file and uncompresses it - let temp_dir = tempfile::TempDir::new().unwrap(); - let output_directory = temp_dir.path().to_str().unwrap(); - let file_name = "espup.zip"; - let url = "https://github.com/esp-rs/espup/releases/latest/download/espup-x86_64-unknown-linux-gnu.zip"; - let result = download_file(url.to_string(), file_name, output_directory, true).await; - assert!(result.is_ok()); - let extracted_file = temp_dir.path().join("espup"); - assert!(extracted_file.exists()); +/// Queries the GitHub API and returns the JSON response. +pub fn github_query(url: &str) -> Result { + info!("{} Querying GitHub API: '{}'", emoji::INFO, url); + let mut headers = header::HeaderMap::new(); + headers.insert(header::USER_AGENT, "espup".parse().unwrap()); + headers.insert( + header::ACCEPT, + "application/vnd.github+json".parse().unwrap(), + ); + headers.insert("X-GitHub-Api-Version", "2022-11-28".parse().unwrap()); + if let Some(token) = env::var_os("GITHUB_TOKEN") { + debug!("{} Auth header added.", emoji::DEBUG); + headers.insert( + "Authorization", + format!("Bearer {}", token.to_string_lossy()) + .parse() + .unwrap(), + ); } + let client = Client::new(); + let json = retry( + Fixed::from_millis(100).take(5), + || -> Result { + let res = client.get(url).headers(headers.clone()).send()?.text()?; + if res.contains( + "https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting", + ) { + warn!("{} GitHub rate limit exceeded", emoji::WARN); + return Err(Error::FailedGithubQuery); + } + let json: serde_json::Value = + serde_json::from_str(&res).map_err(|_| Error::FailedToSerializeJson)?; + Ok(json) + }, + ) + .unwrap(); + Ok(json) } diff --git a/src/toolchain/rust.rs b/src/toolchain/rust.rs index 6fac541..285702c 100644 --- a/src/toolchain/rust.rs +++ b/src/toolchain/rust.rs @@ -5,7 +5,7 @@ use crate::{ emoji, error::Error, host_triple::HostTriple, - toolchain::{download_file, espidf::get_dist_path}, + toolchain::{download_file, espidf::get_dist_path, github_query}, }; use async_trait::async_trait; use directories::BaseDirs; @@ -13,18 +13,23 @@ use embuild::cmd; use log::{debug, info, warn}; use miette::{IntoDiagnostic, Result}; use regex::Regex; -use reqwest::header; use serde::{Deserialize, Serialize}; -use std::{collections::HashSet, fmt::Debug}; -use std::{env, fs::remove_dir_all, path::PathBuf, process::Stdio}; +use std::{ + collections::HashSet, env, fmt::Debug, fs::remove_dir_all, path::PathBuf, process::Stdio, +}; /// Xtensa Rust Toolchain repository const DEFAULT_XTENSA_RUST_REPOSITORY: &str = "https://github.com/esp-rs/rust-build/releases/download"; /// Xtensa Rust Toolchain API URL -const XTENSA_RUST_API_URL: &str = "https://api.github.com/repos/esp-rs/rust-build/releases/latest"; +const XTENSA_RUST_LATEST_API_URL: &str = + "https://api.github.com/repos/esp-rs/rust-build/releases/latest"; +const XTENSA_RUST_API_URL: &str = "https://api.github.com/repos/esp-rs/rust-build/releases"; + /// Xtensa Rust Toolchain version regex. -const RE_TOOLCHAIN_VERSION: &str = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)?$"; +const RE_EXTENDED_SEMANTIC_VERSION: &str = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)?$"; +const RE_SEMANTIC_VERSION: &str = + r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)?$"; #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct XtensaRust { @@ -53,24 +58,7 @@ pub struct XtensaRust { impl XtensaRust { /// Get the latest version of Xtensa Rust toolchain. pub async fn get_latest_version() -> Result { - let mut headers = header::HeaderMap::new(); - headers.insert("Accept", "application/vnd.github.v3+json".parse().unwrap()); - let client = reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .user_agent("espup") - .build() - .unwrap(); - let res = client - .get(XTENSA_RUST_API_URL) - .headers(headers) - .send() - .await - .into_diagnostic()? - .text() - .await - .into_diagnostic()?; - let json: serde_json::Value = - serde_json::from_str(&res).map_err(|_| Error::FailedToSerializeJson)?; + let json = github_query(XTENSA_RUST_LATEST_API_URL)?; let mut version = json["tag_name"].to_string(); version.retain(|c| c != 'v' && c != '"'); @@ -120,13 +108,47 @@ impl XtensaRust { } /// Parses the version of the Xtensa toolchain. - pub fn parse_version(arg: &str) -> Result { + pub fn parse_version(arg: &str) -> Result { debug!("{} Parsing Xtensa Rust version: {}", emoji::DEBUG, arg); - let re = Regex::new(RE_TOOLCHAIN_VERSION).unwrap(); - if !re.is_match(arg) { - return Err(Error::InvalidXtensaToolchanVersion(arg.to_string())).into_diagnostic(); + let re_extended = Regex::new(RE_EXTENDED_SEMANTIC_VERSION).unwrap(); + let re_semver = Regex::new(RE_SEMANTIC_VERSION).unwrap(); + let json = github_query(XTENSA_RUST_API_URL)?; + if re_semver.is_match(arg) { + let mut extended_versions: Vec = Vec::new(); + for release in json.as_array().unwrap() { + let tag_name = release["tag_name"].to_string().replace(['\"', 'v'], ""); + if tag_name.starts_with(arg) { + extended_versions.push(tag_name); + } + } + if extended_versions.is_empty() { + return Err(Error::InvalidXtensaToolchanVersion(arg.to_string())); + } + let mut max_version = extended_versions.pop().unwrap(); + let mut max_subpatch = 0; + for version in extended_versions { + let subpatch: i8 = re_extended + .captures(&version) + .and_then(|cap| { + cap.name("subpatch") + .map(|subpatch| subpatch.as_str().parse().unwrap()) + }) + .unwrap(); + if subpatch > max_subpatch { + max_subpatch = subpatch; + max_version = version; + } + } + return Ok(max_version); + } else if re_extended.is_match(arg) { + for release in json.as_array().unwrap() { + let tag_name = release["tag_name"].to_string().replace(['\"', 'v'], ""); + if tag_name.starts_with(arg) { + return Ok(arg.to_string()); + } + } } - Ok(arg.to_string()) + Err(Error::InvalidXtensaToolchanVersion(arg.to_string())) } /// Removes the Xtensa Rust toolchain. @@ -495,15 +517,24 @@ fn install_rust_nightly(version: &str) -> Result<(), Error> { #[cfg(test)] mod tests { - use crate::toolchain::rust::{get_cargo_home, get_rustup_home, Crate, XtensaRust}; + use crate::{ + logging::initialize_logger, + toolchain::rust::{get_cargo_home, get_rustup_home, Crate, XtensaRust}, + }; use directories::BaseDirs; use std::collections::HashSet; #[test] fn test_xtensa_rust_parse_version() { - assert_eq!(XtensaRust::parse_version("1.45.0.0").unwrap(), "1.45.0.0"); - assert_eq!(XtensaRust::parse_version("1.45.0.1").unwrap(), "1.45.0.1"); - assert_eq!(XtensaRust::parse_version("1.1.1.1").unwrap(), "1.1.1.1"); + initialize_logger("debug"); + assert_eq!(XtensaRust::parse_version("1.65.0.0").unwrap(), "1.65.0.0"); + assert_eq!(XtensaRust::parse_version("1.65.0.1").unwrap(), "1.65.0.1"); + assert_eq!(XtensaRust::parse_version("1.64.0.0").unwrap(), "1.64.0.0"); + assert_eq!(XtensaRust::parse_version("1.63.0").unwrap(), "1.63.0.2"); + assert_eq!(XtensaRust::parse_version("1.65.0").unwrap(), "1.65.0.1"); + assert_eq!(XtensaRust::parse_version("1.64.0").unwrap(), "1.64.0.0"); + assert!(XtensaRust::parse_version("422.0.0").is_err()); + assert!(XtensaRust::parse_version("422.0.0.0").is_err()); assert!(XtensaRust::parse_version("a.1.1.1").is_err()); assert!(XtensaRust::parse_version("1.1.1.1.1").is_err()); assert!(XtensaRust::parse_version("1..1.1").is_err()); diff --git a/test.sh b/test.sh new file mode 100644 index 0000000..9af30b5 --- /dev/null +++ b/test.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -e +set -o pipefail + +for i in {1..100} +do + cargo test test_xtensa_rust_parse_version -- --nocapture +# curl --request GET \ +# --url https://api.github.com/repos/esp-rs/rust-build/releases \ +# --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' +done