mirror of
https://github.com/esp-rs/espup.git
synced 2025-09-27 12:50:54 +00:00
Append header if GITHUB_TOKEN environment variable is present (#128)
* feat: ✨ Append header if GITHUB_TOKEN environment variable is present * build: ⬆️ Add blocking feature of reqwest * feat: ✨ If a semver version is provided, get the latest subpatch and use that version * docs: 📝 Update invalid_version error * perf: ⚡️ Remove unnecesary into_diagnostics calls * feat: ✨ Even if the regex matches, check that the release exists * feat: ✨ Update auth * chore: 🔊 Add log * chore: ⚡️ Add basic auth * Add gh api tests * fix: 🐛 fix typo * feat: 🎨 Remove permissions * test: ✅ Add rust test * test: ✅ Add rust test * ci: ⚡️ Add checkout action * ci: 🧪 Test with permissions * ci: 🧪 Update headers * ci: 🎨 Avoid triggering other CI * ci: ✅ Check if GITHUB TOKEN is present * ci: ✅ Update get call * ci: 🧪 Update headers * style: 🎨 Simplify code * ci: 🧪 Update headers * ci: 🧪 Add curl test * feat: ✨ Add retries * feat: ✨ Add retries * feat: ✨ Improve retries * chore: 🔊 Add logs to tests * feat: ✨ Remove retries * ci: 🧪 Test api limit * chore: 🧪 Test with retries * feat: ✨ Create a github_query fn * ci: 🧪 Update CI rust test * feat: 🔊 Update logging * feat: ✨ Add maximum retries * chore: 🔥 Remove gh_test.yaml * style: 🎨 Remove duplicated log * ci: 👷 Add env to CI * chore: 🔊 Add debug log * ci: 🧪 Test without env * revert: ⏪️ Revert env removal * ci: 👷 Update environment in CI * chore: 🔊 Add logging * chore: 🔥 Remove download test * feat: ✨ Move github_query fn to mod.rs
This commit is contained in:
Sergio Gasquez Arcos
GitHub
parent
e8c4ef1b2a
commit
11becfa4bf
1
.github/workflows/cd.yaml
vendored
1
.github/workflows/cd.yaml
vendored
@ -6,6 +6,7 @@ on:
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
jobs:
|
||||
publish-release:
|
||||
|
||||
7
.github/workflows/ci.yaml
vendored
7
.github/workflows/ci.yaml
vendored
@ -7,14 +7,17 @@ on:
|
||||
- main
|
||||
paths-ignore:
|
||||
- "**/README.md"
|
||||
- "**/cd.yml"
|
||||
- "**/audit.yaml"
|
||||
- "**/cd.yaml"
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- "**/README.md"
|
||||
- "**/cd.yml"
|
||||
- "**/audit.yaml"
|
||||
- "**/cd.yaml"
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
jobs:
|
||||
continuous-integration:
|
||||
|
||||
49
Cargo.lock
generated
49
Cargo.lock
generated
@ -534,6 +534,7 @@ dependencies = [
|
||||
"openssl",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"retry",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
@ -623,6 +624,12 @@ version = "0.3.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.25"
|
||||
@ -642,9 +649,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-task",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1269,6 +1279,12 @@ version = "0.3.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
|
||||
|
||||
[[package]]
|
||||
name = "predicates"
|
||||
version = "2.1.5"
|
||||
@ -1338,11 +1354,35 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
@ -1468,6 +1508,15 @@ dependencies = [
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "retry"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9166d72162de3575f950507683fac47e30f6f2c3836b71b7fbc61aa517c9c5f4"
|
||||
dependencies = [
|
||||
"rand",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.16.20"
|
||||
|
||||
@ -18,7 +18,7 @@ anyhow = "1.0.68"
|
||||
clap = { version = "4.0.32", features = ["derive"] }
|
||||
flate2 = "1.0.25"
|
||||
guess_host_triple = "0.1.3"
|
||||
reqwest = "0.11.12"
|
||||
reqwest = { version = "0.11.12", features = ["blocking"] }
|
||||
tar = "0.4.37"
|
||||
zip = "0.6.3"
|
||||
xz2 = "0.1.6"
|
||||
@ -38,6 +38,8 @@ thiserror = "1.0.38"
|
||||
update-informer = "0.6.0"
|
||||
tokio = { version = "1.24.1", features = ["full"] }
|
||||
async-trait = "0.1.61"
|
||||
retry = "2.0.0"
|
||||
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
openssl = { version = "0.10", features = ["vendored"] }
|
||||
|
||||
@ -38,6 +38,9 @@ pub enum Error {
|
||||
#[error("{} Unsuported file extension: '{0}'", emoji::ERROR)]
|
||||
UnsuportedFileExtension(String),
|
||||
// Toolchain - Rust
|
||||
#[diagnostic(code(espup::toolchain::rust::failed_to_query_github))]
|
||||
#[error("{} Failed To Query GitHub API.", emoji::ERROR)]
|
||||
FailedGithubQuery,
|
||||
#[diagnostic(code(espup::toolchain::rust::failed_to_get_latest_version))]
|
||||
#[error("{} Failed To serialize Json from string.", emoji::ERROR)]
|
||||
FailedToSerializeJson,
|
||||
@ -46,7 +49,7 @@ pub enum Error {
|
||||
XtensaToolchainAlreadyInstalled(String),
|
||||
#[diagnostic(code(espup::toolchain::rust::invalid_version))]
|
||||
#[error(
|
||||
"{} Invalid toolchain version '{0}', must be in the form of '<major>.<minor>.<patch>.<subpatch>'",
|
||||
"{} Invalid toolchain version '{0}'. Verify that the format is correct: '<major>.<minor>.<patch>.<subpatch>' or '<major>.<minor>.<patch>', and that the release exists in https://github.com/esp-rs/rust-build/releases",
|
||||
emoji::ERROR
|
||||
)]
|
||||
InvalidXtensaToolchanVersion(String),
|
||||
|
||||
@ -1,9 +1,13 @@
|
||||
use crate::{emoji, error::Error};
|
||||
use async_trait::async_trait;
|
||||
use flate2::bufread::GzDecoder;
|
||||
use log::info;
|
||||
use log::{debug, info, warn};
|
||||
use miette::Result;
|
||||
use reqwest::blocking::Client;
|
||||
use reqwest::header;
|
||||
use retry::{delay::Fixed, retry};
|
||||
use std::{
|
||||
env,
|
||||
fs::{create_dir_all, File},
|
||||
io::Write,
|
||||
path::Path,
|
||||
@ -95,55 +99,41 @@ pub async fn download_file(
|
||||
Ok(format!("{}/{}", output_directory, file_name))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::toolchain::download_file;
|
||||
use std::{fs::File, io::Write};
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_download_file() {
|
||||
// Returns the correct file path when the file already exists
|
||||
let temp_dir = tempfile::TempDir::new().unwrap();
|
||||
let file_name = "test.txt";
|
||||
let output_directory = temp_dir.path().to_str().unwrap();
|
||||
let file_path = format!("{}/{}", output_directory, file_name);
|
||||
let mut file = File::create(file_path.clone()).unwrap();
|
||||
file.write_all(b"test content").unwrap();
|
||||
|
||||
let url = "https://example.com/test.txt";
|
||||
let result = download_file(url.to_string(), file_name, output_directory, false).await;
|
||||
assert!(result.is_ok());
|
||||
let path = result.unwrap();
|
||||
assert_eq!(path, file_path);
|
||||
|
||||
// Creates the output directory if it does not exist
|
||||
let temp_dir = tempfile::TempDir::new().unwrap();
|
||||
let output_directory = temp_dir.path().join("test");
|
||||
let file_name = "test.txt";
|
||||
|
||||
let url = "https://example.com/test.txt";
|
||||
let result = download_file(
|
||||
url.to_string(),
|
||||
file_name,
|
||||
output_directory.to_str().unwrap(),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_ok());
|
||||
let path = result.unwrap();
|
||||
#[cfg(windows)]
|
||||
let path = path.replace('/', "\\");
|
||||
assert_eq!(path, output_directory.join(file_name).to_str().unwrap());
|
||||
assert!(output_directory.exists());
|
||||
|
||||
// Downloads a ZIP file and uncompresses it
|
||||
let temp_dir = tempfile::TempDir::new().unwrap();
|
||||
let output_directory = temp_dir.path().to_str().unwrap();
|
||||
let file_name = "espup.zip";
|
||||
let url = "https://github.com/esp-rs/espup/releases/latest/download/espup-x86_64-unknown-linux-gnu.zip";
|
||||
let result = download_file(url.to_string(), file_name, output_directory, true).await;
|
||||
assert!(result.is_ok());
|
||||
let extracted_file = temp_dir.path().join("espup");
|
||||
assert!(extracted_file.exists());
|
||||
/// Queries the GitHub API and returns the JSON response.
|
||||
pub fn github_query(url: &str) -> Result<serde_json::Value, Error> {
|
||||
info!("{} Querying GitHub API: '{}'", emoji::INFO, url);
|
||||
let mut headers = header::HeaderMap::new();
|
||||
headers.insert(header::USER_AGENT, "espup".parse().unwrap());
|
||||
headers.insert(
|
||||
header::ACCEPT,
|
||||
"application/vnd.github+json".parse().unwrap(),
|
||||
);
|
||||
headers.insert("X-GitHub-Api-Version", "2022-11-28".parse().unwrap());
|
||||
if let Some(token) = env::var_os("GITHUB_TOKEN") {
|
||||
debug!("{} Auth header added.", emoji::DEBUG);
|
||||
headers.insert(
|
||||
"Authorization",
|
||||
format!("Bearer {}", token.to_string_lossy())
|
||||
.parse()
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
let client = Client::new();
|
||||
let json = retry(
|
||||
Fixed::from_millis(100).take(5),
|
||||
|| -> Result<serde_json::Value, Error> {
|
||||
let res = client.get(url).headers(headers.clone()).send()?.text()?;
|
||||
if res.contains(
|
||||
"https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting",
|
||||
) {
|
||||
warn!("{} GitHub rate limit exceeded", emoji::WARN);
|
||||
return Err(Error::FailedGithubQuery);
|
||||
}
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_str(&res).map_err(|_| Error::FailedToSerializeJson)?;
|
||||
Ok(json)
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
Ok(json)
|
||||
}
|
||||
|
||||
@ -5,7 +5,7 @@ use crate::{
|
||||
emoji,
|
||||
error::Error,
|
||||
host_triple::HostTriple,
|
||||
toolchain::{download_file, espidf::get_dist_path},
|
||||
toolchain::{download_file, espidf::get_dist_path, github_query},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use directories::BaseDirs;
|
||||
@ -13,18 +13,23 @@ use embuild::cmd;
|
||||
use log::{debug, info, warn};
|
||||
use miette::{IntoDiagnostic, Result};
|
||||
use regex::Regex;
|
||||
use reqwest::header;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{collections::HashSet, fmt::Debug};
|
||||
use std::{env, fs::remove_dir_all, path::PathBuf, process::Stdio};
|
||||
use std::{
|
||||
collections::HashSet, env, fmt::Debug, fs::remove_dir_all, path::PathBuf, process::Stdio,
|
||||
};
|
||||
|
||||
/// Xtensa Rust Toolchain repository
|
||||
const DEFAULT_XTENSA_RUST_REPOSITORY: &str =
|
||||
"https://github.com/esp-rs/rust-build/releases/download";
|
||||
/// Xtensa Rust Toolchain API URL
|
||||
const XTENSA_RUST_API_URL: &str = "https://api.github.com/repos/esp-rs/rust-build/releases/latest";
|
||||
const XTENSA_RUST_LATEST_API_URL: &str =
|
||||
"https://api.github.com/repos/esp-rs/rust-build/releases/latest";
|
||||
const XTENSA_RUST_API_URL: &str = "https://api.github.com/repos/esp-rs/rust-build/releases";
|
||||
|
||||
/// Xtensa Rust Toolchain version regex.
|
||||
const RE_TOOLCHAIN_VERSION: &str = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)\.(?P<subpatch>0|[1-9]\d*)?$";
|
||||
const RE_EXTENDED_SEMANTIC_VERSION: &str = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)\.(?P<subpatch>0|[1-9]\d*)?$";
|
||||
const RE_SEMANTIC_VERSION: &str =
|
||||
r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)?$";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct XtensaRust {
|
||||
@ -53,24 +58,7 @@ pub struct XtensaRust {
|
||||
impl XtensaRust {
|
||||
/// Get the latest version of Xtensa Rust toolchain.
|
||||
pub async fn get_latest_version() -> Result<String> {
|
||||
let mut headers = header::HeaderMap::new();
|
||||
headers.insert("Accept", "application/vnd.github.v3+json".parse().unwrap());
|
||||
let client = reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.user_agent("espup")
|
||||
.build()
|
||||
.unwrap();
|
||||
let res = client
|
||||
.get(XTENSA_RUST_API_URL)
|
||||
.headers(headers)
|
||||
.send()
|
||||
.await
|
||||
.into_diagnostic()?
|
||||
.text()
|
||||
.await
|
||||
.into_diagnostic()?;
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_str(&res).map_err(|_| Error::FailedToSerializeJson)?;
|
||||
let json = github_query(XTENSA_RUST_LATEST_API_URL)?;
|
||||
let mut version = json["tag_name"].to_string();
|
||||
|
||||
version.retain(|c| c != 'v' && c != '"');
|
||||
@ -120,13 +108,47 @@ impl XtensaRust {
|
||||
}
|
||||
|
||||
/// Parses the version of the Xtensa toolchain.
|
||||
pub fn parse_version(arg: &str) -> Result<String> {
|
||||
pub fn parse_version(arg: &str) -> Result<String, Error> {
|
||||
debug!("{} Parsing Xtensa Rust version: {}", emoji::DEBUG, arg);
|
||||
let re = Regex::new(RE_TOOLCHAIN_VERSION).unwrap();
|
||||
if !re.is_match(arg) {
|
||||
return Err(Error::InvalidXtensaToolchanVersion(arg.to_string())).into_diagnostic();
|
||||
let re_extended = Regex::new(RE_EXTENDED_SEMANTIC_VERSION).unwrap();
|
||||
let re_semver = Regex::new(RE_SEMANTIC_VERSION).unwrap();
|
||||
let json = github_query(XTENSA_RUST_API_URL)?;
|
||||
if re_semver.is_match(arg) {
|
||||
let mut extended_versions: Vec<String> = Vec::new();
|
||||
for release in json.as_array().unwrap() {
|
||||
let tag_name = release["tag_name"].to_string().replace(['\"', 'v'], "");
|
||||
if tag_name.starts_with(arg) {
|
||||
extended_versions.push(tag_name);
|
||||
}
|
||||
}
|
||||
if extended_versions.is_empty() {
|
||||
return Err(Error::InvalidXtensaToolchanVersion(arg.to_string()));
|
||||
}
|
||||
let mut max_version = extended_versions.pop().unwrap();
|
||||
let mut max_subpatch = 0;
|
||||
for version in extended_versions {
|
||||
let subpatch: i8 = re_extended
|
||||
.captures(&version)
|
||||
.and_then(|cap| {
|
||||
cap.name("subpatch")
|
||||
.map(|subpatch| subpatch.as_str().parse().unwrap())
|
||||
})
|
||||
.unwrap();
|
||||
if subpatch > max_subpatch {
|
||||
max_subpatch = subpatch;
|
||||
max_version = version;
|
||||
}
|
||||
}
|
||||
return Ok(max_version);
|
||||
} else if re_extended.is_match(arg) {
|
||||
for release in json.as_array().unwrap() {
|
||||
let tag_name = release["tag_name"].to_string().replace(['\"', 'v'], "");
|
||||
if tag_name.starts_with(arg) {
|
||||
return Ok(arg.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(arg.to_string())
|
||||
Err(Error::InvalidXtensaToolchanVersion(arg.to_string()))
|
||||
}
|
||||
|
||||
/// Removes the Xtensa Rust toolchain.
|
||||
@ -495,15 +517,24 @@ fn install_rust_nightly(version: &str) -> Result<(), Error> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::toolchain::rust::{get_cargo_home, get_rustup_home, Crate, XtensaRust};
|
||||
use crate::{
|
||||
logging::initialize_logger,
|
||||
toolchain::rust::{get_cargo_home, get_rustup_home, Crate, XtensaRust},
|
||||
};
|
||||
use directories::BaseDirs;
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[test]
|
||||
fn test_xtensa_rust_parse_version() {
|
||||
assert_eq!(XtensaRust::parse_version("1.45.0.0").unwrap(), "1.45.0.0");
|
||||
assert_eq!(XtensaRust::parse_version("1.45.0.1").unwrap(), "1.45.0.1");
|
||||
assert_eq!(XtensaRust::parse_version("1.1.1.1").unwrap(), "1.1.1.1");
|
||||
initialize_logger("debug");
|
||||
assert_eq!(XtensaRust::parse_version("1.65.0.0").unwrap(), "1.65.0.0");
|
||||
assert_eq!(XtensaRust::parse_version("1.65.0.1").unwrap(), "1.65.0.1");
|
||||
assert_eq!(XtensaRust::parse_version("1.64.0.0").unwrap(), "1.64.0.0");
|
||||
assert_eq!(XtensaRust::parse_version("1.63.0").unwrap(), "1.63.0.2");
|
||||
assert_eq!(XtensaRust::parse_version("1.65.0").unwrap(), "1.65.0.1");
|
||||
assert_eq!(XtensaRust::parse_version("1.64.0").unwrap(), "1.64.0.0");
|
||||
assert!(XtensaRust::parse_version("422.0.0").is_err());
|
||||
assert!(XtensaRust::parse_version("422.0.0.0").is_err());
|
||||
assert!(XtensaRust::parse_version("a.1.1.1").is_err());
|
||||
assert!(XtensaRust::parse_version("1.1.1.1.1").is_err());
|
||||
assert!(XtensaRust::parse_version("1..1.1").is_err());
|
||||
|
||||
11
test.sh
Normal file
11
test.sh
Normal file
@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
for i in {1..100}
|
||||
do
|
||||
cargo test test_xtensa_rust_parse_version -- --nocapture
|
||||
# curl --request GET \
|
||||
# --url https://api.github.com/repos/esp-rs/rust-build/releases \
|
||||
# --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}'
|
||||
done
|
||||
Loading…
x
Reference in New Issue
Block a user