mirror of
https://github.com/esp-rs/espup.git
synced 2025-09-26 20:30:28 +00:00
refactor: smoother large file download&proxy (#463)
* refactor: smoother large file download&proxy * chore: more msg * chore: changelog * chore: fmt * logs: make get proxy info so user know it working * refactor: use multiple process bar * chore: fmt * fix: allow log&process bar mix * refactor: per review * chore: fmt
This commit is contained in:
discord9
GitHub
parent
80e1df886e
commit
1e044c5eb4
@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- smoother large file download&proxy support (#463)
|
||||
|
||||
### Fixed
|
||||
- When queriying GitHub for the list of releases, retrieve more items (#462)
|
||||
|
||||
123
Cargo.lock
generated
123
Cargo.lock
generated
@ -323,6 +323,19 @@ version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.15.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
|
||||
dependencies = [
|
||||
"encode_unicode",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"unicode-width 0.1.14",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.3.1"
|
||||
@ -481,6 +494,12 @@ version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
|
||||
|
||||
[[package]]
|
||||
name = "encode_unicode"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.35"
|
||||
@ -556,12 +575,16 @@ version = "0.13.1-dev"
|
||||
dependencies = [
|
||||
"assert_cmd",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"clap",
|
||||
"clap_complete",
|
||||
"directories",
|
||||
"env_logger",
|
||||
"flate2",
|
||||
"guess_host_triple",
|
||||
"indicatif",
|
||||
"indicatif-log-bridge",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"miette",
|
||||
"openssl",
|
||||
@ -575,6 +598,7 @@ dependencies = [
|
||||
"thiserror 2.0.3",
|
||||
"tokio",
|
||||
"tokio-retry",
|
||||
"tokio-stream",
|
||||
"update-informer",
|
||||
"winapi",
|
||||
"winreg 0.52.0",
|
||||
@ -662,6 +686,17 @@ version = "0.3.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.31"
|
||||
@ -682,6 +717,7 @@ checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-macro",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"memchr",
|
||||
@ -1127,6 +1163,29 @@ dependencies = [
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indicatif"
|
||||
version = "0.17.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbf675b85ed934d3c67b5c5469701eec7db22689d0a2139d856e0925fa28b281"
|
||||
dependencies = [
|
||||
"console",
|
||||
"number_prefix",
|
||||
"portable-atomic",
|
||||
"unicode-width 0.2.0",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indicatif-log-bridge"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63703cf9069b85dbe6fe26e1c5230d013dee99d3559cd3d02ba39e099ef7ab02"
|
||||
dependencies = [
|
||||
"indicatif",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inout"
|
||||
version = "0.1.3"
|
||||
@ -1178,6 +1237,12 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.165"
|
||||
@ -1273,7 +1338,7 @@ dependencies = [
|
||||
"terminal_size",
|
||||
"textwrap",
|
||||
"thiserror 1.0.69",
|
||||
"unicode-width",
|
||||
"unicode-width 0.1.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1337,6 +1402,12 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
|
||||
|
||||
[[package]]
|
||||
name = "number_prefix"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.36.5"
|
||||
@ -1495,6 +1566,12 @@ version = "0.3.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6"
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
@ -1712,10 +1789,12 @@ dependencies = [
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
"tokio-socks",
|
||||
"tokio-util",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"wasm-streams",
|
||||
"web-sys",
|
||||
"windows-registry",
|
||||
]
|
||||
@ -2195,7 +2274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
|
||||
dependencies = [
|
||||
"unicode-linebreak",
|
||||
"unicode-width",
|
||||
"unicode-width 0.1.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2350,6 +2429,17 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-stream"
|
||||
version = "0.1.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.7.12"
|
||||
@ -2418,6 +2508,12 @@ version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
@ -2588,6 +2684,19 @@ version = "0.2.95"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-streams"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.72"
|
||||
@ -2598,6 +2707,16 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web-time"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "0.25.4"
|
||||
|
||||
@ -15,16 +15,20 @@ rust-version = "1.74.1"
|
||||
|
||||
[dependencies]
|
||||
async-trait = "0.1.83"
|
||||
bytes = "1.8.0"
|
||||
clap = { version = "4.5.21", features = ["derive", "env"] }
|
||||
clap_complete = "4.5.38"
|
||||
directories = "5.0.1"
|
||||
env_logger = "0.11.5"
|
||||
flate2 = "1.0.35"
|
||||
guess_host_triple = "0.1.4"
|
||||
indicatif = "0.17.9"
|
||||
indicatif-log-bridge = "0.2.3"
|
||||
lazy_static = "1.0"
|
||||
log = "0.4.22"
|
||||
miette = { version = "7.3.0", features = ["fancy"] }
|
||||
regex = "1.11.1"
|
||||
reqwest = { version = "0.12.9", features = ["blocking", "socks"] }
|
||||
reqwest = { version = "0.12.9", features = ["blocking", "socks", "stream"] }
|
||||
retry = "2.0.0"
|
||||
serde_json = "1.0.133"
|
||||
strum = { version = "0.26.3", features = ["derive"] }
|
||||
@ -33,6 +37,7 @@ tempfile = "3.14.0"
|
||||
thiserror = "2.0.3"
|
||||
tokio = { version = "1.41.1", features = ["full"] }
|
||||
tokio-retry = "0.3.0"
|
||||
tokio-stream = "0.1.17"
|
||||
update-informer = "1.1.0"
|
||||
xz2 = "0.1.7"
|
||||
zip = "2.2.1"
|
||||
|
||||
12
src/lib.rs
12
src/lib.rs
@ -8,9 +8,11 @@ pub mod toolchain;
|
||||
pub mod logging {
|
||||
use env_logger::{Builder, Env, WriteStyle};
|
||||
|
||||
use crate::toolchain::PROCESS_BARS;
|
||||
|
||||
/// Initializes the logger
|
||||
pub fn initialize_logger(log_level: &str) {
|
||||
Builder::from_env(Env::default().default_filter_or(log_level))
|
||||
let logger = Builder::from_env(Env::default().default_filter_or(log_level))
|
||||
.format(|buf, record| {
|
||||
use std::io::Write;
|
||||
writeln!(
|
||||
@ -21,7 +23,13 @@ pub mod logging {
|
||||
)
|
||||
})
|
||||
.write_style(WriteStyle::Always)
|
||||
.init();
|
||||
.build();
|
||||
let level = logger.filter();
|
||||
// make logging and process bar no longer mixed up
|
||||
indicatif_log_bridge::LogWrapper::new(PROCESS_BARS.clone(), logger)
|
||||
.try_init()
|
||||
.unwrap();
|
||||
log::set_max_level(level);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -25,10 +25,12 @@ use std::{
|
||||
fs::{create_dir_all, remove_file, File},
|
||||
io::{copy, Write},
|
||||
path::{Path, PathBuf},
|
||||
sync::atomic::{self, AtomicUsize},
|
||||
};
|
||||
use tar::Archive;
|
||||
use tokio::{fs::remove_dir_all, sync::mpsc};
|
||||
use tokio_retry::{strategy::FixedInterval, Retry};
|
||||
use tokio_stream::StreamExt;
|
||||
use xz2::read::XzDecoder;
|
||||
use zip::ZipArchive;
|
||||
|
||||
@ -36,6 +38,11 @@ pub mod gcc;
|
||||
pub mod llvm;
|
||||
pub mod rust;
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref PROCESS_BARS: indicatif::MultiProgress = indicatif::MultiProgress::new();
|
||||
pub static ref DOWNLOAD_CNT: AtomicUsize = AtomicUsize::new(0);
|
||||
}
|
||||
|
||||
pub enum InstallMode {
|
||||
Install,
|
||||
Update,
|
||||
@ -49,6 +56,47 @@ pub trait Installable {
|
||||
fn name(&self) -> String;
|
||||
}
|
||||
|
||||
/// Get https proxy from environment variables(if any)
|
||||
///
|
||||
/// sadly there is not standard on the environment variable name for the proxy, but it seems
|
||||
/// that the most common are:
|
||||
///
|
||||
/// - https_proxy(or http_proxy for http)
|
||||
/// - HTTPS_PROXY(or HTTP_PROXY for http)
|
||||
/// - all_proxy
|
||||
/// - ALL_PROXY
|
||||
///
|
||||
/// hence we will check for all of them
|
||||
fn https_proxy() -> Option<String> {
|
||||
for proxy in ["https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"] {
|
||||
if let Ok(proxy_addr) = std::env::var(proxy) {
|
||||
info!("Get Proxy from env var: {}={}", proxy, proxy_addr);
|
||||
return Some(proxy_addr);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Build a reqwest client with proxy if env var is set
|
||||
fn build_proxy_blocking_client() -> Result<Client, Error> {
|
||||
let mut builder = reqwest::blocking::Client::builder();
|
||||
if let Some(proxy) = https_proxy() {
|
||||
builder = builder.proxy(reqwest::Proxy::https(&proxy).unwrap());
|
||||
}
|
||||
let client = builder.build()?;
|
||||
Ok(client)
|
||||
}
|
||||
|
||||
/// Build a reqwest client with proxy if env var is set
|
||||
fn build_proxy_async_client() -> Result<reqwest::Client, Error> {
|
||||
let mut builder = reqwest::Client::builder();
|
||||
if let Some(proxy) = https_proxy() {
|
||||
builder = builder.proxy(reqwest::Proxy::https(&proxy).unwrap());
|
||||
}
|
||||
let client = builder.build()?;
|
||||
Ok(client)
|
||||
}
|
||||
|
||||
/// Downloads a file from a URL and uncompresses it, if necesary, to the output directory.
|
||||
pub async fn download_file(
|
||||
url: String,
|
||||
@ -69,9 +117,49 @@ pub async fn download_file(
|
||||
create_dir_all(output_directory)
|
||||
.map_err(|_| Error::CreateDirectory(output_directory.to_string()))?;
|
||||
}
|
||||
info!("Downloading '{}'", &file_name);
|
||||
let resp = reqwest::get(&url).await?;
|
||||
let bytes = resp.bytes().await?;
|
||||
|
||||
let resp = {
|
||||
let client = build_proxy_async_client()?;
|
||||
client.get(&url).send().await?
|
||||
};
|
||||
let bytes = {
|
||||
let len = resp.content_length();
|
||||
|
||||
// draw a progress bar
|
||||
let sty = indicatif::ProgressStyle::with_template(
|
||||
"[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
|
||||
)
|
||||
.unwrap()
|
||||
.progress_chars("##-");
|
||||
let bar = len
|
||||
.map(indicatif::ProgressBar::new)
|
||||
.unwrap_or(indicatif::ProgressBar::no_length());
|
||||
let bar = PROCESS_BARS.add(bar);
|
||||
bar.set_style(sty);
|
||||
bar.set_message(file_name.to_string());
|
||||
DOWNLOAD_CNT.fetch_add(1, atomic::Ordering::Relaxed);
|
||||
|
||||
let mut size_downloaded = 0;
|
||||
let mut stream = resp.bytes_stream();
|
||||
let mut bytes = bytes::BytesMut::new();
|
||||
while let Some(chunk_result) = stream.next().await {
|
||||
let chunk = chunk_result?;
|
||||
size_downloaded += chunk.len();
|
||||
bar.set_position(size_downloaded as u64);
|
||||
|
||||
bytes.extend(&chunk);
|
||||
}
|
||||
bar.finish_with_message(format!("{} download complete", file_name));
|
||||
// leave the progress bar after completion
|
||||
if DOWNLOAD_CNT.fetch_sub(1, atomic::Ordering::Relaxed) == 1 {
|
||||
// clear all progress bars
|
||||
PROCESS_BARS.clear().unwrap();
|
||||
info!("All downloads complete");
|
||||
}
|
||||
// wait while DOWNLOAD_CNT is not zero
|
||||
|
||||
bytes.freeze()
|
||||
};
|
||||
if uncompress {
|
||||
let extension = Path::new(file_name).extension().unwrap().to_str().unwrap();
|
||||
match extension {
|
||||
@ -286,7 +374,7 @@ pub fn github_query(url: &str) -> Result<serde_json::Value, Error> {
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
let client = Client::new();
|
||||
let client = build_proxy_blocking_client()?;
|
||||
let json = retry(
|
||||
Fixed::from_millis(100).take(5),
|
||||
|| -> Result<serde_json::Value, Error> {
|
||||
|
||||
@ -73,11 +73,16 @@ pub struct XtensaRust {
|
||||
impl XtensaRust {
|
||||
/// Get the latest version of Xtensa Rust toolchain.
|
||||
pub async fn get_latest_version() -> Result<String> {
|
||||
let json = github_query(XTENSA_RUST_LATEST_API_URL)?;
|
||||
let json = tokio::task::spawn_blocking(|| github_query(XTENSA_RUST_LATEST_API_URL))
|
||||
.await
|
||||
.unwrap()?;
|
||||
let mut version = json["tag_name"].to_string();
|
||||
|
||||
version.retain(|c| c != 'v' && c != '"');
|
||||
Self::parse_version(&version)?;
|
||||
let borrowed = version.clone();
|
||||
tokio::task::spawn_blocking(move || Self::parse_version(&borrowed))
|
||||
.await
|
||||
.expect("Join blocking task error")?;
|
||||
debug!("Latest Xtensa Rust version: {}", version);
|
||||
Ok(version)
|
||||
}
|
||||
@ -228,6 +233,15 @@ impl Installable for XtensaRust {
|
||||
let tmp_dir = tempdir_in(path)?;
|
||||
let tmp_dir_path = &tmp_dir.path().display().to_string();
|
||||
|
||||
download_file(
|
||||
self.src_dist_url.clone(),
|
||||
"rust-src.tar.xz",
|
||||
tmp_dir_path,
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
download_file(
|
||||
self.dist_url.clone(),
|
||||
"rust.tar.xz",
|
||||
@ -261,14 +275,6 @@ impl Installable for XtensaRust {
|
||||
return Err(Error::XtensaRust);
|
||||
}
|
||||
|
||||
download_file(
|
||||
self.src_dist_url.clone(),
|
||||
"rust-src.tar.xz",
|
||||
tmp_dir_path,
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
info!("Installing 'rust-src' component for Xtensa Rust toolchain");
|
||||
if !Command::new("/usr/bin/env")
|
||||
.arg("bash")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user