From 0df8613789bf0d456087601b471514a21a3932fa Mon Sep 17 00:00:00 2001 From: Sunshine Date: Sat, 6 Aug 2022 19:07:39 -1000 Subject: [PATCH] Rewrite part of function retrieve_asset, include support for brotli and deflate (#312) do not crash the app if reqwest throws, add support for deflate & brotli --- Cargo.lock | 60 ++++++++++++++++++++++++++++++++++++++++------------ Cargo.toml | 2 +- src/utils.rs | 41 ++++++++++++++++++++++++++--------- 3 files changed, 79 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d8613a..8d5f93b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ef4730490ad1c4eae5c4325b2a95f521d023e5c885853ff7aca0a6a1631db3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "assert_cmd" version = "2.0.4" @@ -37,6 +52,7 @@ version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "345fd392ab01f746c717b1357165b76f0b67a60192007b234058c9045fdcf695" dependencies = [ + "brotli", "flate2", "futures-core", "memchr", @@ -88,6 +104,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bstr" version = "0.2.17" @@ -197,9 +234,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ "cfg-if", ] @@ -295,13 +332,11 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" dependencies = [ - "cfg-if", "crc32fast", - "libc", "miniz_oxide", ] @@ -369,9 +404,9 @@ checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d" [[package]] name = "futures-io" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" [[package]] name = "futures-sink" @@ -673,12 +708,11 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" [[package]] name = "miniz_oxide" -version = "0.4.4" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" dependencies = [ "adler", - "autocfg 1.1.0", ] [[package]] @@ -757,9 +791,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ "hermit-abi", "libc", diff --git a/Cargo.toml b/Cargo.toml index 0529660..a802213 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ url = "2.2.2" [dependencies.reqwest] version = "0.11.11" default-features = false -features = ["default-tls", "blocking", "gzip"] +features = ["default-tls", "blocking", "gzip", "brotli", "deflate"] [dev-dependencies] assert_cmd = "2.0.4" diff --git a/src/utils.rs b/src/utils.rs index 4985ed4..5d66a53 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -238,8 +238,8 @@ pub fn retrieve_asset( } else { // URL not in cache, we retrieve the file match client.get(url.as_str()).send() { - Ok(mut response) => { - if !options.ignore_errors && response.status() != 200 { + Ok(response) => { + if !options.ignore_errors && response.status() != reqwest::StatusCode::OK { if !options.silent { eprintln!( "{}{}{} ({}){}", @@ -258,19 +258,17 @@ pub fn retrieve_asset( return Err(client.get("").send().unwrap_err()); } + let response_url: Url = response.url().clone(); + if !options.silent { - if url.as_str() == response.url().as_str() { + if url.as_str() == response_url.as_str() { eprintln!("{}{}", indent(depth).as_str(), &url); } else { - eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &response.url()); + eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &response_url); } } - let new_cache_key: String = clean_url(response.url().clone()).to_string(); - - // Convert response into a byte array - let mut data: Vec = vec![]; - response.copy_to(&mut data).unwrap(); + let new_cache_key: String = clean_url(response_url.clone()).to_string(); // Attempt to obtain media type and charset by reading Content-Type header let content_type: &str = response @@ -281,11 +279,34 @@ pub fn retrieve_asset( let (media_type, charset, _is_base64) = parse_content_type(&content_type); + // Convert response into a byte array + let mut data: Vec = vec![]; + match response.bytes() { + Ok(b) => { + data = b.to_vec(); + } + Err(error) => { + if !options.silent { + eprintln!( + "{}{}{}{}", + indent(depth).as_str(), + if options.no_color { "" } else { ANSI_COLOR_RED }, + error, + if options.no_color { + "" + } else { + ANSI_COLOR_RESET + }, + ); + } + } + } + // Add retrieved resource to cache cache.insert(new_cache_key, data.clone()); // Return - Ok((data, response.url().clone(), media_type, charset)) + Ok((data, response_url, media_type, charset)) } Err(error) => { if !options.silent {