Rewrite part of function retrieve_asset, include support for brotli and deflate (#312)

do not crash the app if reqwest throws, add support for deflate & brotli
pull/314/head
Sunshine 2 years ago committed by GitHub
parent 68a1531a11
commit 0df8613789
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

60
Cargo.lock generated

@ -17,6 +17,21 @@ dependencies = [
"memchr",
]
[[package]]
name = "alloc-no-stdlib"
version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35ef4730490ad1c4eae5c4325b2a95f521d023e5c885853ff7aca0a6a1631db3"
[[package]]
name = "alloc-stdlib"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2"
dependencies = [
"alloc-no-stdlib",
]
[[package]]
name = "assert_cmd"
version = "2.0.4"
@ -37,6 +52,7 @@ version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "345fd392ab01f746c717b1357165b76f0b67a60192007b234058c9045fdcf695"
dependencies = [
"brotli",
"flate2",
"futures-core",
"memchr",
@ -88,6 +104,27 @@ dependencies = [
"generic-array",
]
[[package]]
name = "brotli"
version = "3.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"brotli-decompressor",
]
[[package]]
name = "brotli-decompressor"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
]
[[package]]
name = "bstr"
version = "0.2.17"
@ -197,9 +234,9 @@ dependencies = [
[[package]]
name = "crc32fast"
version = "1.2.1"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if",
]
@ -295,13 +332,11 @@ dependencies = [
[[package]]
name = "flate2"
version = "1.0.22"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f"
checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
dependencies = [
"cfg-if",
"crc32fast",
"libc",
"miniz_oxide",
]
@ -369,9 +404,9 @@ checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d"
[[package]]
name = "futures-io"
version = "0.3.17"
version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377"
checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
[[package]]
name = "futures-sink"
@ -673,12 +708,11 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "miniz_oxide"
version = "0.4.4"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc"
dependencies = [
"adler",
"autocfg 1.1.0",
]
[[package]]
@ -757,9 +791,9 @@ dependencies = [
[[package]]
name = "num_cpus"
version = "1.13.0"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
dependencies = [
"hermit-abi",
"libc",

@ -37,7 +37,7 @@ url = "2.2.2"
[dependencies.reqwest]
version = "0.11.11"
default-features = false
features = ["default-tls", "blocking", "gzip"]
features = ["default-tls", "blocking", "gzip", "brotli", "deflate"]
[dev-dependencies]
assert_cmd = "2.0.4"

@ -238,8 +238,8 @@ pub fn retrieve_asset(
} else {
// URL not in cache, we retrieve the file
match client.get(url.as_str()).send() {
Ok(mut response) => {
if !options.ignore_errors && response.status() != 200 {
Ok(response) => {
if !options.ignore_errors && response.status() != reqwest::StatusCode::OK {
if !options.silent {
eprintln!(
"{}{}{} ({}){}",
@ -258,19 +258,17 @@ pub fn retrieve_asset(
return Err(client.get("").send().unwrap_err());
}
let response_url: Url = response.url().clone();
if !options.silent {
if url.as_str() == response.url().as_str() {
if url.as_str() == response_url.as_str() {
eprintln!("{}{}", indent(depth).as_str(), &url);
} else {
eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &response.url());
eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &response_url);
}
}
let new_cache_key: String = clean_url(response.url().clone()).to_string();
// Convert response into a byte array
let mut data: Vec<u8> = vec![];
response.copy_to(&mut data).unwrap();
let new_cache_key: String = clean_url(response_url.clone()).to_string();
// Attempt to obtain media type and charset by reading Content-Type header
let content_type: &str = response
@ -281,11 +279,34 @@ pub fn retrieve_asset(
let (media_type, charset, _is_base64) = parse_content_type(&content_type);
// Convert response into a byte array
let mut data: Vec<u8> = vec![];
match response.bytes() {
Ok(b) => {
data = b.to_vec();
}
Err(error) => {
if !options.silent {
eprintln!(
"{}{}{}{}",
indent(depth).as_str(),
if options.no_color { "" } else { ANSI_COLOR_RED },
error,
if options.no_color {
""
} else {
ANSI_COLOR_RESET
},
);
}
}
}
// Add retrieved resource to cache
cache.insert(new_cache_key, data.clone());
// Return
Ok((data, response.url().clone(), media_type, charset))
Ok((data, response_url, media_type, charset))
}
Err(error) => {
if !options.silent {

Loading…
Cancel
Save