Archives
/
monolith
mirror of https://github.com/Y2Z/monolith
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add flag to ignore errors related to TLS certificates

Browse Source
pull/49/head
Vincent Flyson 5 years ago
parent fada7884dc
commit 02b717ae54
5 changed files with 88 additions and 59 deletions
Show Stats Download Patch File Download Diff File

2
Cargo.toml
Unescape Escape View File

@ -1,6 +1,6 @@
[package] [package]
name = "monolith" name = "monolith"
version = "2.0.14" version = "2.0.16"
authors = [ authors = [
"Sunshine <sunshine@uberspace.net>", "Sunshine <sunshine@uberspace.net>",
"Mahdi Robatipoor <mahdi.robatipoor@gmail.com>", "Mahdi Robatipoor <mahdi.robatipoor@gmail.com>",

1
README.md
Unescape Escape View File

@ -23,6 +23,7 @@ If compared to saving websites with `wget -mpk`, this tool embeds all assets as
### Options ### Options
- `-i`: Remove images - `-i`: Remove images
- `-j`: Exclude JavaScript - `-j`: Exclude JavaScript
- `-k`: Accept invalid X.509 (TLS) certificates
- `-s`: Silent mode - `-s`: Silent mode
- `-u`: Specify custom User-Agent - `-u`: Specify custom User-Agent

128
src/html.rs
Unescape Escape View File

@ -70,18 +70,20 @@ pub fn walk_and_embed_assets(
opt_no_images: bool, opt_no_images: bool,
opt_user_agent: &str, opt_user_agent: &str,
opt_silent: bool, opt_silent: bool,
opt_insecure: bool,
) { ) {
match node.data { match node.data {
NodeData::Document => { NodeData::Document => {
// Dig deeper // Dig deeper
for child in node.children.borrow().iter() { for child in node.children.borrow().iter() {
walk_and_embed_assets( walk_and_embed_assets(
&url, child, &url, child,
opt_no_js, opt_no_js,
opt_no_images, opt_no_images,
opt_user_agent, opt_user_agent,
opt_silent, opt_silent,
); opt_insecure,
);
} }
} }
NodeData::Doctype { .. } => {} NodeData::Doctype { .. } => {}
@ -128,12 +130,14 @@ pub fn walk_and_embed_assets(
) )
.unwrap_or(EMPTY_STRING.clone()); .unwrap_or(EMPTY_STRING.clone());
let favicon_datauri = retrieve_asset( let favicon_datauri = retrieve_asset(
&href_full_url, &href_full_url,
true, true,
"", "",
opt_user_agent, opt_user_agent,
opt_silent, opt_silent,
).unwrap_or(EMPTY_STRING.clone()); opt_insecure,
)
.unwrap_or(EMPTY_STRING.clone());
attr.value.clear(); attr.value.clear();
attr.value.push_slice(favicon_datauri.as_str()); attr.value.push_slice(favicon_datauri.as_str());
} }
@ -148,12 +152,14 @@ pub fn walk_and_embed_assets(
) )
.unwrap_or(EMPTY_STRING.clone()); .unwrap_or(EMPTY_STRING.clone());
let css_datauri = retrieve_asset( let css_datauri = retrieve_asset(
&href_full_url, &href_full_url,
true, true,
"text/css", "text/css",
opt_user_agent, opt_user_agent,
opt_silent, opt_silent,
).unwrap_or(EMPTY_STRING.clone()); opt_insecure,
)
.unwrap_or(EMPTY_STRING.clone());
attr.value.clear(); attr.value.clear();
attr.value.push_slice(css_datauri.as_str()); attr.value.push_slice(css_datauri.as_str());
} }
@ -185,12 +191,14 @@ pub fn walk_and_embed_assets(
) )
.unwrap_or(EMPTY_STRING.clone()); .unwrap_or(EMPTY_STRING.clone());
let img_datauri = retrieve_asset( let img_datauri = retrieve_asset(
&src_full_url, &src_full_url,
true, true,
"", "",
opt_user_agent, opt_user_agent,
opt_silent, opt_silent,
).unwrap_or(EMPTY_STRING.clone()); opt_insecure,
)
.unwrap_or(EMPTY_STRING.clone());
attr.value.clear(); attr.value.clear();
attr.value.push_slice(img_datauri.as_str()); attr.value.push_slice(img_datauri.as_str());
} }
@ -211,12 +219,14 @@ pub fn walk_and_embed_assets(
) )
.unwrap_or(EMPTY_STRING.clone()); .unwrap_or(EMPTY_STRING.clone());
let source_datauri = retrieve_asset( let source_datauri = retrieve_asset(
&srcset_full_url, &srcset_full_url,
true, true,
"", "",
opt_user_agent, opt_user_agent,
opt_silent, opt_silent,
).unwrap_or(EMPTY_STRING.clone()); opt_insecure,
)
.unwrap_or(EMPTY_STRING.clone());
attr.value.clear(); attr.value.clear();
attr.value.push_slice(source_datauri.as_str()); attr.value.push_slice(source_datauri.as_str());
} }
@ -257,12 +267,14 @@ pub fn walk_and_embed_assets(
) )
.unwrap_or(EMPTY_STRING.clone()); .unwrap_or(EMPTY_STRING.clone());
let js_datauri = retrieve_asset( let js_datauri = retrieve_asset(
&src_full_url, &src_full_url,
true, true,
"application/javascript", "application/javascript",
opt_user_agent, opt_user_agent,
opt_silent, opt_silent,
).unwrap_or(EMPTY_STRING.clone()); opt_insecure,
)
.unwrap_or(EMPTY_STRING.clone());
attr.value.clear(); attr.value.clear();
attr.value.push_slice(js_datauri.as_str()); attr.value.push_slice(js_datauri.as_str());
} }
@ -290,21 +302,24 @@ pub fn walk_and_embed_assets(
let src_full_url: String = resolve_url(&url, &attr.value.to_string()) let src_full_url: String = resolve_url(&url, &attr.value.to_string())
.unwrap_or(EMPTY_STRING.clone()); .unwrap_or(EMPTY_STRING.clone());
let iframe_data = retrieve_asset( let iframe_data = retrieve_asset(
&src_full_url, &src_full_url,
false, false,
"text/html", "text/html",
opt_user_agent, opt_user_agent,
opt_silent, opt_silent,
).unwrap_or(EMPTY_STRING.clone()); opt_insecure,
)
.unwrap_or(EMPTY_STRING.clone());
let dom = html_to_dom(&iframe_data); let dom = html_to_dom(&iframe_data);
walk_and_embed_assets( walk_and_embed_assets(
&src_full_url, &src_full_url,
&dom.document, &dom.document,
opt_no_js, opt_no_js,
opt_no_images, opt_no_images,
opt_user_agent, opt_user_agent,
opt_silent, opt_silent,
); opt_insecure,
);
let mut buf: Vec<u8> = Vec::new(); let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap(); serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
let iframe_datauri = data_to_dataurl("text/html", &buf); let iframe_datauri = data_to_dataurl("text/html", &buf);
@ -328,13 +343,14 @@ pub fn walk_and_embed_assets(
// Dig deeper // Dig deeper
for child in node.children.borrow().iter() { for child in node.children.borrow().iter() {
walk_and_embed_assets( walk_and_embed_assets(
&url, &url,
child, child,
opt_no_js, opt_no_js,
opt_no_images, opt_no_images,
opt_user_agent, opt_user_agent,
opt_silent, opt_silent,
); opt_insecure,
);
} }
} }
NodeData::ProcessingInstruction { .. } => unreachable!() NodeData::ProcessingInstruction { .. } => unreachable!()

4
src/http.rs
Unescape Escape View File

@ -36,13 +36,15 @@ pub fn retrieve_asset(
as_dataurl: bool, as_dataurl: bool,
as_mime: &str, as_mime: &str,
opt_user_agent: &str, opt_user_agent: &str,
opt_silent: bool opt_silent: bool,
opt_insecure: bool,
) -> Result<String, reqwest::Error> { ) -> Result<String, reqwest::Error> {
if is_data_url(&url).unwrap() { if is_data_url(&url).unwrap() {
Ok(url.to_string()) Ok(url.to_string())
} else { } else {
let client = Client::builder() let client = Client::builder()
.timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(10))
.danger_accept_invalid_certs(opt_insecure)
.build()?; .build()?;
let mut response = client let mut response = client
.get(url) .get(url)

12
src/main.rs
Unescape Escape View File

@ -23,6 +23,7 @@ fn main() {
) )
.args_from_usage("-i, --no-images 'Removes images'") .args_from_usage("-i, --no-images 'Removes images'")
.args_from_usage("-j, --no-js 'Excludes JavaScript'") .args_from_usage("-j, --no-js 'Excludes JavaScript'")
.args_from_usage("-k, --insecure 'Accept invalid X.509 (TLS) certificates'")
.args_from_usage("-s, --silent 'Suppress verbosity'") .args_from_usage("-s, --silent 'Suppress verbosity'")
.args_from_usage("-u, --user-agent=[Iceweasel] 'Custom User-Agent string'") .args_from_usage("-u, --user-agent=[Iceweasel] 'Custom User-Agent string'")
.get_matches(); .get_matches();
@ -31,11 +32,19 @@ fn main() {
let arg_target = command.value_of("url").unwrap(); let arg_target = command.value_of("url").unwrap();
let opt_no_images = command.is_present("no-images"); let opt_no_images = command.is_present("no-images");
let opt_no_js = command.is_present("no-js"); let opt_no_js = command.is_present("no-js");
let opt_insecure = command.is_present("insecure");
let opt_silent = command.is_present("silent"); let opt_silent = command.is_present("silent");
let opt_user_agent = command.value_of("user-agent").unwrap_or(DEFAULT_USER_AGENT); let opt_user_agent = command.value_of("user-agent").unwrap_or(DEFAULT_USER_AGENT);
if is_valid_url(arg_target) { if is_valid_url(arg_target) {
let data = retrieve_asset(&arg_target, false, "", opt_user_agent, opt_silent).unwrap(); let data = retrieve_asset(
&arg_target,
false,
"",
opt_user_agent,
opt_silent,
opt_insecure,
).unwrap();
let dom = html_to_dom(&data); let dom = html_to_dom(&data);
walk_and_embed_assets( walk_and_embed_assets(
@ -45,6 +54,7 @@ fn main() {
opt_no_images, opt_no_images,
opt_user_agent, opt_user_agent,
opt_silent, opt_silent,
opt_insecure,
); );
print_dom(&dom.document); print_dom(&dom.document);

Write Preview
Loading…
Cancel
Save