mirror of https://github.com/terhechte/postsack
Initial Workspace restructuring compiles
parent
7a0be16578
commit
5dfd1b9630
@ -1,2 +1,3 @@
|
|||||||
/target
|
/target
|
||||||
|
target
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,3 @@
|
|||||||
|
/target
|
||||||
|
target
|
||||||
|
.DS_Store
|
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,29 @@
|
|||||||
|
[package]
|
||||||
|
name = "postsack"
|
||||||
|
version = "0.2.0"
|
||||||
|
edition = "2021"
|
||||||
|
description = "Provides a high level visual overview of swaths of email"
|
||||||
|
|
||||||
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
|
[package.metadata.bundle]
|
||||||
|
name = "Postsack"
|
||||||
|
identifier = "com.stylemac.postsack"
|
||||||
|
icon = ["icons/Icon.icns", "icons/icon-win-256.png", "icons/icon-win-32.png", "icons/icon-win-16.png"]
|
||||||
|
version = "1.0.0"
|
||||||
|
copyright = "Copyright (c) Benedikt Terhechte (2021). All rights reserved."
|
||||||
|
category = "Developer Tool"
|
||||||
|
short_description = "Provides a high level visual overview of swaths of email"
|
||||||
|
osx_minimum_system_version = "10.14"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
ps-gui = { path = "../ps-gui" }
|
||||||
|
ps-core = { path = "../ps-core" }
|
||||||
|
|
||||||
|
[profile.dev]
|
||||||
|
split-debuginfo = "unpacked"
|
||||||
|
|
||||||
|
#[profile.release]
|
||||||
|
#lto = "fat"
|
||||||
|
#codegen-units = 1
|
||||||
|
#panic = "abort"
|
@ -0,0 +1,6 @@
|
|||||||
|
fn main() {
|
||||||
|
#[cfg(debug_assertions)]
|
||||||
|
ps_core::setup_tracing();
|
||||||
|
|
||||||
|
ps_gui::run_ui();
|
||||||
|
}
|
@ -0,0 +1,3 @@
|
|||||||
|
/target
|
||||||
|
target
|
||||||
|
.DS_Store
|
@ -1,13 +1,29 @@
|
|||||||
mod database;
|
mod database;
|
||||||
mod importer;
|
mod importer;
|
||||||
mod model;
|
pub mod message_adapter;
|
||||||
|
pub mod model;
|
||||||
mod types;
|
mod types;
|
||||||
|
|
||||||
pub use database::database_like::DatabaseLike;
|
pub use database::database_like::DatabaseLike;
|
||||||
pub use database::db_message::DBMessage;
|
pub use database::db_message::DBMessage;
|
||||||
pub use database::query::{Field, OtherQuery, Query, ValueField, AMOUNT_FIELD_NAME};
|
pub use database::query::{Field, Filter, OtherQuery, Query, ValueField, AMOUNT_FIELD_NAME};
|
||||||
pub use database::query_result::QueryResult;
|
pub use database::query_result::{QueryResult, QueryRow};
|
||||||
pub use types::{Config, EmailEntry, EmailMeta, FormatType};
|
pub use types::{Config, EmailEntry, EmailMeta, FormatType};
|
||||||
|
|
||||||
pub use crossbeam_channel;
|
pub use crossbeam_channel;
|
||||||
pub use importer::{Importerlike, Message, MessageReceiver, MessageSender};
|
pub use importer::{Importerlike, Message, MessageReceiver, MessageSender};
|
||||||
|
|
||||||
|
// Tracing
|
||||||
|
|
||||||
|
use tracing_subscriber::fmt;
|
||||||
|
use tracing_subscriber::prelude::*;
|
||||||
|
|
||||||
|
pub fn setup_tracing() {
|
||||||
|
if std::env::var("RUST_LOG").is_err() {
|
||||||
|
std::env::set_var("RUST_LOG", "error")
|
||||||
|
}
|
||||||
|
|
||||||
|
let collector = tracing_subscriber::registry().with(fmt::layer().with_writer(std::io::stdout));
|
||||||
|
|
||||||
|
tracing::subscriber::set_global_default(collector).expect("Unable to set a global collector");
|
||||||
|
}
|
||||||
|
@ -0,0 +1,3 @@
|
|||||||
|
/target
|
||||||
|
target
|
||||||
|
.DS_Store
|
@ -0,0 +1,3 @@
|
|||||||
|
/target
|
||||||
|
target
|
||||||
|
.DS_Store
|
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,32 @@
|
|||||||
|
[package]
|
||||||
|
name = "ps-gui"
|
||||||
|
version = "0.2.0"
|
||||||
|
edition = "2021"
|
||||||
|
|
||||||
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
eyre = "0.6.5"
|
||||||
|
thiserror = "1.0.29"
|
||||||
|
tracing = "0.1.29"
|
||||||
|
tracing-subscriber = "0.3.0"
|
||||||
|
once_cell = "1.8.0"
|
||||||
|
crossbeam-channel = "0.5.1"
|
||||||
|
eframe = "0.15.0"
|
||||||
|
num-format = "0.4.0"
|
||||||
|
tinyfiledialogs = "3.0"
|
||||||
|
rand = "0.8.4"
|
||||||
|
image = { version = "0.23", default-features = false, features = ["png"] }
|
||||||
|
chrono = "0.4.19"
|
||||||
|
shellexpand = "2.1.0"
|
||||||
|
ps-core = { path = "../ps-core" }
|
||||||
|
|
||||||
|
[target."cfg(target_os = \"macos\")".dependencies.cocoa]
|
||||||
|
version = "0.24"
|
||||||
|
[target."cfg(target_os = \"macos\")".dependencies.objc]
|
||||||
|
version = "0.2.7"
|
||||||
|
|
||||||
|
[target."cfg(not(target_arch = \"wasm32\"))".dependencies]
|
||||||
|
ps-importer = { path = "../ps-importer" }
|
||||||
|
# Do I need this?
|
||||||
|
ps-database = { path = "../ps-database" }
|
@ -0,0 +1,42 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
# ./setup_web.sh # <- call this first!
|
||||||
|
|
||||||
|
FOLDER_NAME=${PWD##*/}
|
||||||
|
CRATE_NAME=$FOLDER_NAME # assume crate name is the same as the folder name
|
||||||
|
CRATE_NAME_SNAKE_CASE="${CRATE_NAME//-/_}" # for those who name crates with-kebab-case
|
||||||
|
|
||||||
|
# This is required to enable the web_sys clipboard API which egui_web uses
|
||||||
|
# https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html
|
||||||
|
# https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html
|
||||||
|
export RUSTFLAGS=--cfg=web_sys_unstable_apis
|
||||||
|
|
||||||
|
# Clear output from old stuff:
|
||||||
|
rm -f web_demo/${CRATE_NAME_SNAKE_CASE}_bg.wasm
|
||||||
|
|
||||||
|
echo "Building rust…"
|
||||||
|
BUILD=release
|
||||||
|
cargo build --release -p ${CRATE_NAME} --lib --target wasm32-unknown-unknown
|
||||||
|
|
||||||
|
echo "Generating JS bindings for wasm…"
|
||||||
|
TARGET_NAME="${CRATE_NAME_SNAKE_CASE}.wasm"
|
||||||
|
wasm-bindgen "target/wasm32-unknown-unknown/${BUILD}/${TARGET_NAME}" \
|
||||||
|
--out-dir web_demo --no-modules --no-typescript
|
||||||
|
|
||||||
|
# to get wasm-opt: apt/brew/dnf install binaryen
|
||||||
|
# echo "Optimizing wasm…"
|
||||||
|
# wasm-opt web_demo/${CRATE_NAME_SNAKE_CASE}_bg.wasm -O2 --fast-math -o web_demo/${CRATE_NAME_SNAKE_CASE}_bg.wasm # add -g to get debug symbols
|
||||||
|
|
||||||
|
echo "Finished: web_demo/${CRATE_NAME_SNAKE_CASE}.wasm"
|
||||||
|
|
||||||
|
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
|
||||||
|
# Linux, ex: Fedora
|
||||||
|
xdg-open http://localhost:8080/index.html
|
||||||
|
elif [[ "$OSTYPE" == "msys" ]]; then
|
||||||
|
# Windows
|
||||||
|
start http://localhost:8080/index.html
|
||||||
|
else
|
||||||
|
# Darwin/MacOS, or something else
|
||||||
|
open http://localhost:8080/index.html
|
||||||
|
fi
|
@ -0,0 +1,10 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
# Pre-requisites:
|
||||||
|
rustup target add wasm32-unknown-unknown
|
||||||
|
cargo install -f wasm-bindgen-cli
|
||||||
|
cargo update -p wasm-bindgen
|
||||||
|
|
||||||
|
# For local tests with `./start_server`:
|
||||||
|
cargo install basic-http-server
|
@ -1,9 +1,7 @@
|
|||||||
use crate::database::query::Field;
|
|
||||||
use crate::database::query_result::QueryRow;
|
|
||||||
use crate::model::{items, Engine};
|
|
||||||
use chrono::prelude::*;
|
use chrono::prelude::*;
|
||||||
use eframe::egui::{self, Widget};
|
use eframe::egui::{self, Widget};
|
||||||
use eyre::Report;
|
use eyre::Report;
|
||||||
|
use ps_core::{model::items, model::Engine, Field, QueryRow};
|
||||||
|
|
||||||
use super::widgets::Table;
|
use super::widgets::Table;
|
||||||
|
|
@ -1,7 +1,7 @@
|
|||||||
use crate::model::Engine;
|
|
||||||
use eframe::egui::{self, Color32, Label, Widget};
|
use eframe::egui::{self, Color32, Label, Widget};
|
||||||
use eyre::Report;
|
use eyre::Report;
|
||||||
use num_format::{Locale, ToFormattedString};
|
use num_format::{Locale, ToFormattedString};
|
||||||
|
use ps_core::model::Engine;
|
||||||
|
|
||||||
use super::app_state::UIState;
|
use super::app_state::UIState;
|
||||||
use super::platform::navigation_button;
|
use super::platform::navigation_button;
|
Before Width: | Height: | Size: 515 KiB After Width: | Height: | Size: 515 KiB |
@ -1,6 +1,6 @@
|
|||||||
use crate::model::{segmentations, Engine};
|
|
||||||
use eframe::egui::{self, Widget};
|
use eframe::egui::{self, Widget};
|
||||||
use eyre::Report;
|
use eyre::Report;
|
||||||
|
use ps_core::model::{segmentations, Engine};
|
||||||
|
|
||||||
pub struct SegmentationBar<'a> {
|
pub struct SegmentationBar<'a> {
|
||||||
engine: &'a mut Engine,
|
engine: &'a mut Engine,
|
@ -0,0 +1,3 @@
|
|||||||
|
/target
|
||||||
|
target
|
||||||
|
.DS_Store
|
@ -1,61 +0,0 @@
|
|||||||
use std::thread::JoinHandle;
|
|
||||||
|
|
||||||
use super::formats::shared;
|
|
||||||
use ps_core::{
|
|
||||||
crossbeam_channel::{self, unbounded},
|
|
||||||
Config, DatabaseLike, Importerlike, Message, MessageReceiver,
|
|
||||||
};
|
|
||||||
|
|
||||||
use super::formats::ImporterFormat;
|
|
||||||
|
|
||||||
use eyre::Result;
|
|
||||||
|
|
||||||
pub struct Importer<Format: ImporterFormat> {
|
|
||||||
config: Config,
|
|
||||||
format: Format,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<Format: ImporterFormat + 'static> Importer<Format> {
|
|
||||||
pub fn new(config: Config, format: Format) -> Self {
|
|
||||||
Self { config, format }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<Format: ImporterFormat + 'static> Importerlike for Importer<Format> {
|
|
||||||
fn import<Database: DatabaseLike + 'static>(
|
|
||||||
self,
|
|
||||||
database: Database,
|
|
||||||
) -> Result<(MessageReceiver, JoinHandle<Result<()>>)> {
|
|
||||||
let Importer { format, .. } = self;
|
|
||||||
let (sender, receiver) = unbounded();
|
|
||||||
|
|
||||||
let config = self.config;
|
|
||||||
let handle: JoinHandle<Result<()>> = std::thread::spawn(move || {
|
|
||||||
let outer_sender = sender.clone();
|
|
||||||
let processed = move || {
|
|
||||||
let emails = format.emails(&config, sender.clone())?;
|
|
||||||
let processed =
|
|
||||||
shared::database::into_database(&config, emails, sender.clone(), database)?;
|
|
||||||
|
|
||||||
Ok(processed)
|
|
||||||
};
|
|
||||||
let result = processed();
|
|
||||||
|
|
||||||
// Send the error away and map it to a crossbeam channel error
|
|
||||||
match result {
|
|
||||||
Ok(_) => Ok(()),
|
|
||||||
Err(e) => match outer_sender.send(Message::Error(e)) {
|
|
||||||
Ok(_) => Ok(()),
|
|
||||||
Err(e) => Err(eyre::Report::new(e)),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
});
|
|
||||||
Ok((receiver, handle))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// impl<T: Importerlike + Sized> Importerlike for Box<T> {
|
|
||||||
// fn import(self) -> Result<(MessageReceiver, JoinHandle<Result<()>>)> {
|
|
||||||
// (*self).import()
|
|
||||||
// }
|
|
||||||
// }
|
|
Binary file not shown.
@ -1,87 +0,0 @@
|
|||||||
use eyre::Result;
|
|
||||||
|
|
||||||
use std::{
|
|
||||||
io::{stdout, Write},
|
|
||||||
thread::sleep,
|
|
||||||
time::Duration,
|
|
||||||
};
|
|
||||||
|
|
||||||
use postsack::{
|
|
||||||
self,
|
|
||||||
importer::{Adapter, State},
|
|
||||||
types::FormatType,
|
|
||||||
};
|
|
||||||
|
|
||||||
fn main() -> Result<()> {
|
|
||||||
postsack::setup_tracing();
|
|
||||||
|
|
||||||
let config = postsack::make_config();
|
|
||||||
|
|
||||||
let adapter = postsack::importer::Adapter::new();
|
|
||||||
|
|
||||||
// Could not figure out how to build this properly
|
|
||||||
// with dynamic dispatch. (to abstract away the match)
|
|
||||||
// Will try again when I'm online.
|
|
||||||
let handle = match config.format {
|
|
||||||
FormatType::AppleMail => {
|
|
||||||
let importer = postsack::importer::applemail_importer(config);
|
|
||||||
adapter.process(importer)?
|
|
||||||
}
|
|
||||||
FormatType::GmailVault => {
|
|
||||||
let importer = postsack::importer::gmail_importer(config);
|
|
||||||
adapter.process(importer)?
|
|
||||||
}
|
|
||||||
FormatType::Mbox => {
|
|
||||||
let importer = postsack::importer::mbox_importer(config);
|
|
||||||
adapter.process(importer)?
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut stdout = stdout();
|
|
||||||
|
|
||||||
loop {
|
|
||||||
match handle_adapter(&adapter) {
|
|
||||||
Ok(true) => break,
|
|
||||||
Ok(false) => (),
|
|
||||||
Err(e) => {
|
|
||||||
println!("Execution Error:\n{:?}", &e);
|
|
||||||
panic!();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stdout.flush().unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
match handle.join() {
|
|
||||||
Err(e) => println!("Error: {:?}", e),
|
|
||||||
Ok(Err(e)) => println!("Error: {:?}", e),
|
|
||||||
_ => (),
|
|
||||||
}
|
|
||||||
println!("\rDone");
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn handle_adapter(adapter: &Adapter) -> Result<bool> {
|
|
||||||
let State {
|
|
||||||
done, finishing, ..
|
|
||||||
} = adapter.finished()?;
|
|
||||||
if done {
|
|
||||||
return Ok(true);
|
|
||||||
}
|
|
||||||
if finishing {
|
|
||||||
print!("\rFinishing up...");
|
|
||||||
} else {
|
|
||||||
let write = adapter.write_count()?;
|
|
||||||
if write.count > 0 {
|
|
||||||
print!("\rWriting emails to DB {}/{}...", write.count, write.total);
|
|
||||||
} else {
|
|
||||||
let read = adapter.read_count()?;
|
|
||||||
print!(
|
|
||||||
"\rReading Emails {}%...",
|
|
||||||
(read.count as f32 / read.total as f32) * 100.0
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sleep(Duration::from_millis(50));
|
|
||||||
Ok(false)
|
|
||||||
}
|
|
@ -1,12 +0,0 @@
|
|||||||
#[cfg(feature = "gui")]
|
|
||||||
fn main() {
|
|
||||||
#[cfg(debug_assertions)]
|
|
||||||
postsack::setup_tracing();
|
|
||||||
|
|
||||||
postsack::gui::run_gui();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(feature = "gui"))]
|
|
||||||
fn main() {
|
|
||||||
println!("Gui not selected")
|
|
||||||
}
|
|
@ -1,145 +0,0 @@
|
|||||||
use std::collections::HashMap;
|
|
||||||
use std::convert::TryInto;
|
|
||||||
use std::str::FromStr;
|
|
||||||
|
|
||||||
use chrono::prelude::*;
|
|
||||||
use eyre::{bail, eyre, Result};
|
|
||||||
use rusqlite::{self, types, Row};
|
|
||||||
use serde_json::Value;
|
|
||||||
|
|
||||||
use super::query::{Field, ValueField, AMOUNT_FIELD_NAME};
|
|
||||||
use super::query_result::QueryResult;
|
|
||||||
use crate::importer::{EmailEntry, EmailMeta};
|
|
||||||
|
|
||||||
/// rusqlite does offer Serde to Value conversion, but it
|
|
||||||
/// converts everything to strings!
|
|
||||||
pub fn json_to_value(input: &Value) -> Result<types::Value> {
|
|
||||||
let ok = match input {
|
|
||||||
Value::Number(n) if n.is_i64() => {
|
|
||||||
types::Value::Integer(n.as_i64().ok_or_else(|| eyre!("Invalid Number {:?}", n))?)
|
|
||||||
}
|
|
||||||
Value::Number(n) if n.is_u64() => {
|
|
||||||
let value = n.as_u64().ok_or_else(|| eyre!("Invalid Number {:?}", n))?;
|
|
||||||
let converted: i64 = value.try_into()?;
|
|
||||||
types::Value::Integer(converted)
|
|
||||||
}
|
|
||||||
Value::Number(n) if n.is_f64() => {
|
|
||||||
types::Value::Real(n.as_f64().ok_or_else(|| eyre!("Invalid Number {:?}", n))?)
|
|
||||||
}
|
|
||||||
Value::Bool(n) => types::Value::Integer(*n as i64),
|
|
||||||
Value::String(n) => types::Value::Text(n.clone()),
|
|
||||||
_ => bail!("Invalid type: {}", &input),
|
|
||||||
};
|
|
||||||
Ok(ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait RowConversion<'a>: Sized {
|
|
||||||
fn grouped_from_row<'stmt>(field: &'a Field, row: &Row<'stmt>) -> Result<Self>;
|
|
||||||
fn from_row<'stmt>(fields: &'a [Field], row: &Row<'stmt>) -> Result<Self>;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a> RowConversion<'a> for QueryResult {
|
|
||||||
fn grouped_from_row<'stmt>(field: &'a Field, row: &Row<'stmt>) -> Result<Self> {
|
|
||||||
let amount: usize = row.get(AMOUNT_FIELD_NAME)?;
|
|
||||||
let values = values_from_fields(&[*field], row)?;
|
|
||||||
|
|
||||||
Ok(QueryResult::Grouped {
|
|
||||||
count: amount,
|
|
||||||
value: values[field].clone(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
fn from_row<'stmt>(fields: &'a [Field], row: &Row<'stmt>) -> Result<Self> {
|
|
||||||
let values = values_from_fields(fields, row)?;
|
|
||||||
Ok(QueryResult::Normal(values))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn values_from_fields<'stmt>(
|
|
||||||
fields: &[Field],
|
|
||||||
row: &Row<'stmt>,
|
|
||||||
) -> Result<HashMap<Field, ValueField>> {
|
|
||||||
let mut values: HashMap<Field, ValueField> = HashMap::default();
|
|
||||||
for field in fields {
|
|
||||||
values.insert(*field, value_from_field(field, row)?);
|
|
||||||
}
|
|
||||||
Ok(values)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn value_from_field<'stmt>(field: &Field, row: &Row<'stmt>) -> Result<ValueField> {
|
|
||||||
use Field::*;
|
|
||||||
// Use type safety when unpacking
|
|
||||||
match field {
|
|
||||||
Path | SenderDomain | SenderLocalPart | SenderName | ToGroup | ToName | ToAddress
|
|
||||||
| Subject => {
|
|
||||||
let string: String = row.get::<&str, String>(field.as_str())?;
|
|
||||||
Ok(ValueField::string(field, &string))
|
|
||||||
}
|
|
||||||
Year | Month | Day | Timestamp => {
|
|
||||||
return Ok(ValueField::usize(
|
|
||||||
field,
|
|
||||||
row.get::<&str, usize>(field.as_str())?,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
MetaTags => {
|
|
||||||
let tag_string = row.get::<&str, String>(field.as_str())?;
|
|
||||||
let tags =
|
|
||||||
crate::importer::formats::shared::email::EmailMeta::tags_from_string(&tag_string);
|
|
||||||
Ok(ValueField::array(
|
|
||||||
field,
|
|
||||||
tags.into_iter().map(Value::String).collect(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
IsReply | IsSend | MetaIsSeen => {
|
|
||||||
return Ok(ValueField::bool(
|
|
||||||
field,
|
|
||||||
row.get::<&str, bool>(field.as_str())?,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl EmailEntry {
|
|
||||||
#[allow(unused)]
|
|
||||||
fn from_row(row: &Row<'_>) -> Result<Self> {
|
|
||||||
let path: String = row.get("path")?;
|
|
||||||
let path = std::path::PathBuf::from_str(&path)?;
|
|
||||||
let sender_domain: String = row.get("sender_domain")?;
|
|
||||||
let sender_local_part: String = row.get("sender_local_part")?;
|
|
||||||
let sender_name: String = row.get("sender_name")?;
|
|
||||||
let timestamp: i64 = row.get("timestamp")?;
|
|
||||||
let datetime = Utc.timestamp(timestamp, 0);
|
|
||||||
let subject: String = row.get("subject")?;
|
|
||||||
let to_count: usize = row.get("to_count")?;
|
|
||||||
let to_group: Option<String> = row.get("to_group")?;
|
|
||||||
let to_name: Option<String> = row.get("to_name")?;
|
|
||||||
let to_address: Option<String> = row.get("to_address")?;
|
|
||||||
|
|
||||||
let to_first = to_address.map(|a| (to_name.unwrap_or_default(), a));
|
|
||||||
|
|
||||||
let is_reply: bool = row.get("is_reply")?;
|
|
||||||
let is_send: bool = row.get("is_send")?;
|
|
||||||
|
|
||||||
// Parse EmailMeta
|
|
||||||
let meta_tags: Option<String> = row.get("meta_tags")?;
|
|
||||||
let meta_is_seen: Option<bool> = row.get("meta_is_seen")?;
|
|
||||||
let meta = match (meta_tags, meta_is_seen) {
|
|
||||||
(Some(a), Some(b)) => Some(EmailMeta::from(b, &a)),
|
|
||||||
_ => None,
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(EmailEntry {
|
|
||||||
path,
|
|
||||||
sender_domain,
|
|
||||||
sender_local_part,
|
|
||||||
sender_name,
|
|
||||||
datetime,
|
|
||||||
subject,
|
|
||||||
to_count,
|
|
||||||
to_group,
|
|
||||||
to_first,
|
|
||||||
is_reply,
|
|
||||||
is_send,
|
|
||||||
meta,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,255 +0,0 @@
|
|||||||
use chrono::Datelike;
|
|
||||||
use crossbeam_channel::{unbounded, Sender};
|
|
||||||
use eyre::{bail, Report, Result};
|
|
||||||
use rusqlite::{self, params, Connection, Statement};
|
|
||||||
|
|
||||||
use core::panic;
|
|
||||||
use std::{collections::HashMap, path::Path, thread::JoinHandle};
|
|
||||||
|
|
||||||
use super::{query::Query, query_result::QueryResult, sql::*, DBMessage};
|
|
||||||
use crate::database::query::OtherQuery;
|
|
||||||
use crate::types::Config;
|
|
||||||
use crate::{
|
|
||||||
database::{value_from_field, RowConversion},
|
|
||||||
importer::EmailEntry,
|
|
||||||
};
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct Database {
|
|
||||||
connection: Option<Connection>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Database {
|
|
||||||
/// Open a database and try to retrieve a config from the information stored in there
|
|
||||||
pub fn config<P: AsRef<Path>>(path: P) -> Result<Config> {
|
|
||||||
let database = Self::new(path.as_ref())?;
|
|
||||||
let fields = database.select_config_fields()?;
|
|
||||||
Config::from_fields(path.as_ref(), fields)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Open database at path `Path`.
|
|
||||||
pub fn new<P: AsRef<Path>>(path: P) -> Result<Self> {
|
|
||||||
#[allow(unused_mut)]
|
|
||||||
let mut connection = Connection::open(path.as_ref())?;
|
|
||||||
|
|
||||||
// Improve the insertion performance.
|
|
||||||
connection.pragma_update(None, "journal_mode", &"memory")?;
|
|
||||||
connection.pragma_update(None, "synchronous", &"OFF")?;
|
|
||||||
|
|
||||||
Self::create_tables(&connection)?;
|
|
||||||
|
|
||||||
#[cfg(feature = "trace-sql")]
|
|
||||||
connection.trace(Some(|query| {
|
|
||||||
tracing::trace!("SQL: {}", &query);
|
|
||||||
}));
|
|
||||||
|
|
||||||
Ok(Database {
|
|
||||||
connection: Some(connection),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn total_mails(&self) -> Result<usize> {
|
|
||||||
let connection = match &self.connection {
|
|
||||||
Some(n) => n,
|
|
||||||
None => bail!("No connection to database available in query"),
|
|
||||||
};
|
|
||||||
let mut stmt = connection.prepare(QUERY_COUNT_MAILS)?;
|
|
||||||
let count: usize = stmt.query_row([], |q| q.get(0))?;
|
|
||||||
Ok(count)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn save_config(&self, config: Config) -> Result<()> {
|
|
||||||
let fields = config
|
|
||||||
.into_fields()
|
|
||||||
.ok_or_else(|| eyre::eyre!("Could not create fields from config"))?;
|
|
||||||
self.insert_config_fields(fields)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn query(&self, query: &super::query::Query) -> Result<Vec<QueryResult>> {
|
|
||||||
use rusqlite::params_from_iter;
|
|
||||||
let c = match &self.connection {
|
|
||||||
Some(n) => n,
|
|
||||||
None => bail!("No connection to database available in query"),
|
|
||||||
};
|
|
||||||
let (sql, values) = query.to_sql();
|
|
||||||
let mut stmt = c.prepare(&sql)?;
|
|
||||||
let mut query_results = Vec::new();
|
|
||||||
let mut converted = Vec::new();
|
|
||||||
for value in values {
|
|
||||||
converted.push(super::conversion::json_to_value(&value)?);
|
|
||||||
}
|
|
||||||
|
|
||||||
let p = params_from_iter(converted.iter());
|
|
||||||
|
|
||||||
let mut rows = stmt.query(p)?;
|
|
||||||
while let Some(row) = rows.next()? {
|
|
||||||
match query {
|
|
||||||
Query::Grouped { group_by, .. } => {
|
|
||||||
let result = QueryResult::grouped_from_row(group_by, row)?;
|
|
||||||
query_results.push(result);
|
|
||||||
}
|
|
||||||
Query::Normal { fields, .. } => {
|
|
||||||
let result = QueryResult::from_row(fields, row)?;
|
|
||||||
query_results.push(result);
|
|
||||||
}
|
|
||||||
Query::Other {
|
|
||||||
query: OtherQuery::All(field),
|
|
||||||
} => query_results.push(QueryResult::Other(value_from_field(field, row)?)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(query_results)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Begin the data import.
|
|
||||||
/// This will consume the `Database`. A new one has to be opened
|
|
||||||
/// afterwards in order to support multi-threading.
|
|
||||||
/// Returns an input `Sender` and a `JoinHandle`.
|
|
||||||
/// The `Sender` is used to submit work to the database via `DBMessage`
|
|
||||||
/// cases. The `JoinHandle` is used to wait for database completion.
|
|
||||||
///
|
|
||||||
/// # Examples
|
|
||||||
///
|
|
||||||
/// ``` ignore
|
|
||||||
/// let db = Database::new("db.sqlite").unwrap();
|
|
||||||
/// let (sender, handle) = db.import();
|
|
||||||
/// sender.send(DBMessage::Mail(m1)).unwrap();
|
|
||||||
/// sender.send(DBMessage::Mail(m2)).unwrap();
|
|
||||||
/// handle.join().unwrap();
|
|
||||||
/// ```
|
|
||||||
pub fn import(mut self) -> (Sender<DBMessage>, JoinHandle<Result<usize>>) {
|
|
||||||
let (sender, receiver) = unbounded();
|
|
||||||
|
|
||||||
// Import can only be called *once* on a database created with `new`.
|
|
||||||
// Therefore there should always be a value to unwrap;
|
|
||||||
let mut connection = self.connection.take().unwrap();
|
|
||||||
let handle = std::thread::spawn(move || {
|
|
||||||
let mut counter = 0;
|
|
||||||
{
|
|
||||||
let transaction = connection.transaction()?;
|
|
||||||
{
|
|
||||||
let mut mail_prepared = transaction.prepare(QUERY_EMAILS)?;
|
|
||||||
let mut error_prepared = transaction.prepare(QUERY_ERRORS)?;
|
|
||||||
loop {
|
|
||||||
let next = match receiver.recv() {
|
|
||||||
Ok(n) => n,
|
|
||||||
Err(e) => {
|
|
||||||
println!("Receiver error: {:?}", &e);
|
|
||||||
panic!("should not happen");
|
|
||||||
}
|
|
||||||
};
|
|
||||||
match next {
|
|
||||||
DBMessage::Mail(mail) => {
|
|
||||||
counter += 1;
|
|
||||||
insert_mail(&mut mail_prepared, &mail)
|
|
||||||
}
|
|
||||||
DBMessage::Error(report) => insert_error(&mut error_prepared, &report),
|
|
||||||
DBMessage::Done => {
|
|
||||||
tracing::trace!("Received DBMessage::Done");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Err(e) = transaction.commit() {
|
|
||||||
return Err(eyre::eyre!("Transaction Error: {:?}", &e));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// In case closing the database fails, we try again until we succeed
|
|
||||||
let mut c = connection;
|
|
||||||
loop {
|
|
||||||
tracing::trace!("Attempting close");
|
|
||||||
match c.close() {
|
|
||||||
Ok(_n) => break,
|
|
||||||
Err((a, _b)) => c = a,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tracing::trace!("Finished SQLITE: {}", &counter);
|
|
||||||
Ok(counter)
|
|
||||||
});
|
|
||||||
(sender, handle)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn create_tables(connection: &Connection) -> Result<()> {
|
|
||||||
connection.execute(TBL_EMAILS, params![])?;
|
|
||||||
connection.execute(TBL_ERRORS, params![])?;
|
|
||||||
connection.execute(TBL_META, params![])?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn select_config_fields(&self) -> Result<HashMap<String, serde_json::Value>> {
|
|
||||||
let connection = match &self.connection {
|
|
||||||
Some(n) => n,
|
|
||||||
None => bail!("No connection to database available in query"),
|
|
||||||
};
|
|
||||||
let mut stmt = connection.prepare(QUERY_SELECT_META)?;
|
|
||||||
let mut query_results = HashMap::new();
|
|
||||||
let mut rows = stmt.query([])?;
|
|
||||||
while let Some(row) = rows.next()? {
|
|
||||||
let (k, v) = match (
|
|
||||||
row.get::<_, String>("key"),
|
|
||||||
row.get::<_, serde_json::Value>("value"),
|
|
||||||
) {
|
|
||||||
(Ok(k), Ok(v)) => (k, v),
|
|
||||||
(a, b) => {
|
|
||||||
tracing::error!("Invalid row data. Missing fields key and or value:\nkey: {:?}\nvalue: {:?}\n", a, b);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
query_results.insert(k, v);
|
|
||||||
}
|
|
||||||
Ok(query_results)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn insert_config_fields(&self, fields: HashMap<String, serde_json::Value>) -> Result<()> {
|
|
||||||
let connection = match &self.connection {
|
|
||||||
Some(n) => n,
|
|
||||||
None => bail!("No connection to database available in query"),
|
|
||||||
};
|
|
||||||
let mut stmt = connection.prepare(QUERY_INSERT_META)?;
|
|
||||||
for (key, value) in fields {
|
|
||||||
stmt.execute(params![key, value])?;
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn insert_mail(statement: &mut Statement, entry: &EmailEntry) -> Result<()> {
|
|
||||||
let path = entry.path.display().to_string();
|
|
||||||
let year = entry.datetime.date().year();
|
|
||||||
let month = entry.datetime.date().month();
|
|
||||||
let day = entry.datetime.date().day();
|
|
||||||
let timestamp = entry.datetime.timestamp();
|
|
||||||
let e = entry;
|
|
||||||
let to_name = e.to_first.as_ref().map(|e| &e.0);
|
|
||||||
let to_address = e.to_first.as_ref().map(|e| &e.1);
|
|
||||||
let meta_tags = e.meta.as_ref().map(|e| e.tags_string());
|
|
||||||
let meta_is_seen = e.meta.as_ref().map(|e| e.is_seen);
|
|
||||||
let p = params![
|
|
||||||
path,
|
|
||||||
e.sender_domain,
|
|
||||||
e.sender_local_part,
|
|
||||||
e.sender_name,
|
|
||||||
year,
|
|
||||||
month,
|
|
||||||
day,
|
|
||||||
timestamp,
|
|
||||||
e.subject,
|
|
||||||
e.to_count,
|
|
||||||
e.to_group,
|
|
||||||
to_name,
|
|
||||||
to_address,
|
|
||||||
e.is_reply,
|
|
||||||
e.is_send,
|
|
||||||
meta_tags,
|
|
||||||
meta_is_seen
|
|
||||||
];
|
|
||||||
statement.execute(p)?;
|
|
||||||
tracing::trace!("Insert Mail {}", &path);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn insert_error(statement: &mut Statement, message: &Report) -> Result<()> {
|
|
||||||
statement.execute(params![message.to_string()])?;
|
|
||||||
tracing::trace!("Insert Error {}", message);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
@ -1,14 +0,0 @@
|
|||||||
use eyre::Report;
|
|
||||||
|
|
||||||
use crate::importer::EmailEntry;
|
|
||||||
|
|
||||||
/// Parameter for sending work to the database during `import`.
|
|
||||||
pub enum DBMessage {
|
|
||||||
/// Send for a successfuly parsed mail
|
|
||||||
Mail(Box<EmailEntry>),
|
|
||||||
/// Send for any kind of error during reading / parsing
|
|
||||||
Error(Report),
|
|
||||||
/// Send once all parsing is done.
|
|
||||||
/// This is used to break out of the receiving loop
|
|
||||||
Done,
|
|
||||||
}
|
|
@ -1,10 +0,0 @@
|
|||||||
mod conversion;
|
|
||||||
mod db;
|
|
||||||
mod db_message;
|
|
||||||
pub mod query;
|
|
||||||
pub mod query_result;
|
|
||||||
mod sql;
|
|
||||||
|
|
||||||
pub use conversion::{value_from_field, RowConversion};
|
|
||||||
pub use db::Database;
|
|
||||||
pub use db_message::DBMessage;
|
|
@ -1,242 +0,0 @@
|
|||||||
use rsql_builder;
|
|
||||||
use serde_json;
|
|
||||||
pub use serde_json::Value;
|
|
||||||
use strum::{self, IntoEnumIterator};
|
|
||||||
use strum_macros::{EnumIter, IntoStaticStr};
|
|
||||||
|
|
||||||
use std::ops::Range;
|
|
||||||
|
|
||||||
pub const AMOUNT_FIELD_NAME: &str = "amount";
|
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
pub enum Filter {
|
|
||||||
/// A database Like Operation
|
|
||||||
Like(ValueField),
|
|
||||||
NotLike(ValueField),
|
|
||||||
/// A extended like that implies:
|
|
||||||
/// - wildcards on both sides (like '%test%')
|
|
||||||
/// - case in-sensitive comparison
|
|
||||||
/// - Trying to handle values as strings
|
|
||||||
Contains(ValueField),
|
|
||||||
Is(ValueField),
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, IntoStaticStr, EnumIter)]
|
|
||||||
#[strum(serialize_all = "snake_case")]
|
|
||||||
pub enum Field {
|
|
||||||
Path,
|
|
||||||
SenderDomain,
|
|
||||||
SenderLocalPart,
|
|
||||||
SenderName,
|
|
||||||
Year,
|
|
||||||
Month,
|
|
||||||
Day,
|
|
||||||
Timestamp,
|
|
||||||
ToGroup,
|
|
||||||
ToName,
|
|
||||||
ToAddress,
|
|
||||||
IsReply,
|
|
||||||
IsSend,
|
|
||||||
Subject,
|
|
||||||
MetaIsSeen,
|
|
||||||
MetaTags,
|
|
||||||
}
|
|
||||||
|
|
||||||
const INVALID_FIELDS: &[Field] = &[
|
|
||||||
Field::Path,
|
|
||||||
Field::Subject,
|
|
||||||
Field::Timestamp,
|
|
||||||
Field::IsReply,
|
|
||||||
Field::IsSend,
|
|
||||||
Field::MetaIsSeen,
|
|
||||||
Field::MetaTags,
|
|
||||||
];
|
|
||||||
|
|
||||||
impl Field {
|
|
||||||
pub fn all_cases() -> impl Iterator<Item = Field> {
|
|
||||||
Field::iter().filter(|f| !INVALID_FIELDS.contains(f))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Just a wrapper to offer `into` without the type ambiguity
|
|
||||||
/// that sometimes arises
|
|
||||||
pub fn as_str(&self) -> &'static str {
|
|
||||||
self.into()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A human readable name
|
|
||||||
pub fn name(&self) -> &str {
|
|
||||||
use Field::*;
|
|
||||||
match self {
|
|
||||||
SenderDomain => "Domain",
|
|
||||||
SenderLocalPart => "Address",
|
|
||||||
SenderName => "Name",
|
|
||||||
ToGroup => "Group",
|
|
||||||
ToName => "To name",
|
|
||||||
ToAddress => "To address",
|
|
||||||
Year => "Year",
|
|
||||||
Month => "Month",
|
|
||||||
Day => "Day",
|
|
||||||
Subject => "Subject",
|
|
||||||
_ => self.as_str(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::fmt::Display for Field {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
||||||
write!(f, "{}", self.name())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, PartialEq, Eq, Clone)]
|
|
||||||
pub struct ValueField {
|
|
||||||
field: Field,
|
|
||||||
value: Value,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ValueField {
|
|
||||||
pub fn string<S: AsRef<str>>(field: &Field, value: S) -> ValueField {
|
|
||||||
ValueField {
|
|
||||||
field: *field,
|
|
||||||
value: Value::String(value.as_ref().to_string()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn bool(field: &Field, value: bool) -> ValueField {
|
|
||||||
ValueField {
|
|
||||||
field: *field,
|
|
||||||
value: Value::Bool(value),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn usize(field: &Field, value: usize) -> ValueField {
|
|
||||||
ValueField {
|
|
||||||
field: *field,
|
|
||||||
value: Value::Number(value.into()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn array(field: &Field, value: Vec<Value>) -> ValueField {
|
|
||||||
ValueField {
|
|
||||||
field: *field,
|
|
||||||
value: Value::Array(value),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn field(&self) -> &Field {
|
|
||||||
&self.field
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn value(&self) -> &Value {
|
|
||||||
&self.value
|
|
||||||
}
|
|
||||||
|
|
||||||
#[allow(clippy::inherent_to_string)]
|
|
||||||
pub fn to_string(&self) -> String {
|
|
||||||
match &self.value {
|
|
||||||
Value::String(s) => s.clone(),
|
|
||||||
_ => format!("{}", &self.value),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub enum OtherQuery {
|
|
||||||
/// Get all contents of a specific field
|
|
||||||
All(Field),
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
pub enum Query {
|
|
||||||
Grouped {
|
|
||||||
filters: Vec<Filter>,
|
|
||||||
group_by: Field,
|
|
||||||
},
|
|
||||||
Normal {
|
|
||||||
fields: Vec<Field>,
|
|
||||||
filters: Vec<Filter>,
|
|
||||||
range: Range<usize>,
|
|
||||||
},
|
|
||||||
Other {
|
|
||||||
query: OtherQuery,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Query {
|
|
||||||
fn filters(&self) -> &[Filter] {
|
|
||||||
match self {
|
|
||||||
Query::Grouped { ref filters, .. } => filters,
|
|
||||||
Query::Normal { ref filters, .. } => filters,
|
|
||||||
Query::Other { .. } => &[],
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Query {
|
|
||||||
pub fn to_sql(&self) -> (String, Vec<serde_json::Value>) {
|
|
||||||
let mut conditions = {
|
|
||||||
let mut whr = rsql_builder::B::new_where();
|
|
||||||
for filter in self.filters() {
|
|
||||||
match filter {
|
|
||||||
Filter::Like(f) => whr.like(f.field.into(), f.value()),
|
|
||||||
Filter::NotLike(f) => whr.not_like(f.field.into(), f.value()),
|
|
||||||
Filter::Contains(f) => whr.like(
|
|
||||||
f.field.into(),
|
|
||||||
&format!("%{}%", f.to_string().to_lowercase()),
|
|
||||||
),
|
|
||||||
Filter::Is(f) => whr.eq(f.field.into(), f.value()),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
whr
|
|
||||||
};
|
|
||||||
|
|
||||||
let (header, group_by) = match self {
|
|
||||||
Query::Grouped { group_by, .. } => (
|
|
||||||
format!(
|
|
||||||
"SELECT count(path) as {}, {} FROM emails",
|
|
||||||
AMOUNT_FIELD_NAME,
|
|
||||||
group_by.as_str()
|
|
||||||
),
|
|
||||||
format!("GROUP BY {}", group_by.as_str()),
|
|
||||||
),
|
|
||||||
Query::Normal { fields, range, .. } => {
|
|
||||||
let fields: Vec<&str> = fields.iter().map(|e| e.into()).collect();
|
|
||||||
(
|
|
||||||
format!("SELECT {} FROM emails", fields.join(", ")),
|
|
||||||
format!("LIMIT {}, {}", range.start, range.end - range.start),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
Query::Other {
|
|
||||||
query: OtherQuery::All(field),
|
|
||||||
} => (
|
|
||||||
format!("SELECT {} FROM emails", field.as_str()),
|
|
||||||
format!(""),
|
|
||||||
),
|
|
||||||
};
|
|
||||||
|
|
||||||
let (sql, values) = rsql_builder::B::prepare(
|
|
||||||
rsql_builder::B::new_sql(&header)
|
|
||||||
.push_build(&mut conditions)
|
|
||||||
.push_sql(&group_by),
|
|
||||||
);
|
|
||||||
|
|
||||||
(sql, values)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_test() {
|
|
||||||
let query = Query::Grouped {
|
|
||||||
filters: vec![
|
|
||||||
Filter::Like(ValueField::string(&Field::SenderDomain, "gmail.com")),
|
|
||||||
Filter::Is(ValueField::usize(&Field::Year, 2021)),
|
|
||||||
],
|
|
||||||
group_by: Field::Month,
|
|
||||||
};
|
|
||||||
dbg!(&query.to_sql());
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,17 +0,0 @@
|
|||||||
use super::query::{Field, ValueField};
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
pub type QueryRow = HashMap<Field, ValueField>;
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub enum QueryResult {
|
|
||||||
Grouped {
|
|
||||||
/// How many items did we find?
|
|
||||||
count: usize,
|
|
||||||
/// All the itmes that we grouped by including their values.
|
|
||||||
/// So that we can use each of them to limit the next query.
|
|
||||||
value: ValueField,
|
|
||||||
},
|
|
||||||
Normal(QueryRow),
|
|
||||||
Other(ValueField),
|
|
||||||
}
|
|
@ -1,71 +0,0 @@
|
|||||||
pub const TBL_EMAILS: &str = r#"
|
|
||||||
CREATE TABLE IF NOT EXISTS emails (
|
|
||||||
path TEXT NOT NULL,
|
|
||||||
sender_domain TEXT NOT NULL,
|
|
||||||
sender_local_part TEXT NOT NULL,
|
|
||||||
sender_name TEXT NOT NULL,
|
|
||||||
year INTEGER NOT NULL,
|
|
||||||
month INTEGER NOT NULL,
|
|
||||||
day INTEGER NOT NULL,
|
|
||||||
timestamp INTEGER NOT NULL,
|
|
||||||
subject TEXT NOT NULL,
|
|
||||||
to_count INTEGER NOT NULL,
|
|
||||||
to_group TEXT NULL,
|
|
||||||
to_name TEXT NULL,
|
|
||||||
to_address TEXT NULL,
|
|
||||||
is_reply BOOL,
|
|
||||||
is_send BOOL,
|
|
||||||
meta_tags TEXT NULL,
|
|
||||||
meta_is_seen BOOL NULL
|
|
||||||
);"#;
|
|
||||||
|
|
||||||
pub const QUERY_EMAILS: &str = r#"
|
|
||||||
INSERT INTO emails
|
|
||||||
(
|
|
||||||
path, sender_domain, sender_local_part, sender_name,
|
|
||||||
year, month, day, timestamp, subject,
|
|
||||||
to_count, to_group, to_name, to_address,
|
|
||||||
is_reply, is_send,
|
|
||||||
meta_tags, meta_is_seen
|
|
||||||
)
|
|
||||||
VALUES
|
|
||||||
(
|
|
||||||
?, ?, ?, ?,
|
|
||||||
?, ?, ?, ?, ?,
|
|
||||||
?, ?, ?, ?,
|
|
||||||
?, ?,
|
|
||||||
?, ?
|
|
||||||
)
|
|
||||||
"#;
|
|
||||||
|
|
||||||
pub const TBL_ERRORS: &str = r#"
|
|
||||||
CREATE TABLE IF NOT EXISTS errors (
|
|
||||||
message TEXT NOT NULL
|
|
||||||
);"#;
|
|
||||||
|
|
||||||
pub const QUERY_ERRORS: &str = r#"
|
|
||||||
INSERT INTO errors
|
|
||||||
(message)
|
|
||||||
VALUES
|
|
||||||
(?)
|
|
||||||
"#;
|
|
||||||
|
|
||||||
pub const TBL_META: &str = r#"
|
|
||||||
CREATE TABLE IF NOT EXISTS meta (
|
|
||||||
key TEXT NOT NULL,
|
|
||||||
value TEXT NOT NULL
|
|
||||||
);"#;
|
|
||||||
|
|
||||||
pub const QUERY_INSERT_META: &str = r#"
|
|
||||||
INSERT INTO meta
|
|
||||||
(key, value)
|
|
||||||
VALUES
|
|
||||||
(?, ?)
|
|
||||||
"#;
|
|
||||||
|
|
||||||
pub const QUERY_SELECT_META: &str = r#"
|
|
||||||
SELECT key, value FROM meta"#;
|
|
||||||
|
|
||||||
pub const QUERY_COUNT_MAILS: &str = r#"
|
|
||||||
SELECT count(path) FROM emails
|
|
||||||
"#;
|
|
Binary file not shown.
@ -1,88 +0,0 @@
|
|||||||
//! We use a stubbornly stupid algorithm where we just
|
|
||||||
//! recursively drill down into the appropriate folder
|
|
||||||
//! until we find `emlx` files and return those.
|
|
||||||
|
|
||||||
use eyre::{eyre, Result};
|
|
||||||
use rayon::prelude::*;
|
|
||||||
use walkdir::WalkDir;
|
|
||||||
|
|
||||||
use super::super::shared::filesystem::emails_in;
|
|
||||||
use super::super::{Message, MessageSender};
|
|
||||||
use crate::types::Config;
|
|
||||||
|
|
||||||
use super::mail::Mail;
|
|
||||||
use std::path::PathBuf;
|
|
||||||
|
|
||||||
pub fn read_emails(config: &Config, sender: MessageSender) -> Result<Vec<Mail>> {
|
|
||||||
// on macOS, we might need permission for the `Library` folder...
|
|
||||||
match std::fs::read_dir(&config.emails_folder_path) {
|
|
||||||
Ok(_) => (),
|
|
||||||
Err(e) => match e.kind() {
|
|
||||||
#[cfg(target_os = "macos")]
|
|
||||||
std::io::ErrorKind::PermissionDenied => {
|
|
||||||
tracing::info!("Could not read folder: {}", e);
|
|
||||||
if let Err(e) = sender.send(Message::MissingPermissions) {
|
|
||||||
tracing::error!("Error sending: {}", e);
|
|
||||||
}
|
|
||||||
// We should return early now, otherwise the code below will send a different
|
|
||||||
// error
|
|
||||||
return Ok(Vec::new());
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
if let Err(e) = sender.send(Message::Error(eyre!("Error: {:?}", &e))) {
|
|
||||||
tracing::error!("Error sending: {}", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// As `walkdir` does not support `par_iter` (see https://www.reddit.com/r/rust/comments/6eif7r/walkdir_users_we_need_you/)
|
|
||||||
// - -we first collect all folders,
|
|
||||||
// then all sub-folders in those ending in mboxending in .mbox and then iterate over them in paralell
|
|
||||||
let folders: Vec<PathBuf> = WalkDir::new(&config.emails_folder_path)
|
|
||||||
.into_iter()
|
|
||||||
.filter_map(|e| match e {
|
|
||||||
Ok(n)
|
|
||||||
if n.path().is_dir()
|
|
||||||
&& n.path()
|
|
||||||
.to_str()
|
|
||||||
.map(|e| e.contains(".mbox"))
|
|
||||||
.unwrap_or(false) =>
|
|
||||||
{
|
|
||||||
tracing::trace!("Found folder {}", n.path().display());
|
|
||||||
Some(n.path().to_path_buf())
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::info!("Could not read folder: {}", e);
|
|
||||||
if let Err(e) = sender.send(Message::Error(eyre!("Could not read folder: {:?}", e)))
|
|
||||||
{
|
|
||||||
tracing::error!("Error sending error {}", e);
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
|
||||||
_ => None,
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
sender.send(Message::ReadTotal(folders.len()))?;
|
|
||||||
let mails: Vec<Mail> = folders
|
|
||||||
.into_par_iter()
|
|
||||||
.filter_map(
|
|
||||||
|path| match emails_in(path.clone(), sender.clone(), Mail::new) {
|
|
||||||
Ok(n) => Some(n),
|
|
||||||
Err(e) => {
|
|
||||||
tracing::error!("{} {:?}", path.display(), &e);
|
|
||||||
if let Err(e) = sender.send(Message::Error(eyre!(
|
|
||||||
"Could read mails in {}: {:?}",
|
|
||||||
path.display(),
|
|
||||||
e
|
|
||||||
))) {
|
|
||||||
tracing::error!("Error sending error {}", e);
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.flatten()
|
|
||||||
.collect();
|
|
||||||
Ok(mails)
|
|
||||||
}
|
|
@ -1,69 +0,0 @@
|
|||||||
use emlx::parse_emlx;
|
|
||||||
use eyre::Result;
|
|
||||||
|
|
||||||
use std::borrow::Cow;
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
|
|
||||||
use super::super::shared::email::EmailMeta;
|
|
||||||
use super::super::shared::parse::ParseableEmail;
|
|
||||||
|
|
||||||
pub struct Mail {
|
|
||||||
path: PathBuf,
|
|
||||||
// This is parsed out of the `emlx` as it is parsed
|
|
||||||
is_seen: bool,
|
|
||||||
// This is parsed out of the `path`
|
|
||||||
label: Option<String>,
|
|
||||||
// Maildata
|
|
||||||
data: Vec<u8>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Mail {
|
|
||||||
pub fn new<P: AsRef<Path>>(path: P) -> Option<Self> {
|
|
||||||
let path = path.as_ref();
|
|
||||||
let name = path.file_name()?.to_str()?;
|
|
||||||
if !name.ends_with(".emlx") {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
// find the folder ending with `.mbox` in the path
|
|
||||||
let ext = ".mbox";
|
|
||||||
let label = path
|
|
||||||
.iter()
|
|
||||||
.map(|e| e.to_str())
|
|
||||||
.flatten()
|
|
||||||
.find(|s| s.ends_with(ext))
|
|
||||||
.map(|s| s.replace(ext, ""));
|
|
||||||
Some(Self {
|
|
||||||
path: path.to_path_buf(),
|
|
||||||
is_seen: false,
|
|
||||||
label,
|
|
||||||
data: Vec::new(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ParseableEmail for Mail {
|
|
||||||
fn prepare(&mut self) -> Result<()> {
|
|
||||||
let data = std::fs::read(self.path.as_path())?;
|
|
||||||
let parsed = parse_emlx(&data)?;
|
|
||||||
self.is_seen = !parsed.flags.is_read;
|
|
||||||
self.data = parsed.message.to_vec();
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
fn message(&self) -> Result<Cow<'_, [u8]>> {
|
|
||||||
Ok(Cow::Borrowed(self.data.as_slice()))
|
|
||||||
}
|
|
||||||
fn path(&self) -> &Path {
|
|
||||||
self.path.as_path()
|
|
||||||
}
|
|
||||||
fn meta(&self) -> Result<Option<EmailMeta>> {
|
|
||||||
let tags = match self.label {
|
|
||||||
Some(ref n) => vec![n.clone()],
|
|
||||||
None => vec![],
|
|
||||||
};
|
|
||||||
let meta = EmailMeta {
|
|
||||||
tags,
|
|
||||||
is_seen: self.is_seen,
|
|
||||||
};
|
|
||||||
Ok(Some(meta))
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,23 +0,0 @@
|
|||||||
mod filesystem;
|
|
||||||
mod mail;
|
|
||||||
|
|
||||||
use shellexpand;
|
|
||||||
use std::{path::PathBuf, str::FromStr};
|
|
||||||
|
|
||||||
use super::{Config, ImporterFormat, MessageSender, Result};
|
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
pub struct AppleMail {}
|
|
||||||
|
|
||||||
impl ImporterFormat for AppleMail {
|
|
||||||
type Item = mail::Mail;
|
|
||||||
|
|
||||||
fn default_path() -> Option<PathBuf> {
|
|
||||||
let path = shellexpand::tilde("~/Library/Mail");
|
|
||||||
Some(PathBuf::from_str(&path.to_string()).unwrap())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn emails(&self, config: &Config, sender: MessageSender) -> Result<Vec<Self::Item>> {
|
|
||||||
filesystem::read_emails(config, sender)
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,46 +0,0 @@
|
|||||||
use chrono::prelude::*;
|
|
||||||
|
|
||||||
use eyre::{bail, Result};
|
|
||||||
use serde::Deserialize;
|
|
||||||
use serde_json;
|
|
||||||
|
|
||||||
use super::super::shared::email::EmailMeta;
|
|
||||||
use super::raw_email::RawEmailEntry;
|
|
||||||
|
|
||||||
#[derive(Deserialize, Debug, Clone)]
|
|
||||||
pub struct Meta {
|
|
||||||
pub msg_id: String,
|
|
||||||
pub subject: String,
|
|
||||||
pub labels: Vec<String>,
|
|
||||||
pub flags: Vec<String>,
|
|
||||||
internal_date: i64,
|
|
||||||
|
|
||||||
#[serde(skip, default = "Utc::now")]
|
|
||||||
pub created: DateTime<Utc>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Meta {
|
|
||||||
pub fn is_seen(&self) -> bool {
|
|
||||||
self.labels.contains(&"\\seen".to_owned())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<Meta> for EmailMeta {
|
|
||||||
fn from(meta: Meta) -> Self {
|
|
||||||
let is_seen = meta.is_seen();
|
|
||||||
EmailMeta {
|
|
||||||
tags: meta.labels,
|
|
||||||
is_seen,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn parse_meta(raw_entry: &RawEmailEntry) -> Result<Meta> {
|
|
||||||
let content = match raw_entry.read_gmail_meta() {
|
|
||||||
None => bail!("No Gmail Meta Information Available"),
|
|
||||||
Some(content) => content?,
|
|
||||||
};
|
|
||||||
let mut meta: Meta = serde_json::from_slice(&content)?;
|
|
||||||
meta.created = Utc.timestamp(meta.internal_date, 0);
|
|
||||||
Ok(meta)
|
|
||||||
}
|
|
@ -1,23 +0,0 @@
|
|||||||
mod meta;
|
|
||||||
mod raw_email;
|
|
||||||
|
|
||||||
use super::shared::filesystem::{emails_in, folders_in};
|
|
||||||
use super::{Config, ImporterFormat, MessageSender, Result};
|
|
||||||
use raw_email::RawEmailEntry;
|
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
pub struct Gmail {}
|
|
||||||
|
|
||||||
impl ImporterFormat for Gmail {
|
|
||||||
type Item = raw_email::RawEmailEntry;
|
|
||||||
|
|
||||||
fn default_path() -> Option<std::path::PathBuf> {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
fn emails(&self, config: &Config, sender: MessageSender) -> Result<Vec<Self::Item>> {
|
|
||||||
folders_in(&config.emails_folder_path, sender, |path, sender| {
|
|
||||||
emails_in(path, sender, RawEmailEntry::new)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,118 +0,0 @@
|
|||||||
use eyre::{eyre, Result};
|
|
||||||
use flate2::read::GzDecoder;
|
|
||||||
|
|
||||||
use std::borrow::Cow;
|
|
||||||
use std::io::Read;
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
|
|
||||||
use super::super::shared::email::EmailMeta;
|
|
||||||
use super::super::shared::parse::ParseableEmail;
|
|
||||||
|
|
||||||
/// Raw representation of an email.
|
|
||||||
/// Contains the paths to the relevant files as well
|
|
||||||
/// as the name of the folder the email was in.
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct RawEmailEntry {
|
|
||||||
#[allow(unused)]
|
|
||||||
folder_name: String,
|
|
||||||
eml_path: PathBuf,
|
|
||||||
gmail_meta_path: Option<PathBuf>,
|
|
||||||
is_compressed: bool,
|
|
||||||
#[allow(unused)]
|
|
||||||
size: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl RawEmailEntry {
|
|
||||||
pub fn path(&self) -> &Path {
|
|
||||||
self.eml_path.as_path()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn read(&self) -> Result<Vec<u8>> {
|
|
||||||
if self.is_compressed {
|
|
||||||
let reader = std::fs::File::open(&self.eml_path)?;
|
|
||||||
let mut decoder = GzDecoder::new(reader);
|
|
||||||
let mut buffer = Vec::new();
|
|
||||||
decoder.read_to_end(&mut buffer)?;
|
|
||||||
Ok(buffer)
|
|
||||||
} else {
|
|
||||||
std::fs::read(&self.eml_path).map_err(|e| eyre!("IO Error: {}", &e))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn has_gmail_meta(&self) -> bool {
|
|
||||||
self.gmail_meta_path.is_some()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn read_gmail_meta(&self) -> Option<Result<Vec<u8>>> {
|
|
||||||
// Just using map here returns a `&Option` whereas we want `Option`
|
|
||||||
#[allow(clippy::manual_map)]
|
|
||||||
match &self.gmail_meta_path {
|
|
||||||
Some(p) => Some(std::fs::read(p).map_err(|e| eyre!("IO Error: {}", &e))),
|
|
||||||
None => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl RawEmailEntry {
|
|
||||||
pub(super) fn new<P: AsRef<std::path::Path>>(path: P) -> Option<RawEmailEntry> {
|
|
||||||
let path = path.as_ref();
|
|
||||||
let stem = path.file_stem()?.to_str()?;
|
|
||||||
let name = path.file_name()?.to_str()?;
|
|
||||||
let is_eml_gz = name.ends_with(".eml.gz");
|
|
||||||
let is_eml = name.ends_with(".eml");
|
|
||||||
if !is_eml_gz && !is_eml {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let is_compressed = is_eml_gz;
|
|
||||||
let folder_name = path.parent()?.file_name()?.to_str()?.to_owned();
|
|
||||||
let eml_path = path.to_path_buf();
|
|
||||||
|
|
||||||
let file_metadata = path.metadata().ok()?;
|
|
||||||
|
|
||||||
// Build a meta path
|
|
||||||
let meta_path = path
|
|
||||||
.parent()?
|
|
||||||
.join(format!("{}.meta", stem.replace(".eml", "")));
|
|
||||||
|
|
||||||
// Only embed it, if it exists
|
|
||||||
let gmail_meta_path = if meta_path.exists() {
|
|
||||||
Some(meta_path)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
tracing::trace!(
|
|
||||||
"Email [c?: {}] {} {:?}",
|
|
||||||
is_compressed,
|
|
||||||
eml_path.display(),
|
|
||||||
gmail_meta_path
|
|
||||||
);
|
|
||||||
Some(RawEmailEntry {
|
|
||||||
folder_name,
|
|
||||||
eml_path,
|
|
||||||
gmail_meta_path,
|
|
||||||
is_compressed,
|
|
||||||
size: file_metadata.len(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ParseableEmail for RawEmailEntry {
|
|
||||||
fn prepare(&mut self) -> Result<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
fn message(&self) -> Result<Cow<'_, [u8]>> {
|
|
||||||
Ok(Cow::Owned(self.read()?))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn path(&self) -> &Path {
|
|
||||||
self.eml_path.as_path()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn meta(&self) -> Result<Option<EmailMeta>> {
|
|
||||||
if self.has_gmail_meta() {
|
|
||||||
Ok(Some(super::meta::parse_meta(self)?.into()))
|
|
||||||
} else {
|
|
||||||
Ok(None)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,133 +0,0 @@
|
|||||||
//! FIXME: Implement our own Mailbox reader that better implements the spec.
|
|
||||||
//! use jetsci for efficient searching:
|
|
||||||
//! https://github.com/shepmaster/jetscii
|
|
||||||
//! (or aho corasick)
|
|
||||||
//! MBox parsing is also not particularly fast as it currently doesn't use parallelism
|
|
||||||
|
|
||||||
use eyre::eyre;
|
|
||||||
use mbox_reader;
|
|
||||||
use rayon::prelude::*;
|
|
||||||
use tracing;
|
|
||||||
use walkdir::WalkDir;
|
|
||||||
|
|
||||||
use super::{Config, ImporterFormat, Message, MessageSender, Result};
|
|
||||||
|
|
||||||
use super::shared::email::EmailMeta;
|
|
||||||
use super::shared::parse::ParseableEmail;
|
|
||||||
|
|
||||||
use std::borrow::Cow;
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
|
|
||||||
pub struct Mail {
|
|
||||||
path: PathBuf,
|
|
||||||
/// For now, we go with a very simple implementation:
|
|
||||||
/// Each mal will have a heap-allocated vec of the corresponding
|
|
||||||
/// bytes in the mbox.
|
|
||||||
/// This wastes a lot of allocations and shows the limits of our current abstraction.
|
|
||||||
/// It would be better to just save the headers and ignore the rest.
|
|
||||||
content: Vec<u8>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
pub struct Mbox;
|
|
||||||
|
|
||||||
/// The inner parsing code
|
|
||||||
fn inner_emails(config: &Config, sender: MessageSender) -> Result<Vec<Mail>> {
|
|
||||||
// find all files ending in .mbox
|
|
||||||
let mboxes: Vec<PathBuf> = WalkDir::new(&config.emails_folder_path)
|
|
||||||
.into_iter()
|
|
||||||
.filter_map(|e| match e {
|
|
||||||
Ok(n)
|
|
||||||
if n.path().is_file()
|
|
||||||
&& n.path()
|
|
||||||
.to_str()
|
|
||||||
.map(|e| e.contains(".mbox"))
|
|
||||||
.unwrap_or(false) =>
|
|
||||||
{
|
|
||||||
tracing::trace!("Found mbox file {}", n.path().display());
|
|
||||||
Some(n.path().to_path_buf())
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::info!("Could not read folder: {}", e);
|
|
||||||
if let Err(e) = sender.send(Message::Error(eyre!("Could not read folder: {:?}", e)))
|
|
||||||
{
|
|
||||||
tracing::error!("Error sending error {}", e);
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
|
||||||
_ => None,
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
let mails: Vec<Mail> = mboxes
|
|
||||||
.into_par_iter()
|
|
||||||
.filter_map(|mbox_file| {
|
|
||||||
let mbox = match mbox_reader::MboxFile::from_file(&mbox_file) {
|
|
||||||
Ok(n) => n,
|
|
||||||
Err(e) => {
|
|
||||||
tracing::error!(
|
|
||||||
"Could not open mbox file at {}: {}",
|
|
||||||
&mbox_file.display(),
|
|
||||||
e
|
|
||||||
);
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let inner_mails: Vec<Mail> = mbox
|
|
||||||
.iter()
|
|
||||||
.filter_map(|e| {
|
|
||||||
let content = match e.message() {
|
|
||||||
Some(n) => n,
|
|
||||||
None => {
|
|
||||||
tracing::error!("Could not parse mail at offset {}", e.offset());
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
Some(Mail {
|
|
||||||
path: mbox_file.clone(),
|
|
||||||
content: content.to_owned(),
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
Some(inner_mails)
|
|
||||||
})
|
|
||||||
.flatten()
|
|
||||||
.collect();
|
|
||||||
Ok(mails)
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ImporterFormat for Mbox {
|
|
||||||
type Item = Mail;
|
|
||||||
|
|
||||||
fn default_path() -> Option<std::path::PathBuf> {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
fn emails(&self, config: &Config, sender: MessageSender) -> Result<Vec<Self::Item>> {
|
|
||||||
inner_emails(config, sender)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ParseableEmail for Mail {
|
|
||||||
fn prepare(&mut self) -> Result<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
fn message(&self) -> Result<Cow<'_, [u8]>> {
|
|
||||||
Ok(self.content.as_slice().into())
|
|
||||||
}
|
|
||||||
fn path(&self) -> &Path {
|
|
||||||
self.path.as_path()
|
|
||||||
}
|
|
||||||
fn meta(&self) -> Result<Option<EmailMeta>> {
|
|
||||||
// The filename is a tag, e.g. `INBOX.mbox`, `WORK.mbox`
|
|
||||||
if let Some(prefix) = self.path.file_stem() {
|
|
||||||
if let Some(s) = prefix.to_str().map(|s| s.to_owned()) {
|
|
||||||
return Ok(Some(EmailMeta {
|
|
||||||
tags: vec![s],
|
|
||||||
is_seen: false,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(None)
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,31 +0,0 @@
|
|||||||
use std::path::PathBuf;
|
|
||||||
|
|
||||||
pub use eyre::Result;
|
|
||||||
|
|
||||||
mod apple_mail;
|
|
||||||
mod gmailbackup;
|
|
||||||
mod mbox;
|
|
||||||
pub mod shared;
|
|
||||||
|
|
||||||
pub use apple_mail::AppleMail;
|
|
||||||
pub use gmailbackup::Gmail;
|
|
||||||
pub use mbox::Mbox;
|
|
||||||
|
|
||||||
pub use crate::types::Config;
|
|
||||||
use shared::parse::ParseableEmail;
|
|
||||||
|
|
||||||
pub use super::{Message, MessageReceiver, MessageSender};
|
|
||||||
|
|
||||||
/// This is implemented by the various formats
|
|
||||||
/// to define how they return email data.
|
|
||||||
pub trait ImporterFormat: Send + Sync {
|
|
||||||
type Item: ParseableEmail;
|
|
||||||
|
|
||||||
/// The default location path where the data for this format resides
|
|
||||||
/// on system. If there is none (such as for mbox) return `None`
|
|
||||||
fn default_path() -> Option<PathBuf>;
|
|
||||||
|
|
||||||
/// Return all the emails in this format.
|
|
||||||
/// Use the sneder to give progress updates via the `ReadProgress` case.
|
|
||||||
fn emails(&self, config: &Config, sender: MessageSender) -> Result<Vec<Self::Item>>;
|
|
||||||
}
|
|
@ -1,83 +0,0 @@
|
|||||||
use super::parse::{parse_email, ParseableEmail};
|
|
||||||
use crate::database::{DBMessage, Database};
|
|
||||||
use crate::types::Config;
|
|
||||||
|
|
||||||
use super::super::{Message, MessageSender};
|
|
||||||
|
|
||||||
use eyre::{bail, Result};
|
|
||||||
use rayon::prelude::*;
|
|
||||||
|
|
||||||
pub fn into_database<Mail: ParseableEmail + 'static>(
|
|
||||||
config: &Config,
|
|
||||||
mut emails: Vec<Mail>,
|
|
||||||
tx: MessageSender,
|
|
||||||
) -> Result<usize> {
|
|
||||||
let total = emails.len();
|
|
||||||
tracing::info!("Loaded {} emails", &total);
|
|
||||||
|
|
||||||
// First, communicate the total amount of mails received
|
|
||||||
if let Err(e) = tx.send(Message::WriteTotal(total)) {
|
|
||||||
bail!("Channel Failure {:?}", &e);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create a new database connection, just for writing
|
|
||||||
let database = Database::new(config.database_path.clone()).unwrap();
|
|
||||||
|
|
||||||
// Save the config into the database
|
|
||||||
if let Err(e) = database.save_config(config.clone()) {
|
|
||||||
bail!("Could not save config to database {:?}", &e);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Consume the connection to begin the import. It will return the `handle` to use for
|
|
||||||
// waiting for the database to finish importing, and the `sender` to submit work.
|
|
||||||
let (sender, handle) = database.import();
|
|
||||||
|
|
||||||
// Iterate over the mails..
|
|
||||||
emails
|
|
||||||
// in paralell..
|
|
||||||
.par_iter_mut()
|
|
||||||
// parsing them
|
|
||||||
.map(|raw_mail| parse_email(raw_mail, &config.sender_emails))
|
|
||||||
// and inserting them into SQLite
|
|
||||||
.for_each(|entry| {
|
|
||||||
// Try to write the message into the database
|
|
||||||
if let Err(e) = match entry {
|
|
||||||
Ok(mail) => sender.send(DBMessage::Mail(Box::new(mail))),
|
|
||||||
Err(e) => sender.send(DBMessage::Error(e)),
|
|
||||||
} {
|
|
||||||
tracing::error!("Error Inserting into Database: {:?}", &e);
|
|
||||||
}
|
|
||||||
// Signal the write
|
|
||||||
if let Err(e) = tx.send(Message::WriteOne) {
|
|
||||||
tracing::error!("Channel Failure: {:?}", &e);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Tell SQLite there's no more work coming. This will exit the listening loop
|
|
||||||
if let Err(e) = sender.send(DBMessage::Done) {
|
|
||||||
bail!("Channel Failure {:?}", &e);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait for SQLite to finish parsing
|
|
||||||
tracing::info!("Waiting for SQLite to finish");
|
|
||||||
|
|
||||||
if let Err(e) = tx.send(Message::FinishingUp) {
|
|
||||||
bail!("Channel Failure {:?}", &e);
|
|
||||||
}
|
|
||||||
|
|
||||||
tracing::trace!("Waiting for database handle...");
|
|
||||||
let output = match handle.join() {
|
|
||||||
Ok(Ok(count)) => Ok(count),
|
|
||||||
Ok(Err(e)) => Err(e),
|
|
||||||
Err(e) => Err(eyre::eyre!("Join Error: {:?}", &e)),
|
|
||||||
};
|
|
||||||
|
|
||||||
// Tell the caller that we're done processing. This will allow leaving the
|
|
||||||
// display loop
|
|
||||||
tracing::trace!("Messaging Done");
|
|
||||||
if let Err(e) = tx.send(Message::Done) {
|
|
||||||
bail!("Channel Failure {:?}", &e);
|
|
||||||
}
|
|
||||||
|
|
||||||
output
|
|
||||||
}
|
|
@ -1,50 +0,0 @@
|
|||||||
use chrono::prelude::*;
|
|
||||||
use std::path::PathBuf;
|
|
||||||
|
|
||||||
pub type Tag = String;
|
|
||||||
|
|
||||||
/// This is based on additional information in some systems such as
|
|
||||||
/// Gmail labels or Apple Mail tags or Apple XML
|
|
||||||
#[derive(Debug, Default)]
|
|
||||||
pub struct EmailMeta {
|
|
||||||
pub tags: Vec<Tag>,
|
|
||||||
pub is_seen: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
const TAG_SEP: &str = ":|:";
|
|
||||||
|
|
||||||
impl EmailMeta {
|
|
||||||
pub fn tags_from_string(tag_string: &str) -> Vec<String> {
|
|
||||||
tag_string.split(TAG_SEP).map(|e| e.to_string()).collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn from(is_seen: bool, tag_string: &str) -> Self {
|
|
||||||
let tags = EmailMeta::tags_from_string(tag_string);
|
|
||||||
EmailMeta { tags, is_seen }
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn tags_string(&self) -> String {
|
|
||||||
self.tags.join(TAG_SEP)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Representation of an email
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct EmailEntry {
|
|
||||||
pub path: PathBuf,
|
|
||||||
pub sender_domain: String,
|
|
||||||
pub sender_local_part: String,
|
|
||||||
pub sender_name: String,
|
|
||||||
pub datetime: chrono::DateTime<Utc>,
|
|
||||||
pub subject: String,
|
|
||||||
/// The amount of `to:` adresses
|
|
||||||
pub to_count: usize,
|
|
||||||
/// When this email was send to a group, the group name
|
|
||||||
pub to_group: Option<String>,
|
|
||||||
/// The first address and name in `To`, if any
|
|
||||||
pub to_first: Option<(String, String)>,
|
|
||||||
pub is_reply: bool,
|
|
||||||
/// Was this email send from the account we're importing?
|
|
||||||
pub is_send: bool,
|
|
||||||
pub meta: Option<EmailMeta>,
|
|
||||||
}
|
|
@ -1,76 +0,0 @@
|
|||||||
use eyre::{bail, Result};
|
|
||||||
use rayon::prelude::*;
|
|
||||||
use tracing::trace;
|
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
|
|
||||||
use super::super::{Message, MessageSender};
|
|
||||||
|
|
||||||
/// Call `FolderAction` on all files in all sub folders in
|
|
||||||
/// folder `folder`.
|
|
||||||
pub fn folders_in<FolderAction, ActionResult, P>(
|
|
||||||
folder: P,
|
|
||||||
sender: MessageSender,
|
|
||||||
action: FolderAction,
|
|
||||||
) -> Result<Vec<ActionResult>>
|
|
||||||
where
|
|
||||||
P: AsRef<Path>,
|
|
||||||
FolderAction: Fn(PathBuf, MessageSender) -> Result<Vec<ActionResult>> + Send + Sync,
|
|
||||||
ActionResult: Send,
|
|
||||||
{
|
|
||||||
let folder = folder.as_ref();
|
|
||||||
if !folder.exists() {
|
|
||||||
bail!("Folder {} does not exist", &folder.display());
|
|
||||||
}
|
|
||||||
// For progress reporting, we collect the iterator in order to
|
|
||||||
// know how many items there are.
|
|
||||||
let items: Vec<_> = std::fs::read_dir(&folder)?.collect();
|
|
||||||
let total = items.len();
|
|
||||||
sender.send(Message::ReadTotal(total))?;
|
|
||||||
Ok(items
|
|
||||||
.into_iter()
|
|
||||||
.par_bridge()
|
|
||||||
.filter_map(|entry| {
|
|
||||||
let path = entry
|
|
||||||
.map_err(|e| tracing::error!("{} {:?}", &folder.display(), &e))
|
|
||||||
.ok()?
|
|
||||||
.path();
|
|
||||||
if !path.is_dir() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let sender = sender.clone();
|
|
||||||
trace!("Reading folder {}", path.display());
|
|
||||||
action(path.clone(), sender)
|
|
||||||
.map_err(|e| tracing::error!("{} {:?}", path.display(), &e))
|
|
||||||
.ok()
|
|
||||||
})
|
|
||||||
.flatten()
|
|
||||||
.collect())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn emails_in<O, F, P: AsRef<Path>>(path: P, sender: MessageSender, make: F) -> Result<Vec<O>>
|
|
||||||
where
|
|
||||||
F: Fn(PathBuf) -> Option<O>,
|
|
||||||
F: Send + Sync + 'static,
|
|
||||||
O: Send + Sync,
|
|
||||||
{
|
|
||||||
let path = path.as_ref();
|
|
||||||
let result = Ok(std::fs::read_dir(path)?
|
|
||||||
.into_iter()
|
|
||||||
.par_bridge()
|
|
||||||
.filter_map(|entry| {
|
|
||||||
let path = entry
|
|
||||||
.map_err(|e| tracing::error!("{} {:?}", &path.display(), &e))
|
|
||||||
.ok()?
|
|
||||||
.path();
|
|
||||||
if path.is_dir() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
trace!("Reading {}", &path.display());
|
|
||||||
make(path)
|
|
||||||
})
|
|
||||||
.collect());
|
|
||||||
// We're done reading the folder
|
|
||||||
sender.send(Message::ReadOne).unwrap();
|
|
||||||
result
|
|
||||||
}
|
|
@ -1,4 +0,0 @@
|
|||||||
pub mod database;
|
|
||||||
pub mod email;
|
|
||||||
pub mod filesystem;
|
|
||||||
pub mod parse;
|
|
@ -1,155 +0,0 @@
|
|||||||
use chrono::prelude::*;
|
|
||||||
use email_parser::address::{Address, EmailAddress, Mailbox};
|
|
||||||
use eyre::{eyre, Result};
|
|
||||||
|
|
||||||
use std::borrow::Cow;
|
|
||||||
use std::collections::HashSet;
|
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
use super::email::{EmailEntry, EmailMeta};
|
|
||||||
|
|
||||||
/// Different `importer`s can implement this trait to provide the necessary
|
|
||||||
/// data to parse their data into a `EmailEntry`.
|
|
||||||
pub trait ParseableEmail: Send + Sized + Sync {
|
|
||||||
/// This will be called once before `message`, `path` and `meta`
|
|
||||||
/// are called. It can be used to perform parsing operations
|
|
||||||
fn prepare(&mut self) -> Result<()>;
|
|
||||||
/// The message content as bytes
|
|
||||||
fn message(&self) -> Result<Cow<'_, [u8]>>;
|
|
||||||
/// The original path of the email in the filesystem
|
|
||||||
fn path(&self) -> &Path;
|
|
||||||
/// Optional meta information if they're available.
|
|
||||||
/// (Depending on the `importer` capabilities and system)
|
|
||||||
fn meta(&self) -> Result<Option<EmailMeta>>;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn parse_email<Entry: ParseableEmail>(
|
|
||||||
entry: &mut Entry,
|
|
||||||
sender_emails: &HashSet<String>,
|
|
||||||
) -> Result<EmailEntry> {
|
|
||||||
if let Err(e) = entry.prepare() {
|
|
||||||
tracing::error!("Prepare Error: {:?}", e);
|
|
||||||
return Err(e);
|
|
||||||
}
|
|
||||||
let content = entry.message()?;
|
|
||||||
match email_parser::email::Email::parse(&content) {
|
|
||||||
Ok(email) => {
|
|
||||||
let path = entry.path();
|
|
||||||
tracing::trace!("Parsing {}", path.display());
|
|
||||||
let (sender_name, _, sender_local_part, sender_domain) =
|
|
||||||
mailbox_to_string(&email.sender);
|
|
||||||
|
|
||||||
let datetime = emaildatetime_to_chrono(&email.date);
|
|
||||||
let subject = email.subject.map(|e| e.to_string()).unwrap_or_default();
|
|
||||||
|
|
||||||
let to_count = match email.to.as_ref() {
|
|
||||||
Some(n) => n.len(),
|
|
||||||
None => 0,
|
|
||||||
};
|
|
||||||
let to = match email.to.as_ref().map(|v| v.first()).flatten() {
|
|
||||||
Some(n) => address_to_name_string(n),
|
|
||||||
None => None,
|
|
||||||
};
|
|
||||||
let to_group = to.as_ref().map(|e| e.0.clone()).flatten();
|
|
||||||
let to_first = to.as_ref().map(|e| (e.1.clone(), e.2.clone()));
|
|
||||||
|
|
||||||
let is_reply = email.in_reply_to.map(|v| !v.is_empty()).unwrap_or(false);
|
|
||||||
|
|
||||||
let meta = entry.meta()?;
|
|
||||||
|
|
||||||
// In order to determine the sender, we have to
|
|
||||||
// build up the address again :-(
|
|
||||||
let is_send = {
|
|
||||||
let email = format!("{}@{}", sender_local_part, sender_domain);
|
|
||||||
sender_emails.contains(&email)
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(EmailEntry {
|
|
||||||
path: path.to_path_buf(),
|
|
||||||
sender_domain,
|
|
||||||
sender_local_part,
|
|
||||||
sender_name,
|
|
||||||
datetime,
|
|
||||||
subject,
|
|
||||||
meta,
|
|
||||||
is_reply,
|
|
||||||
to_count,
|
|
||||||
to_group,
|
|
||||||
to_first,
|
|
||||||
is_send,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
let error = eyre!(
|
|
||||||
"Could not parse email (trace to see contents): {:?} [{}]",
|
|
||||||
&error,
|
|
||||||
entry.path().display()
|
|
||||||
);
|
|
||||||
tracing::error!("{:?}", &error);
|
|
||||||
if let Ok(content_string) = String::from_utf8(content.into_owned()) {
|
|
||||||
tracing::trace!("Contents:\n{}\n---\n", content_string);
|
|
||||||
} else {
|
|
||||||
tracing::trace!("Contents:\nInvalid UTF8\n---\n");
|
|
||||||
}
|
|
||||||
Err(error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns a conversion from address to the fields we care about:
|
|
||||||
/// ([group name], display name, email address)
|
|
||||||
fn address_to_name_string(address: &Address) -> Option<(Option<String>, String, String)> {
|
|
||||||
match address {
|
|
||||||
Address::Group((names, boxes)) => match (names.first(), boxes.first()) {
|
|
||||||
(group_name, Some(mailbox)) => {
|
|
||||||
let group = group_name.map(|e| e.to_string());
|
|
||||||
let (display_name, address, _, _) = mailbox_to_string(mailbox);
|
|
||||||
Some((group, display_name, address))
|
|
||||||
}
|
|
||||||
_ => None,
|
|
||||||
},
|
|
||||||
Address::Mailbox(mailbox) => {
|
|
||||||
let (display_name, address, _, _) = mailbox_to_string(mailbox);
|
|
||||||
Some((None, display_name, address))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns (display name, email address, local part, domain)
|
|
||||||
fn mailbox_to_string(mailbox: &Mailbox) -> (String, String, String, String) {
|
|
||||||
let names = match mailbox.name.as_ref() {
|
|
||||||
Some(n) => n
|
|
||||||
.iter()
|
|
||||||
.map(|e| e.as_ref())
|
|
||||||
.collect::<Vec<&str>>()
|
|
||||||
.join(" "),
|
|
||||||
None => "".to_owned(),
|
|
||||||
};
|
|
||||||
(
|
|
||||||
names,
|
|
||||||
emailaddress_to_string(&mailbox.address),
|
|
||||||
mailbox.address.local_part.to_string(),
|
|
||||||
mailbox.address.domain.to_string(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn emailaddress_to_string(address: &EmailAddress) -> String {
|
|
||||||
format!(
|
|
||||||
"{}@{}",
|
|
||||||
address.local_part.to_string(),
|
|
||||||
address.domain.to_string()
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn emaildatetime_to_chrono(dt: &email_parser::time::DateTime) -> chrono::DateTime<Utc> {
|
|
||||||
Utc.ymd(
|
|
||||||
dt.date.year as i32,
|
|
||||||
dt.date.month_number() as u32,
|
|
||||||
dt.date.day as u32,
|
|
||||||
)
|
|
||||||
.and_hms(
|
|
||||||
dt.time.time.hour as u32,
|
|
||||||
dt.time.time.minute as u32,
|
|
||||||
dt.time.time.second as u32,
|
|
||||||
)
|
|
||||||
}
|
|
@ -1,58 +0,0 @@
|
|||||||
use super::formats::shared;
|
|
||||||
use super::{Config, ImporterFormat};
|
|
||||||
|
|
||||||
use super::{Message, MessageReceiver};
|
|
||||||
|
|
||||||
use crossbeam_channel::{self, unbounded};
|
|
||||||
use eyre::Result;
|
|
||||||
use std::thread::JoinHandle;
|
|
||||||
|
|
||||||
pub trait Importerlike {
|
|
||||||
fn import(self) -> Result<(MessageReceiver, JoinHandle<Result<()>>)>;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct Importer<Format: ImporterFormat> {
|
|
||||||
config: Config,
|
|
||||||
format: Format,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<Format: ImporterFormat + 'static> Importer<Format> {
|
|
||||||
pub fn new(config: Config, format: Format) -> Self {
|
|
||||||
Self { config, format }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<Format: ImporterFormat + 'static> Importerlike for Importer<Format> {
|
|
||||||
fn import(self) -> Result<(MessageReceiver, JoinHandle<Result<()>>)> {
|
|
||||||
let Importer { format, .. } = self;
|
|
||||||
let (sender, receiver) = unbounded();
|
|
||||||
|
|
||||||
let config = self.config;
|
|
||||||
let handle: JoinHandle<Result<()>> = std::thread::spawn(move || {
|
|
||||||
let outer_sender = sender.clone();
|
|
||||||
let processed = move || {
|
|
||||||
let emails = format.emails(&config, sender.clone())?;
|
|
||||||
let processed = shared::database::into_database(&config, emails, sender.clone())?;
|
|
||||||
|
|
||||||
Ok(processed)
|
|
||||||
};
|
|
||||||
let result = processed();
|
|
||||||
|
|
||||||
// Send the error away and map it to a crossbeam channel error
|
|
||||||
match result {
|
|
||||||
Ok(_) => Ok(()),
|
|
||||||
Err(e) => match outer_sender.send(Message::Error(e)) {
|
|
||||||
Ok(_) => Ok(()),
|
|
||||||
Err(e) => Err(eyre::Report::new(e)),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
});
|
|
||||||
Ok((receiver, handle))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T: Importerlike + Sized> Importerlike for Box<T> {
|
|
||||||
fn import(self) -> Result<(MessageReceiver, JoinHandle<Result<()>>)> {
|
|
||||||
(*self).import()
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,151 +0,0 @@
|
|||||||
use eyre::{bail, eyre, Report, Result};
|
|
||||||
|
|
||||||
use std::sync::{Arc, RwLock};
|
|
||||||
use std::thread::JoinHandle;
|
|
||||||
|
|
||||||
use super::formats::ImporterFormat;
|
|
||||||
use super::importer::Importerlike;
|
|
||||||
use super::Message;
|
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
|
||||||
struct Data {
|
|
||||||
total_read: usize,
|
|
||||||
read: usize,
|
|
||||||
total_write: usize,
|
|
||||||
write: usize,
|
|
||||||
finishing: bool,
|
|
||||||
done: bool,
|
|
||||||
error: Option<Report>,
|
|
||||||
#[cfg(target_os = "macos")]
|
|
||||||
missing_permissions: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, Copy)]
|
|
||||||
pub struct Progress {
|
|
||||||
pub total: usize,
|
|
||||||
pub count: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, Copy)]
|
|
||||||
pub struct State {
|
|
||||||
pub finishing: bool,
|
|
||||||
pub done: bool,
|
|
||||||
pub written: usize,
|
|
||||||
#[cfg(target_os = "macos")]
|
|
||||||
pub missing_permissions: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// This can be initialized with a [`MessageSender`] and it will
|
|
||||||
/// automatically tally up the information into a thread-safe
|
|
||||||
/// datastructure
|
|
||||||
pub struct Adapter {
|
|
||||||
producer_lock: Arc<RwLock<Data>>,
|
|
||||||
consumer_lock: Arc<RwLock<Data>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Adapter {
|
|
||||||
#[allow(clippy::new_without_default)]
|
|
||||||
pub fn new() -> Self {
|
|
||||||
let rw_lock = Arc::new(RwLock::default());
|
|
||||||
// FIXME: Look up this warning. It looks like the clones are necessary?
|
|
||||||
#[allow(clippy::redundant_clone)]
|
|
||||||
let producer_lock = rw_lock.clone();
|
|
||||||
#[allow(clippy::redundant_clone)]
|
|
||||||
let consumer_lock = rw_lock.clone();
|
|
||||||
Self {
|
|
||||||
producer_lock,
|
|
||||||
consumer_lock,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Starts up a thread that handles the `MessageReceiver` messages
|
|
||||||
/// into state that can be accessed via [`read_count`], [`write_count`] and [`finished`]
|
|
||||||
pub fn process<Format: ImporterFormat + 'static>(
|
|
||||||
&self,
|
|
||||||
importer: super::importer::Importer<Format>,
|
|
||||||
) -> Result<JoinHandle<Result<()>>> {
|
|
||||||
let (receiver, handle) = importer.import()?;
|
|
||||||
let lock = self.producer_lock.clone();
|
|
||||||
let handle = std::thread::spawn(move || {
|
|
||||||
'outer: loop {
|
|
||||||
let mut write_guard = match lock.write() {
|
|
||||||
Ok(n) => n,
|
|
||||||
Err(e) => bail!("RwLock Error: {:?}", e),
|
|
||||||
};
|
|
||||||
for entry in receiver.try_iter() {
|
|
||||||
match entry {
|
|
||||||
Message::ReadTotal(n) => write_guard.total_read = n,
|
|
||||||
Message::ReadOne => {
|
|
||||||
write_guard.read += 1;
|
|
||||||
// Depending on the implementation, we may receive read calls before
|
|
||||||
// the total size is known. We prevent division by zero by
|
|
||||||
// always setting the total to read + 1 in these cases
|
|
||||||
if write_guard.total_read <= write_guard.read {
|
|
||||||
write_guard.total_read = write_guard.read + 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Message::WriteTotal(n) => write_guard.total_write = n,
|
|
||||||
Message::WriteOne => write_guard.write += 1,
|
|
||||||
Message::FinishingUp => write_guard.finishing = true,
|
|
||||||
Message::Done => {
|
|
||||||
write_guard.done = true;
|
|
||||||
break 'outer;
|
|
||||||
}
|
|
||||||
Message::Error(e) => {
|
|
||||||
write_guard.error = Some(e);
|
|
||||||
}
|
|
||||||
#[cfg(target_os = "macos")]
|
|
||||||
Message::MissingPermissions => {
|
|
||||||
write_guard.missing_permissions = true;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let _ = handle.join().map_err(|op| eyre::eyre!("{:?}", &op))??;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
});
|
|
||||||
Ok(handle)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn read_count(&self) -> Result<Progress> {
|
|
||||||
let item = self.consumer_lock.read().map_err(|e| eyre!("{:?}", &e))?;
|
|
||||||
Ok(Progress {
|
|
||||||
total: item.total_read,
|
|
||||||
count: item.read,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn write_count(&self) -> Result<Progress> {
|
|
||||||
let item = self.consumer_lock.read().map_err(|e| eyre!("{:?}", &e))?;
|
|
||||||
Ok(Progress {
|
|
||||||
total: item.total_write,
|
|
||||||
count: item.write,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn finished(&self) -> Result<State> {
|
|
||||||
let item = self.consumer_lock.read().map_err(|e| eyre!("{:?}", &e))?;
|
|
||||||
Ok(State {
|
|
||||||
finishing: item.finishing,
|
|
||||||
done: item.done,
|
|
||||||
written: item.write,
|
|
||||||
#[cfg(target_os = "macos")]
|
|
||||||
missing_permissions: item.missing_permissions,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn error(&self) -> Result<Option<Report>> {
|
|
||||||
// We take the error of out of the write lock only if there is an error.
|
|
||||||
let item = self.consumer_lock.read().map_err(|e| eyre!("{:?}", &e))?;
|
|
||||||
let is_error = item.error.is_some();
|
|
||||||
drop(item);
|
|
||||||
if is_error {
|
|
||||||
let mut item = self.producer_lock.write().map_err(|e| eyre!("{:?}", &e))?;
|
|
||||||
Ok(item.error.take())
|
|
||||||
} else {
|
|
||||||
Ok(None)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,67 +0,0 @@
|
|||||||
use crossbeam_channel;
|
|
||||||
|
|
||||||
pub(crate) mod formats;
|
|
||||||
#[allow(clippy::module_inception)]
|
|
||||||
mod importer;
|
|
||||||
mod message_adapter;
|
|
||||||
|
|
||||||
use crate::types::Config;
|
|
||||||
pub use formats::shared::email::{EmailEntry, EmailMeta};
|
|
||||||
pub use importer::Importerlike;
|
|
||||||
pub use message_adapter::*;
|
|
||||||
|
|
||||||
use formats::ImporterFormat;
|
|
||||||
|
|
||||||
/// The message that informs of the importers progress
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub enum Message {
|
|
||||||
/// How much progress are we making on reading the contents
|
|
||||||
/// of the emails.
|
|
||||||
/// The `usize` parameter marks the total amount of items to read - if it is known.
|
|
||||||
/// The values here can vary wildly based on the type of Importer `Format` in use.
|
|
||||||
/// A Gmail backup will list the folders and how many of them
|
|
||||||
/// are already read. A mbox format will list other things as there
|
|
||||||
/// no folders.
|
|
||||||
ReadTotal(usize),
|
|
||||||
/// Whenever an item out of the total is read, this message will be emitted
|
|
||||||
ReadOne,
|
|
||||||
/// Similar to [`ReadTotal`]
|
|
||||||
WriteTotal(usize),
|
|
||||||
/// Similar to `ReadOne`
|
|
||||||
WriteOne,
|
|
||||||
/// Once everything has been written, we need to wait for the database
|
|
||||||
/// to sync
|
|
||||||
FinishingUp,
|
|
||||||
/// Finally, this indicates that we're done.
|
|
||||||
Done,
|
|
||||||
/// An error happened during processing
|
|
||||||
Error(eyre::Report),
|
|
||||||
/// A special case for macOS, where a permission error means we have to grant this app
|
|
||||||
/// the right to see the mail folder
|
|
||||||
#[cfg(target_os = "macos")]
|
|
||||||
MissingPermissions,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub type MessageSender = crossbeam_channel::Sender<Message>;
|
|
||||||
pub type MessageReceiver = crossbeam_channel::Receiver<Message>;
|
|
||||||
|
|
||||||
pub fn importer(config: &Config) -> Box<dyn importer::Importerlike> {
|
|
||||||
use crate::types::FormatType::*;
|
|
||||||
match config.format {
|
|
||||||
AppleMail => Box::new(applemail_importer(config.clone())),
|
|
||||||
GmailVault => Box::new(gmail_importer(config.clone())),
|
|
||||||
Mbox => Box::new(gmail_importer(config.clone())),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn gmail_importer(config: Config) -> importer::Importer<formats::Gmail> {
|
|
||||||
importer::Importer::new(config, formats::Gmail::default())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn applemail_importer(config: Config) -> importer::Importer<formats::AppleMail> {
|
|
||||||
importer::Importer::new(config, formats::AppleMail::default())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn mbox_importer(config: Config) -> importer::Importer<formats::Mbox> {
|
|
||||||
importer::Importer::new(config, formats::Mbox::default())
|
|
||||||
}
|
|
@ -1,70 +0,0 @@
|
|||||||
#[cfg(target_os = "macos")]
|
|
||||||
#[macro_use]
|
|
||||||
extern crate objc;
|
|
||||||
|
|
||||||
use tracing_subscriber::fmt;
|
|
||||||
use tracing_subscriber::prelude::*;
|
|
||||||
|
|
||||||
pub mod database;
|
|
||||||
#[cfg(feature = "gui")]
|
|
||||||
pub mod gui;
|
|
||||||
pub mod importer;
|
|
||||||
pub mod model;
|
|
||||||
pub mod types;
|
|
||||||
|
|
||||||
pub fn setup_tracing() {
|
|
||||||
if std::env::var("RUST_LOG").is_err() {
|
|
||||||
std::env::set_var("RUST_LOG", "error")
|
|
||||||
}
|
|
||||||
|
|
||||||
let collector = tracing_subscriber::registry().with(fmt::layer().with_writer(std::io::stdout));
|
|
||||||
|
|
||||||
tracing::subscriber::set_global_default(collector).expect("Unable to set a global collector");
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Create a config for the `cli` and validate the input
|
|
||||||
pub fn make_config() -> types::Config {
|
|
||||||
use std::path::Path;
|
|
||||||
use types::FormatType;
|
|
||||||
let arguments: Vec<String> = std::env::args().collect();
|
|
||||||
let folder = arguments
|
|
||||||
.get(1)
|
|
||||||
.unwrap_or_else(|| usage("Missing email folder argument"));
|
|
||||||
let database = arguments
|
|
||||||
.get(2)
|
|
||||||
.unwrap_or_else(|| usage("Missing database path argument"));
|
|
||||||
let sender = arguments
|
|
||||||
.get(3)
|
|
||||||
.unwrap_or_else(|| usage("Missing sender email address argument"));
|
|
||||||
let format: FormatType = arguments
|
|
||||||
.get(4)
|
|
||||||
.unwrap_or_else(|| usage("Missing sender email address argument"))
|
|
||||||
.into();
|
|
||||||
|
|
||||||
let database_path = Path::new(database);
|
|
||||||
if database_path.is_dir() {
|
|
||||||
panic!(
|
|
||||||
"Database Path can't be a directory: {}",
|
|
||||||
&database_path.display()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
let emails_folder_path = Path::new(folder);
|
|
||||||
// For non-mbox files, we make sure we have a directory
|
|
||||||
if !emails_folder_path.is_dir() {
|
|
||||||
panic!(
|
|
||||||
"Emails Folder Path is not a directory: {}",
|
|
||||||
&emails_folder_path.display()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
match crate::types::Config::new(Some(database), folder, vec![sender.to_string()], format) {
|
|
||||||
Ok(n) => n,
|
|
||||||
Err(r) => panic!("Error: {:?}", &r),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn usage(error: &'static str) -> ! {
|
|
||||||
println!("Usage: cli [email-folder] [database-path] [sender-email-address] [format]");
|
|
||||||
println!("\tExample: cli ~/Library/Mails/V9/ ./db.sqlite my-address@gmail.com apple");
|
|
||||||
panic!("{}", error);
|
|
||||||
}
|
|
@ -1,243 +0,0 @@
|
|||||||
//! The `Engine` is the entry point to the data that should be
|
|
||||||
//! displayed in Segmentations.
|
|
||||||
//! See [`Engine`] for more information.
|
|
||||||
//! See also:
|
|
||||||
//! - [`segmentations::`]
|
|
||||||
//! - [`items::`]
|
|
||||||
use eyre::{bail, Result};
|
|
||||||
use lru::LruCache;
|
|
||||||
|
|
||||||
use crate::database::query::{Field, Filter, OtherQuery, Query, ValueField};
|
|
||||||
use crate::model::link::Response;
|
|
||||||
use crate::types::Config;
|
|
||||||
|
|
||||||
use super::link::Link;
|
|
||||||
use super::segmentations;
|
|
||||||
use super::types::{LoadingState, Segment, Segmentation};
|
|
||||||
|
|
||||||
/// This signifies the action we're currently evaluating
|
|
||||||
/// It is used for sending requests and receiving responses
|
|
||||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
|
||||||
pub(super) enum Action {
|
|
||||||
/// Recalculate the current `Segmentation` based on a changed aggregation
|
|
||||||
RecalculateSegmentation,
|
|
||||||
/// Push a new `Segmentation`
|
|
||||||
PushSegmentation,
|
|
||||||
/// Load the mails for the current `Segmentation`
|
|
||||||
LoadItems,
|
|
||||||
/// Load all tags
|
|
||||||
AllTags,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Interact with the `Database`, operate on `Segmentations`, `Segments`, and `Items`.
|
|
||||||
/// `Engine` is used as the input for almost all operations in the
|
|
||||||
/// `items::` and `segmentation::` modules.
|
|
||||||
pub struct Engine {
|
|
||||||
pub(super) search_stack: Vec<ValueField>,
|
|
||||||
pub(super) group_by_stack: Vec<Field>,
|
|
||||||
pub(super) link: Link<Action>,
|
|
||||||
pub(super) segmentations: Vec<Segmentation>,
|
|
||||||
/// Additional filters. See [`segmentations::set_filters`]
|
|
||||||
pub(super) filters: Vec<Filter>,
|
|
||||||
/// This is a very simple cache from ranges to rows.
|
|
||||||
/// It doesn't account for overlapping ranges.
|
|
||||||
/// There's a lot of room for improvement here.
|
|
||||||
pub(super) item_cache: LruCache<usize, LoadingState>,
|
|
||||||
pub(super) known_tags: Vec<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Engine {
|
|
||||||
pub fn new(config: &Config) -> Result<Self> {
|
|
||||||
let link = super::link::run(config)?;
|
|
||||||
let engine = Engine {
|
|
||||||
link,
|
|
||||||
search_stack: Vec::new(),
|
|
||||||
group_by_stack: vec![default_group_by_stack(0).unwrap()],
|
|
||||||
segmentations: Vec::new(),
|
|
||||||
filters: Vec::new(),
|
|
||||||
item_cache: LruCache::new(10000),
|
|
||||||
known_tags: Vec::new(),
|
|
||||||
};
|
|
||||||
Ok(engine)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Start the `Engine`. This will create a thread to
|
|
||||||
/// asynchronously communicate with the underlying backend
|
|
||||||
/// in a non-blocking manner.
|
|
||||||
pub fn start(&mut self) -> Result<()> {
|
|
||||||
// The initial segmentation
|
|
||||||
self.link
|
|
||||||
.request(&segmentations::make_query(self)?, Action::PushSegmentation)?;
|
|
||||||
// Get all tags
|
|
||||||
self.link.request(
|
|
||||||
&Query::Other {
|
|
||||||
query: OtherQuery::All(Field::MetaTags),
|
|
||||||
},
|
|
||||||
Action::AllTags,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Information on the underlying `Format`. Does it have tags
|
|
||||||
pub fn format_has_tags(&self) -> bool {
|
|
||||||
!self.known_tags.is_empty()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Information on the underlying `Format`. Does it have `seen` information
|
|
||||||
pub fn format_has_seen(&self) -> bool {
|
|
||||||
// FIXME: The current implementation just assumes that the existance of meta tags also implies is_seen
|
|
||||||
!self.known_tags.is_empty()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// All the known tags in the current emails
|
|
||||||
pub fn known_tags(&self) -> &[String] {
|
|
||||||
&self.known_tags
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Return the current stack of `Segmentations`
|
|
||||||
pub fn segmentations(&self) -> &[Segmentation] {
|
|
||||||
&self.segmentations
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Push a new `Segment` to select a more specific `Segmentation`.
|
|
||||||
///
|
|
||||||
/// Pushing will create an additional `Aggregation` based on the selected
|
|
||||||
/// `Segment`, retrieve the data from the backend, and add it to the
|
|
||||||
/// current stack of `Segmentations`.
|
|
||||||
/// It allows to **drill down** into the data.
|
|
||||||
pub fn push(&mut self, segment: Segment) -> Result<()> {
|
|
||||||
// Assign the segmentation
|
|
||||||
let current = match self.segmentations.last_mut() {
|
|
||||||
Some(n) => n,
|
|
||||||
None => return Ok(()),
|
|
||||||
};
|
|
||||||
current.selected = Some(segment);
|
|
||||||
|
|
||||||
// Create the new search stack
|
|
||||||
self.search_stack = self
|
|
||||||
.segmentations
|
|
||||||
.iter()
|
|
||||||
.filter_map(|e| e.selected.as_ref())
|
|
||||||
.map(|p| p.field.clone())
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
// Add the next group by
|
|
||||||
let index = self.group_by_stack.len();
|
|
||||||
let next = default_group_by_stack(index)
|
|
||||||
.ok_or_else(|| eyre::eyre!("default group by stack out of bounds"))?;
|
|
||||||
self.group_by_stack.push(next);
|
|
||||||
|
|
||||||
// Block UI & Wait for updates
|
|
||||||
self.link
|
|
||||||
.request(&segmentations::make_query(self)?, Action::PushSegmentation)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Pop the current `Segmentation` from the stack.
|
|
||||||
/// The opposite of [`engine::push`]
|
|
||||||
pub fn pop(&mut self) {
|
|
||||||
if self.group_by_stack.is_empty()
|
|
||||||
|| self.segmentations.is_empty()
|
|
||||||
|| self.search_stack.is_empty()
|
|
||||||
{
|
|
||||||
tracing::error!(
|
|
||||||
"Invalid state. Not everything has the same length: {:?}, {:?}, {:?}",
|
|
||||||
&self.group_by_stack,
|
|
||||||
self.segmentations,
|
|
||||||
self.search_stack
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove the last entry of everything
|
|
||||||
self.group_by_stack.remove(self.group_by_stack.len() - 1);
|
|
||||||
self.segmentations.remove(self.segmentations.len() - 1);
|
|
||||||
self.search_stack.remove(self.search_stack.len() - 1);
|
|
||||||
|
|
||||||
// Remove the selection in the last segmentation
|
|
||||||
if let Some(e) = self.segmentations.last_mut() {
|
|
||||||
e.selected = None
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove any rows that were cached for this segmentation
|
|
||||||
self.item_cache.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Call this continously to retrieve calculation results and apply them.
|
|
||||||
/// Any mutating function on [`Engine`], such as [`Engine::push`] or [`items::items`]
|
|
||||||
/// require calling this method to apply there results once they're
|
|
||||||
/// available from the asynchronous backend.
|
|
||||||
/// This method is specifically non-blocking for usage in
|
|
||||||
/// `Eventloop` based UI frameworks such as `egui`.
|
|
||||||
pub fn process(&mut self) -> Result<()> {
|
|
||||||
let response = match self.link.receive()? {
|
|
||||||
Some(n) => n,
|
|
||||||
None => return Ok(()),
|
|
||||||
};
|
|
||||||
|
|
||||||
match response {
|
|
||||||
Response::Grouped(_, Action::PushSegmentation, p) => {
|
|
||||||
self.segmentations.push(p);
|
|
||||||
// Remove any rows that were cached for this segmentation
|
|
||||||
self.item_cache.clear();
|
|
||||||
}
|
|
||||||
Response::Grouped(_, Action::RecalculateSegmentation, p) => {
|
|
||||||
let len = self.segmentations.len();
|
|
||||||
self.segmentations[len - 1] = p;
|
|
||||||
// Remove any rows that were cached for this segmentation
|
|
||||||
self.item_cache.clear();
|
|
||||||
}
|
|
||||||
Response::Normal(Query::Normal { range, .. }, Action::LoadItems, r) => {
|
|
||||||
for (index, row) in range.zip(r) {
|
|
||||||
let entry = LoadingState::Loaded(row.clone());
|
|
||||||
self.item_cache.put(index, entry);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Response::Other(Query::Other { .. }, Action::AllTags, r) => {
|
|
||||||
self.known_tags = r;
|
|
||||||
}
|
|
||||||
_ => bail!("Invalid Query / Response combination"),
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns true if there're currently calculations open and `process`
|
|
||||||
/// needs to be called. This can be used in `Eventloop` based frameworks
|
|
||||||
/// such as `egui` to know when to continue calling `process` in the `loop`
|
|
||||||
/// ```ignore
|
|
||||||
/// loop {
|
|
||||||
/// self.engine.process().unwrap();
|
|
||||||
/// if self.engine.is_busy() {
|
|
||||||
/// // Call the library function to run the event-loop again.
|
|
||||||
/// ctx.request_repaint();
|
|
||||||
/// }
|
|
||||||
/// }
|
|
||||||
/// ```
|
|
||||||
pub fn is_busy(&self) -> bool {
|
|
||||||
self.link.is_processing() || self.segmentations.is_empty()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Blocking waiting until the current operation is done
|
|
||||||
/// This is useful for usage on a commandline or in unit tests
|
|
||||||
#[allow(unused)]
|
|
||||||
pub fn wait(&mut self) -> Result<()> {
|
|
||||||
loop {
|
|
||||||
self.process()?;
|
|
||||||
if !self.link.is_processing() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Return the default aggregation fields for each segmentation stack level
|
|
||||||
pub fn default_group_by_stack(index: usize) -> Option<Field> {
|
|
||||||
match index {
|
|
||||||
0 => Some(Field::Year),
|
|
||||||
1 => Some(Field::SenderDomain),
|
|
||||||
2 => Some(Field::SenderLocalPart),
|
|
||||||
3 => Some(Field::Month),
|
|
||||||
4 => Some(Field::Day),
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,93 +0,0 @@
|
|||||||
//! Operations related to retrieving `items` from the current `Segmentation`
|
|
||||||
//!
|
|
||||||
//! A `Segmentation` is a aggregation of items into many `Segments`.
|
|
||||||
//! These operations allow retreiving the individual items for all
|
|
||||||
//! segments in the `Segmentation.
|
|
||||||
|
|
||||||
use eyre::Result;
|
|
||||||
|
|
||||||
use super::types::LoadingState;
|
|
||||||
use super::{engine::Action, Engine};
|
|
||||||
use crate::database::{
|
|
||||||
query::{Field, Filter, Query},
|
|
||||||
query_result::QueryRow,
|
|
||||||
};
|
|
||||||
|
|
||||||
use std::ops::Range;
|
|
||||||
|
|
||||||
/// Return the `items` in the current `Segmentation`
|
|
||||||
///
|
|
||||||
/// If the items don't exist in the cache, they will be queried
|
|
||||||
/// asynchronously from the database. The return value distinguishes
|
|
||||||
/// between `Loaded` and `Loading` items.
|
|
||||||
///
|
|
||||||
/// # Arguments
|
|
||||||
///
|
|
||||||
/// * `engine` - The engine to use for retrieving data
|
|
||||||
/// * `range` - The range of items to retrieve. If `None` then all items will be retrieved
|
|
||||||
pub fn items(engine: &mut Engine, range: Option<Range<usize>>) -> Result<Vec<Option<QueryRow>>> {
|
|
||||||
// build an array with either empty values or values from our cache.
|
|
||||||
let mut rows = Vec::new();
|
|
||||||
|
|
||||||
// The given range or all items
|
|
||||||
let range = range.unwrap_or_else(|| Range {
|
|
||||||
start: 0,
|
|
||||||
end: count(engine),
|
|
||||||
});
|
|
||||||
|
|
||||||
let mut missing_data = false;
|
|
||||||
for index in range.clone() {
|
|
||||||
let entry = engine.item_cache.get(&index);
|
|
||||||
let entry = match entry {
|
|
||||||
Some(LoadingState::Loaded(n)) => Some((*n).clone()),
|
|
||||||
Some(LoadingState::Loading) => None,
|
|
||||||
None => {
|
|
||||||
// for simplicity, we keep the "something is missing" state separate
|
|
||||||
missing_data = true;
|
|
||||||
|
|
||||||
// Mark the row as being loaded
|
|
||||||
engine.item_cache.put(index, LoadingState::Loading);
|
|
||||||
None
|
|
||||||
}
|
|
||||||
};
|
|
||||||
rows.push(entry);
|
|
||||||
}
|
|
||||||
// Only if at least some data is missing do we perform the request
|
|
||||||
if missing_data && !range.is_empty() {
|
|
||||||
let request = make_query(engine, range);
|
|
||||||
engine.link.request(&request, Action::LoadItems)?;
|
|
||||||
}
|
|
||||||
Ok(rows)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The total amount of elements in the current `Segmentation`
|
|
||||||
///
|
|
||||||
/// # Arguments
|
|
||||||
///
|
|
||||||
/// * `engine` - The engine to use for retrieving data
|
|
||||||
pub fn count(engine: &Engine) -> usize {
|
|
||||||
let segmentation = match engine.segmentations.last() {
|
|
||||||
Some(n) => n,
|
|
||||||
None => return 0,
|
|
||||||
};
|
|
||||||
segmentation.element_count()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Make the query for retrieving items
|
|
||||||
fn make_query(engine: &Engine, range: Range<usize>) -> Query {
|
|
||||||
let mut filters = Vec::new();
|
|
||||||
for entry in &engine.search_stack {
|
|
||||||
filters.push(Filter::Like(entry.clone()));
|
|
||||||
}
|
|
||||||
Query::Normal {
|
|
||||||
filters,
|
|
||||||
fields: vec![
|
|
||||||
Field::SenderDomain,
|
|
||||||
Field::SenderLocalPart,
|
|
||||||
Field::Subject,
|
|
||||||
Field::Path,
|
|
||||||
Field::Timestamp,
|
|
||||||
],
|
|
||||||
range,
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,163 +0,0 @@
|
|||||||
//! Abstraction to perform asynchronous calculations & queries without blocking UI
|
|
||||||
//!
|
|
||||||
//! This opens a `crossbeam` `channel` to communicate with a backend.
|
|
||||||
//! Each backend operation is send and retrieved in a loop on a thread.
|
|
||||||
//! This allows sending operations into `Link` and retrieving the contents
|
|
||||||
//! asynchronously without blocking the UI.
|
|
||||||
|
|
||||||
use std::sync::{
|
|
||||||
atomic::{AtomicUsize, Ordering},
|
|
||||||
Arc,
|
|
||||||
};
|
|
||||||
use std::{collections::HashSet, convert::TryInto};
|
|
||||||
|
|
||||||
use crossbeam_channel::{unbounded, Receiver, Sender};
|
|
||||||
use eyre::Result;
|
|
||||||
use serde_json::Value;
|
|
||||||
|
|
||||||
use crate::database::{
|
|
||||||
query::Query,
|
|
||||||
query_result::{QueryResult, QueryRow},
|
|
||||||
Database,
|
|
||||||
};
|
|
||||||
use crate::types::Config;
|
|
||||||
|
|
||||||
use super::types::Segmentation;
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub enum Response<Context: Send + 'static> {
|
|
||||||
Grouped(Query, Context, Segmentation),
|
|
||||||
Normal(Query, Context, Vec<QueryRow>),
|
|
||||||
/// FIXME: OtherQuery results are currently limited to strings as that's enough right now.
|
|
||||||
Other(Query, Context, Vec<String>),
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) type InputSender<Context> = Sender<(Query, Context)>;
|
|
||||||
pub(super) type OutputReciever<Context> = Receiver<Result<Response<Context>>>;
|
|
||||||
|
|
||||||
pub(super) struct Link<Context: Send + 'static> {
|
|
||||||
pub input_sender: InputSender<Context>,
|
|
||||||
pub output_receiver: OutputReciever<Context>,
|
|
||||||
// We need to account for the brief moment where the processing channel is empty
|
|
||||||
// but we're applying the results. If there is a UI update in this window,
|
|
||||||
// the UI will not update again after the changes were applied because an empty
|
|
||||||
// channel indicates completed processing.
|
|
||||||
// There's also a delay between a request taken out of the input channel and being
|
|
||||||
// put into the output channel. In order to account for all of this, we employ a
|
|
||||||
// request counter to know how many requests are currently in the pipeline
|
|
||||||
request_counter: Arc<AtomicUsize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<Context: Send + Sync + 'static> Link<Context> {
|
|
||||||
pub fn request(&mut self, query: &Query, context: Context) -> Result<()> {
|
|
||||||
self.request_counter.fetch_add(1, Ordering::Relaxed);
|
|
||||||
self.input_sender.send((query.clone(), context))?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn receive(&mut self) -> Result<Option<Response<Context>>> {
|
|
||||||
match self.output_receiver.try_recv() {
|
|
||||||
// We received something
|
|
||||||
Ok(Ok(response)) => {
|
|
||||||
// Only subtract if we successfuly received a value
|
|
||||||
self.request_counter.fetch_sub(1, Ordering::Relaxed);
|
|
||||||
Ok(Some(response))
|
|
||||||
}
|
|
||||||
// We received nothing
|
|
||||||
Err(_) => Ok(None),
|
|
||||||
// There was an error, we forward it
|
|
||||||
Ok(Err(e)) => Err(e),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn is_processing(&self) -> bool {
|
|
||||||
self.request_counter.load(Ordering::Relaxed) > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
/// This can be used to track the `link` from a different thread.
|
|
||||||
#[allow(unused)]
|
|
||||||
pub fn request_counter(&self) -> Arc<AtomicUsize> {
|
|
||||||
self.request_counter.clone()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn run<Context: Send + Sync + 'static>(config: &Config) -> Result<Link<Context>> {
|
|
||||||
// Create a new database connection, just for reading
|
|
||||||
let database = Database::new(&config.database_path)?;
|
|
||||||
let (input_sender, input_receiver) = unbounded();
|
|
||||||
let (output_sender, output_receiver) = unbounded();
|
|
||||||
let _ = std::thread::spawn(move || inner_loop(database, input_receiver, output_sender));
|
|
||||||
Ok(Link {
|
|
||||||
input_sender,
|
|
||||||
output_receiver,
|
|
||||||
request_counter: Arc::new(AtomicUsize::new(0)),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn inner_loop<Context: Send + Sync + 'static>(
|
|
||||||
database: Database,
|
|
||||||
input_receiver: Receiver<(Query, Context)>,
|
|
||||||
output_sender: Sender<Result<Response<Context>>>,
|
|
||||||
) -> Result<()> {
|
|
||||||
loop {
|
|
||||||
let (query, context) = input_receiver.recv()?;
|
|
||||||
let result = database.query(&query)?;
|
|
||||||
let response = match query {
|
|
||||||
Query::Grouped { .. } => {
|
|
||||||
let segmentations = calculate_segmentations(&result)?;
|
|
||||||
Response::Grouped(query, context, segmentations)
|
|
||||||
}
|
|
||||||
Query::Normal { .. } => {
|
|
||||||
let converted = calculate_rows(&result)?;
|
|
||||||
Response::Normal(query, context, converted)
|
|
||||||
}
|
|
||||||
Query::Other { .. } => {
|
|
||||||
let mut results = HashSet::new();
|
|
||||||
for entry in result {
|
|
||||||
match entry {
|
|
||||||
QueryResult::Other(field) => match field.value() {
|
|
||||||
Value::Array(s) => {
|
|
||||||
for n in s {
|
|
||||||
if let Value::String(s) = n {
|
|
||||||
if !results.contains(s) {
|
|
||||||
results.insert(s.to_owned());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => panic!("Should not end up here"),
|
|
||||||
},
|
|
||||||
_ => panic!("Should not end up here"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Response::Other(query, context, results.into_iter().collect())
|
|
||||||
}
|
|
||||||
};
|
|
||||||
output_sender.send(Ok(response))?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn calculate_segmentations(result: &[QueryResult]) -> Result<Segmentation> {
|
|
||||||
let mut segmentations = Vec::new();
|
|
||||||
for r in result.iter() {
|
|
||||||
let segmentation = r.try_into()?;
|
|
||||||
segmentations.push(segmentation);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(Segmentation::new(segmentations))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn calculate_rows(result: &[QueryResult]) -> Result<Vec<QueryRow>> {
|
|
||||||
Ok(result
|
|
||||||
.iter()
|
|
||||||
.map(|r| {
|
|
||||||
let values = match r {
|
|
||||||
QueryResult::Normal(values) => values,
|
|
||||||
_ => {
|
|
||||||
panic!("Invalid result type, expected `Normal`")
|
|
||||||
}
|
|
||||||
};
|
|
||||||
values.clone()
|
|
||||||
})
|
|
||||||
.collect())
|
|
||||||
}
|
|
@ -1,8 +0,0 @@
|
|||||||
mod engine;
|
|
||||||
pub mod items;
|
|
||||||
mod link;
|
|
||||||
pub mod segmentations;
|
|
||||||
mod types;
|
|
||||||
|
|
||||||
pub use engine::Engine;
|
|
||||||
pub use types::Segment;
|
|
@ -1,206 +0,0 @@
|
|||||||
//! Operations on `Segmentations`
|
|
||||||
//!
|
|
||||||
//! `Segmentations` are collections of `Segments` based on an aggregation of `Items`.
|
|
||||||
//!
|
|
||||||
//! A `Segmentation` can be changed to be aggregated on a different `Field.
|
|
||||||
//! - [`aggregations`]
|
|
||||||
//! - [`aggregated_by`]
|
|
||||||
//! - [`set_aggregation`]
|
|
||||||
//! A `Segmentation` can be changed to only return a `Range` of segments.
|
|
||||||
//! - [`current_range`]
|
|
||||||
//! - [`set_current_range`]
|
|
||||||
//! A `Segmentation` has multiple `Segments` which each can be layouted
|
|
||||||
//! to fit into a rectangle.
|
|
||||||
//! - [`layouted_segments]
|
|
||||||
|
|
||||||
use eyre::{eyre, Result};
|
|
||||||
|
|
||||||
use super::engine::Action;
|
|
||||||
use super::{
|
|
||||||
types::{Aggregation, Segment},
|
|
||||||
Engine,
|
|
||||||
};
|
|
||||||
use crate::database::query::{Field, Filter, Query};
|
|
||||||
use std::ops::RangeInclusive;
|
|
||||||
|
|
||||||
/// Filter the `Range` of segments of the current `Segmentation`
|
|
||||||
///
|
|
||||||
/// Returns the `Range` and the total number of segments.
|
|
||||||
/// If no custom range has been set with [`set_segments_range`], returns
|
|
||||||
/// the full range of items, otherwise the custom range.
|
|
||||||
///
|
|
||||||
/// Returns `None` if no current `Segmentation` exists.
|
|
||||||
///
|
|
||||||
/// # Arguments
|
|
||||||
///
|
|
||||||
/// * `engine` - The engine to use for retrieving data
|
|
||||||
/// * `aggregation` - The aggregation to return the fields for. Required to also return the current aggregation field.
|
|
||||||
pub fn segments_range(engine: &Engine) -> Option<(RangeInclusive<usize>, usize)> {
|
|
||||||
let segmentation = engine.segmentations.last()?;
|
|
||||||
let len = segmentation.len();
|
|
||||||
Some(match &segmentation.range {
|
|
||||||
Some(n) => (0..=len, *n.end()),
|
|
||||||
None => (0..=len, len),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Set the `Range` of segments of the current `Segmentation`
|
|
||||||
///
|
|
||||||
/// # Arguments
|
|
||||||
///
|
|
||||||
/// * `engine` - The engine to use for setting data
|
|
||||||
/// * `range` - The range to apply. `None` to reset it to all `Segments`
|
|
||||||
pub fn set_segments_range(engine: &mut Engine, range: Option<RangeInclusive<usize>>) {
|
|
||||||
if let Some(n) = engine.segmentations.last_mut() {
|
|
||||||
// Make sure the range does not go beyond the current semgents count
|
|
||||||
if let Some(r) = range {
|
|
||||||
let len = n.len();
|
|
||||||
if len > *r.start() && *r.end() < len {
|
|
||||||
n.range = Some(r);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
n.range = None;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Additional filters to use in the query
|
|
||||||
///
|
|
||||||
/// These filters will be evaluated in addition to the `segmentation` conditions
|
|
||||||
/// in the query.
|
|
||||||
/// Setting this value will recalculate the current segmentations.
|
|
||||||
pub fn set_filters(engine: &mut Engine, filters: &[Filter]) -> Result<()> {
|
|
||||||
engine.filters = filters.to_vec();
|
|
||||||
|
|
||||||
// Remove any rows that were cached for this Segmentation
|
|
||||||
engine.item_cache.clear();
|
|
||||||
engine
|
|
||||||
.link
|
|
||||||
.request(&make_query(engine)?, Action::RecalculateSegmentation)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The fields available for the given aggregation
|
|
||||||
///
|
|
||||||
/// As the user `pushes` Segmentations and dives into the data,
|
|
||||||
/// less fields become available to aggregate by. It is inconsequential
|
|
||||||
/// to aggregate, say, by year, then by month, and then again by year.
|
|
||||||
/// This method returns the possible fields still available for aggregation.
|
|
||||||
///
|
|
||||||
/// # Arguments
|
|
||||||
///
|
|
||||||
/// * `engine` - The engine to use for retrieving data
|
|
||||||
/// * `aggregation` - The aggregation to return the fields for. Required to also return the current aggregation field.
|
|
||||||
pub fn aggregation_fields(engine: &Engine, aggregation: &Aggregation) -> Vec<Field> {
|
|
||||||
#[allow(clippy::unnecessary_filter_map)]
|
|
||||||
Field::all_cases()
|
|
||||||
.filter_map(|f| {
|
|
||||||
if f == aggregation.field {
|
|
||||||
return Some(f);
|
|
||||||
}
|
|
||||||
if engine.group_by_stack.contains(&f) {
|
|
||||||
None
|
|
||||||
} else {
|
|
||||||
Some(f)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Return all `Aggregation`s applied for the current `Segmentation`
|
|
||||||
///
|
|
||||||
/// E.g. if we're first aggregating by Year, and then by Month, this
|
|
||||||
/// will return a `Vec` of `[Year, Month]`.
|
|
||||||
///
|
|
||||||
/// # Arguments
|
|
||||||
///
|
|
||||||
/// * `engine` - The engine to use for retrieving data
|
|
||||||
pub fn aggregated_by(engine: &Engine) -> Vec<Aggregation> {
|
|
||||||
let mut result = Vec::new();
|
|
||||||
// for everything in the current stack
|
|
||||||
let len = engine.group_by_stack.len();
|
|
||||||
for (index, field) in engine.group_by_stack.iter().enumerate() {
|
|
||||||
let value = match (
|
|
||||||
len,
|
|
||||||
engine.segmentations.get(index).map(|e| e.selected.as_ref()),
|
|
||||||
) {
|
|
||||||
(n, Some(Some(segment))) if len == n => Some(segment.field.clone()),
|
|
||||||
_ => None,
|
|
||||||
};
|
|
||||||
result.push(Aggregation {
|
|
||||||
value,
|
|
||||||
field: *field,
|
|
||||||
index,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
result
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Change the `Field` in the given `Aggregation` to the new one.
|
|
||||||
///
|
|
||||||
/// The `Aggregation` will identify the `Segmentation` to use. So this function
|
|
||||||
/// can be used to change the way a `Segmentation` is the aggregated.
|
|
||||||
///
|
|
||||||
/// Retrieve the available aggregations with [`segmentation::aggregated_by`].
|
|
||||||
///
|
|
||||||
/// # Arguments
|
|
||||||
///
|
|
||||||
/// * `engine` - The engine to use for retrieving data
|
|
||||||
/// * `aggregation` - The aggregation to change
|
|
||||||
/// * `field` - The field to aggregate the `aggregation` by.
|
|
||||||
pub fn set_aggregation(
|
|
||||||
engine: &mut Engine,
|
|
||||||
aggregation: &Aggregation,
|
|
||||||
field: &Field,
|
|
||||||
) -> Result<()> {
|
|
||||||
if let Some(e) = engine.group_by_stack.get_mut(aggregation.index) {
|
|
||||||
*e = *field;
|
|
||||||
}
|
|
||||||
// Remove any rows that were cached for this Segmentation
|
|
||||||
engine.item_cache.clear();
|
|
||||||
engine
|
|
||||||
.link
|
|
||||||
.request(&make_query(engine)?, Action::RecalculateSegmentation)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Return the `Segment`s in the current `Segmentation`. Apply layout based on `Rect`.
|
|
||||||
///
|
|
||||||
/// It will perform the calculations so that all segments fit into bounds.
|
|
||||||
/// The results will be applied to each `Segment`.
|
|
||||||
///
|
|
||||||
/// Returns the layouted segments.
|
|
||||||
///
|
|
||||||
/// # Arguments
|
|
||||||
///
|
|
||||||
/// * `engine` - The engine to use for retrieving data
|
|
||||||
/// * `Rect` - The bounds into which the segments have to fit.
|
|
||||||
pub fn layouted_segments(engine: &mut Engine, bounds: eframe::egui::Rect) -> Option<&[Segment]> {
|
|
||||||
let segmentation = engine.segmentations.last_mut()?;
|
|
||||||
segmentation.update_layout(bounds);
|
|
||||||
Some(segmentation.items())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Can another level of aggregation be performed? Based on
|
|
||||||
/// [`Engine::default_group_by_stack`]
|
|
||||||
pub fn can_aggregate_more(engine: &Engine) -> bool {
|
|
||||||
let index = engine.group_by_stack.len();
|
|
||||||
super::engine::default_group_by_stack(index).is_some()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Perform the query that returns an aggregated `Segmentation`
|
|
||||||
pub(super) fn make_query(engine: &Engine) -> Result<Query> {
|
|
||||||
let mut filters = Vec::new();
|
|
||||||
for entry in &engine.search_stack {
|
|
||||||
filters.push(Filter::Like(entry.clone()));
|
|
||||||
}
|
|
||||||
for entry in &engine.filters {
|
|
||||||
filters.push(entry.clone());
|
|
||||||
}
|
|
||||||
let last = engine
|
|
||||||
.group_by_stack
|
|
||||||
.last()
|
|
||||||
.ok_or_else(|| eyre!("Invalid Segmentation state"))?;
|
|
||||||
Ok(Query::Grouped {
|
|
||||||
filters,
|
|
||||||
group_by: *last,
|
|
||||||
})
|
|
||||||
}
|
|
@ -1,27 +0,0 @@
|
|||||||
use crate::database::query::{Field, ValueField};
|
|
||||||
|
|
||||||
/// A aggregation field.
|
|
||||||
/// Contains the `Field` to aggregate by, the `Value` used for aggregation
|
|
||||||
/// As well as the index in the stack of Segmentations that this relates to.
|
|
||||||
pub struct Aggregation {
|
|
||||||
pub(in super::super) value: Option<ValueField>,
|
|
||||||
pub(in super::super) field: Field,
|
|
||||||
pub(in super::super) index: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Aggregation {
|
|
||||||
/// Return the value in this aggregation as a string
|
|
||||||
pub fn value(&self) -> Option<String> {
|
|
||||||
self.value.as_ref().map(|e| e.value().to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The name of the field as a `String`
|
|
||||||
pub fn name(&self) -> &str {
|
|
||||||
self.field.name()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The indes of the field within the given fields
|
|
||||||
pub fn index(&self, in_fields: &[Field]) -> Option<usize> {
|
|
||||||
in_fields.iter().position(|p| p == &self.field)
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,8 +0,0 @@
|
|||||||
use crate::database::query_result::QueryRow;
|
|
||||||
|
|
||||||
/// Is a individual row/item being loaded or already loaded.
|
|
||||||
/// Used in a cache to improve the loading of data for the UI.
|
|
||||||
pub enum LoadingState {
|
|
||||||
Loaded(QueryRow),
|
|
||||||
Loading,
|
|
||||||
}
|
|
@ -1,9 +0,0 @@
|
|||||||
mod aggregation;
|
|
||||||
mod loading_state;
|
|
||||||
mod segment;
|
|
||||||
mod segmentation;
|
|
||||||
|
|
||||||
pub use aggregation::Aggregation;
|
|
||||||
pub use loading_state::LoadingState;
|
|
||||||
pub use segment::*;
|
|
||||||
pub use segmentation::*;
|
|
@ -1,61 +0,0 @@
|
|||||||
use std::convert::TryFrom;
|
|
||||||
|
|
||||||
use eframe::egui::Rect as EguiRect;
|
|
||||||
use eyre::{Report, Result};
|
|
||||||
use treemap::{Mappable, Rect};
|
|
||||||
|
|
||||||
use crate::database::{query::ValueField, query_result::QueryResult};
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct Segment {
|
|
||||||
pub field: ValueField,
|
|
||||||
pub count: usize,
|
|
||||||
/// A TreeMap Rect
|
|
||||||
pub rect: Rect,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Segment {
|
|
||||||
/// Perform rect conversion from TreeMap to Egui
|
|
||||||
pub fn layout_rect(&self) -> EguiRect {
|
|
||||||
use eframe::egui::pos2;
|
|
||||||
EguiRect {
|
|
||||||
min: pos2(self.rect.x as f32, self.rect.y as f32),
|
|
||||||
max: pos2(
|
|
||||||
self.rect.x as f32 + self.rect.w as f32,
|
|
||||||
self.rect.y as f32 + self.rect.h as f32,
|
|
||||||
),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Mappable for Segment {
|
|
||||||
fn size(&self) -> f64 {
|
|
||||||
self.count as f64
|
|
||||||
}
|
|
||||||
|
|
||||||
fn bounds(&self) -> &Rect {
|
|
||||||
&self.rect
|
|
||||||
}
|
|
||||||
|
|
||||||
fn set_bounds(&mut self, bounds: Rect) {
|
|
||||||
self.rect = bounds;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a> TryFrom<&'a QueryResult> for Segment {
|
|
||||||
type Error = Report;
|
|
||||||
fn try_from(result: &'a QueryResult) -> Result<Self> {
|
|
||||||
let (count, field) = match result {
|
|
||||||
QueryResult::Grouped { count, value } => (count, value),
|
|
||||||
_ => return Err(eyre::eyre!("Invalid result type, expected `Grouped`")),
|
|
||||||
};
|
|
||||||
// so far we can only support one group by at a time.
|
|
||||||
// at least in here. The queries support it
|
|
||||||
|
|
||||||
Ok(Segment {
|
|
||||||
field: field.clone(),
|
|
||||||
count: *count,
|
|
||||||
rect: Rect::new(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,58 +0,0 @@
|
|||||||
use eframe::egui::Rect as EguiRect;
|
|
||||||
use treemap::{Rect, TreemapLayout};
|
|
||||||
|
|
||||||
use super::segment::Segment;
|
|
||||||
|
|
||||||
/// A small NewType so that we can keep all the `TreeMap` code in here and don't
|
|
||||||
/// have to do the layout calculation in a widget.
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct Segmentation {
|
|
||||||
items: Vec<Segment>,
|
|
||||||
pub selected: Option<Segment>,
|
|
||||||
pub range: Option<std::ops::RangeInclusive<usize>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Segmentation {
|
|
||||||
pub fn new(items: Vec<Segment>) -> Self {
|
|
||||||
Self {
|
|
||||||
items,
|
|
||||||
selected: None,
|
|
||||||
range: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn len(&self) -> usize {
|
|
||||||
self.items.len()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Update the layout information in the Segments
|
|
||||||
/// based on the current size
|
|
||||||
pub fn update_layout(&mut self, rect: EguiRect) {
|
|
||||||
let layout = TreemapLayout::new();
|
|
||||||
let bounds = Rect::from_points(
|
|
||||||
rect.left() as f64,
|
|
||||||
rect.top() as f64,
|
|
||||||
rect.width() as f64,
|
|
||||||
rect.height() as f64,
|
|
||||||
);
|
|
||||||
layout.layout_items(self.items(), bounds);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The total amount of items in all the `Segments`.
|
|
||||||
/// E.g. the sum of the count of the `Segments`
|
|
||||||
pub fn element_count(&self) -> usize {
|
|
||||||
self.items.iter().map(|e| e.count).sum::<usize>()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The items in this `Segmentation`, with range applied
|
|
||||||
pub fn items(&mut self) -> &mut [Segment] {
|
|
||||||
match &self.range {
|
|
||||||
Some(n) => {
|
|
||||||
// we reverse the range
|
|
||||||
let reversed_range = (self.len() - n.end())..=(self.len() - 1);
|
|
||||||
&mut self.items[reversed_range]
|
|
||||||
}
|
|
||||||
None => self.items.as_mut_slice(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,200 +0,0 @@
|
|||||||
use eyre::{eyre, Result};
|
|
||||||
use rand::Rng;
|
|
||||||
use serde_json::Value;
|
|
||||||
use strum::{self, IntoEnumIterator};
|
|
||||||
use strum_macros::{EnumIter, IntoStaticStr};
|
|
||||||
|
|
||||||
use std::collections::{HashMap, HashSet};
|
|
||||||
use std::iter::FromIterator;
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
use std::str::FromStr;
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, IntoStaticStr, EnumIter)]
|
|
||||||
pub enum FormatType {
|
|
||||||
AppleMail,
|
|
||||||
GmailVault,
|
|
||||||
Mbox,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl FormatType {
|
|
||||||
pub fn all_cases() -> impl Iterator<Item = FormatType> {
|
|
||||||
FormatType::iter()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn name(&self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
FormatType::AppleMail => "Apple Mail",
|
|
||||||
FormatType::GmailVault => "Gmail Vault Download",
|
|
||||||
FormatType::Mbox => "Mbox",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Forward the importer format location
|
|
||||||
pub fn default_path(&self) -> Option<PathBuf> {
|
|
||||||
use crate::importer::formats::{self, ImporterFormat};
|
|
||||||
match self {
|
|
||||||
FormatType::AppleMail => formats::AppleMail::default_path(),
|
|
||||||
FormatType::GmailVault => formats::Gmail::default_path(),
|
|
||||||
FormatType::Mbox => formats::Mbox::default_path(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for FormatType {
|
|
||||||
/// We return a different default, based on the platform we're on
|
|
||||||
/// FIXME: We don't have support for Outlook yet, so on windows we go with Mbox as well
|
|
||||||
fn default() -> Self {
|
|
||||||
#[cfg(target_os = "macos")]
|
|
||||||
return FormatType::AppleMail;
|
|
||||||
|
|
||||||
#[cfg(not(target_os = "macos"))]
|
|
||||||
return FormatType::Mbox;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<&String> for FormatType {
|
|
||||||
fn from(format: &String) -> Self {
|
|
||||||
FormatType::from(format.as_str())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<&str> for FormatType {
|
|
||||||
fn from(format: &str) -> Self {
|
|
||||||
match format {
|
|
||||||
"apple" => FormatType::AppleMail,
|
|
||||||
"gmailvault" => FormatType::GmailVault,
|
|
||||||
"mbox" => FormatType::Mbox,
|
|
||||||
_ => panic!("Unknown format: {}", &format),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<FormatType> for String {
|
|
||||||
fn from(format: FormatType) -> Self {
|
|
||||||
match format {
|
|
||||||
FormatType::AppleMail => "apple".to_owned(),
|
|
||||||
FormatType::GmailVault => "gmailvault".to_owned(),
|
|
||||||
FormatType::Mbox => "mbox".to_owned(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct Config {
|
|
||||||
/// The path to where the database should be stored
|
|
||||||
pub database_path: PathBuf,
|
|
||||||
/// The path where the emails are
|
|
||||||
pub emails_folder_path: PathBuf,
|
|
||||||
/// The addresses used to send emails
|
|
||||||
pub sender_emails: HashSet<String>,
|
|
||||||
/// The importer format we're using
|
|
||||||
pub format: FormatType,
|
|
||||||
/// Did the user intend to keep the database
|
|
||||||
/// (e.g. is the database path temporary?)
|
|
||||||
pub persistent: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Config {
|
|
||||||
/// Construct a config from a hashmap of field values.
|
|
||||||
/// For missing fields, take a reasonable default value,
|
|
||||||
/// in order to be somewhat backwards compatible.
|
|
||||||
pub fn from_fields<P: AsRef<Path>>(path: P, fields: HashMap<String, Value>) -> Result<Config> {
|
|
||||||
// The following fields are of version 1.0, so they should aways exist
|
|
||||||
let emails_folder_path_str = fields
|
|
||||||
.get("emails_folder_path")
|
|
||||||
.ok_or_else(|| eyre!("Missing config field emails_folder_path"))?
|
|
||||||
.as_str()
|
|
||||||
.ok_or_else(|| eyre!("Invalid field type for emails_folder_path"))?;
|
|
||||||
let emails_folder_path = PathBuf::from_str(emails_folder_path_str).map_err(|e| {
|
|
||||||
eyre!(
|
|
||||||
"Invalid emails_folder_path: {}: {}",
|
|
||||||
&emails_folder_path_str,
|
|
||||||
e
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
#[allow(clippy::needless_collect)]
|
|
||||||
let sender_emails: Vec<String> = fields
|
|
||||||
.get("sender_emails")
|
|
||||||
.map(|v| v.as_str().map(|e| e.to_string()))
|
|
||||||
.flatten()
|
|
||||||
.ok_or_else(|| eyre!("Missing config field sender_emails"))?
|
|
||||||
.split(',')
|
|
||||||
.map(|e| e.trim().to_owned())
|
|
||||||
.collect();
|
|
||||||
let format = fields
|
|
||||||
.get("format")
|
|
||||||
.map(|e| e.as_str())
|
|
||||||
.flatten()
|
|
||||||
.map(FormatType::from)
|
|
||||||
.ok_or_else(|| eyre!("Missing config field format_type"))?;
|
|
||||||
let persistent = fields
|
|
||||||
.get("persistent")
|
|
||||||
.map(|e| e.as_bool())
|
|
||||||
.flatten()
|
|
||||||
.ok_or_else(|| eyre!("Missing config field persistent"))?;
|
|
||||||
Ok(Config {
|
|
||||||
database_path: path.as_ref().to_path_buf(),
|
|
||||||
emails_folder_path,
|
|
||||||
sender_emails: HashSet::from_iter(sender_emails.into_iter()),
|
|
||||||
format,
|
|
||||||
persistent,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new<A: AsRef<Path>>(
|
|
||||||
db: Option<A>,
|
|
||||||
mails: A,
|
|
||||||
sender_emails: Vec<String>,
|
|
||||||
format: FormatType,
|
|
||||||
) -> eyre::Result<Self> {
|
|
||||||
// If we don't have a database path, we use a temporary folder.
|
|
||||||
let persistent = db.is_some();
|
|
||||||
let database_path = match db {
|
|
||||||
Some(n) => n.as_ref().to_path_buf(),
|
|
||||||
None => {
|
|
||||||
let number: u32 = rand::thread_rng().gen();
|
|
||||||
let folder = "postsack";
|
|
||||||
let filename = format!("{}.sqlite", number);
|
|
||||||
let mut temp_dir = std::env::temp_dir();
|
|
||||||
temp_dir.push(folder);
|
|
||||||
// the folder has to be created
|
|
||||||
std::fs::create_dir_all(&temp_dir)?;
|
|
||||||
temp_dir.push(filename);
|
|
||||||
temp_dir
|
|
||||||
}
|
|
||||||
};
|
|
||||||
Ok(Config {
|
|
||||||
database_path,
|
|
||||||
emails_folder_path: mails.as_ref().to_path_buf(),
|
|
||||||
sender_emails: HashSet::from_iter(sender_emails.into_iter()),
|
|
||||||
format,
|
|
||||||
persistent,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn into_fields(&self) -> Option<HashMap<String, Value>> {
|
|
||||||
let mut new = HashMap::new();
|
|
||||||
new.insert(
|
|
||||||
"database_path".to_owned(),
|
|
||||||
self.database_path.to_str()?.into(),
|
|
||||||
);
|
|
||||||
new.insert(
|
|
||||||
"emails_folder_path".to_owned(),
|
|
||||||
self.emails_folder_path.to_str()?.into(),
|
|
||||||
);
|
|
||||||
new.insert("persistent".to_owned(), self.persistent.into());
|
|
||||||
new.insert(
|
|
||||||
"sender_emails".to_owned(),
|
|
||||||
self.sender_emails
|
|
||||||
.iter()
|
|
||||||
.cloned()
|
|
||||||
.collect::<Vec<String>>()
|
|
||||||
.join(",")
|
|
||||||
.into(),
|
|
||||||
);
|
|
||||||
let format: String = self.format.into();
|
|
||||||
new.insert("format".to_owned(), format.into());
|
|
||||||
|
|
||||||
Some(new)
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,3 +0,0 @@
|
|||||||
mod config;
|
|
||||||
|
|
||||||
pub use config::{Config, FormatType};
|
|
Loading…
Reference in New Issue