Add search support (#131)

pull/137/head
Chip Senkbeil 2 years ago committed by GitHub
parent 5130ee3b5f
commit 01610a3ac7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -13,6 +13,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `SystemInfo` via ssh backend now reports os when windows detected
- `Capabilities` request/response for server and manager that report back the
capabilities (and descriptions) supported by the server or manager
- `Search` and `CancelSearch` request/response for server that performs a
search using `grep` crate against paths or file contents, returning results
back as a stream
- New `Searcher` available as part of distant client interface to support
performing a search and getting back results
- Updated `DistantChannelExt` to support creating a `Searcher` and canceling
an ongoing search query
- `distant client action search` now supported, waiting for results and
printing them out
### Changed
@ -37,9 +46,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- `shutdown-after` replaced with `shutdown` that supports three options:
1. `never` - server will never shutdown automatically
2. `after=N` - server will shutdown after N seconds
3. `lonely=N` - server will shutdown N seconds after no connections
1. `never` - server will never shutdown automatically
2. `after=N` - server will shutdown after N seconds
3. `lonely=N` - server will shutdown N seconds after no connections
## [0.17.6] - 2022-08-18
### Fixed

112
Cargo.lock generated

@ -761,6 +761,7 @@ dependencies = [
"distant-net",
"flexi_logger",
"futures",
"grep",
"hex",
"indoc",
"log",
@ -905,6 +906,24 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "encoding_rs"
version = "0.8.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"
dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "encoding_rs_io"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83"
dependencies = [
"encoding_rs",
]
[[package]]
name = "err-derive"
version = "0.3.1"
@ -1228,6 +1247,90 @@ dependencies = [
"walkdir",
]
[[package]]
name = "grep"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cea81f81c4c120466aef365c225a05c707b002a20f2b9fe3287124b809a8d4f"
dependencies = [
"grep-cli",
"grep-matcher",
"grep-printer",
"grep-regex",
"grep-searcher",
]
[[package]]
name = "grep-cli"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dd110c34bb4460d0de5062413b773e385cbf8a85a63fc535590110a09e79e8a"
dependencies = [
"atty",
"bstr 0.2.17",
"globset",
"lazy_static",
"log",
"regex",
"same-file",
"termcolor",
"winapi-util",
]
[[package]]
name = "grep-matcher"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d27563c33062cd33003b166ade2bb4fd82db1fd6a86db764dfdad132d46c1cc"
dependencies = [
"memchr",
]
[[package]]
name = "grep-printer"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05c271a24daedf5675b61a275a1d0af06e03312ab7856d15433ae6cde044dc72"
dependencies = [
"base64",
"bstr 0.2.17",
"grep-matcher",
"grep-searcher",
"serde",
"serde_json",
"termcolor",
]
[[package]]
name = "grep-regex"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1345f8d33c89f2d5b081f2f2a41175adef9fd0bed2fea6a26c96c2deb027e58e"
dependencies = [
"aho-corasick",
"bstr 0.2.17",
"grep-matcher",
"log",
"regex",
"regex-syntax",
"thread_local",
]
[[package]]
name = "grep-searcher"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48852bd08f9b4eb3040ecb6d2f4ade224afe880a9a0909c5563cc59fa67932cc"
dependencies = [
"bstr 0.2.17",
"bytecount",
"encoding_rs",
"encoding_rs_io",
"grep-matcher",
"log",
"memmap2",
]
[[package]]
name = "group"
version = "0.12.0"
@ -1513,6 +1616,15 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "memmap2"
version = "0.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95af15f345b17af2efc8ead6080fb8bc376f8cec1b35277b935637595fe77498"
dependencies = [
"libc",
]
[[package]]
name = "memmem"
version = "0.1.1"

@ -27,7 +27,9 @@ talk to the server.
[RustCrypto/ChaCha20Poly1305](https://github.com/RustCrypto/AEADs/tree/master/chacha20poly1305)
Additionally, the core of the distant client and server codebase can be pulled
in to be used with your own Rust crates via the `distant-core` crate.
in to be used with your own Rust crates via the `distant-core` crate. The
networking library, which is agnostic of `distant` protocols, can be used via
the `distant-net` crate.
## Installation
@ -48,6 +50,41 @@ cargo install distant
Alternatively, you can clone this repository and build from source following
the [build guide](./BUILDING.md).
## Backend Feature Matrix
Distant supports multiple backends to facilitate remote communication with
another server. Today, these backends include:
* `distant` - a standalone server acting as the reference implementation
* `ssh` - a wrapper around an `ssh` client that translates the distant protocol
into ssh server requests
Not every backend supports every feature of distant. Below is a table outlining
the available features and which backend supports each feature:
| Feature | distant | ssh |
| --------------------- | --------| ----|
| Capabilities | ✅ | ✅ |
| Filesystem I/O | ✅ | ✅ |
| Filesystem Watching | ✅ | ✅ |
| Process Execution | ✅ | ✅ |
| Search | ✅ | ❌ |
| System Information | ✅ | ⚠ |
* ✅ means full support
* ⚠ means partial support
* ❌ means no support
### Feature Details
* `Capabilities` - able to report back what it is capable of performing
* `Filesystem I/O` - able to read from and write to the filesystem
* `Filesystem Watching` - able to receive notifications when changes to the
filesystem occur
* `Process Execution` - able to execute processes
* `Search` - able to search the filesystem
* `System Information` - able to retrieve information about the system
## Example
### Starting the manager

@ -21,6 +21,7 @@ bytes = "1.2.1"
derive_more = { version = "0.99.17", default-features = false, features = ["as_mut", "as_ref", "deref", "deref_mut", "display", "from", "error", "into", "into_iterator", "is_variant", "try_into"] }
distant-net = { version = "=0.18.0", path = "../distant-net" }
futures = "0.3.21"
grep = "0.2.10"
hex = "0.4.3"
log = "0.4.17"
notify = { version = "=5.0.0-pre.15", features = ["serde"] }

@ -1,7 +1,7 @@
use crate::{
data::{
Capabilities, ChangeKind, DirEntry, Environment, Error, Metadata, ProcessId, PtySize,
SystemInfo,
SearchId, SearchQuery, SystemInfo,
},
ConnectionId, DistantMsg, DistantRequestData, DistantResponseData,
};
@ -317,6 +317,34 @@ pub trait DistantApi {
unsupported("metadata")
}
/// Searches files for matches based on a query.
///
/// * `query` - the specific query to perform
///
/// *Override this, otherwise it will return "unsupported" as an error.*
#[allow(unused_variables)]
async fn search(
&self,
ctx: DistantCtx<Self::LocalData>,
query: SearchQuery,
) -> io::Result<SearchId> {
unsupported("search")
}
/// Cancels an actively-ongoing search.
///
/// * `id` - the id of the search to cancel
///
/// *Override this, otherwise it will return "unsupported" as an error.*
#[allow(unused_variables)]
async fn cancel_search(
&self,
ctx: DistantCtx<Self::LocalData>,
id: SearchId,
) -> io::Result<()> {
unsupported("cancel_search")
}
/// Spawns a new process, returning its id.
///
/// * `cmd` - the full command to run as a new process (including arguments)
@ -613,6 +641,18 @@ where
.await
.map(DistantResponseData::Metadata)
.unwrap_or_else(DistantResponseData::from),
DistantRequestData::Search { query } => server
.api
.search(ctx, query)
.await
.map(|id| DistantResponseData::SearchStarted { id })
.unwrap_or_else(DistantResponseData::from),
DistantRequestData::CancelSearch { id } => server
.api
.cancel_search(ctx, id)
.await
.map(|_| DistantResponseData::Ok)
.unwrap_or_else(DistantResponseData::from),
DistantRequestData::ProcSpawn {
cmd,
environment,

@ -1,7 +1,7 @@
use crate::{
data::{
Capabilities, ChangeKind, ChangeKindSet, DirEntry, Environment, FileType, Metadata,
ProcessId, PtySize, SystemInfo,
ProcessId, PtySize, SearchId, SearchQuery, SystemInfo,
},
DistantApi, DistantCtx,
};
@ -427,6 +427,29 @@ impl DistantApi for LocalDistantApi {
Metadata::read(path, canonicalize, resolve_file_type).await
}
async fn search(
&self,
ctx: DistantCtx<Self::LocalData>,
query: SearchQuery,
) -> io::Result<SearchId> {
debug!(
"[Conn {}] Performing search via {query:?}",
ctx.connection_id,
);
self.state.search.start(query, ctx.reply).await
}
async fn cancel_search(
&self,
ctx: DistantCtx<Self::LocalData>,
id: SearchId,
) -> io::Result<()> {
debug!("[Conn {}] Cancelling search {id}", ctx.connection_id,);
self.state.search.cancel(id).await
}
async fn proc_spawn(
&self,
ctx: DistantCtx<Self::LocalData>,

@ -1,9 +1,15 @@
use crate::{data::ProcessId, ConnectionId};
use crate::{
data::{ProcessId, SearchId},
ConnectionId,
};
use std::{io, path::PathBuf};
mod process;
pub use process::*;
mod search;
pub use search::*;
mod watcher;
pub use watcher::*;
@ -12,6 +18,9 @@ pub struct GlobalState {
/// State that holds information about processes running on the server
pub process: ProcessState,
/// State that holds information about searches running on the server
pub search: SearchState,
/// Watcher used for filesystem events
pub watcher: WatcherState,
}
@ -20,6 +29,7 @@ impl GlobalState {
pub fn initialize() -> io::Result<Self> {
Ok(Self {
process: ProcessState::new(),
search: SearchState::new(),
watcher: WatcherState::initialize()?,
})
}
@ -34,6 +44,9 @@ pub struct ConnectionState {
/// Channel connected to global process state
pub(crate) process_channel: ProcessChannel,
/// Channel connected to global search state
pub(crate) search_channel: SearchChannel,
/// Channel connected to global watcher state
pub(crate) watcher_channel: WatcherChannel,
@ -42,6 +55,9 @@ pub struct ConnectionState {
/// Contains paths being watched that will be unwatched when the connection is closed
paths: Vec<PathBuf>,
/// Contains ids of searches that will be terminated when the connection is closed
searches: Vec<SearchId>,
}
impl Drop for ConnectionState {
@ -49,8 +65,10 @@ impl Drop for ConnectionState {
let id = self.id;
let processes: Vec<ProcessId> = self.processes.drain(..).collect();
let paths: Vec<PathBuf> = self.paths.drain(..).collect();
let searches: Vec<SearchId> = self.searches.drain(..).collect();
let process_channel = self.process_channel.clone();
let search_channel = self.search_channel.clone();
let watcher_channel = self.watcher_channel.clone();
// NOTE: We cannot (and should not) block during drop to perform cleanup,
@ -60,6 +78,10 @@ impl Drop for ConnectionState {
let _ = process_channel.kill(id).await;
}
for id in searches {
let _ = search_channel.cancel(id).await;
}
for path in paths {
let _ = watcher_channel.unwatch(id, path).await;
}

File diff suppressed because it is too large Load Diff

@ -4,6 +4,7 @@ use distant_net::{Channel, Client};
mod ext;
mod lsp;
mod process;
mod searcher;
mod watcher;
/// Represents a [`Client`] that communicates using the distant protocol
@ -15,4 +16,5 @@ pub type DistantChannel = Channel<DistantMsg<DistantRequestData>, DistantMsg<Dis
pub use ext::*;
pub use lsp::*;
pub use process::*;
pub use searcher::*;
pub use watcher::*;

@ -1,10 +1,11 @@
use crate::{
client::{
RemoteCommand, RemoteLspCommand, RemoteLspProcess, RemoteOutput, RemoteProcess, Watcher,
RemoteCommand, RemoteLspCommand, RemoteLspProcess, RemoteOutput, RemoteProcess, Searcher,
Watcher,
},
data::{
Capabilities, ChangeKindSet, DirEntry, DistantRequestData, DistantResponseData,
Environment, Error as Failure, Metadata, PtySize, SystemInfo,
Environment, Error as Failure, Metadata, PtySize, SearchId, SearchQuery, SystemInfo,
},
DistantMsg,
};
@ -18,7 +19,7 @@ fn mismatched_response() -> io::Error {
io::Error::new(io::ErrorKind::Other, "Mismatched response")
}
/// Provides convenience functions on top of a [`SessionChannel`]
/// Provides convenience functions on top of a [`Channel`]
pub trait DistantChannelExt {
/// Appends to a remote file using the data from a collection of bytes
fn append_file(
@ -53,6 +54,12 @@ pub trait DistantChannelExt {
resolve_file_type: bool,
) -> AsyncReturn<'_, Metadata>;
/// Perform a search
fn search(&mut self, query: impl Into<SearchQuery>) -> AsyncReturn<'_, Searcher>;
/// Cancel an active search query
fn cancel_search(&mut self, id: SearchId) -> AsyncReturn<'_, ()>;
/// Reads entries from a directory, returning a tuple of directory entries and failures
fn read_dir(
&mut self,
@ -249,6 +256,19 @@ impl DistantChannelExt
)
}
fn search(&mut self, query: impl Into<SearchQuery>) -> AsyncReturn<'_, Searcher> {
let query = query.into();
Box::pin(async move { Searcher::search(self.clone(), query).await })
}
fn cancel_search(&mut self, id: SearchId) -> AsyncReturn<'_, ()> {
make_body!(
self,
DistantRequestData::CancelSearch { id },
@ok
)
}
fn read_dir(
&mut self,
path: impl Into<PathBuf>,

@ -0,0 +1,626 @@
use crate::{
client::{DistantChannel, DistantChannelExt},
constants::CLIENT_SEARCHER_CAPACITY,
data::{DistantRequestData, DistantResponseData, SearchId, SearchQuery, SearchQueryMatch},
DistantMsg,
};
use distant_net::Request;
use log::*;
use std::{fmt, io};
use tokio::{sync::mpsc, task::JoinHandle};
/// Represents a searcher for files, directories, and symlinks on the filesystem
pub struct Searcher {
channel: DistantChannel,
id: SearchId,
query: SearchQuery,
task: JoinHandle<()>,
rx: mpsc::Receiver<SearchQueryMatch>,
}
impl fmt::Debug for Searcher {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Searcher")
.field("id", &self.id)
.field("query", &self.query)
.finish()
}
}
impl Searcher {
/// Creates a searcher for some query
pub async fn search(mut channel: DistantChannel, query: SearchQuery) -> io::Result<Self> {
trace!("Searching using {query:?}",);
// Submit our run request and get back a mailbox for responses
let mut mailbox = channel
.mail(Request::new(DistantMsg::Single(
DistantRequestData::Search {
query: query.clone(),
},
)))
.await?;
let (tx, rx) = mpsc::channel(CLIENT_SEARCHER_CAPACITY);
// Wait to get the confirmation of watch as either ok or error
let mut queue: Vec<SearchQueryMatch> = Vec::new();
let mut search_id = None;
while let Some(res) = mailbox.next().await {
for data in res.payload.into_vec() {
match data {
// If we get results before the started indicator, queue them up
DistantResponseData::SearchResults { matches, .. } => {
queue.extend(matches);
}
// Once we get the started indicator, mark as ready to go
DistantResponseData::SearchStarted { id } => {
trace!("[Query {id}] Searcher has started");
search_id = Some(id);
}
// If we get an explicit error, convert and return it
DistantResponseData::Error(x) => return Err(io::Error::from(x)),
// Otherwise, we got something unexpected, and report as such
x => {
return Err(io::Error::new(
io::ErrorKind::Other,
format!("Unexpected response: {:?}", x),
))
}
}
}
// Exit if we got the confirmation
// NOTE: Doing this later because we want to make sure the entire payload is processed
// first before exiting the loop
if search_id.is_some() {
break;
}
}
let search_id = match search_id {
// Send out any of our queued changes that we got prior to the acknowledgement
Some(id) => {
trace!("[Query {id}] Forwarding {} queued matches", queue.len());
for r#match in queue.drain(..) {
if tx.send(r#match).await.is_err() {
return Err(io::Error::new(
io::ErrorKind::Other,
format!("[Query {id}] Queue search match dropped"),
));
}
}
id
}
// If we never received an acknowledgement of search before the mailbox closed,
// fail with a missing confirmation error
None => {
return Err(io::Error::new(
io::ErrorKind::Other,
"Search query missing started confirmation",
))
}
};
// Spawn a task that continues to look for search result events and the conclusion of the
// search, discarding anything else that it gets
let task = tokio::spawn({
async move {
while let Some(res) = mailbox.next().await {
let mut done = false;
for data in res.payload.into_vec() {
match data {
DistantResponseData::SearchResults { matches, .. } => {
// If we can't queue up a match anymore, we've
// been closed and therefore want to quit
if tx.is_closed() {
break;
}
// Otherwise, send over the matches
for r#match in matches {
if let Err(x) = tx.send(r#match).await {
error!(
"[Query {search_id}] Searcher failed to send match {:?}",
x.0
);
break;
}
}
}
// Received completion indicator, so close out
DistantResponseData::SearchDone { .. } => {
trace!("[Query {search_id}] Searcher has finished");
done = true;
break;
}
_ => continue,
}
}
if done {
break;
}
}
}
});
Ok(Self {
id: search_id,
query,
channel,
task,
rx,
})
}
/// Returns a reference to the query this searcher is running
pub fn query(&self) -> &SearchQuery {
&self.query
}
/// Returns true if the searcher is still actively searching
pub fn is_active(&self) -> bool {
!self.task.is_finished()
}
/// Returns the next match detected by the searcher, or none if the searcher has concluded
pub async fn next(&mut self) -> Option<SearchQueryMatch> {
self.rx.recv().await
}
/// Cancels the search being performed by the watcher
pub async fn cancel(&mut self) -> io::Result<()> {
trace!("[Query {}] Cancelling search", self.id);
self.channel.cancel_search(self.id).await?;
// Kill our task that processes inbound matches if we have successfully stopped searching
self.task.abort();
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::data::{
SearchQueryCondition, SearchQueryMatchData, SearchQueryOptions, SearchQueryPathMatch,
SearchQuerySubmatch, SearchQueryTarget,
};
use crate::DistantClient;
use distant_net::{
Client, FramedTransport, InmemoryTransport, IntoSplit, PlainCodec, Response,
TypedAsyncRead, TypedAsyncWrite,
};
use std::{path::PathBuf, sync::Arc};
use tokio::sync::Mutex;
fn make_session() -> (
FramedTransport<InmemoryTransport, PlainCodec>,
DistantClient,
) {
let (t1, t2) = FramedTransport::pair(100);
let (writer, reader) = t2.into_split();
(t1, Client::new(writer, reader).unwrap())
}
#[tokio::test]
async fn searcher_should_have_query_reflect_ongoing_query() {
let (mut transport, session) = make_session();
let test_query = SearchQuery {
path: PathBuf::from("/some/test/path"),
target: SearchQueryTarget::Path,
condition: SearchQueryCondition::Regex {
value: String::from("."),
},
options: SearchQueryOptions::default(),
};
// Create a task for searcher as we need to handle the request and a response
// in a separate async block
let search_task = {
let test_query = test_query.clone();
tokio::spawn(async move { Searcher::search(session.clone_channel(), test_query).await })
};
// Wait until we get the request from the session
let req: Request<DistantRequestData> = transport.read().await.unwrap().unwrap();
// Send back an acknowledgement that a search was started
transport
.write(Response::new(
req.id,
DistantResponseData::SearchStarted { id: rand::random() },
))
.await
.unwrap();
// Get the searcher and verify the query
let searcher = search_task.await.unwrap().unwrap();
assert_eq!(searcher.query(), &test_query);
}
#[tokio::test]
async fn searcher_should_support_getting_next_match() {
let (mut transport, session) = make_session();
let test_query = SearchQuery {
path: PathBuf::from("/some/test/path"),
target: SearchQueryTarget::Path,
condition: SearchQueryCondition::Regex {
value: String::from("."),
},
options: SearchQueryOptions::default(),
};
// Create a task for searcher as we need to handle the request and a response
// in a separate async block
let search_task =
tokio::spawn(
async move { Searcher::search(session.clone_channel(), test_query).await },
);
// Wait until we get the request from the session
let req: Request<DistantRequestData> = transport.read().await.unwrap().unwrap();
// Send back an acknowledgement that a searcher was created
let id = rand::random::<SearchId>();
transport
.write(Response::new(
req.id.clone(),
DistantResponseData::SearchStarted { id },
))
.await
.unwrap();
// Get the searcher
let mut searcher = search_task.await.unwrap().unwrap();
// Send some matches related to the file
transport
.write(Response::new(
req.id,
vec![
DistantResponseData::SearchResults {
id,
matches: vec![
SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/1"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match".to_string()),
start: 3,
end: 7,
}],
}),
SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/2"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match 2".to_string()),
start: 88,
end: 99,
}],
}),
],
},
DistantResponseData::SearchResults {
id,
matches: vec![SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/3"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match 3".to_string()),
start: 5,
end: 9,
}],
})],
},
],
))
.await
.unwrap();
// Verify that the searcher gets the matches, one at a time
let m = searcher.next().await.expect("Searcher closed unexpectedly");
assert_eq!(
m,
SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/1"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match".to_string()),
start: 3,
end: 7,
}],
})
);
let m = searcher.next().await.expect("Searcher closed unexpectedly");
assert_eq!(
m,
SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/2"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match 2".to_string()),
start: 88,
end: 99,
}],
}),
);
let m = searcher.next().await.expect("Searcher closed unexpectedly");
assert_eq!(
m,
SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/3"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match 3".to_string()),
start: 5,
end: 9,
}],
})
);
}
#[tokio::test]
async fn searcher_should_distinguish_match_events_and_only_receive_matches_for_itself() {
let (mut transport, session) = make_session();
let test_query = SearchQuery {
path: PathBuf::from("/some/test/path"),
target: SearchQueryTarget::Path,
condition: SearchQueryCondition::Regex {
value: String::from("."),
},
options: SearchQueryOptions::default(),
};
// Create a task for searcher as we need to handle the request and a response
// in a separate async block
let search_task =
tokio::spawn(
async move { Searcher::search(session.clone_channel(), test_query).await },
);
// Wait until we get the request from the session
let req: Request<DistantRequestData> = transport.read().await.unwrap().unwrap();
// Send back an acknowledgement that a searcher was created
let id = rand::random();
transport
.write(Response::new(
req.id.clone(),
DistantResponseData::SearchStarted { id },
))
.await
.unwrap();
// Get the searcher
let mut searcher = search_task.await.unwrap().unwrap();
// Send a match from the appropriate origin
transport
.write(Response::new(
req.id.clone(),
DistantResponseData::SearchResults {
id,
matches: vec![SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/1"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match".to_string()),
start: 3,
end: 7,
}],
})],
},
))
.await
.unwrap();
// Send a chanmatchge from a different origin
transport
.write(Response::new(
req.id.clone() + "1",
DistantResponseData::SearchResults {
id,
matches: vec![SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/2"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match 2".to_string()),
start: 88,
end: 99,
}],
})],
},
))
.await
.unwrap();
// Send a chanmatchge from the appropriate origin
transport
.write(Response::new(
req.id,
DistantResponseData::SearchResults {
id,
matches: vec![SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/3"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match 3".to_string()),
start: 5,
end: 9,
}],
})],
},
))
.await
.unwrap();
// Verify that the searcher gets the matches, one at a time
let m = searcher.next().await.expect("Searcher closed unexpectedly");
assert_eq!(
m,
SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/1"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match".to_string()),
start: 3,
end: 7,
}],
})
);
let m = searcher.next().await.expect("Watcher closed unexpectedly");
assert_eq!(
m,
SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/3"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match 3".to_string()),
start: 5,
end: 9,
}],
})
);
}
#[tokio::test]
async fn searcher_should_stop_receiving_events_if_cancelled() {
let (mut transport, session) = make_session();
let test_query = SearchQuery {
path: PathBuf::from("/some/test/path"),
target: SearchQueryTarget::Path,
condition: SearchQueryCondition::Regex {
value: String::from("."),
},
options: SearchQueryOptions::default(),
};
// Create a task for searcher as we need to handle the request and a response
// in a separate async block
let search_task =
tokio::spawn(
async move { Searcher::search(session.clone_channel(), test_query).await },
);
// Wait until we get the request from the session
let req: Request<DistantRequestData> = transport.read().await.unwrap().unwrap();
// Send back an acknowledgement that a watcher was created
let id = rand::random::<SearchId>();
transport
.write(Response::new(
req.id.clone(),
DistantResponseData::SearchStarted { id },
))
.await
.unwrap();
// Send some matches from the appropriate origin
transport
.write(Response::new(
req.id,
DistantResponseData::SearchResults {
id,
matches: vec![
SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/1"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match".to_string()),
start: 3,
end: 7,
}],
}),
SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/2"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match 2".to_string()),
start: 88,
end: 99,
}],
}),
],
},
))
.await
.unwrap();
// Wait a little bit for all matches to be queued
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
// Create a task for for cancelling as we need to handle the request and a response
// in a separate async block
let searcher = Arc::new(Mutex::new(search_task.await.unwrap().unwrap()));
// Verify that the searcher gets the first match
let m = searcher
.lock()
.await
.next()
.await
.expect("Searcher closed unexpectedly");
assert_eq!(
m,
SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/1"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match".to_string()),
start: 3,
end: 7,
}],
}),
);
// Cancel the search, verify the request is sent out, and respond with ok
let searcher_2 = Arc::clone(&searcher);
let cancel_task = tokio::spawn(async move { searcher_2.lock().await.cancel().await });
let req: Request<DistantRequestData> = transport.read().await.unwrap().unwrap();
transport
.write(Response::new(req.id.clone(), DistantResponseData::Ok))
.await
.unwrap();
// Wait for the cancel to complete
cancel_task.await.unwrap().unwrap();
// Send a match that will get ignored
transport
.write(Response::new(
req.id,
DistantResponseData::SearchResults {
id,
matches: vec![SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/3"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match 3".to_string()),
start: 5,
end: 9,
}],
})],
},
))
.await
.unwrap();
// Verify that we get any remaining matches that were received before cancel,
// but nothing new after that
assert_eq!(
searcher.lock().await.next().await,
Some(SearchQueryMatch::Path(SearchQueryPathMatch {
path: PathBuf::from("/some/path/2"),
submatches: vec![SearchQuerySubmatch {
r#match: SearchQueryMatchData::Text("test match 2".to_string()),
start: 88,
end: 99,
}],
}))
);
assert_eq!(searcher.lock().await.next().await, None);
}
}

@ -4,6 +4,9 @@ pub const CLIENT_PIPE_CAPACITY: usize = 10000;
/// Capacity associated with a client watcher receiving changes
pub const CLIENT_WATCHER_CAPACITY: usize = 100;
/// Capacity associated with a client searcher receiving matches
pub const CLIENT_SEARCHER_CAPACITY: usize = 10000;
/// Capacity associated with the server's file watcher to pass events outbound
pub const SERVER_WATCHER_CAPACITY: usize = 10000;

@ -33,6 +33,9 @@ pub use metadata::*;
mod pty;
pub use pty::*;
mod search;
pub use search::*;
mod system;
pub use system::*;
@ -145,6 +148,7 @@ impl<T: schemars::JsonSchema> DistantMsg<T> {
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "clap", derive(clap::Subcommand))]
#[strum_discriminants(derive(
AsRefStr,
strum::Display,
EnumIter,
EnumMessage,
@ -381,6 +385,22 @@ pub enum DistantRequestData {
resolve_file_type: bool,
},
/// Searches filesystem using the provided query
#[strum_discriminants(strum(message = "Supports searching filesystem using queries"))]
Search {
/// Query to perform against the filesystem
query: SearchQuery,
},
/// Cancels an active search being run against the filesystem
#[strum_discriminants(strum(
message = "Supports canceling an active search against the filesystem"
))]
CancelSearch {
/// Id of the search to cancel
id: SearchId,
},
/// Spawns a new process on the remote machine
#[cfg_attr(feature = "clap", clap(visible_aliases = &["spawn", "run"]))]
#[strum_discriminants(strum(message = "Supports spawning a process"))]
@ -499,6 +519,27 @@ pub enum DistantResponseData {
/// Represents metadata about some filesystem object (file, directory, symlink) on remote machine
Metadata(Metadata),
/// Represents a search being started
SearchStarted {
/// Arbitrary id associated with search
id: SearchId,
},
/// Represents some subset of results for a search query (may not be all of them)
SearchResults {
/// Arbitrary id associated with search
id: SearchId,
/// Collection of matches from performing a query
matches: Vec<SearchQueryMatch>,
},
/// Represents a search being completed
SearchDone {
/// Arbitrary id associated with search
id: SearchId,
},
/// Response to starting a new process
ProcSpawned {
/// Arbitrary id associated with running process

@ -53,6 +53,15 @@ impl Capabilities {
self.0.take(&cap)
}
/// Removes the capability with the described kind, returning true if it existed
pub fn remove(&mut self, kind: impl AsRef<str>) -> bool {
let cap = Capability {
kind: kind.as_ref().to_string(),
description: String::new(),
};
self.0.remove(&cap)
}
/// Converts into vec of capabilities sorted by kind
pub fn into_sorted_vec(self) -> Vec<Capability> {
let mut this = self.0.into_iter().collect::<Vec<_>>();

@ -1,6 +1,6 @@
use derive_more::IsVariant;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use std::{fs::FileType as StdFileType, path::PathBuf};
use strum::AsRefStr;
/// Represents information about a single entry within a directory
@ -27,7 +27,7 @@ impl DirEntry {
}
/// Represents the type associated with a dir entry
#[derive(Copy, Clone, Debug, PartialEq, Eq, AsRefStr, IsVariant, Serialize, Deserialize)]
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, AsRefStr, IsVariant, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
#[strum(serialize_all = "snake_case")]
@ -37,6 +37,18 @@ pub enum FileType {
Symlink,
}
impl From<StdFileType> for FileType {
fn from(ft: StdFileType) -> Self {
if ft.is_dir() {
Self::Dir
} else if ft.is_symlink() {
Self::Symlink
} else {
Self::File
}
}
}
#[cfg(feature = "schemars")]
impl FileType {
pub fn root_schema() -> schemars::schema::RootSchema {

@ -0,0 +1,335 @@
use super::FileType;
use serde::{Deserialize, Serialize};
use std::{borrow::Cow, collections::HashSet, path::PathBuf, str::FromStr};
/// Id associated with a search
pub type SearchId = u32;
/// Represents a query to perform against the filesystem
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct SearchQuery {
/// Path in which to perform the query
pub path: PathBuf,
/// Kind of data to example using conditions
pub target: SearchQueryTarget,
/// Condition to meet to be considered a match
pub condition: SearchQueryCondition,
/// Options to apply to the query
#[serde(default)]
pub options: SearchQueryOptions,
}
#[cfg(feature = "schemars")]
impl SearchQuery {
pub fn root_schema() -> schemars::schema::RootSchema {
schemars::schema_for!(SearchQuery)
}
}
impl FromStr for SearchQuery {
type Err = serde_json::error::Error;
/// Parses search query from a JSON string
fn from_str(s: &str) -> Result<Self, Self::Err> {
serde_json::from_str(s)
}
}
/// Kind of data to examine using conditions
#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum SearchQueryTarget {
/// Checks path of file, directory, or symlink
Path,
/// Checks contents of files
Contents,
}
#[cfg(feature = "schemars")]
impl SearchQueryTarget {
pub fn root_schema() -> schemars::schema::RootSchema {
schemars::schema_for!(SearchQueryTarget)
}
}
/// Condition used to find a match in a search query
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case", deny_unknown_fields, tag = "type")]
pub enum SearchQueryCondition {
/// Begins with some text
EndsWith { value: String },
/// Matches some text exactly
Equals { value: String },
/// Matches some regex
Regex { value: String },
/// Begins with some text
StartsWith { value: String },
}
impl SearchQueryCondition {
/// Creates a new instance with `EndsWith` variant
pub fn ends_with(value: impl Into<String>) -> Self {
Self::EndsWith {
value: value.into(),
}
}
/// Creates a new instance with `Equals` variant
pub fn equals(value: impl Into<String>) -> Self {
Self::Equals {
value: value.into(),
}
}
/// Creates a new instance with `Regex` variant
pub fn regex(value: impl Into<String>) -> Self {
Self::Regex {
value: value.into(),
}
}
/// Creates a new instance with `StartsWith` variant
pub fn starts_with(value: impl Into<String>) -> Self {
Self::StartsWith {
value: value.into(),
}
}
/// Converts the condition in a regex string
pub fn to_regex_string(&self) -> String {
match self {
Self::EndsWith { value } => format!(r"{value}$"),
Self::Equals { value } => format!(r"^{value}$"),
Self::Regex { value } => value.to_string(),
Self::StartsWith { value } => format!(r"^{value}"),
}
}
}
#[cfg(feature = "schemars")]
impl SearchQueryCondition {
pub fn root_schema() -> schemars::schema::RootSchema {
schemars::schema_for!(SearchQueryCondition)
}
}
/// Options associated with a search query
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct SearchQueryOptions {
/// Restrict search to only these file types (otherwise all are allowed)
#[serde(default)]
pub allowed_file_types: HashSet<FileType>,
/// Regex to use to filter paths being searched to only those that match the include condition
#[serde(default)]
pub include: Option<SearchQueryCondition>,
/// Regex to use to filter paths being searched to only those that do not match the exclude
/// condition
#[serde(default)]
pub exclude: Option<SearchQueryCondition>,
/// Search should follow symbolic links
#[serde(default)]
pub follow_symbolic_links: bool,
/// Maximum results to return before stopping the query
#[serde(default)]
pub limit: Option<u64>,
/// Minimum depth (directories) to search
///
/// The smallest depth is 0 and always corresponds to the path given to the new function on
/// this type. Its direct descendents have depth 1, and their descendents have depth 2, and so
/// on.
#[serde(default)]
pub min_depth: Option<u64>,
/// Maximum depth (directories) to search
///
/// The smallest depth is 0 and always corresponds to the path given to the new function on
/// this type. Its direct descendents have depth 1, and their descendents have depth 2, and so
/// on.
///
/// Note that this will not simply filter the entries of the iterator, but it will actually
/// avoid descending into directories when the depth is exceeded.
#[serde(default)]
pub max_depth: Option<u64>,
/// Amount of results to batch before sending back excluding final submission that will always
/// include the remaining results even if less than pagination request
#[serde(default)]
pub pagination: Option<u64>,
}
#[cfg(feature = "schemars")]
impl SearchQueryOptions {
pub fn root_schema() -> schemars::schema::RootSchema {
schemars::schema_for!(SearchQueryOptions)
}
}
/// Represents a match for a search query
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case", deny_unknown_fields, tag = "type")]
pub enum SearchQueryMatch {
/// Matches part of a file's path
Path(SearchQueryPathMatch),
/// Matches part of a file's contents
Contents(SearchQueryContentsMatch),
}
impl SearchQueryMatch {
pub fn into_path_match(self) -> Option<SearchQueryPathMatch> {
match self {
Self::Path(x) => Some(x),
_ => None,
}
}
pub fn into_contents_match(self) -> Option<SearchQueryContentsMatch> {
match self {
Self::Contents(x) => Some(x),
_ => None,
}
}
}
#[cfg(feature = "schemars")]
impl SearchQueryMatch {
pub fn root_schema() -> schemars::schema::RootSchema {
schemars::schema_for!(SearchQueryMatch)
}
}
/// Represents details for a match on a path
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct SearchQueryPathMatch {
/// Path associated with the match
pub path: PathBuf,
/// Collection of matches tied to `path` where each submatch's byte offset is relative to
/// `path`
pub submatches: Vec<SearchQuerySubmatch>,
}
#[cfg(feature = "schemars")]
impl SearchQueryPathMatch {
pub fn root_schema() -> schemars::schema::RootSchema {
schemars::schema_for!(SearchQueryPathMatch)
}
}
/// Represents details for a match on a file's contents
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct SearchQueryContentsMatch {
/// Path to file whose contents match
pub path: PathBuf,
/// Line(s) that matched
pub lines: SearchQueryMatchData,
/// Line number where match starts (base index 1)
pub line_number: u64,
/// Absolute byte offset corresponding to the start of `lines` in the data being searched
pub absolute_offset: u64,
/// Collection of matches tied to `lines` where each submatch's byte offset is relative to
/// `lines` and not the overall content
pub submatches: Vec<SearchQuerySubmatch>,
}
#[cfg(feature = "schemars")]
impl SearchQueryContentsMatch {
pub fn root_schema() -> schemars::schema::RootSchema {
schemars::schema_for!(SearchQueryContentsMatch)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct SearchQuerySubmatch {
/// Content matched by query
pub r#match: SearchQueryMatchData,
/// Byte offset representing start of submatch (inclusive)
pub start: u64,
/// Byte offset representing end of submatch (exclusive)
pub end: u64,
}
#[cfg(feature = "schemars")]
impl SearchQuerySubmatch {
pub fn root_schema() -> schemars::schema::RootSchema {
schemars::schema_for!(SearchQuerySubmatch)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[serde(
rename_all = "snake_case",
deny_unknown_fields,
tag = "type",
content = "value"
)]
pub enum SearchQueryMatchData {
/// Match represented as UTF-8 text
Text(String),
/// Match represented as bytes
Bytes(Vec<u8>),
}
impl SearchQueryMatchData {
/// Creates a new instance with `Text` variant
pub fn text(value: impl Into<String>) -> Self {
Self::Text(value.into())
}
/// Creates a new instance with `Bytes` variant
pub fn bytes(value: impl Into<Vec<u8>>) -> Self {
Self::Bytes(value.into())
}
/// Returns the UTF-8 str reference to the data, if is valid UTF-8
pub fn to_str(&self) -> Option<&str> {
match self {
Self::Text(x) => Some(x),
Self::Bytes(x) => std::str::from_utf8(x).ok(),
}
}
/// Converts data to a UTF-8 string, replacing any invalid UTF-8 sequences with
/// [`U+FFFD REPLACEMENT CHARACTER`](https://doc.rust-lang.org/nightly/core/char/const.REPLACEMENT_CHARACTER.html)
pub fn to_string_lossy(&self) -> Cow<'_, str> {
match self {
Self::Text(x) => Cow::Borrowed(x),
Self::Bytes(x) => String::from_utf8_lossy(x),
}
}
}
#[cfg(feature = "schemars")]
impl SearchQueryMatchData {
pub fn root_schema() -> schemars::schema::RootSchema {
schemars::schema_for!(SearchQueryMatchData)
}
}

@ -57,6 +57,15 @@ impl ManagerCapabilities {
self.0.take(&cap)
}
/// Removes the capability with the described kind, returning true if it existed
pub fn remove(&mut self, kind: impl AsRef<str>) -> bool {
let cap = ManagerCapability {
kind: kind.as_ref().to_string(),
description: String::new(),
};
self.0.remove(&cap)
}
/// Converts into vec of capabilities sorted by kind
pub fn into_sorted_vec(self) -> Vec<ManagerCapability> {
let mut this = self.0.into_iter().collect::<Vec<_>>();

@ -3,11 +3,13 @@ use crate::{DistantMsg, DistantRequestData, Map};
use derive_more::IsVariant;
use distant_net::Request;
use serde::{Deserialize, Serialize};
use strum::{EnumDiscriminants, EnumIter, EnumMessage, EnumString};
use strum::{AsRefStr, EnumDiscriminants, EnumIter, EnumMessage, EnumString};
#[allow(clippy::large_enum_variant)]
#[derive(Clone, Debug, EnumDiscriminants, Serialize, Deserialize)]
#[cfg_attr(feature = "clap", derive(clap::Subcommand))]
#[strum_discriminants(derive(
AsRefStr,
strum::Display,
EnumIter,
EnumMessage,

@ -7,8 +7,8 @@ use async_once_cell::OnceCell;
use async_trait::async_trait;
use distant_core::{
data::{
Capabilities, DirEntry, Environment, FileType, Metadata, ProcessId, PtySize, SystemInfo,
UnixMetadata,
Capabilities, CapabilityKind, DirEntry, Environment, FileType, Metadata, ProcessId,
PtySize, SystemInfo, UnixMetadata,
},
DistantApi, DistantCtx,
};
@ -82,7 +82,14 @@ impl DistantApi for SshDistantApi {
async fn capabilities(&self, ctx: DistantCtx<Self::LocalData>) -> io::Result<Capabilities> {
debug!("[Conn {}] Querying capabilities", ctx.connection_id);
Ok(Capabilities::all())
let mut capabilities = Capabilities::all();
// Searching is not supported by ssh implementation
// TODO: Could we have external search using ripgrep's JSON lines API?
capabilities.take(CapabilityKind::Search);
capabilities.take(CapabilityKind::CancelSearch);
Ok(capabilities)
}
async fn read_file(

@ -5,6 +5,7 @@ mod generate;
mod manager;
mod server;
#[allow(clippy::large_enum_variant)]
#[derive(Debug, Subcommand)]
pub enum DistantSubcommand {
/// Perform client commands

@ -17,7 +17,7 @@ use distant_core::{
data::{ChangeKindSet, Environment},
net::{IntoSplit, Request, Response, TypedAsyncRead, TypedAsyncWrite},
ConnectionId, Destination, DistantManagerClient, DistantMsg, DistantRequestData,
DistantResponseData, Host, Map, RemoteCommand, Watcher,
DistantResponseData, Host, Map, RemoteCommand, Searcher, Watcher,
};
use log::*;
use serde_json::{json, Value};
@ -279,7 +279,7 @@ impl ClientSubcommand {
}
);
let formatter = Formatter::shell();
let mut formatter = Formatter::shell();
debug!("Sending request {:?}", request);
match request {
@ -320,6 +320,26 @@ impl ClientSubcommand {
}
}
}
DistantRequestData::Search { query } => {
debug!("Special request creating searcher for {:?}", query);
let mut searcher = Searcher::search(channel, query)
.await
.context("Failed to start search")?;
// Continue to receive and process matches
while let Some(m) = searcher.next().await {
// TODO: Provide a cleaner way to print just a match
let res = Response::new(
"".to_string(),
DistantMsg::Single(DistantResponseData::SearchResults {
id: 0,
matches: vec![m],
}),
);
formatter.print(res).context("Failed to print match")?;
}
}
DistantRequestData::Watch {
path,
recursive,

@ -1,10 +1,17 @@
use clap::ValueEnum;
use distant_core::{
data::{ChangeKind, DistantMsg, DistantResponseData, Error, FileType, Metadata, SystemInfo},
data::{
ChangeKind, DistantMsg, DistantResponseData, Error, FileType, Metadata,
SearchQueryContentsMatch, SearchQueryMatch, SearchQueryPathMatch, SystemInfo,
},
net::Response,
};
use log::*;
use std::io::{self, Write};
use std::{
collections::HashMap,
io::{self, Write},
path::PathBuf,
};
use tabled::{object::Rows, style::Style, Alignment, Disable, Modify, Table, Tabled};
#[derive(Copy, Clone, Debug, PartialEq, Eq, ValueEnum)]
@ -31,14 +38,24 @@ impl Default for Format {
}
}
#[derive(Default)]
struct FormatterState {
/// Last seen path during search
pub last_searched_path: Option<PathBuf>,
}
pub struct Formatter {
format: Format,
state: FormatterState,
}
impl Formatter {
/// Create a new output message for the given response based on the specified format
pub fn new(format: Format) -> Self {
Self { format }
Self {
format,
state: Default::default(),
}
}
/// Creates a new [`Formatter`] using [`Format`] of `Format::Shell`
@ -47,7 +64,7 @@ impl Formatter {
}
/// Consumes the output message, printing it based on its configuration
pub fn print(&self, res: Response<DistantMsg<DistantResponseData>>) -> io::Result<()> {
pub fn print(&mut self, res: Response<DistantMsg<DistantResponseData>>) -> io::Result<()> {
let output = match self.format {
Format::Json => Output::StdoutLine(
serde_json::to_vec(&res)
@ -61,7 +78,7 @@ impl Formatter {
"Shell does not support batch responses",
))
}
Format::Shell => format_shell(res.payload.into_single().unwrap()),
Format::Shell => format_shell(&mut self.state, res.payload.into_single().unwrap()),
};
match output {
@ -127,7 +144,7 @@ enum Output {
None,
}
fn format_shell(data: DistantResponseData) -> Output {
fn format_shell(state: &mut FormatterState, data: DistantResponseData) -> Output {
match data {
DistantResponseData::Ok => Output::None,
DistantResponseData::Error(Error { description, .. }) => {
@ -283,6 +300,68 @@ fn format_shell(data: DistantResponseData) -> Output {
)
.into_bytes(),
),
DistantResponseData::SearchStarted { id } => {
Output::StdoutLine(format!("Query {id} started").into_bytes())
}
DistantResponseData::SearchDone { .. } => Output::None,
DistantResponseData::SearchResults { matches, .. } => {
let mut files: HashMap<_, Vec<String>> = HashMap::new();
let mut is_targeting_paths = false;
for m in matches {
match m {
SearchQueryMatch::Path(SearchQueryPathMatch { path, .. }) => {
// Create the entry with no lines called out
files.entry(path).or_default();
is_targeting_paths = true;
}
SearchQueryMatch::Contents(SearchQueryContentsMatch {
path,
lines,
line_number,
..
}) => {
let file_matches = files.entry(path).or_default();
file_matches.push(format!(
"{line_number}:{}",
lines.to_string_lossy().trim_end()
));
}
}
}
let mut output = String::new();
for (path, lines) in files {
use std::fmt::Write;
// If we are seening a new path, print it out
if state.last_searched_path.as_deref() != Some(path.as_path()) {
// If we have already seen some path before, we would have printed it, and
// we want to add a space between it and the current path, but only if we are
// printing out file content matches and not paths
if state.last_searched_path.is_some() && !is_targeting_paths {
writeln!(&mut output).unwrap();
}
writeln!(&mut output, "{}", path.to_string_lossy()).unwrap();
}
for line in lines {
writeln!(&mut output, "{line}").unwrap();
}
// Update our last seen path
state.last_searched_path = Some(path);
}
if !output.is_empty() {
Output::Stdout(output.into_bytes())
} else {
Output::None
}
}
DistantResponseData::ProcSpawned { .. } => Output::None,
DistantResponseData::ProcStdout { data, .. } => Output::Stdout(data),
DistantResponseData::ProcStderr { data, .. } => Output::Stderr(data),

@ -7,6 +7,8 @@ const EXPECTED_TABLE: &str = indoc! {"
+------------------+------------------------------------------------------------------+
| kind | description |
+------------------+------------------------------------------------------------------+
| cancel_search | Supports canceling an active search against the filesystem |
+------------------+------------------------------------------------------------------+
| capabilities | Supports retrieving capabilities |
+------------------+------------------------------------------------------------------+
| copy | Supports copying files, directories, and symlinks |
@ -43,6 +45,8 @@ const EXPECTED_TABLE: &str = indoc! {"
+------------------+------------------------------------------------------------------+
| rename | Supports renaming files, directories, and symlinks |
+------------------+------------------------------------------------------------------+
| search | Supports searching filesystem using queries |
+------------------+------------------------------------------------------------------+
| system_info | Supports retrieving system information |
+------------------+------------------------------------------------------------------+
| unwatch | Supports unwatching filesystem for changes |

@ -13,5 +13,6 @@ mod metadata;
mod proc_spawn;
mod remove;
mod rename;
mod search;
mod system_info;
mod watch;

@ -0,0 +1,62 @@
use crate::cli::fixtures::*;
use assert_cmd::Command;
use assert_fs::prelude::*;
use indoc::indoc;
use predicates::Predicate;
use rstest::*;
use serde_json::json;
const SEARCH_RESULTS_REGEX: &str = indoc! {r"
.*?[\\/]file1.txt
1:some file text
.*?[\\/]file2.txt
3:textual
.*?[\\/]file3.txt
1:more content
"};
#[rstest]
fn should_search_filesystem_using_query(mut action_cmd: CtxCommand<Command>) {
let root = assert_fs::TempDir::new().unwrap();
root.child("file1.txt").write_str("some file text").unwrap();
root.child("file2.txt")
.write_str("lines\nof\ntextual\ninformation")
.unwrap();
root.child("file3.txt").write_str("more content").unwrap();
let query = json!({
"path": root.path().to_string_lossy(),
"target": "contents",
"condition": {"type": "regex", "value": "te[a-z]*\\b"},
});
let stdout_predicate_fn = predicates::function::function(|s: &[u8]| {
let s = std::str::from_utf8(s).unwrap();
// Split by empty line, sort, and then rejoin with empty line inbetween
let mut lines = s
.split("\n\n")
.map(|lines| lines.trim_end())
.collect::<Vec<_>>();
lines.sort_unstable();
// Put together sorted text lines
let full_text = lines.join("\n\n");
// Verify that it matches our search results regex
let regex_fn = predicates::str::is_match(SEARCH_RESULTS_REGEX).unwrap();
regex_fn.eval(&full_text)
});
// distant action system-info
action_cmd
.arg("search")
.arg(&serde_json::to_string(&query).unwrap())
.assert()
.success()
.stdout(stdout_predicate_fn)
.stderr("");
}

@ -13,5 +13,6 @@ mod metadata;
mod proc_spawn;
mod remove;
mod rename;
mod search;
mod system_info;
mod watch;

@ -0,0 +1,82 @@
use crate::cli::fixtures::*;
use assert_fs::prelude::*;
use rstest::*;
use serde_json::json;
#[rstest]
#[tokio::test]
async fn should_support_json_search_filesystem_using_query(mut json_repl: CtxCommand<Repl>) {
let root = assert_fs::TempDir::new().unwrap();
root.child("file1.txt").write_str("some file text").unwrap();
root.child("file2.txt")
.write_str("lines\nof\ntextual\ninformation")
.unwrap();
root.child("file3.txt").write_str("more content").unwrap();
let id = rand::random::<u64>().to_string();
let req = json!({
"id": id,
"payload": {
"type": "search",
"query": {
"path": root.path().to_string_lossy(),
"target": "contents",
"condition": {"type": "regex", "value": "ua"},
},
},
});
// Submit search request and get back started confirmation
let res = json_repl.write_and_read_json(req).await.unwrap().unwrap();
// Get id from started confirmation
assert_eq!(res["origin_id"], id);
assert_eq!(res["payload"]["type"], "search_started");
let search_id = res["payload"]["id"]
.as_u64()
.expect("id missing or not number");
// Get search results back
let res = json_repl.read_json_from_stdout().await.unwrap().unwrap();
assert_eq!(res["origin_id"], id);
assert_eq!(
res["payload"],
json!({
"type": "search_results",
"id": search_id,
"matches": [
{
"type": "contents",
"path": root.child("file2.txt").to_string_lossy(),
"lines": {
"type": "text",
"value": "textual\n",
},
"line_number": 3,
"absolute_offset": 9,
"submatches": [
{
"match": {
"type": "text",
"value": "ua",
},
"start": 4,
"end": 6,
}
],
},
]
})
);
// Get search completion confirmation
let res = json_repl.read_json_from_stdout().await.unwrap().unwrap();
assert_eq!(res["origin_id"], id);
assert_eq!(
res["payload"],
json!({
"type": "search_done",
"id": search_id,
})
);
}
Loading…
Cancel
Save