[docs] pass for next release

docs/source/cli.rst

@@ -92,7 +92,7 @@ Options
 
 .. option:: -q
 
-   Do not print the log messages after executing all of the commands.
+   Do not print informational messages.
 
 
 .. _management_cli:

docs/source/sqltab.rst

@@ -7,6 +7,7 @@ In addition to the tables generated for each log format, **lnav** includes
 the following tables/views:
 
 * `environ`_
+* `fstat(<path|pattern>)`_
 * `lnav_events`_
 * `lnav_file`_
 * `lnav_file_metadata`_
@@ -31,7 +32,7 @@ by executing the '.schema' SQL command, like so::
 environ
 -------
 
-The **environ** table gives you access to the **lnav** process' environment
+The :code:`environ` table gives you access to the **lnav** process' environment
 variables.  You can :code:`SELECT`, :code:`INSERT`, and :code:`UPDATE`
 environment variables, like so:
 
@@ -55,6 +56,19 @@ named "FILENAME" and then open it in **lnav** by referencing it with
     :open $FILENAME
 
 
+fstat(<path|pattern>)
+---------------------
+
+The :code:`fstat` table-valued function provides access to the local
+file system.  The function takes a file path or a glob pattern and
+returns the results of :code:`lstat(2)` for the matching files.  If
+the parameter is a pattern that matches nothing, no rows will be
+returned.  If the parameter is a path for a non-existent file, a
+row will be returned with the :code:`error` column set and the
+stat columns as :code:`NULL`.  To read the contents of a file, you
+can :code:`SELECT` the hidden :code:`data` column.
+
+
 .. _table_lnav_events:
 
 lnav_events

docs/source/usage.rst

@@ -162,6 +162,22 @@ it needs to open the destination for viewing in **lnav**.  For example,
 the docker URL handler uses the :ref:`:sh<sh>` command to run
 :code:`docker logs` with the container.
 
+Using as a PAGER
+^^^^^^^^^^^^^^^^
+
+Setting **lnav** as your :envvar:`PAGER` can have some advantages, like
+basic syntax highlighting and discovering sections in a document.  For
+example, when viewing a man page, the current section is displayed in
+the breadcrumb bar and you can jump to a section with the
+:ref:`:goto<goto>` command.
+
+You will probably want to pass the :option:`-q` option to suppress the
+message showing the path to the captured input.
+
+.. prompt:: bash
+
+   export PAGER="man -q"
+
 Searching
 ---------
 

src/data_scanner_re.re

@@ -145,7 +145,7 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
            RET(DT_ZERO_WIDTH_SPACE);
        }
 
-       ("f"|"u"|"r")?'"'('\\'.|[^\x00\x16\x1b"\\]|'""')*'"' {
+       ("f"|"u"|"r")?'"'('\\'.|[^\x00\x16\x1b\n"\\]|'""')*'"' {
            CAPTURE(DT_QUOTED_STRING);
            switch (this->ds_input[cap_inner.c_begin]) {
            case 'f':
@@ -180,7 +180,7 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
        [a-qstv-zA-QSTV-Z]"'" {
            CAPTURE(DT_WORD);
        }
-       ("f"|"u"|"r")?"'"('\\'.|"''"|[^\x00\x16\x1b'\\])*"'"/[^sS] {
+       ("f"|"u"|"r")?"'"('\\'.|"''"|[^\x00\x16\x1b\n'\\])*"'"/[^sS] {
            CAPTURE(DT_QUOTED_STRING);
            if (tf == text_format_t::TF_RUST) {
                auto sf = this->to_string_fragment(cap_all);

src/document.sections.cc

@@ -214,6 +214,18 @@ discover_metadata_int(const attr_line_t& al, metadata_builder& mb)
             interval.stop += stop_off_iter->sa_value.get<int64_t>();
         }
     }
+    for (auto& interval : mb.mb_type_intervals) {
+        auto start_off_iter = find_string_attr_containing(
+            orig_attrs, &SA_ORIGIN_OFFSET, interval.start);
+        if (start_off_iter != orig_attrs.end()) {
+            interval.start += start_off_iter->sa_value.get<int64_t>();
+        }
+        auto stop_off_iter = find_string_attr_containing(
+            orig_attrs, &SA_ORIGIN_OFFSET, interval.stop - 1);
+        if (stop_off_iter != orig_attrs.end()) {
+            interval.stop += stop_off_iter->sa_value.get<int64_t>();
+        }
+    }
 
     hier_node::depth_first(root_node.get(), [&orig_attrs](hier_node* node) {
         auto off_opt

src/formats/redis_log.json

@@ -1,6 +1,7 @@
 {
     "$schema": "https://lnav.org/schemas/format-v1.schema.json",
     "redis_log": {
+        "title": "Redis",
         "url": [
             "https://redis.com",
             "https://build47.com/redis-log-format-levels/"

src/fstat_vtab.cc

@@ -42,6 +42,7 @@
 #include "bound_tags.hh"
 #include "config.h"
 #include "ghc/filesystem.hpp"
+#include "sql_help.hh"
 #include "sql_util.hh"
 #include "vtab_module.hh"
 
@@ -75,6 +76,7 @@ enum {
 struct fstat_table {
     static constexpr const char* NAME = "fstat";
     static constexpr const char* CREATE_STMT = R"(
+-- The fstat() table-valued function allows you to query the file system.
 CREATE TABLE fstat (
     st_parent TEXT,
     st_name TEXT,
@@ -472,6 +474,51 @@ int
 register_fstat_vtab(sqlite3* db)
 {
     static vtab_module<tvt_no_update<fstat_table>> FSTAT_MODULE;
+    static auto fstat_help
+        = help_text("fstat",
+                    "A table-valued function for getting information about "
+                    "file paths/globs")
+              .sql_table_valued_function()
+              .with_parameter(
+                  {"pattern", "The file path or glob pattern to query."})
+              .with_result(
+                  {"st_parent", "The parent path of the directory entry"})
+              .with_result({"st_name", "The name of the directory entry"})
+              .with_result({"st_dev", "The device number"})
+              .with_result({"st_ino", "The inode number"})
+              .with_result(help_text{"st_type", "The type of the entry"}
+                               .with_enum_values({
+                                   "reg",
+                                   "blk",
+                                   "chr",
+                                   "dir",
+                                   "fifo",
+                                   "lnk",
+                                   "sock",
+                               }))
+              .with_result({"st_mode", "The protection mode"})
+              .with_result(
+                  {"st_nlink", "The number of hard links to the entry"})
+              .with_result({"st_uid", "The ID of the owning user"})
+              .with_result({"st_user", "The user name"})
+              .with_result({"st_gid", "The ID of the owning group"})
+              .with_result({"st_group", "The group name"})
+              .with_result({"st_rdev", "The device type"})
+              .with_result({"st_size", "The size of the entry in bytes"})
+              .with_result({"st_blksize", "The optimal size for I/O"})
+              .with_result({"st_blocks", "Blocks allocated for the file"})
+              .with_result({"st_atime", "The last access time"})
+              .with_result({"st_mtime", "The last modified time"})
+              .with_result({"st_ctime", "The creation time"})
+              .with_result({"error",
+                            "Error message if there was a problem looking up "
+                            "the entry"})
+              .with_result({"data", "The contents of the file"})
+              .with_example(help_example{
+                  "To read a file and raise an error if there is a problem",
+                  "SELECT ifnull(data, raise_error('cannot read: ' || st_name, "
+                  "error)) FROM fstat('/non-existent')",
+              });
 
     int rc;
 
@@ -485,6 +532,8 @@ register_fstat_vtab(sqlite3* db)
     }
 
     rc = FSTAT_MODULE.create(db, "fstat");
+    sqlite_function_help.emplace("fstat", &fstat_help);
+    fstat_help.index_tags();
 
     ensure(rc == SQLITE_OK);
 

src/internals/sql-ref.rst

@@ -1277,6 +1277,31 @@ floor(*num*)
 ----
 
 
+.. _fstat:
+
+fstat(*pattern*)
+^^^^^^^^^^^^^^^^
+
+  A table-valued function for getting information about file paths/globs
+
+  **Parameters**
+    * **pattern\*** --- The file path or glob pattern to query.
+
+  **Examples**
+    To read a file and raise an error if there is a problem:
+
+    .. code-block::  custsqlite
+
+      ;SELECT ifnull(data, raise_error('cannot read: ' || st_name, error)) FROM fstat('/non-existent')
+      ✘ error: cannot read: non-existent
+       reason: No such file or directory
+       --> command:1
+      
+
+
+----
+
+
 .. _generate_series:
 
 generate_series(*start*, *stop*, *\[step\]*)
@@ -3136,6 +3161,17 @@ raise_error(*msg*, *\[reason\]*)
     * **msg\*** --- The error message
     * **reason** --- The reason the error occurred
 
+  **Examples**
+    To raise an error if a variable is not set:
+
+    .. code-block::  custsqlite
+
+      ;SELECT ifnull($val, raise_error('please set $val', 'because'))
+      ✘ error: please set $val
+       reason: because
+       --> command:1
+      
+
 
 ----
 

src/lnav.cc

@@ -652,10 +652,7 @@ make it easier to navigate through files quickly.
         .append("  ")
         .append("-q"_symbol)
         .append("         ")
-        .append(
-            R"(Do not print the log messages after executing all
-             of the commands.
-)")
+        .append("Do not print informational messages.\n")
         .append("\n")
         .append("Optional arguments"_h2)
         .append("\n")
@@ -2186,6 +2183,10 @@ main(int argc, char* argv[])
         lnav_data.ld_flags |= LNF_SECURE_MODE;
     }
 
+    // Set PAGER so that stuff run from `:sh` will just dump their
+    // output for lnav to display.  One example would be `man`, as
+    // in `:sh man ls`.
+    setenv("PAGER", "cat", 1);
     setenv("LNAV_HOME_DIR", lnav::paths::dotlnav().c_str(), 1);
     setenv("LNAV_WORK_DIR", lnav::paths::workdir().c_str(), 1);
 
@@ -3427,7 +3428,9 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
 
         // When reading from stdin, tell the user where the capture file is
         // stored so they can look at it later.
-        if (stdin_url && !(lnav_data.ld_flags & LNF_HEADLESS)) {
+        if (stdin_url && !(lnav_data.ld_flags & LNF_HEADLESS)
+            && verbosity != verbosity_t::quiet)
+        {
             file_size_t stdin_size = 0;
             for (const auto& ent :
                  ghc::filesystem::directory_iterator(stdin_dir))

src/lnav.indexing.cc

@@ -183,6 +183,11 @@ public:
         }
     }
 
+    void renamed_file(const std::shared_ptr<logfile>& lf) override
+    {
+        lnav_data.ld_active_files.regenerate_unique_file_names();
+    }
+
     std::shared_ptr<logfile> front_file;
     file_location_t front_top;
     bool did_promotion{false};

src/logfile.hh

@@ -294,6 +294,11 @@ public:
         };
     }
 
+    file_off_t get_line_content_offset(const_iterator ll)
+    {
+        return ll->get_offset() + (this->lf_line_buffer.is_piper() ? 22 : 0);
+    }
+
     void read_full_message(const_iterator ll,
                            shared_buffer_ref& msg_out,
                            int max_lines = 50);

src/state-extension-functions.cc

@@ -163,7 +163,12 @@ state_extension_functions(struct FuncDef** basic_funcs,
                 .with_parameter({"msg", "The error message"})
                 .with_parameter(
                     help_text("reason", "The reason the error occurred")
-                        .optional()))
+                        .optional())
+                .with_example({
+                    "To raise an error if a variable is not set",
+                    "SELECT ifnull($val, raise_error('please set $val', "
+                    "'because'))",
+                }))
             .with_flags(SQLITE_UTF8),
 
         sqlite_func_adapter<decltype(&sql_echoln), sql_echoln>::builder(

src/text_format.cc

@@ -203,3 +203,31 @@ detect_text_format(string_fragment sf,
 
     return text_format_t::TF_UNKNOWN;
 }
+
+nonstd::optional<text_format_meta_t>
+extract_text_meta(string_fragment sf, text_format_t tf)
+{
+    static const auto MAN_NAME = lnav::pcre2pp::code::from_const(
+        R"(^([A-Za-z][A-Za-z\-_\+0-9]+\(\d\))\s+)", PCRE2_MULTILINE);
+
+    switch (tf) {
+        case text_format_t::TF_MAN: {
+            static thread_local auto md
+                = lnav::pcre2pp::match_data::unitialized();
+
+            auto find_res
+                = MAN_NAME.capture_from(sf).into(md).matches().ignore_error();
+
+            if (find_res) {
+                return text_format_meta_t{
+                    md.to_string(),
+                };
+            }
+            break;
+        }
+        default:
+            break;
+    }
+
+    return nonstd::nullopt;
+}

src/text_format.hh

@@ -126,4 +126,11 @@ text_format_t detect_text_format(string_fragment sf,
                                  nonstd::optional<ghc::filesystem::path> path
                                  = nonstd::nullopt);
 
+struct text_format_meta_t {
+    std::string tfm_filename;
+};
+
+nonstd::optional<text_format_meta_t> extract_text_meta(string_fragment sf,
+                                                       text_format_t tf);
+
 #endif

src/textfile_sub_source.cc

@@ -182,17 +182,18 @@ textfile_sub_source::text_attrs_for_line(textview_curses& tc,
                     : ll_next_iter->get_offset() - 1;
                 const auto& meta = meta_opt.value().get();
                 meta.ms_metadata.m_section_types_tree.visit_overlapping(
-                    ll->get_offset(),
+                    lf->get_line_content_offset(ll),
                     end_offset,
-                    [&value_out, &ll, end_offset](const auto& iv) {
+                    [&value_out, &ll, &lf, end_offset](const auto& iv) {
+                        auto ll_offset = lf->get_line_content_offset(ll);
                         auto lr = line_range{0, -1};
-                        if (iv.start > ll->get_offset()) {
-                            lr.lr_start = iv.start - ll->get_offset();
+                        if (iv.start > ll_offset) {
+                            lr.lr_start = iv.start - ll_offset;
                         }
                         if (iv.stop < end_offset) {
-                            lr.lr_end = iv.stop - ll->get_offset();
+                            lr.lr_end = iv.stop - ll_offset;
                         } else {
-                            lr.lr_end = end_offset - ll->get_offset();
+                            lr.lr_end = end_offset - ll_offset;
                         }
                         auto role = role_t::VCR_NONE;
                         switch (iv.value) {
@@ -501,7 +502,7 @@ textfile_sub_source::text_crumbs_for_line(
         const auto initial_size = crumbs.size();
 
         meta_iter->second.ms_metadata.m_sections_tree.visit_overlapping(
-            ll_iter->get_offset(),
+            lf->get_line_content_offset(ll_iter),
             end_offset,
             [&crumbs,
              initial_size,
@@ -814,6 +815,14 @@ textfile_sub_source::rescan_files(
                                  lf->get_filename().c_str());
                         scrub_ansi_string(content.get_string(),
                                           &content.get_attrs());
+
+                        auto text_meta = extract_text_meta(
+                            content.get_string(), lf->get_text_format());
+                        if (text_meta) {
+                            lf->set_filename(text_meta->tfm_filename);
+                            callback.renamed_file(lf);
+                        }
+
                         this->tss_doc_metadata[lf->get_filename()]
                             = metadata_state{
                                 st.st_mtime,

src/textfile_sub_source.hh

@@ -115,6 +115,7 @@ public:
             = 0;
         virtual void promote_file(const std::shared_ptr<logfile>& lf) = 0;
         virtual void scanned_file(const std::shared_ptr<logfile>& lf) = 0;
+        virtual void renamed_file(const std::shared_ptr<logfile>& lf) = 0;
     };
 
     struct rescan_result_t {

src/view_helpers.cc

@@ -880,6 +880,10 @@ execute_examples()
                 continue;
             }
 
+            if (EXAMPLE_RESULTS.count(ex.he_cmd)) {
+                continue;
+            }
+
             switch (ht.ht_context) {
                 case help_context_t::HC_SQL_KEYWORD:
                 case help_context_t::HC_SQL_INFIX:
@@ -887,9 +891,13 @@ execute_examples()
                 case help_context_t::HC_SQL_TABLE_VALUED_FUNCTION: {
                     exec_context ec;
 
-                    execute_sql(ec, ex.he_cmd, alt_msg);
+                    auto exec_res = execute_sql(ec, ex.he_cmd, alt_msg);
 
-                    if (dls.dls_rows.size() == 1 && dls.dls_rows[0].size() == 1)
+                    if (exec_res.isErr()) {
+                        auto um = exec_res.unwrapErr();
+                        result.append(um.to_attr_line());
+                    } else if (dls.dls_rows.size() == 1
+                               && dls.dls_rows[0].size() == 1)
                     {
                         result.append(dls.dls_rows[0][0]);
                     } else {

test/expected/expected.am

@@ -104,6 +104,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_cmds.sh_53a9686102f69b07b034df291f554a00b265ed20.out \
     $(srcdir)/%reldir%/test_cmds.sh_55c2fd15ec2c7d96dbef7b36a42a1b7b42f90dbc.err \
     $(srcdir)/%reldir%/test_cmds.sh_55c2fd15ec2c7d96dbef7b36a42a1b7b42f90dbc.out \
+    $(srcdir)/%reldir%/test_cmds.sh_5630626e6f68c3d4a2c3e5f27d024df5950b88b5.err \
+    $(srcdir)/%reldir%/test_cmds.sh_5630626e6f68c3d4a2c3e5f27d024df5950b88b5.out \
     $(srcdir)/%reldir%/test_cmds.sh_5bfd08c1639701476d7b9348c36afd46fdbe6f2a.err \
     $(srcdir)/%reldir%/test_cmds.sh_5bfd08c1639701476d7b9348c36afd46fdbe6f2a.out \
     $(srcdir)/%reldir%/test_cmds.sh_624a41e152675575f4b07c19b2cf0e3a028429a2.err \

test/expected/test_cmds.sh_5630626e6f68c3d4a2c3e5f27d024df5950b88b5.out

@@ -0,0 +1,2 @@
+Hello
+World

test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out

@@ -2429,6 +2429,42 @@ For support questions, email:
    
 
 
+fstat(pattern)
+══════════════════════════════════════════════════════════════════════
+  A table-valued function for getting information about file
+  paths/globs
+Parameter
+  pattern   The file path or glob pattern to query.
+Results
+  st_parent    The parent path of the directory entry
+  st_name      The name of the directory entry
+  st_dev       The device number
+  st_ino       The inode number
+  st_type      The type of the entry
+  st_mode      The protection mode
+  st_nlink     The number of hard links to the entry
+  st_uid       The ID of the owning user
+  st_user      The user name
+  st_gid       The ID of the owning group
+  st_group     The group name
+  st_rdev      The device type
+  st_size      The size of the entry in bytes
+  st_blksize   The optimal size for I/O
+  st_blocks    Blocks allocated for the file
+  st_atime     The last access time
+  st_mtime     The last modified time
+  st_ctime     The creation time
+  error        Error message if there was a problem
+               looking up the entry
+  data         The contents of the file
+
+Example
+#1 To read a file and raise an error if there is a problem:
+   ;SELECT ifnull(data, raise_error('cannot read: ' || st_name, error)) FROM
+      fstat('/non-existent')
+   
+
+
 generate_series(start, stop, [step])
 ══════════════════════════════════════════════════════════════════════
   A table-valued-function that returns the whole numbers between a
@@ -3781,6 +3817,11 @@ For support questions, email:
   msg      The error message
   reason   The reason the error occurred
 
+Example
+#1 To raise an error if a variable is not set:
+   ;SELECT ifnull($val, raise_error('please set $val', 'because'))
+   
+
 
 random()
 ══════════════════════════════════════════════════════════════════════

test/expected/test_text_file.sh_5e9320f18d066e6fc930dbbffc357af64312bd4b.out

@@ -1,14 +1,14 @@
 [
     {
         "top_meta": {
-            "file": "{test_dir}/man_echo.txt",
+            "file": "ECHO(1)",
             "breadcrumbs": [
                 {
-                    "display_value": "man_echo.txt",
+                    "display_value": "ECHO(1)",
                     "search_placeholder": "",
                     "possibilities": [
                         {
-                            "display_value": "man_echo.txt"
+                            "display_value": "ECHO(1)"
                         }
                     ]
                 },

test/expected/test_text_file.sh_73f69c883f60761bff9f8874f61d21a189e92912.out

@@ -1,15 +1,15 @@
 [
     {
         "top_meta": {
-            "file": "stdin",
+            "file": "ECHO(1)",
             "anchor": "#description",
             "breadcrumbs": [
                 {
-                    "display_value": "stdin",
+                    "display_value": "ECHO(1)",
                     "search_placeholder": "",
                     "possibilities": [
                         {
-                            "display_value": "stdin"
+                            "display_value": "ECHO(1)"
                         }
                     ]
                 },

test/test_cmds.sh

@@ -4,6 +4,10 @@ export TZ=UTC
 export YES_COLOR=1
 export DUMP_CRASH=1
 
+run_cap_test ${lnav_test} -nN \
+    -c ";SELECT ':echo Hello' || char(10) || ':echo World' AS cmds" \
+    -c ':eval ${cmds}'
+
 run_cap_test ${lnav_test} -nN \
     -c ":cd /bad-dir"