From 16e1904096678356b727d89bae10cf83a215af02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Menu?= Date: Sun, 14 Nov 2021 09:50:13 +0100 Subject: [PATCH] Add a command to produce a graph of the indexed notes (#106) --- CHANGELOG.md | 1 + README.md | 4 +- internal/adapter/sqlite/db.go | 11 ++ internal/adapter/sqlite/db_test.go | 2 +- internal/adapter/sqlite/link_dao.go | 156 ++++++++++++++++ internal/adapter/sqlite/link_dao_test.go | 53 ++++++ internal/adapter/sqlite/note_dao.go | 139 +++----------- internal/adapter/sqlite/note_dao_test.go | 206 --------------------- internal/adapter/sqlite/note_index.go | 66 ++++++- internal/adapter/sqlite/note_index_test.go | 170 +++++++++++++++++ internal/adapter/sqlite/testdata/sample.db | 2 +- internal/adapter/sqlite/util.go | 41 +++- internal/cli/cmd/graph.go | 97 ++++++++++ internal/core/link.go | 25 ++- internal/core/note_index.go | 3 + internal/core/note_new_test.go | 3 + internal/core/notebook.go | 5 + internal/util/strings/strings.go | 16 ++ internal/util/strings/strings_test.go | 13 ++ main.go | 16 +- 20 files changed, 683 insertions(+), 346 deletions(-) create mode 100644 internal/adapter/sqlite/link_dao.go create mode 100644 internal/adapter/sqlite/link_dao_test.go create mode 100644 internal/cli/cmd/graph.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b9eed1f..b9b9333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Added +* New `zk graph --format json` command which produces a JSON graph of the notes matching the given criteria. * New template variables `filename` and `filename-stem` when formatting notes (e.g. with `zk list --format`) and for the [`fzf-line`](docs/tool-fzf.md) config key. * Customize how LSP completion items appear in your editor when auto-completing links with the [`[lsp.completion]` configuration section](docs/config-lsp.md). ```toml diff --git a/README.md b/README.md index b1af989..2cbd726 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ * [Creating notes from templates](docs/note-creation.md) * [Advanced search and filtering capabilities](docs/note-filtering.md) including [tags](docs/tags.md), links and mentions * [Integration with your favorite editors](docs/editors-integration.md): - * [`zk.nvim`](https://github.com/megalithic/zk.nvim) for Neovim 0.5+, maintained by [Seth Messer](https://github.com/megalithic) - * [`zk-vscode`](https://github.com/mickael-menu/zk-vscode) for Visual Studio Code * [Any LSP-compatible editor](docs/editors-integration.md) + * [`zk-vscode`](https://github.com/mickael-menu/zk-vscode) for Visual Studio Code + * (*unmaintained*) [`zk.nvim`](https://github.com/megalithic/zk.nvim) for Neovim 0.5+ by [Seth Messer](https://github.com/megalithic) * [Interactive browser](docs/tool-fzf.md), powered by `fzf` * [Git-style command aliases](docs/config-alias.md) and [named filters](docs/config-filter.md) * [Made with automation in mind](docs/automation.md) diff --git a/internal/adapter/sqlite/db.go b/internal/adapter/sqlite/db.go index 3a83a48..2d51f0b 100644 --- a/internal/adapter/sqlite/db.go +++ b/internal/adapter/sqlite/db.go @@ -196,6 +196,17 @@ func (db *DB) migrate() error { }, NeedsReindexing: true, }, + + { // 6 + SQL: []string{ + // View of links with the source and target notes metadata, for simpler queries. + `CREATE VIEW resolved_links AS + SELECT l.*, s.path AS source_path, s.title AS source_title, t.path AS target_path, t.title AS target_title + FROM links l + LEFT JOIN notes s ON l.source_id = s.id + LEFT JOIN notes t ON l.target_id = t.id`, + }, + }, } needsReindexing := false diff --git a/internal/adapter/sqlite/db_test.go b/internal/adapter/sqlite/db_test.go index 086e5ec..2c5c650 100644 --- a/internal/adapter/sqlite/db_test.go +++ b/internal/adapter/sqlite/db_test.go @@ -27,7 +27,7 @@ func TestMigrateFrom0(t *testing.T) { var version int err := tx.QueryRow("PRAGMA user_version").Scan(&version) assert.Nil(t, err) - assert.Equal(t, version, 5) + assert.Equal(t, version, 6) _, err = tx.Exec(` INSERT INTO notes (path, sortable_path, title, body, word_count, checksum) diff --git a/internal/adapter/sqlite/link_dao.go b/internal/adapter/sqlite/link_dao.go new file mode 100644 index 0000000..4746bef --- /dev/null +++ b/internal/adapter/sqlite/link_dao.go @@ -0,0 +1,156 @@ +package sqlite + +import ( + "database/sql" + "fmt" + + "github.com/mickael-menu/zk/internal/core" + "github.com/mickael-menu/zk/internal/util" +) + +// LinkDAO persists links in the SQLite database. +type LinkDAO struct { + tx Transaction + logger util.Logger + + // Prepared SQL statements + addLinkStmt *LazyStmt + setLinksTargetStmt *LazyStmt + removeLinksStmt *LazyStmt +} + +// NewLinkDAO creates a new instance of a DAO working on the given database +// transaction. +func NewLinkDAO(tx Transaction, logger util.Logger) *LinkDAO { + return &LinkDAO{ + tx: tx, + logger: logger, + + // Add a new link. + addLinkStmt: tx.PrepareLazy(` + INSERT INTO links (source_id, target_id, title, href, type, external, rels, snippet, snippet_start, snippet_end) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `), + + // Set links matching a given href and missing a target ID to the given + // target ID. + setLinksTargetStmt: tx.PrepareLazy(` + UPDATE links + SET target_id = ? + WHERE target_id IS NULL AND external = 0 AND ? LIKE href || '%' + `), + + // Remove all the outbound links of a note. + removeLinksStmt: tx.PrepareLazy(` + DELETE FROM links + WHERE source_id = ? + `), + } +} + +// Add inserts all the outbound links of the given note. +func (d *LinkDAO) Add(links []core.ResolvedLink) error { + for _, link := range links { + sourceID := noteIDToSQL(link.SourceID) + targetID := noteIDToSQL(link.TargetID) + + _, err := d.addLinkStmt.Exec(sourceID, targetID, link.Title, link.Href, link.Type, link.IsExternal, joinLinkRels(link.Rels), link.Snippet, link.SnippetStart, link.SnippetEnd) + if err != nil { + return err + } + } + + return nil +} + +// RemoveAll removes all the outbound links of the given note. +func (d *LinkDAO) RemoveAll(id core.NoteID) error { + _, err := d.removeLinksStmt.Exec(noteIDToSQL(id)) + return err +} + +// SetTargetID updates the missing target_id for links matching the given href. +// FIXME: Probably doesn't work for all type of href (partial, wikilinks, etc.) +func (d *LinkDAO) SetTargetID(href string, id core.NoteID) error { + _, err := d.setLinksTargetStmt.Exec(int64(id), href) + return err +} + +// joinLinkRels will concatenate a list of rels into a SQLite ready string. +// Each rel is delimited by \x01 for easy matching in queries. +func joinLinkRels(rels []core.LinkRelation) string { + if len(rels) == 0 { + return "" + } + delimiter := "\x01" + res := delimiter + for _, rel := range rels { + res += string(rel) + delimiter + } + return res +} + +func (d *LinkDAO) FindBetweenNotes(ids []core.NoteID) ([]core.ResolvedLink, error) { + links := make([]core.ResolvedLink, 0) + + idsString := joinNoteIDs(ids, ",") + rows, err := d.tx.Query(fmt.Sprintf(` + SELECT id, source_id, source_path, target_id, target_path, title, href, type, external, rels, snippet, snippet_start, snippet_end + FROM resolved_links + WHERE source_id IN (%s) AND target_id IN (%s) + `, idsString, idsString)) + if err != nil { + return links, err + } + defer rows.Close() + + for rows.Next() { + link, err := d.scanLink(rows) + if err != nil { + d.logger.Err(err) + continue + } + if link != nil { + links = append(links, *link) + } + } + + return links, nil +} + +func (d *LinkDAO) scanLink(row RowScanner) (*core.ResolvedLink, error) { + var ( + id, sourceID, targetID, snippetStart, snippetEnd int + sourcePath, targetPath, title, href, linkType, snippet string + external bool + rels sql.NullString + ) + + err := row.Scan( + &id, &sourceID, &sourcePath, &targetID, &targetPath, &title, &href, + &linkType, &external, &rels, &snippet, &snippetStart, &snippetEnd, + ) + switch { + case err == sql.ErrNoRows: + return nil, nil + case err != nil: + return nil, err + default: + return &core.ResolvedLink{ + SourceID: core.NoteID(sourceID), + SourcePath: sourcePath, + TargetID: core.NoteID(targetID), + TargetPath: targetPath, + Link: core.Link{ + Title: title, + Href: href, + Type: core.LinkType(linkType), + IsExternal: external, + Rels: core.LinkRels(parseListFromNullString(rels)...), + Snippet: snippet, + SnippetStart: snippetStart, + SnippetEnd: snippetEnd, + }, + }, nil + } +} diff --git a/internal/adapter/sqlite/link_dao_test.go b/internal/adapter/sqlite/link_dao_test.go new file mode 100644 index 0000000..98c1398 --- /dev/null +++ b/internal/adapter/sqlite/link_dao_test.go @@ -0,0 +1,53 @@ +package sqlite + +import ( + "fmt" + "testing" + + "github.com/mickael-menu/zk/internal/core" + "github.com/mickael-menu/zk/internal/util" + "github.com/mickael-menu/zk/internal/util/test/assert" +) + +func testLinkDAO(t *testing.T, callback func(tx Transaction, dao *LinkDAO)) { + testTransaction(t, func(tx Transaction) { + callback(tx, NewLinkDAO(tx, &util.NullLogger)) + }) +} + +type linkRow struct { + SourceId core.NoteID + TargetId *core.NoteID + Href, Type, Title, Rels, Snippet string + SnippetStart, SnippetEnd int + IsExternal bool +} + +func queryLinkRows(t *testing.T, q RowQuerier, where string) []linkRow { + links := make([]linkRow, 0) + + rows, err := q.Query(fmt.Sprintf(` + SELECT source_id, target_id, title, href, type, external, rels, snippet, snippet_start, snippet_end + FROM links + WHERE %v + ORDER BY id + `, where)) + assert.Nil(t, err) + + for rows.Next() { + var row linkRow + var sourceId int64 + var targetId *int64 + err = rows.Scan(&sourceId, &targetId, &row.Title, &row.Href, &row.Type, &row.IsExternal, &row.Rels, &row.Snippet, &row.SnippetStart, &row.SnippetEnd) + assert.Nil(t, err) + row.SourceId = core.NoteID(sourceId) + if targetId != nil { + row.TargetId = idPointer(*targetId) + } + links = append(links, row) + } + rows.Close() + assert.Nil(t, rows.Err()) + + return links +} diff --git a/internal/adapter/sqlite/note_dao.go b/internal/adapter/sqlite/note_dao.go index b41d0a2..5487022 100644 --- a/internal/adapter/sqlite/note_dao.go +++ b/internal/adapter/sqlite/note_dao.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "regexp" - "strconv" "strings" "time" @@ -25,15 +24,12 @@ type NoteDAO struct { logger util.Logger // Prepared SQL statements - indexedStmt *LazyStmt - addStmt *LazyStmt - updateStmt *LazyStmt - removeStmt *LazyStmt - findIdByPathStmt *LazyStmt - findByIdStmt *LazyStmt - addLinkStmt *LazyStmt - setLinksTargetStmt *LazyStmt - removeLinksStmt *LazyStmt + indexedStmt *LazyStmt + addStmt *LazyStmt + updateStmt *LazyStmt + removeStmt *LazyStmt + findIdByPathStmt *LazyStmt + findByIdStmt *LazyStmt } // NewNoteDAO creates a new instance of a DAO working on the given database @@ -80,26 +76,6 @@ func NewNoteDAO(tx Transaction, logger util.Logger) *NoteDAO { FROM notes_with_metadata WHERE id = ? `), - - // Add a new link. - addLinkStmt: tx.PrepareLazy(` - INSERT INTO links (source_id, target_id, title, href, type, external, rels, snippet, snippet_start, snippet_end) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - - // Set links matching a given href and missing a target ID to the given - // target ID. - setLinksTargetStmt: tx.PrepareLazy(` - UPDATE links - SET target_id = ? - WHERE target_id IS NULL AND external = 0 AND ? LIKE href || '%' - `), - - // Remove all the outbound links of a note. - removeLinksStmt: tx.PrepareLazy(` - DELETE FROM links - WHERE source_id = ? - `), } } @@ -165,14 +141,12 @@ func (d *NoteDAO) Add(note core.Note) (core.NoteID, error) { return 0, err } - id := core.NoteID(lastId) - err = d.addLinks(id, note) - return id, err + return core.NoteID(lastId), err } // Update modifies an existing note. func (d *NoteDAO) Update(note core.Note) (core.NoteID, error) { - id, err := d.findIdByPath(note.Path) + id, err := d.FindIdByPath(note.Path) if err != nil { return 0, err } @@ -185,16 +159,6 @@ func (d *NoteDAO) Update(note core.Note) (core.NoteID, error) { note.Title, note.Lead, note.Body, note.RawContent, note.WordCount, metadata, note.Checksum, note.Modified, note.Path, ) - if err != nil { - return id, err - } - - _, err = d.removeLinksStmt.Exec(d.idToSql(id)) - if err != nil { - return id, err - } - - err = d.addLinks(id, note) return id, err } @@ -209,43 +173,9 @@ func (d *NoteDAO) metadataToJSON(note core.Note) string { return string(json) } -// addLinks inserts all the outbound links of the given note. -func (d *NoteDAO) addLinks(id core.NoteID, note core.Note) error { - for _, link := range note.Links { - allowPartialMatch := (link.Type == core.LinkTypeWikiLink) - targetId, err := d.findIdByHref(link.Href, allowPartialMatch) - - if err != nil { - return err - } - - _, err = d.addLinkStmt.Exec(id, d.idToSql(targetId), link.Title, link.Href, link.Type, link.IsExternal, joinLinkRels(link.Rels), link.Snippet, link.SnippetStart, link.SnippetEnd) - if err != nil { - return err - } - } - - _, err := d.setLinksTargetStmt.Exec(int64(id), note.Path) - return err -} - -// joinLinkRels will concatenate a list of rels into a SQLite ready string. -// Each rel is delimited by \x01 for easy matching in queries. -func joinLinkRels(rels []core.LinkRelation) string { - if len(rels) == 0 { - return "" - } - delimiter := "\x01" - res := delimiter - for _, rel := range rels { - res += string(rel) + delimiter - } - return res -} - // Remove deletes the note with the given path from the index. func (d *NoteDAO) Remove(path string) error { - id, err := d.findIdByPath(path) + id, err := d.FindIdByPath(path) if err != nil { return err } @@ -257,7 +187,7 @@ func (d *NoteDAO) Remove(path string) error { return err } -func (d *NoteDAO) findIdByPath(path string) (core.NoteID, error) { +func (d *NoteDAO) FindIdByPath(path string) (core.NoteID, error) { row, err := d.findIdByPathStmt.QueryRow(path) if err != nil { return core.NoteID(0), err @@ -265,10 +195,10 @@ func (d *NoteDAO) findIdByPath(path string) (core.NoteID, error) { return idForRow(row) } -func (d *NoteDAO) findIdsByHrefs(hrefs []string, allowPartialMatch bool) ([]core.NoteID, error) { +func (d *NoteDAO) FindIdsByHrefs(hrefs []string, allowPartialMatch bool) ([]core.NoteID, error) { ids := make([]core.NoteID, 0) for _, href := range hrefs { - id, err := d.findIdByHref(href, allowPartialMatch) + id, err := d.FindIdByHref(href, allowPartialMatch) if err != nil { return ids, err } @@ -283,9 +213,9 @@ func (d *NoteDAO) findIdsByHrefs(hrefs []string, allowPartialMatch bool) ([]core return ids, nil } -func (d *NoteDAO) findIdByHref(href string, allowPartialMatch bool) (core.NoteID, error) { +func (d *NoteDAO) FindIdByHref(href string, allowPartialMatch bool) (core.NoteID, error) { if allowPartialMatch { - id, err := d.findIdByHref(href, false) + id, err := d.FindIdByHref(href, false) if id.IsValid() || err != nil { return id, err } @@ -382,6 +312,7 @@ func parseListFromNullString(str sql.NullString) []string { if str.Valid && str.String != "" { list = strings.Split(str.String, "\x01") list = strutil.RemoveDuplicates(list) + list = strutil.RemoveBlank(list) } return list } @@ -397,7 +328,7 @@ func (d *NoteDAO) expandMentionsIntoMatch(opts core.NoteFindOpts) (core.NoteFind } // Find the IDs for the mentioned paths. - ids, err := d.findIdsByHrefs(opts.Mention, true /* allowPartialMatch */) + ids, err := d.FindIdsByHrefs(opts.Mention, true /* allowPartialMatch */) if err != nil { return opts, err } @@ -408,7 +339,7 @@ func (d *NoteDAO) expandMentionsIntoMatch(opts core.NoteFindOpts) (core.NoteFind } // Find their titles. - titlesQuery := "SELECT title, metadata FROM notes WHERE id IN (" + d.joinIds(ids, ",") + ")" + titlesQuery := "SELECT title, metadata FROM notes WHERE id IN (" + joinNoteIDs(ids, ",") + ")" rows, err := d.tx.Query(titlesQuery) if err != nil { return opts, err @@ -460,14 +391,14 @@ func (d *NoteDAO) findRows(opts core.NoteFindOpts, selection noteSelection) (*sq maxDistance := 0 setupLinkFilter := func(hrefs []string, direction int, negate, recursive bool) error { - ids, err := d.findIdsByHrefs(hrefs, true /* allowPartialMatch */) + ids, err := d.FindIdsByHrefs(hrefs, true /* allowPartialMatch */) if err != nil { return err } if len(ids) == 0 { return nil } - idsList := "(" + d.joinIds(ids, ",") + ")" + idsList := "(" + joinNoteIDs(ids, ",") + ")" linksSrc := "links" @@ -614,7 +545,7 @@ WHERE collection_id IN (SELECT id FROM collections t WHERE kind = '%s' AND (%s)) } if opts.MentionedBy != nil { - ids, err := d.findIdsByHrefs(opts.MentionedBy, true /* allowPartialMatch */) + ids, err := d.FindIdsByHrefs(opts.MentionedBy, true /* allowPartialMatch */) if err != nil { return nil, err } @@ -625,7 +556,7 @@ WHERE collection_id IN (SELECT id FROM collections t WHERE kind = '%s' AND (%s)) } snippetCol = `snippet(nsrc.notes_fts, 2, '', '', '…', 20)` - joinClauses = append(joinClauses, "JOIN notes_fts nsrc ON nsrc.rowid IN ("+d.joinIds(ids, ",")+") AND nsrc.notes_fts MATCH mention_query(n.title, n.metadata)") + joinClauses = append(joinClauses, "JOIN notes_fts nsrc ON nsrc.rowid IN ("+joinNoteIDs(ids, ",")+") AND nsrc.notes_fts MATCH mention_query(n.title, n.metadata)") } if opts.LinkedBy != nil { @@ -682,7 +613,7 @@ WHERE collection_id IN (SELECT id FROM collections t WHERE kind = '%s' AND (%s)) } if opts.ExcludeIDs != nil { - whereExprs = append(whereExprs, "n.id NOT IN ("+d.joinIds(opts.ExcludeIDs, ",")+")") + whereExprs = append(whereExprs, "n.id NOT IN ("+joinNoteIDs(opts.ExcludeIDs, ",")+")") } orderTerms := []string{} @@ -876,28 +807,6 @@ func pathRegex(path string) string { return path + "[^/]*|" + path + "/.+" } -func (d *NoteDAO) idToSql(id core.NoteID) sql.NullInt64 { - if id.IsValid() { - return sql.NullInt64{Int64: int64(id), Valid: true} - } else { - return sql.NullInt64{} - } -} - -func (d *NoteDAO) joinIds(ids []core.NoteID, delimiter string) string { - strs := make([]string, 0) - for _, i := range ids { - strs = append(strs, strconv.FormatInt(int64(i), 10)) - } - return strings.Join(strs, delimiter) -} - -func unmarshalMetadata(metadataJSON string) (metadata map[string]interface{}, err error) { - err = json.Unmarshal([]byte(metadataJSON), &metadata) - err = errors.Wrapf(err, "cannot parse note metadata from JSON: %s", metadataJSON) - return -} - // buildMentionQuery creates an FTS5 predicate to match the given note's title // (or aliases from the metadata) in the content of another note. // @@ -939,7 +848,3 @@ func buildMentionQuery(title, metadataJSON string) string { return "(" + strings.Join(titles, " OR ") + ")" } - -type RowScanner interface { - Scan(dest ...interface{}) error -} diff --git a/internal/adapter/sqlite/note_dao_test.go b/internal/adapter/sqlite/note_dao_test.go index 8129ac6..4911082 100644 --- a/internal/adapter/sqlite/note_dao_test.go +++ b/internal/adapter/sqlite/note_dao_test.go @@ -135,108 +135,6 @@ func TestNoteDAOAdd(t *testing.T) { }) } -func TestNoteDAOAddWithLinks(t *testing.T) { - testNoteDAO(t, func(tx Transaction, dao *NoteDAO) { - id, err := dao.Add(core.Note{ - Path: "log/added.md", - Links: []core.Link{ - { - Title: "Same dir", - Href: "log/2021-01-04", - Rels: core.LinkRels("rel-1", "rel-2"), - }, - { - Title: "Relative", - Href: "f39c8", - Snippet: "[Relative](f39c8) link", - SnippetStart: 50, - SnippetEnd: 100, - }, - { - Title: "Second is added", - Href: "f39c8#anchor", - Rels: core.LinkRels("second"), - }, - { - Title: "Unknown", - Href: "unknown", - }, - { - Title: "URL", - Href: "http://example.com", - IsExternal: true, - Snippet: "External [URL](http://example.com)", - }, - }, - }) - assert.Nil(t, err) - - rows := queryLinkRows(t, tx, fmt.Sprintf("source_id = %d", id)) - assert.Equal(t, rows, []linkRow{ - { - SourceId: id, - TargetId: idPointer(2), - Title: "Same dir", - Href: "log/2021-01-04", - Rels: "\x01rel-1\x01rel-2\x01", - }, - { - SourceId: id, - TargetId: idPointer(4), - Title: "Relative", - Href: "f39c8", - Rels: "", - Snippet: "[Relative](f39c8) link", - SnippetStart: 50, - SnippetEnd: 100, - }, - { - SourceId: id, - TargetId: idPointer(4), - Title: "Second is added", - Href: "f39c8#anchor", - Rels: "\x01second\x01", - }, - { - SourceId: id, - TargetId: nil, - Title: "Unknown", - Href: "unknown", - Rels: "", - }, - { - SourceId: id, - TargetId: nil, - Title: "URL", - Href: "http://example.com", - IsExternal: true, - Rels: "", - Snippet: "External [URL](http://example.com)", - }, - }) - }) -} - -func TestNoteDAOAddFillsLinksMissingTargetId(t *testing.T) { - testNoteDAO(t, func(tx Transaction, dao *NoteDAO) { - id, err := dao.Add(core.Note{ - Path: "missing_target.md", - }) - assert.Nil(t, err) - - rows := queryLinkRows(t, tx, fmt.Sprintf("target_id = %d", id)) - assert.Equal(t, rows, []linkRow{ - { - SourceId: 3, - TargetId: &id, - Title: "Missing target", - Href: "missing", - Snippet: "There's a Missing target", - }, - }) - }) -} - // Check that we can't add a duplicate note with an existing path. func TestNoteDAOAddExistingNote(t *testing.T) { testNoteDAO(t, func(tx Transaction, dao *NoteDAO) { @@ -288,73 +186,6 @@ func TestNoteDAOUpdateUnknown(t *testing.T) { }) } -func TestNoteDAOUpdateWithLinks(t *testing.T) { - testNoteDAO(t, func(tx Transaction, dao *NoteDAO) { - links := queryLinkRows(t, tx, "source_id = 1") - assert.Equal(t, links, []linkRow{ - { - SourceId: 1, - TargetId: idPointer(2), - Title: "An internal link", - Href: "log/2021-01-04.md", - Snippet: "[[An internal link]]", - }, - { - SourceId: 1, - TargetId: nil, - Title: "An external link", - Href: "https://domain.com", - IsExternal: true, - Snippet: "[[An external link]]", - }, - }) - - _, err := dao.Update(core.Note{ - Path: "log/2021-01-03.md", - Links: []core.Link{ - { - Title: "A new link", - Href: "index", - Type: core.LinkTypeWikiLink, - IsExternal: false, - Rels: core.LinkRels("rel"), - Snippet: "[[A new link]]", - }, - { - Title: "An external link", - Href: "https://domain.com", - Type: core.LinkTypeMarkdown, - IsExternal: true, - Snippet: "[[An external link]]", - }, - }, - }) - assert.Nil(t, err) - - links = queryLinkRows(t, tx, "source_id = 1") - assert.Equal(t, links, []linkRow{ - { - SourceId: 1, - TargetId: idPointer(3), - Title: "A new link", - Href: "index", - Type: "wiki-link", - Rels: "\x01rel\x01", - Snippet: "[[A new link]]", - }, - { - SourceId: 1, - TargetId: nil, - Title: "An external link", - Href: "https://domain.com", - Type: "markdown", - IsExternal: true, - Snippet: "[[An external link]]", - }, - }) - }) -} - func TestNoteDAORemove(t *testing.T) { testNoteDAO(t, func(tx Transaction, dao *NoteDAO) { _, err := queryNoteRow(tx, `path = "ref/test/a.md"`) @@ -1160,43 +991,6 @@ func queryNoteRow(tx Transaction, where string) (noteRow, error) { return row, err } -type linkRow struct { - SourceId core.NoteID - TargetId *core.NoteID - Href, Type, Title, Rels, Snippet string - SnippetStart, SnippetEnd int - IsExternal bool -} - -func queryLinkRows(t *testing.T, tx Transaction, where string) []linkRow { - links := make([]linkRow, 0) - - rows, err := tx.Query(fmt.Sprintf(` - SELECT source_id, target_id, title, href, type, external, rels, snippet, snippet_start, snippet_end - FROM links - WHERE %v - ORDER BY id - `, where)) - assert.Nil(t, err) - - for rows.Next() { - var row linkRow - var sourceId int64 - var targetId *int64 - err = rows.Scan(&sourceId, &targetId, &row.Title, &row.Href, &row.Type, &row.IsExternal, &row.Rels, &row.Snippet, &row.SnippetStart, &row.SnippetEnd) - assert.Nil(t, err) - row.SourceId = core.NoteID(sourceId) - if targetId != nil { - row.TargetId = idPointer(*targetId) - } - links = append(links, row) - } - rows.Close() - assert.Nil(t, rows.Err()) - - return links -} - func idPointer(i int64) *core.NoteID { id := core.NoteID(i) return &id diff --git a/internal/adapter/sqlite/note_index.go b/internal/adapter/sqlite/note_index.go index 13e6e0f..6d65d68 100644 --- a/internal/adapter/sqlite/note_index.go +++ b/internal/adapter/sqlite/note_index.go @@ -17,6 +17,7 @@ type NoteIndex struct { type dao struct { notes *NoteDAO + links *LinkDAO collections *CollectionDAO metadata *MetadataDAO } @@ -46,6 +47,15 @@ func (ni *NoteIndex) FindMinimal(opts core.NoteFindOpts) (notes []core.MinimalNo return } +// FindLinksBetweenNotes implements core.NoteIndex. +func (ni *NoteIndex) FindLinksBetweenNotes(ids []core.NoteID) (links []core.ResolvedLink, err error) { + err = ni.commit(func(dao *dao) error { + links, err = dao.links.FindBetweenNotes(ids) + return err + }) + return +} + // FindCollections implements core.NoteIndex. func (ni *NoteIndex) FindCollections(kind core.CollectionKind, sorters []core.CollectionSorter) (collections []core.Collection, err error) { err = ni.commit(func(dao *dao) error { @@ -73,6 +83,11 @@ func (ni *NoteIndex) Add(note core.Note) (id core.NoteID, err error) { return err } + err = ni.addLinks(dao, id, note) + if err != nil { + return err + } + return ni.associateTags(dao.collections, id, note.Tags) }) @@ -83,17 +98,27 @@ func (ni *NoteIndex) Add(note core.Note) (id core.NoteID, err error) { // Update implements core.NoteIndex. func (ni *NoteIndex) Update(note core.Note) error { err := ni.commit(func(dao *dao) error { - noteId, err := dao.notes.Update(note) + id, err := dao.notes.Update(note) if err != nil { return err } - err = dao.collections.RemoveAssociations(noteId) + // Reset links + err = dao.links.RemoveAll(id) + if err != nil { + return err + } + err = ni.addLinks(dao, id, note) if err != nil { return err } - return ni.associateTags(dao.collections, noteId, note.Tags) + // Reset tags + err = dao.collections.RemoveAssociations(id) + if err != nil { + return err + } + return ni.associateTags(dao.collections, id, note.Tags) }) return errors.Wrapf(err, "%v: failed to update note index", note.Path) @@ -114,6 +139,40 @@ func (ni *NoteIndex) associateTags(collections *CollectionDAO, noteId core.NoteI return nil } +func (ni *NoteIndex) addLinks(dao *dao, id core.NoteID, note core.Note) error { + links, err := ni.resolveLinkNoteIDs(dao, id, note.Links) + if err != nil { + return err + } + + err = dao.links.Add(links) + if err != nil { + return err + } + + return dao.links.SetTargetID(note.Path, id) +} + +func (ni *NoteIndex) resolveLinkNoteIDs(dao *dao, sourceID core.NoteID, links []core.Link) ([]core.ResolvedLink, error) { + resolvedLinks := []core.ResolvedLink{} + + for _, link := range links { + allowPartialMatch := (link.Type == core.LinkTypeWikiLink) + targetID, err := dao.notes.FindIdByHref(link.Href, allowPartialMatch) + if err != nil { + return resolvedLinks, err + } + + resolvedLinks = append(resolvedLinks, core.ResolvedLink{ + Link: link, + SourceID: sourceID, + TargetID: targetID, + }) + } + + return resolvedLinks, nil +} + // Remove implements core.NoteIndex func (ni *NoteIndex) Remove(path string) error { err := ni.commit(func(dao *dao) error { @@ -162,6 +221,7 @@ func (ni *NoteIndex) commit(transaction func(dao *dao) error) error { return ni.db.WithTransaction(func(tx Transaction) error { dao := dao{ notes: NewNoteDAO(tx, ni.logger), + links: NewLinkDAO(tx, ni.logger), collections: NewCollectionDAO(tx, ni.logger), metadata: NewMetadataDAO(tx), } diff --git a/internal/adapter/sqlite/note_index_test.go b/internal/adapter/sqlite/note_index_test.go index 13fb26b..cf3efa7 100644 --- a/internal/adapter/sqlite/note_index_test.go +++ b/internal/adapter/sqlite/note_index_test.go @@ -1,6 +1,7 @@ package sqlite import ( + "fmt" "testing" "github.com/mickael-menu/zk/internal/core" @@ -10,6 +11,175 @@ import ( // FIXME: Missing tests +func TestNoteIndexAddWithLinks(t *testing.T) { + db, index := testNoteIndex(t) + + id, err := index.Add(core.Note{ + Path: "log/added.md", + Links: []core.Link{ + { + Title: "Same dir", + Href: "log/2021-01-04", + Rels: core.LinkRels("rel-1", "rel-2"), + }, + { + Title: "Relative", + Href: "f39c8", + Snippet: "[Relative](f39c8) link", + SnippetStart: 50, + SnippetEnd: 100, + }, + { + Title: "Second is added", + Href: "f39c8#anchor", + Rels: core.LinkRels("second"), + }, + { + Title: "Unknown", + Href: "unknown", + }, + { + Title: "URL", + Href: "http://example.com", + IsExternal: true, + Snippet: "External [URL](http://example.com)", + }, + }, + }) + assert.Nil(t, err) + + rows := queryLinkRows(t, db.db, fmt.Sprintf("source_id = %d", id)) + assert.Equal(t, rows, []linkRow{ + { + SourceId: id, + TargetId: idPointer(2), + Title: "Same dir", + Href: "log/2021-01-04", + Rels: "\x01rel-1\x01rel-2\x01", + }, + { + SourceId: id, + TargetId: idPointer(4), + Title: "Relative", + Href: "f39c8", + Rels: "", + Snippet: "[Relative](f39c8) link", + SnippetStart: 50, + SnippetEnd: 100, + }, + { + SourceId: id, + TargetId: idPointer(4), + Title: "Second is added", + Href: "f39c8#anchor", + Rels: "\x01second\x01", + }, + { + SourceId: id, + TargetId: nil, + Title: "Unknown", + Href: "unknown", + Rels: "", + }, + { + SourceId: id, + TargetId: nil, + Title: "URL", + Href: "http://example.com", + IsExternal: true, + Rels: "", + Snippet: "External [URL](http://example.com)", + }, + }) +} + +func TestNoteIndexAddFillsLinksMissingTargetId(t *testing.T) { + db, index := testNoteIndex(t) + + id, err := index.Add(core.Note{ + Path: "missing_target.md", + }) + assert.Nil(t, err) + + rows := queryLinkRows(t, db.db, fmt.Sprintf("target_id = %d", id)) + assert.Equal(t, rows, []linkRow{ + { + SourceId: 3, + TargetId: &id, + Title: "Missing target", + Href: "missing", + Snippet: "There's a Missing target", + }, + }) +} + +func TestNoteIndexUpdateWithLinks(t *testing.T) { + db, index := testNoteIndex(t) + + links := queryLinkRows(t, db.db, "source_id = 1") + assert.Equal(t, links, []linkRow{ + { + SourceId: 1, + TargetId: idPointer(2), + Title: "An internal link", + Href: "log/2021-01-04.md", + Snippet: "[[An internal link]]", + }, + { + SourceId: 1, + TargetId: nil, + Title: "An external link", + Href: "https://domain.com", + IsExternal: true, + Snippet: "[[An external link]]", + }, + }) + + err := index.Update(core.Note{ + Path: "log/2021-01-03.md", + Links: []core.Link{ + { + Title: "A new link", + Href: "index", + Type: core.LinkTypeWikiLink, + IsExternal: false, + Rels: core.LinkRels("rel"), + Snippet: "[[A new link]]", + }, + { + Title: "An external link", + Href: "https://domain.com", + Type: core.LinkTypeMarkdown, + IsExternal: true, + Snippet: "[[An external link]]", + }, + }, + }) + assert.Nil(t, err) + + links = queryLinkRows(t, db.db, "source_id = 1") + assert.Equal(t, links, []linkRow{ + { + SourceId: 1, + TargetId: idPointer(3), + Title: "A new link", + Href: "index", + Type: "wiki-link", + Rels: "\x01rel\x01", + Snippet: "[[A new link]]", + }, + { + SourceId: 1, + TargetId: nil, + Title: "An external link", + Href: "https://domain.com", + Type: "markdown", + IsExternal: true, + Snippet: "[[An external link]]", + }, + }) +} + func TestNoteIndexAddWithTags(t *testing.T) { db, index := testNoteIndex(t) diff --git a/internal/adapter/sqlite/testdata/sample.db b/internal/adapter/sqlite/testdata/sample.db index 133b9db..f458ebd 100644 --- a/internal/adapter/sqlite/testdata/sample.db +++ b/internal/adapter/sqlite/testdata/sample.db @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ac4ece96d790615233beec6e47eb094200eba7178d65f4803a8df6232b9719c4 +oid sha256:4ed584a5c1177888066b7b63eb3b5a256c5a9a404669134f722144a30314959b size 86016 diff --git a/internal/adapter/sqlite/util.go b/internal/adapter/sqlite/util.go index cd32acd..9aa7b1c 100644 --- a/internal/adapter/sqlite/util.go +++ b/internal/adapter/sqlite/util.go @@ -1,6 +1,23 @@ package sqlite -import "strings" +import ( + "database/sql" + "encoding/json" + "strconv" + "strings" + + "github.com/mickael-menu/zk/internal/core" + "github.com/mickael-menu/zk/internal/util/errors" +) + +type RowScanner interface { + Scan(dest ...interface{}) error +} + +type RowQuerier interface { + Query(query string, args ...interface{}) (*sql.Rows, error) + QueryRow(query string, args ...interface{}) *sql.Row +} // escapeLikeTerm returns the given term after escaping any LIKE-significant // characters with the given escapeChar. @@ -12,3 +29,25 @@ func escapeLikeTerm(term string, escapeChar rune) string { } return escape(escape(escape(term, string(escapeChar)), "%"), "_") } + +func noteIDToSQL(id core.NoteID) sql.NullInt64 { + if id.IsValid() { + return sql.NullInt64{Int64: int64(id), Valid: true} + } else { + return sql.NullInt64{} + } +} + +func joinNoteIDs(ids []core.NoteID, delimiter string) string { + strs := make([]string, 0) + for _, i := range ids { + strs = append(strs, strconv.FormatInt(int64(i), 10)) + } + return strings.Join(strs, delimiter) +} + +func unmarshalMetadata(metadataJSON string) (metadata map[string]interface{}, err error) { + err = json.Unmarshal([]byte(metadataJSON), &metadata) + err = errors.Wrapf(err, "cannot parse note metadata from JSON: %s", metadataJSON) + return +} diff --git a/internal/cli/cmd/graph.go b/internal/cli/cmd/graph.go new file mode 100644 index 0000000..cffe33e --- /dev/null +++ b/internal/cli/cmd/graph.go @@ -0,0 +1,97 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/mickael-menu/zk/internal/adapter/fzf" + "github.com/mickael-menu/zk/internal/cli" + "github.com/mickael-menu/zk/internal/core" + "github.com/mickael-menu/zk/internal/util/errors" + "github.com/mickael-menu/zk/internal/util/strings" +) + +// Graph produces a directed graph of the notes matching a set of criteria. +type Graph struct { + Format string `group:format short:f help:"Format of the graph among: json." enum:"json" required` + Quiet bool `group:format short:q help:"Do not print the total number of notes found."` + cli.Filtering +} + +func (cmd *Graph) Run(container *cli.Container) error { + notebook, err := container.CurrentNotebook() + if err != nil { + return err + } + + format, err := notebook.NewNoteFormatter("{{json .}}") + if err != nil { + return err + } + + findOpts, err := cmd.Filtering.NewNoteFindOpts(notebook) + if err != nil { + return errors.Wrapf(err, "incorrect criteria") + } + + notes, err := notebook.FindNotes(findOpts) + if err != nil { + return err + } + noteIDs := []core.NoteID{} + for _, note := range notes { + noteIDs = append(noteIDs, note.ID) + } + links, err := notebook.FindLinksBetweenNotes(noteIDs) + if err != nil { + return err + } + + filter := container.NewNoteFilter(fzf.NoteFilterOpts{ + Interactive: cmd.Interactive, + AlwaysFilter: false, + NotebookDir: notebook.Path, + }) + + notes, err = filter.Apply(notes) + if err != nil { + if err == fzf.ErrCancelled { + return nil + } + return err + } + + fmt.Print("{\n \"notes\": [\n") + for i, note := range notes { + if i > 0 { + fmt.Print(",\n") + } + ft, err := format(note) + if err != nil { + return err + } + fmt.Printf(" %s", ft) + } + + fmt.Print("\n ],\n \"links\": [\n") + for i, link := range links { + if i > 0 { + fmt.Print(",\n") + } + ft, err := json.Marshal(link) + if err != nil { + return err + } + fmt.Printf(" %s", string(ft)) + } + + fmt.Print("\n ]\n}\n") + + if err == nil && !cmd.Quiet { + count := len(notes) + fmt.Fprintf(os.Stderr, "\n\nFound %d %s\n", count, strings.Pluralize("note", count)) + } + + return err +} diff --git a/internal/core/link.go b/internal/core/link.go index 0626024..5eb5441 100644 --- a/internal/core/link.go +++ b/internal/core/link.go @@ -3,21 +3,30 @@ package core // Link represents a link in a note to another note or an external resource. type Link struct { // Label of the link. - Title string + Title string `json:"title"` // Destination URI of the link. - Href string + Href string `json:"href"` // Type of link, e.g. wiki link. - Type LinkType + Type LinkType `json:"type"` // Indicates whether the target is a remote (e.g. HTTP) resource. - IsExternal bool + IsExternal bool `json:"isExternal"` // Relationships between the note and the linked target. - Rels []LinkRelation + Rels []LinkRelation `json:"rels"` // Excerpt of the paragraph containing the note. - Snippet string + Snippet string `json:"snippet"` // Start byte offset of the snippet in the note content. - SnippetStart int + SnippetStart int `json:"snippetStart"` // End byte offset of the snippet in the note content. - SnippetEnd int + SnippetEnd int `json:"snippetEnd"` +} + +// ResolvedLink represents a link between two indexed notes. +type ResolvedLink struct { + Link + SourceID NoteID `json:"sourceId"` + SourcePath string `json:"sourcePath"` + TargetID NoteID `json:"targetId"` + TargetPath string `json:"targetPath"` } // LinkType represents the kind of link, e.g. wiki link. diff --git a/internal/core/note_index.go b/internal/core/note_index.go index 7bb2144..140f374 100644 --- a/internal/core/note_index.go +++ b/internal/core/note_index.go @@ -19,6 +19,9 @@ type NoteIndex interface { // given filtering and sorting criteria. FindMinimal(opts NoteFindOpts) ([]MinimalNote, error) + // FindLinksBetweenNotes retrieves the links between the given notes. + FindLinksBetweenNotes(ids []NoteID) ([]ResolvedLink, error) + // FindCollections retrieves all the collections of the given kind. FindCollections(kind CollectionKind, sorters []CollectionSorter) ([]Collection, error) diff --git a/internal/core/note_new_test.go b/internal/core/note_new_test.go index 5aa0ed3..7f64c63 100644 --- a/internal/core/note_new_test.go +++ b/internal/core/note_new_test.go @@ -504,6 +504,9 @@ type noteIndexAddMock struct { func (m *noteIndexAddMock) Find(opts NoteFindOpts) ([]ContextualNote, error) { return nil, nil } func (m *noteIndexAddMock) FindMinimal(opts NoteFindOpts) ([]MinimalNote, error) { return nil, nil } +func (m *noteIndexAddMock) FindLinksBetweenNotes(ids []NoteID) ([]ResolvedLink, error) { + return nil, nil +} func (m *noteIndexAddMock) FindCollections(kind CollectionKind, sorters []CollectionSorter) ([]Collection, error) { return nil, nil } diff --git a/internal/core/notebook.go b/internal/core/notebook.go index 1436e76..35ecdae 100644 --- a/internal/core/notebook.go +++ b/internal/core/notebook.go @@ -236,6 +236,11 @@ func (n *Notebook) FindMatching(terms string) (*MinimalNote, error) { }) } +// FindLinksBetweenNotes retrieves the links between the given notes. +func (n *Notebook) FindLinksBetweenNotes(ids []NoteID) ([]ResolvedLink, error) { + return n.index.FindLinksBetweenNotes(ids) +} + // FindCollections retrieves all the collections of the given kind. func (n *Notebook) FindCollections(kind CollectionKind, sorters []CollectionSorter) ([]Collection, error) { return n.index.FindCollections(kind, sorters) diff --git a/internal/util/strings/strings.go b/internal/util/strings/strings.go index beff6c7..f204b00 100644 --- a/internal/util/strings/strings.go +++ b/internal/util/strings/strings.go @@ -92,6 +92,22 @@ func RemoveDuplicates(strings []string) []string { return res } +// RemoveBlank keeps only non-empty strings in the source. +func RemoveBlank(strs []string) []string { + if strs == nil { + return nil + } + + res := make([]string, 0) + for _, val := range strs { + if strings.TrimSpace(val) != "" { + res = append(res, val) + } + } + + return res +} + // InList returns whether the string is part of the given list of strings. func InList(strings []string, s string) bool { for _, c := range strings { diff --git a/internal/util/strings/strings_test.go b/internal/util/strings/strings_test.go index 3ce6a24..f4ca2c5 100644 --- a/internal/util/strings/strings_test.go +++ b/internal/util/strings/strings_test.go @@ -94,6 +94,19 @@ func TestRemoveDuplicates(t *testing.T) { test([]string{"One", "Two", "OneTwo"}, []string{"One", "Two", "OneTwo"}) } +func TestRemoveBlank(t *testing.T) { + test := func(items []string, expected []string) { + assert.Equal(t, RemoveBlank(items), expected) + } + + test([]string{}, []string{}) + test([]string{"One"}, []string{"One"}) + test([]string{"One", "Two"}, []string{"One", "Two"}) + test([]string{"One", "Two", ""}, []string{"One", "Two"}) + test([]string{"Two", "One", " "}, []string{"Two", "One"}) + test([]string{"One", "Two", " "}, []string{"One", "Two"}) +} + func TestInList(t *testing.T) { test := func(items []string, s string, expected bool) { assert.Equal(t, InList(items, s), expected) diff --git a/main.go b/main.go index efdba59..029e17b 100644 --- a/main.go +++ b/main.go @@ -22,10 +22,11 @@ var root struct { Init cmd.Init `cmd group:"zk" help:"Create a new notebook in the given directory."` Index cmd.Index `cmd group:"zk" help:"Index the notes to be searchable."` - New cmd.New `cmd group:"notes" help:"Create a new note in the given notebook directory."` - List cmd.List `cmd group:"notes" help:"List notes matching the given criteria."` - Edit cmd.Edit `cmd group:"notes" help:"Edit notes matching the given criteria."` - Tag cmd.Tag `cmd group:"notes" help:"Manage the note tags."` + New cmd.New `cmd group:"notes" help:"Create a new note in the given notebook directory."` + List cmd.List `cmd group:"notes" help:"List notes matching the given criteria."` + Graph cmd.Graph `cmd group:"notes" help:"Produce a graph of the notes matching the given criteria."` + Edit cmd.Edit `cmd group:"notes" help:"Edit notes matching the given criteria."` + Tag cmd.Tag `cmd group:"notes" help:"Manage the note tags."` NotebookDir string `type:path placeholder:PATH help:"Turn off notebook auto-discovery and set manually the notebook where commands are run."` WorkingDir string `short:W type:path placeholder:PATH help:"Run as if zk was started in instead of the current working directory."` @@ -104,9 +105,10 @@ func options(container *cli.Container) []kong.Option { kong.Name("zk"), kong.UsageOnError(), kong.HelpOptions{ - Compact: true, - FlagsLast: true, - WrapUpperBound: 100, + Compact: true, + FlagsLast: true, + WrapUpperBound: 100, + NoExpandSubcommands: true, }, kong.Vars{ "version": "zk " + strings.TrimPrefix(Version, "v"),