diff --git a/adapter/fzf/fzf.go b/adapter/fzf/fzf.go index f09618b..da637d6 100644 --- a/adapter/fzf/fzf.go +++ b/adapter/fzf/fzf.go @@ -71,7 +71,7 @@ func New(opts Opts) (*Fzf, error) { "--no-hscroll", // Don't highlight search terms "--color", "hl:-1,hl+:-1", - // "--preview-window", "noborder:wrap", + "--preview-window", "wrap", } if !opts.PreviewCmd.IsNull() { args = append(args, "--preview", opts.PreviewCmd.String()) diff --git a/adapter/markdown/markdown.go b/adapter/markdown/markdown.go index 0a1c1e7..23225b1 100644 --- a/adapter/markdown/markdown.go +++ b/adapter/markdown/markdown.go @@ -7,6 +7,7 @@ import ( "github.com/mickael-menu/zk/core/note" "github.com/mickael-menu/zk/util/opt" + strutil "github.com/mickael-menu/zk/util/strings" "github.com/yuin/goldmark" meta "github.com/yuin/goldmark-meta" "github.com/yuin/goldmark/ast" @@ -127,12 +128,13 @@ func parseLinks(root ast.Node, source []byte) ([]note.Link, error) { err := ast.Walk(root, func(n ast.Node, entering bool) (ast.WalkStatus, error) { if link, ok := n.(*ast.Link); ok && entering { - target := string(link.Destination) - if target != "" { + href := string(link.Destination) + if href != "" { links = append(links, note.Link{ - Title: string(link.Text(source)), - Target: target, - Rels: strings.Fields(string(link.Title)), + Title: string(link.Text(source)), + Href: href, + Rels: strings.Fields(string(link.Title)), + External: strutil.IsURL(href), }) } } diff --git a/adapter/markdown/markdown_test.go b/adapter/markdown/markdown_test.go index 987f5a7..0da92ae 100644 --- a/adapter/markdown/markdown_test.go +++ b/adapter/markdown/markdown_test.go @@ -165,38 +165,52 @@ func TestParseLinks(t *testing.T) { test(` # Heading with a [link](heading) -Paragraph containing [multiple **links**](stripped-formatting) some [external like this one](http://example.com), and other [relative](../other). +Paragraph containing [multiple **links**](stripped-formatting), here's one [relative](../other). A link can have [one relation](one "rel-1") or [several relations](several "rel-1 rel-2"). + +[External links](http://example.com) are marked [as such](ftp://domain). `, []note.Link{ { - Title: "link", - Target: "heading", - Rels: []string{}, + Title: "link", + Href: "heading", + Rels: []string{}, + External: false, + }, + { + Title: "multiple links", + Href: "stripped-formatting", + Rels: []string{}, + External: false, }, { - Title: "multiple links", - Target: "stripped-formatting", - Rels: []string{}, + Title: "relative", + Href: "../other", + Rels: []string{}, + External: false, }, { - Title: "external like this one", - Target: "http://example.com", - Rels: []string{}, + Title: "one relation", + Href: "one", + Rels: []string{"rel-1"}, + External: false, }, { - Title: "relative", - Target: "../other", - Rels: []string{}, + Title: "several relations", + Href: "several", + Rels: []string{"rel-1", "rel-2"}, + External: false, }, { - Title: "one relation", - Target: "one", - Rels: []string{"rel-1"}, + Title: "External links", + Href: "http://example.com", + Rels: []string{}, + External: true, }, { - Title: "several relations", - Target: "several", - Rels: []string{"rel-1", "rel-2"}, + Title: "as such", + Href: "ftp://domain", + Rels: []string{}, + External: true, }, }) } diff --git a/adapter/sqlite/db.go b/adapter/sqlite/db.go index e992c2f..7bb71cb 100644 --- a/adapter/sqlite/db.go +++ b/adapter/sqlite/db.go @@ -23,10 +23,20 @@ func OpenInMemory() (*DB, error) { } func open(uri string) (*DB, error) { + wrap := errors.Wrapper("failed to open the database") + db, err := sql.Open("sqlite3", uri) if err != nil { - return nil, errors.Wrap(err, "failed to open the database") + return nil, wrap(err) } + + // Make sure that CASCADE statements are properly applied by enabling + // foreign keys. + _, err = db.Exec("PRAGMA foreign_keys = ON") + if err != nil { + return nil, wrap(err) + } + return &DB{db}, nil } @@ -47,6 +57,7 @@ func (db *DB) Migrate() error { if version == 0 { err = tx.ExecStmts([]string{ + // Notes `CREATE TABLE IF NOT EXISTS notes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, path TEXT NOT NULL, @@ -63,6 +74,22 @@ func (db *DB) Migrate() error { )`, `CREATE INDEX IF NOT EXISTS index_notes_checksum ON notes (checksum)`, `CREATE INDEX IF NOT EXISTS index_notes_path ON notes (path)`, + + // Links + `CREATE TABLE IF NOT EXISTS links ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + source_id INTEGER NOT NULL REFERENCES notes(id) + ON DELETE CASCADE, + target_id INTEGER REFERENCES notes(id) + ON DELETE SET NULL, + title TEXT DEFAULT('') NOT NULL, + href TEXT NOT NULL, + external INT DEFAULT(0) NOT NULL, + rels TEXT DEFAULT('') NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS index_links_source_id_target_id ON links (source_id, target_id)`, + + // FTS index `CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5( path, title, body, content = notes, diff --git a/adapter/sqlite/fixtures/default/links.yml b/adapter/sqlite/fixtures/default/links.yml new file mode 100644 index 0000000..fcf41b4 --- /dev/null +++ b/adapter/sqlite/fixtures/default/links.yml @@ -0,0 +1,27 @@ +- id: 1 + source_id: 3 + target_id: null + title: "Missing target" + href: "missing" + external: false + +- id: 2 + source_id: 1 + target_id: 2 + title: "An internal link" + href: "log/2021-01-04.md" + external: false + +- id: 3 + source_id: 1 + target_id: null + title: "An external link" + href: "https://domain.com" + external: true + +- id: 4 + source_id: 4 + target_id: 1 + title: "Another link" + href: "log/2021-01-03.md" + external: false diff --git a/adapter/sqlite/note_dao.go b/adapter/sqlite/note_dao.go index 61662a8..806598f 100644 --- a/adapter/sqlite/note_dao.go +++ b/adapter/sqlite/note_dao.go @@ -20,40 +20,84 @@ type NoteDAO struct { logger util.Logger // Prepared SQL statements - indexedStmt *LazyStmt - addStmt *LazyStmt - updateStmt *LazyStmt - removeStmt *LazyStmt - existsStmt *LazyStmt + indexedStmt *LazyStmt + addStmt *LazyStmt + updateStmt *LazyStmt + removeStmt *LazyStmt + findIdByPathStmt *LazyStmt + findIdByPathPrefixStmt *LazyStmt + addLinkStmt *LazyStmt + setLinksTargetStmt *LazyStmt + removeLinksStmt *LazyStmt } +// NewNoteDAO creates a new instance of a DAO working on the given database +// transaction. func NewNoteDAO(tx Transaction, logger util.Logger) *NoteDAO { return &NoteDAO{ tx: tx, logger: logger, + + // Get file info about all indexed notes. indexedStmt: tx.PrepareLazy(` SELECT path, modified from notes ORDER BY sortable_path ASC `), + + // Add a new note to the index. addStmt: tx.PrepareLazy(` INSERT INTO notes (path, sortable_path, title, lead, body, raw_content, word_count, checksum, created, modified) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `), + + // Update the content of a note. updateStmt: tx.PrepareLazy(` UPDATE notes SET title = ?, lead = ?, body = ?, raw_content = ?, word_count = ?, checksum = ?, modified = ? WHERE path = ? `), + + // Remove a note. removeStmt: tx.PrepareLazy(` DELETE FROM notes + WHERE id = ? + `), + + // Find a note ID from its exact path. + findIdByPathStmt: tx.PrepareLazy(` + SELECT id FROM notes WHERE path = ? `), - existsStmt: tx.PrepareLazy(` - SELECT EXISTS (SELECT 1 FROM notes WHERE path = ?) + + // Find a note ID from a prefix of its path. + findIdByPathPrefixStmt: tx.PrepareLazy(` + SELECT id FROM notes + WHERE path LIKE ? || '%' + `), + + // Add a new link. + addLinkStmt: tx.PrepareLazy(` + INSERT INTO links (source_id, target_id, title, href, external, rels) + VALUES (?, ?, ?, ?, ?, ?) + `), + + // Set links matching a given href and missing a target ID to the given + // target ID. + setLinksTargetStmt: tx.PrepareLazy(` + UPDATE links + SET target_id = ? + WHERE target_id IS NULL AND external = 0 AND ? LIKE href || '%' + `), + + // Remove all the outbound links of a note. + removeLinksStmt: tx.PrepareLazy(` + DELETE FROM links + WHERE source_id = ? `), } } +// Indexed returns file info of all indexed notes. func (d *NoteDAO) Indexed() (<-chan paths.Metadata, error) { wrap := errors.Wrapper("failed to get indexed notes") @@ -92,7 +136,10 @@ func (d *NoteDAO) Indexed() (<-chan paths.Metadata, error) { return c, nil } -func (d *NoteDAO) Add(note note.Metadata) error { +// Add inserts a new note to the index. +func (d *NoteDAO) Add(note note.Metadata) (int64, error) { + wrap := errors.Wrapperf("%v: can't add note to the index", note.Path) + // For sortable_path, we replace in path / by the shortest non printable // character available to make it sortable. Without this, sorting by the // path would be a lexicographical sort instead of being the same order @@ -101,21 +148,32 @@ func (d *NoteDAO) Add(note note.Metadata) error { // string. sortablePath := strings.ReplaceAll(note.Path, "/", "\x01") - _, err := d.addStmt.Exec( + res, err := d.addStmt.Exec( note.Path, sortablePath, note.Title, note.Lead, note.Body, note.RawContent, note.WordCount, note.Checksum, note.Created, note.Modified, ) - return errors.Wrapf(err, "%v: can't add note to the index", note.Path) + if err != nil { + return 0, wrap(err) + } + + id, err := res.LastInsertId() + if err != nil { + return 0, wrap(err) + } + + err = d.addLinks(id, note) + return id, err } +// Update modifies an existing note. func (d *NoteDAO) Update(note note.Metadata) error { wrap := errors.Wrapperf("%v: failed to update note index", note.Path) - exists, err := d.exists(note.Path) + id, err := d.findIdByPath(note.Path) if err != nil { return wrap(err) } - if !exists { + if !id.Valid { return wrap(errors.New("note not found in the index")) } @@ -123,37 +181,94 @@ func (d *NoteDAO) Update(note note.Metadata) error { note.Title, note.Lead, note.Body, note.RawContent, note.WordCount, note.Checksum, note.Modified, note.Path, ) - return errors.Wrapf(err, "%v: failed to update note index", note.Path) + if err != nil { + return wrap(err) + } + + _, err = d.removeLinksStmt.Exec(id.Int64) + if err != nil { + return wrap(err) + } + + err = d.addLinks(id.Int64, note) + return wrap(err) +} + +// addLinks inserts all the outbound links of the given note. +func (d *NoteDAO) addLinks(id int64, note note.Metadata) error { + for _, link := range note.Links { + targetId, err := d.findIdByPathPrefix(link.Href) + if err != nil { + return err + } + + _, err = d.addLinkStmt.Exec(id, targetId, link.Title, link.Href, link.External, joinLinkRels(link.Rels)) + if err != nil { + return err + } + } + + _, err := d.setLinksTargetStmt.Exec(id, note.Path) + return err +} + +// joinLinkRels will concatenate a list of rels into a SQLite ready string. +// Each rel is delimited by \x01 for easy matching in queries. +func joinLinkRels(rels []string) string { + if len(rels) == 0 { + return "" + } + delimiter := "\x01" + return delimiter + strings.Join(rels, delimiter) + delimiter } +// Remove deletes the note with the given path from the index. func (d *NoteDAO) Remove(path string) error { wrap := errors.Wrapperf("%v: failed to remove note index", path) - exists, err := d.exists(path) + id, err := d.findIdByPath(path) if err != nil { return wrap(err) } - if !exists { + if !id.Valid { return wrap(errors.New("note not found in the index")) } - _, err = d.removeStmt.Exec(path) + _, err = d.removeStmt.Exec(id) return wrap(err) } -func (d *NoteDAO) exists(path string) (bool, error) { - row, err := d.existsStmt.QueryRow(path) +func (d *NoteDAO) findIdByPath(path string) (sql.NullInt64, error) { + row, err := d.findIdByPathStmt.QueryRow(path) if err != nil { - return false, err + return sql.NullInt64{}, err } - var exists bool - row.Scan(&exists) + return idForRow(row) +} + +func (d *NoteDAO) findIdByPathPrefix(path string) (sql.NullInt64, error) { + row, err := d.findIdByPathPrefixStmt.QueryRow(path) if err != nil { - return false, err + return sql.NullInt64{}, err + } + return idForRow(row) +} + +func idForRow(row *sql.Row) (sql.NullInt64, error) { + var id sql.NullInt64 + err := row.Scan(&id) + + switch { + case err == sql.ErrNoRows: + return id, nil + case err != nil: + return id, err + default: + return id, err } - return exists, nil } +// Find returns all the notes matching the given criteria. func (d *NoteDAO) Find(opts note.FinderOpts) ([]note.Match, error) { matches := make([]note.Match, 0) diff --git a/adapter/sqlite/note_dao_test.go b/adapter/sqlite/note_dao_test.go index f47bd9e..cff085a 100644 --- a/adapter/sqlite/note_dao_test.go +++ b/adapter/sqlite/note_dao_test.go @@ -1,6 +1,7 @@ package sqlite import ( + "database/sql" "fmt" "testing" "time" @@ -47,7 +48,8 @@ func TestNoteDAOIndexed(t *testing.T) { Modified: time.Date(2019, 11, 12, 20, 34, 6, 0, time.UTC), }, } { - assert.Nil(t, dao.Add(note)) + _, err := dao.Add(note) + assert.Nil(t, err) } // We check that the metadata are sorted by the path but not @@ -101,7 +103,7 @@ func TestNoteDAOIndexed(t *testing.T) { func TestNoteDAOAdd(t *testing.T) { testNoteDAO(t, func(tx Transaction, dao *NoteDAO) { - err := dao.Add(note.Metadata{ + _, err := dao.Add(note.Metadata{ Path: "log/added.md", Title: "Added note", Lead: "Note", @@ -130,10 +132,103 @@ func TestNoteDAOAdd(t *testing.T) { }) } +func TestNoteDAOAddWithLinks(t *testing.T) { + testNoteDAO(t, func(tx Transaction, dao *NoteDAO) { + id, err := dao.Add(note.Metadata{ + Path: "log/added.md", + Links: []note.Link{ + { + Title: "Same dir", + Href: "log/2021-01-04", + Rels: []string{"rel-1", "rel-2"}, + }, + { + Title: "Relative", + Href: "f39c8", + }, + { + Title: "Second is added", + Href: "f39c8", + Rels: []string{"second"}, + }, + { + Title: "Unknown", + Href: "unknown", + }, + { + Title: "URL", + Href: "http://example.com", + External: true, + }, + }, + }) + assert.Nil(t, err) + + rows := queryLinkRows(t, tx, fmt.Sprintf("source_id = %d", id)) + assert.Equal(t, rows, []linkRow{ + { + SourceId: id, + TargetId: intPointer(2), + Title: "Same dir", + Href: "log/2021-01-04", + Rels: "\x01rel-1\x01rel-2\x01", + }, + { + SourceId: id, + TargetId: intPointer(4), + Title: "Relative", + Href: "f39c8", + Rels: "", + }, + { + SourceId: id, + TargetId: intPointer(4), + Title: "Second is added", + Href: "f39c8", + Rels: "\x01second\x01", + }, + { + SourceId: id, + TargetId: nil, + Title: "Unknown", + Href: "unknown", + Rels: "", + }, + { + SourceId: id, + TargetId: nil, + Title: "URL", + Href: "http://example.com", + External: true, + Rels: "", + }, + }) + }) +} + +func TestNoteDAOAddFillsLinksMissingTargetId(t *testing.T) { + testNoteDAO(t, func(tx Transaction, dao *NoteDAO) { + id, err := dao.Add(note.Metadata{ + Path: "missing_target.md", + }) + assert.Nil(t, err) + + rows := queryLinkRows(t, tx, fmt.Sprintf("target_id = %d", id)) + assert.Equal(t, rows, []linkRow{ + { + SourceId: 3, + TargetId: &id, + Title: "Missing target", + Href: "missing", + }, + }) + }) +} + // Check that we can't add a duplicate note with an existing path. func TestNoteDAOAddExistingNote(t *testing.T) { testNoteDAO(t, func(tx Transaction, dao *NoteDAO) { - err := dao.Add(note.Metadata{Path: "ref/test/a.md"}) + _, err := dao.Add(note.Metadata{Path: "ref/test/a.md"}) assert.Err(t, err, "ref/test/a.md: can't add note to the index: UNIQUE constraint failed: notes.path") }) } @@ -178,10 +273,73 @@ func TestNoteDAOUpdateUnknown(t *testing.T) { }) } +func TestNoteDAOUpdateWithLinks(t *testing.T) { + testNoteDAO(t, func(tx Transaction, dao *NoteDAO) { + links := queryLinkRows(t, tx, "source_id = 1") + assert.Equal(t, links, []linkRow{ + { + SourceId: 1, + TargetId: intPointer(2), + Title: "An internal link", + Href: "log/2021-01-04.md", + }, + { + SourceId: 1, + TargetId: nil, + Title: "An external link", + Href: "https://domain.com", + External: true, + }, + }) + + err := dao.Update(note.Metadata{ + Path: "log/2021-01-03.md", + Links: []note.Link{ + { + Title: "A new link", + Href: "index", + External: false, + Rels: []string{"rel"}, + }, + { + Title: "An external link", + Href: "https://domain.com", + External: true, + }, + }, + }) + assert.Nil(t, err) + + links = queryLinkRows(t, tx, "source_id = 1") + assert.Equal(t, links, []linkRow{ + { + SourceId: 1, + TargetId: intPointer(3), + Title: "A new link", + Href: "index", + Rels: "\x01rel\x01", + }, + { + SourceId: 1, + TargetId: nil, + Title: "An external link", + Href: "https://domain.com", + External: true, + }, + }) + }) +} + func TestNoteDAORemove(t *testing.T) { testNoteDAO(t, func(tx Transaction, dao *NoteDAO) { - err := dao.Remove("ref/test/a.md") + _, err := queryNoteRow(tx, `path = "ref/test/a.md"`) + assert.Nil(t, err) + + err = dao.Remove("ref/test/a.md") assert.Nil(t, err) + + _, err = queryNoteRow(tx, `path = "ref/test/a.md"`) + assert.Equal(t, err, sql.ErrNoRows) }) } @@ -192,6 +350,26 @@ func TestNoteDAORemoveUnknown(t *testing.T) { }) } +// Also remove the outbound links, and set the target_id of inbound links to NULL. +func TestNoteDAORemoveCascadeLinks(t *testing.T) { + testNoteDAO(t, func(tx Transaction, dao *NoteDAO) { + links := queryLinkRows(t, tx, `source_id = 1`) + assert.Equal(t, len(links) > 0, true) + + links = queryLinkRows(t, tx, `id = 4`) + assert.Equal(t, *links[0].TargetId, int64(1)) + + err := dao.Remove("log/2021-01-03.md") + assert.Nil(t, err) + + links = queryLinkRows(t, tx, `source_id = 1`) + assert.Equal(t, len(links), 0) + + links = queryLinkRows(t, tx, `id = 4`) + assert.Nil(t, links[0].TargetId) + }) +} + func TestNoteDAOFindAll(t *testing.T) { testNoteDAOFindPaths(t, note.FinderOpts{}, []string{ "ref/test/b.md", @@ -515,3 +693,37 @@ func queryNoteRow(tx Transaction, where string) (noteRow, error) { `, where)).Scan(&row.Path, &row.Title, &row.Lead, &row.Body, &row.RawContent, &row.WordCount, &row.Checksum, &row.Created, &row.Modified) return row, err } + +type linkRow struct { + SourceId int64 + TargetId *int64 + Href, Title, Rels string + External bool +} + +func queryLinkRows(t *testing.T, tx Transaction, where string) []linkRow { + links := make([]linkRow, 0) + + rows, err := tx.Query(fmt.Sprintf(` + SELECT source_id, target_id, title, href, external, rels + FROM links + WHERE %v + ORDER BY id + `, where)) + assert.Nil(t, err) + + for rows.Next() { + var row linkRow + err = rows.Scan(&row.SourceId, &row.TargetId, &row.Title, &row.Href, &row.External, &row.Rels) + assert.Nil(t, err) + links = append(links, row) + } + rows.Close() + assert.Nil(t, rows.Err()) + + return links +} + +func intPointer(i int64) *int64 { + return &i +} diff --git a/cmd/index.go b/cmd/index.go index 962d12f..7bd92c0 100644 --- a/cmd/index.go +++ b/cmd/index.go @@ -13,9 +13,8 @@ import ( // Index indexes the content of all the notes in the slip box. type Index struct { - Directory string `arg optional type:"path" default:"." help:"Directory containing the notes to index"` - Force bool `help:"Force indexing all the notes" short:"f"` - Quiet bool `help:"Do not print statistics nor progress" short:"q"` + Force bool `help:"Force indexing all the notes" short:"f"` + Quiet bool `help:"Do not print statistics nor progress" short:"q"` } func (cmd *Index) Run(container *Container) error { @@ -24,48 +23,35 @@ func (cmd *Index) Run(container *Container) error { return err } - dir, err := zk.RequireDirAt(cmd.Directory) - if err != nil { - return err - } - db, err := container.Database(zk.DBPath()) if err != nil { return err } - var bar *progressbar.ProgressBar - if !cmd.Quiet { - bar = progressbar.NewOptions(-1, - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionThrottle(100*time.Millisecond), - progressbar.OptionSpinnerType(14), - ) - } + var bar = progressbar.NewOptions(-1, + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionThrottle(100*time.Millisecond), + progressbar.OptionSpinnerType(14), + ) var stats note.IndexingStats err = db.WithTransaction(func(tx sqlite.Transaction) error { notes := sqlite.NewNoteDAO(tx, container.Logger) stats, err = note.Index( - *dir, + zk, cmd.Force, container.Parser(), notes, container.Logger, func(change paths.DiffChange) { - if bar != nil { - bar.Add(1) - bar.Describe(change.String()) - } + bar.Add(1) + bar.Describe(change.String()) }, ) return err }) - - if bar != nil { - bar.Clear() - } + bar.Clear() if err == nil && !cmd.Quiet { fmt.Println(stats) diff --git a/cmd/list.go b/cmd/list.go index 5fab2b2..d7a131e 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -16,6 +16,7 @@ import ( type List struct { Format string `help:"Pretty prints the list using the given format" short:"f" placeholder:"