diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fd23be..f538e46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,10 @@ All notable changes to this project will be documented in this file. * Support for tags. * Filter notes by their tags using `--tag "history, europe"`. * To match notes associated with either tags, use a pipe `|` or `OR` (all caps), e.g. `--tag "inbox OR todo"`. - * If you want to exclude notes having a particular tag, prefix it with `-` or `NOT` (all caps), e.g. `--tag "NOT done"` + * If you want to exclude notes having a particular tag, prefix it with `-` or `NOT` (all caps), e.g. `--tag "NOT done"`. + * Use glob patterns to match multiple tags, e.g. `--tag "book-*"`. * Many tag flavors are supported: `#hashtags`, `:colon:separated:tags:` and even Bear's [`#multi-word tags#`](https://blog.bear.app/2017/11/bear-tips-how-to-create-multi-word-tags/). If you prefer to use a YAML frontmatter, list your tags with the key `tags` or `keywords`. +* Print metadata from the YAML frontmatter in `list` output using `{{metadata.}}`, e.g. `{{metadata.description}}`. Keys are normalized to lower case. ### Changed diff --git a/adapter/markdown/markdown.go b/adapter/markdown/markdown.go index c5917ab..06fcdb7 100644 --- a/adapter/markdown/markdown.go +++ b/adapter/markdown/markdown.go @@ -91,11 +91,12 @@ func (p *Parser) Parse(source string) (*note.Content, error) { } return ¬e.Content{ - Title: title, - Body: body, - Lead: parseLead(body), - Links: links, - Tags: tags, + Title: title, + Body: body, + Lead: parseLead(body), + Links: links, + Tags: tags, + Metadata: frontmatter.values, }, nil } @@ -215,23 +216,29 @@ func parseLinks(root ast.Node, source []byte) ([]note.Link, error) { case *ast.Link: href := string(link.Destination) if href != "" { + snippet, snStart, snEnd := extractLines(n.Parent(), source) links = append(links, note.Link{ - Title: string(link.Text(source)), - Href: href, - Rels: strings.Fields(string(link.Title)), - External: strutil.IsURL(href), - Snippet: extractLines(n.Parent(), source), + Title: string(link.Text(source)), + Href: href, + Rels: strings.Fields(string(link.Title)), + External: strutil.IsURL(href), + Snippet: snippet, + SnippetStart: snStart, + SnippetEnd: snEnd, }) } case *ast.AutoLink: if href := string(link.URL(source)); href != "" && link.AutoLinkType == ast.AutoLinkURL { + snippet, snStart, snEnd := extractLines(n.Parent(), source) links = append(links, note.Link{ - Title: string(link.Label(source)), - Href: href, - Rels: []string{}, - External: true, - Snippet: extractLines(n.Parent(), source), + Title: string(link.Label(source)), + Href: href, + Rels: []string{}, + External: true, + Snippet: snippet, + SnippetStart: snStart, + SnippetEnd: snEnd, }) } } @@ -241,17 +248,18 @@ func parseLinks(root ast.Node, source []byte) ([]note.Link, error) { return links, err } -func extractLines(n ast.Node, source []byte) string { +func extractLines(n ast.Node, source []byte) (content string, start, end int) { if n == nil { - return "" + return } segs := n.Lines() if segs.Len() == 0 { - return "" + return } - start := segs.At(0).Start - end := segs.At(segs.Len() - 1).Stop - return string(source[start:end]) + start = segs.At(0).Start + end = segs.At(segs.Len() - 1).Stop + content = string(source[start:end]) + return } // frontmatter contains metadata parsed from a YAML frontmatter. @@ -265,6 +273,7 @@ var frontmatterRegex = regexp.MustCompile(`(?ms)^\s*-+\s*$.*?^\s*-+\s*$`) func parseFrontmatter(context parser.Context, source []byte) (frontmatter, error) { var front frontmatter + front.values = map[string]interface{}{} index := frontmatterRegex.FindIndex(source) if index == nil { @@ -273,7 +282,6 @@ func parseFrontmatter(context parser.Context, source []byte) (frontmatter, error front.start = index[0] front.end = index[1] - front.values = map[string]interface{}{} values, err := meta.TryGet(context) if err != nil { diff --git a/adapter/markdown/markdown_test.go b/adapter/markdown/markdown_test.go index ca876ae..f5c1cc2 100644 --- a/adapter/markdown/markdown_test.go +++ b/adapter/markdown/markdown_test.go @@ -377,11 +377,13 @@ Neuron links with titles: [[trailing|Trailing link]]# #[[leading | Leading link [External links](http://example.com) are marked [as such](ftp://domain). `, []note.Link{ { - Title: "link", - Href: "heading", - Rels: []string{}, - External: false, - Snippet: "Heading with a [link](heading)", + Title: "link", + Href: "heading", + Rels: []string{}, + External: false, + Snippet: "Heading with a [link](heading)", + SnippetStart: 3, + SnippetEnd: 33, }, { Title: "multiple links", @@ -390,6 +392,8 @@ Neuron links with titles: [[trailing|Trailing link]]# #[[leading | Leading link External: false, Snippet: `Paragraph containing [multiple **links**](stripped-formatting), here's one [relative](../other). A link can have [one relation](one "rel-1") or [several relations](several "rel-1 rel-2").`, + SnippetStart: 35, + SnippetEnd: 222, }, { Title: "relative", @@ -398,6 +402,8 @@ A link can have [one relation](one "rel-1") or [several relations](several "rel- External: false, Snippet: `Paragraph containing [multiple **links**](stripped-formatting), here's one [relative](../other). A link can have [one relation](one "rel-1") or [several relations](several "rel-1 rel-2").`, + SnippetStart: 35, + SnippetEnd: 222, }, { Title: "one relation", @@ -406,6 +412,8 @@ A link can have [one relation](one "rel-1") or [several relations](several "rel- External: false, Snippet: `Paragraph containing [multiple **links**](stripped-formatting), here's one [relative](../other). A link can have [one relation](one "rel-1") or [several relations](several "rel-1 rel-2").`, + SnippetStart: 35, + SnippetEnd: 222, }, { Title: "several relations", @@ -414,94 +422,143 @@ A link can have [one relation](one "rel-1") or [several relations](several "rel- External: false, Snippet: `Paragraph containing [multiple **links**](stripped-formatting), here's one [relative](../other). A link can have [one relation](one "rel-1") or [several relations](several "rel-1 rel-2").`, + SnippetStart: 35, + SnippetEnd: 222, }, { - Title: "https://inline-link.com", - Href: "https://inline-link.com", - External: true, - Rels: []string{}, - Snippet: "An https://inline-link.com and http://another-inline-link.com.", + Title: "https://inline-link.com", + Href: "https://inline-link.com", + External: true, + Rels: []string{}, + Snippet: "An https://inline-link.com and http://another-inline-link.com.", + SnippetStart: 224, + SnippetEnd: 286, }, { - Title: "http://another-inline-link.com", - Href: "http://another-inline-link.com", - External: true, - Rels: []string{}, - Snippet: "An https://inline-link.com and http://another-inline-link.com.", + Title: "http://another-inline-link.com", + Href: "http://another-inline-link.com", + External: true, + Rels: []string{}, + Snippet: "An https://inline-link.com and http://another-inline-link.com.", + SnippetStart: 224, + SnippetEnd: 286, }, { - Title: "Wiki link", - Href: "Wiki link", - External: false, - Rels: []string{}, - Snippet: "A [[Wiki link]] is surrounded by [[2-brackets | two brackets]].", + Title: "Wiki link", + Href: "Wiki link", + External: false, + Rels: []string{}, + Snippet: "A [[Wiki link]] is surrounded by [[2-brackets | two brackets]].", + SnippetStart: 288, + SnippetEnd: 351, }, { - Title: "two brackets", - Href: "2-brackets", - External: false, - Rels: []string{}, - Snippet: "A [[Wiki link]] is surrounded by [[2-brackets | two brackets]].", + Title: "two brackets", + Href: "2-brackets", + External: false, + Rels: []string{}, + Snippet: "A [[Wiki link]] is surrounded by [[2-brackets | two brackets]].", + SnippetStart: 288, + SnippetEnd: 351, }, { - Title: `esca]]ped [chara\cters`, - Href: `esca]]ped [chara\cters`, - External: false, - Rels: []string{}, - Snippet: `It can contain [[esca]\]ped \[chara\\cters]].`, + Title: `esca]]ped [chara\cters`, + Href: `esca]]ped [chara\cters`, + External: false, + Rels: []string{}, + Snippet: `It can contain [[esca]\]ped \[chara\\cters]].`, + SnippetStart: 353, + SnippetEnd: 398, }, { - Title: "Folgezettel link", - Href: "Folgezettel link", - External: false, - Rels: []string{"down"}, - Snippet: "A [[[Folgezettel link]]] is surrounded by three brackets.", + Title: "Folgezettel link", + Href: "Folgezettel link", + External: false, + Rels: []string{"down"}, + Snippet: "A [[[Folgezettel link]]] is surrounded by three brackets.", + SnippetStart: 400, + SnippetEnd: 457, }, { - Title: "trailing hash", - Href: "trailing hash", - External: false, - Rels: []string{"down"}, - Snippet: "Neuron also supports a [[trailing hash]]# for Folgezettel links.", + Title: "trailing hash", + Href: "trailing hash", + External: false, + Rels: []string{"down"}, + Snippet: "Neuron also supports a [[trailing hash]]# for Folgezettel links.", + SnippetStart: 459, + SnippetEnd: 523, }, { - Title: "leading hash", - Href: "leading hash", - External: false, - Rels: []string{"up"}, - Snippet: "A #[[leading hash]] is used for #uplinks.", + Title: "leading hash", + Href: "leading hash", + External: false, + Rels: []string{"up"}, + Snippet: "A #[[leading hash]] is used for #uplinks.", + SnippetStart: 525, + SnippetEnd: 566, }, { - Title: "Trailing link", - Href: "trailing", - External: false, - Rels: []string{"down"}, - Snippet: "Neuron links with titles: [[trailing|Trailing link]]# #[[leading | Leading link]]", + Title: "Trailing link", + Href: "trailing", + External: false, + Rels: []string{"down"}, + Snippet: "Neuron links with titles: [[trailing|Trailing link]]# #[[leading | Leading link]]", + SnippetStart: 568, + SnippetEnd: 650, }, { - Title: "Leading link", - Href: "leading", - External: false, - Rels: []string{"up"}, - Snippet: "Neuron links with titles: [[trailing|Trailing link]]# #[[leading | Leading link]]", + Title: "Leading link", + Href: "leading", + External: false, + Rels: []string{"up"}, + Snippet: "Neuron links with titles: [[trailing|Trailing link]]# #[[leading | Leading link]]", + SnippetStart: 568, + SnippetEnd: 650, }, { - Title: "External links", - Href: "http://example.com", - Rels: []string{}, - External: true, - Snippet: `[External links](http://example.com) are marked [as such](ftp://domain).`, + Title: "External links", + Href: "http://example.com", + Rels: []string{}, + External: true, + Snippet: `[External links](http://example.com) are marked [as such](ftp://domain).`, + SnippetStart: 652, + SnippetEnd: 724, }, { - Title: "as such", - Href: "ftp://domain", - Rels: []string{}, - External: true, - Snippet: `[External links](http://example.com) are marked [as such](ftp://domain).`, + Title: "as such", + Href: "ftp://domain", + Rels: []string{}, + External: true, + Snippet: `[External links](http://example.com) are marked [as such](ftp://domain).`, + SnippetStart: 652, + SnippetEnd: 724, }, }) } +func TestParseMetadataFromFrontmatter(t *testing.T) { + test := func(source string, expectedMetadata map[string]interface{}) { + content := parse(t, source) + assert.Equal(t, content.Metadata, expectedMetadata) + } + + test("", map[string]interface{}{}) + test("# A title", map[string]interface{}{}) + test("---\n---\n# A title", map[string]interface{}{}) + test(`--- +title: A title +tags: + - tag1 + - "tag 2" +--- + +Paragraph +`, map[string]interface{}{ + "title": "A title", + "tags": []interface{}{"tag1", "tag 2"}, + }) +} + func parse(t *testing.T, source string) note.Content { return parseWithOptions(t, source, ParserOpts{ HashtagEnabled: true, diff --git a/adapter/sqlite/db.go b/adapter/sqlite/db.go index cf947eb..b6d2a88 100644 --- a/adapter/sqlite/db.go +++ b/adapter/sqlite/db.go @@ -48,8 +48,8 @@ func (db *DB) Close() error { } // Migrate upgrades the SQL schema of the database. -func (db *DB) Migrate() error { - err := db.WithTransaction(func(tx Transaction) error { +func (db *DB) Migrate() (needsReindexing bool, err error) { + err = db.WithTransaction(func(tx Transaction) error { var version int err := tx.QueryRow("PRAGMA user_version").Scan(&version) if err != nil { @@ -154,8 +154,28 @@ func (db *DB) Migrate() error { } } + if version <= 2 { + err = tx.ExecStmts([]string{ + // Add a `metadata` column to `notes` + `ALTER TABLE notes ADD COLUMN metadata TEXT DEFAULT('{}') NOT NULL`, + + // Add snippet's start and end offsets to `links` + `ALTER TABLE links ADD COLUMN snippet_start INTEGER DEFAULT(0) NOT NULL`, + `ALTER TABLE links ADD COLUMN snippet_end INTEGER DEFAULT(0) NOT NULL`, + + `PRAGMA user_version = 3`, + }) + + if err != nil { + return err + } + + needsReindexing = true + } + return nil }) - return errors.Wrap(err, "database migration failed") + err = errors.Wrap(err, "database migration failed") + return } diff --git a/adapter/sqlite/db_test.go b/adapter/sqlite/db_test.go index 73fbfe8..f19ae95 100644 --- a/adapter/sqlite/db_test.go +++ b/adapter/sqlite/db_test.go @@ -23,17 +23,17 @@ func TestMigrateFrom0(t *testing.T) { db, err := OpenInMemory() assert.Nil(t, err) - err = db.Migrate() + _, err = db.Migrate() assert.Nil(t, err) // Should be able to migrate twice in a row - err = db.Migrate() + _, err = db.Migrate() assert.Nil(t, err) err = db.WithTransaction(func(tx Transaction) error { var version int err := tx.QueryRow("PRAGMA user_version").Scan(&version) assert.Nil(t, err) - assert.Equal(t, version, 2) + assert.Equal(t, version, 3) _, err = tx.Exec(` INSERT INTO notes (path, sortable_path, title, body, word_count, checksum) diff --git a/adapter/sqlite/fixtures/default/notes.yml b/adapter/sqlite/fixtures/default/notes.yml index e5b18ad..26f49df 100644 --- a/adapter/sqlite/fixtures/default/notes.yml +++ b/adapter/sqlite/fixtures/default/notes.yml @@ -9,6 +9,7 @@ checksum: "qwfpgj" created: "2020-11-22T16:27:45Z" modified: "2020-11-22T16:27:45Z" + metadata: '{"author":"Dom"}' - id: 2 path: "log/2021-01-04.md" @@ -21,6 +22,7 @@ checksum: "arstde" created: "2020-11-29T08:20:18Z" modified: "2020-11-29T08:20:18Z" + metadata: "{}" - id: 3 path: "index.md" @@ -33,6 +35,7 @@ checksum: "iaefhv" created: "2019-12-04T11:59:11Z" modified: "2019-12-04T12:17:21Z" + metadata: "{}" - id: 4 path: "f39c8.md" @@ -45,6 +48,7 @@ checksum: "irkwyc" created: "2020-01-19T10:58:41Z" modified: "2020-01-20T08:52:42Z" + metadata: "{}" - id: 5 path: "ref/test/b.md" @@ -57,6 +61,7 @@ checksum: "yvwbae" created: "2019-11-20T20:32:56Z" modified: "2019-11-20T20:34:06Z" + metadata: "{}" - id: 6 path: "ref/test/a.md" @@ -69,6 +74,7 @@ checksum: "iecywst" created: "2019-11-20T20:32:56Z" modified: "2019-11-20T20:34:06Z" + metadata: '{"alias":"a.md"}' - id: 7 path: "log/2021-02-04.md" @@ -81,3 +87,4 @@ checksum: "earkte" created: "2020-11-29T08:20:18Z" modified: "2020-11-10T08:20:18Z" + metadata: "{}" diff --git a/adapter/sqlite/note_dao.go b/adapter/sqlite/note_dao.go index 0328a4e..f58ef32 100644 --- a/adapter/sqlite/note_dao.go +++ b/adapter/sqlite/note_dao.go @@ -2,6 +2,7 @@ package sqlite import ( "database/sql" + "encoding/json" "fmt" "regexp" "strconv" @@ -51,14 +52,14 @@ func NewNoteDAO(tx Transaction, logger util.Logger) *NoteDAO { // Add a new note to the index. addStmt: tx.PrepareLazy(` - INSERT INTO notes (path, sortable_path, title, lead, body, raw_content, word_count, checksum, created, modified) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO notes (path, sortable_path, title, lead, body, raw_content, word_count, metadata, checksum, created, modified) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `), // Update the content of a note. updateStmt: tx.PrepareLazy(` UPDATE notes - SET title = ?, lead = ?, body = ?, raw_content = ?, word_count = ?, checksum = ?, modified = ? + SET title = ?, lead = ?, body = ?, raw_content = ?, word_count = ?, metadata = ?, checksum = ?, modified = ? WHERE path = ? `), @@ -82,8 +83,8 @@ func NewNoteDAO(tx Transaction, logger util.Logger) *NoteDAO { // Add a new link. addLinkStmt: tx.PrepareLazy(` - INSERT INTO links (source_id, target_id, title, href, external, rels, snippet) - VALUES (?, ?, ?, ?, ?, ?, ?) + INSERT INTO links (source_id, target_id, title, href, external, rels, snippet, snippet_start, snippet_end) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) `), // Set links matching a given href and missing a target ID to the given @@ -149,9 +150,15 @@ func (d *NoteDAO) Add(note note.Metadata) (core.NoteId, error) { // string. sortablePath := strings.ReplaceAll(note.Path, "/", "\x01") + metadata, err := d.metadataToJson(note) + if err != nil { + return 0, err + } + res, err := d.addStmt.Exec( - note.Path, sortablePath, note.Title, note.Lead, note.Body, note.RawContent, note.WordCount, note.Checksum, - note.Created, note.Modified, + note.Path, sortablePath, note.Title, note.Lead, note.Body, + note.RawContent, note.WordCount, metadata, note.Checksum, note.Created, + note.Modified, ) if err != nil { return 0, err @@ -177,9 +184,14 @@ func (d *NoteDAO) Update(note note.Metadata) (core.NoteId, error) { return 0, errors.New("note not found in the index") } + metadata, err := d.metadataToJson(note) + if err != nil { + return 0, err + } + _, err = d.updateStmt.Exec( - note.Title, note.Lead, note.Body, note.RawContent, note.WordCount, note.Checksum, note.Modified, - note.Path, + note.Title, note.Lead, note.Body, note.RawContent, note.WordCount, + metadata, note.Checksum, note.Modified, note.Path, ) if err != nil { return id, err @@ -194,6 +206,14 @@ func (d *NoteDAO) Update(note note.Metadata) (core.NoteId, error) { return id, err } +func (d *NoteDAO) metadataToJson(note note.Metadata) (string, error) { + json, err := json.Marshal(note.Metadata) + if err != nil { + return "", errors.Wrapf(err, "cannot serialize note metadata to JSON: %s", note.Path) + } + return string(json), nil +} + // addLinks inserts all the outbound links of the given note. func (d *NoteDAO) addLinks(id core.NoteId, note note.Metadata) error { for _, link := range note.Links { @@ -202,7 +222,7 @@ func (d *NoteDAO) addLinks(id core.NoteId, note note.Metadata) error { return err } - _, err = d.addLinkStmt.Exec(id, d.idToSql(targetId), link.Title, link.Href, link.External, joinLinkRels(link.Rels), link.Snippet) + _, err = d.addLinkStmt.Exec(id, d.idToSql(targetId), link.Title, link.Href, link.External, joinLinkRels(link.Rels), link.Snippet, link.SnippetStart, link.SnippetEnd) if err != nil { return err } @@ -295,16 +315,25 @@ func (d *NoteDAO) Find(opts note.FinderOpts) ([]note.Match, error) { id, wordCount int title, lead, body, rawContent string snippets, tags sql.NullString - path, checksum string + path, metadataJSON, checksum string created, modified time.Time ) - err := rows.Scan(&id, &path, &title, &lead, &body, &rawContent, &wordCount, &created, &modified, &checksum, &tags, &snippets) + err := rows.Scan( + &id, &path, &title, &lead, &body, &rawContent, &wordCount, + &created, &modified, &metadataJSON, &checksum, &tags, &snippets, + ) if err != nil { d.logger.Err(err) continue } + var metadata map[string]interface{} + err = json.Unmarshal([]byte(metadataJSON), &metadata) + if err != nil { + d.logger.Err(errors.Wrapf(err, "cannot parse note metadata from JSON: %s", path)) + } + matches = append(matches, note.Match{ Snippets: parseListFromNullString(snippets), Metadata: note.Metadata{ @@ -316,6 +345,7 @@ func (d *NoteDAO) Find(opts note.FinderOpts) ([]note.Match, error) { WordCount: wordCount, Links: []note.Link{}, Tags: parseListFromNullString(tags), + Metadata: metadata, Created: created, Modified: modified, Checksum: checksum, @@ -585,7 +615,7 @@ SELECT note_id FROM notes_collections query += "\n)\n" } - query += fmt.Sprintf("SELECT n.id, n.path, n.title, n.lead, n.body, n.raw_content, n.word_count, n.created, n.modified, n.checksum, n.tags, %s AS snippet\n", snippetCol) + query += fmt.Sprintf("SELECT n.id, n.path, n.title, n.lead, n.body, n.raw_content, n.word_count, n.created, n.modified, n.metadata, n.checksum, n.tags, %s AS snippet\n", snippetCol) query += "FROM notes_with_metadata n\n" diff --git a/adapter/sqlite/note_dao_test.go b/adapter/sqlite/note_dao_test.go index c7fcd18..22a34f0 100644 --- a/adapter/sqlite/note_dao_test.go +++ b/adapter/sqlite/note_dao_test.go @@ -111,6 +111,7 @@ func TestNoteDAOAdd(t *testing.T) { Body: "Note body", RawContent: "# Added note\nNote body", WordCount: 2, + Metadata: map[string]interface{}{"key": "value"}, Created: time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC), Modified: time.Date(2020, 11, 22, 16, 49, 47, 0, time.UTC), Checksum: "check", @@ -129,6 +130,7 @@ func TestNoteDAOAdd(t *testing.T) { Checksum: "check", Created: time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC), Modified: time.Date(2020, 11, 22, 16, 49, 47, 0, time.UTC), + Metadata: `{"key":"value"}`, }) }) } @@ -144,9 +146,11 @@ func TestNoteDAOAddWithLinks(t *testing.T) { Rels: []string{"rel-1", "rel-2"}, }, { - Title: "Relative", - Href: "f39c8", - Snippet: "[Relative](f39c8) link", + Title: "Relative", + Href: "f39c8", + Snippet: "[Relative](f39c8) link", + SnippetStart: 50, + SnippetEnd: 100, }, { Title: "Second is added", @@ -177,12 +181,14 @@ func TestNoteDAOAddWithLinks(t *testing.T) { Rels: "\x01rel-1\x01rel-2\x01", }, { - SourceId: id, - TargetId: idPointer(4), - Title: "Relative", - Href: "f39c8", - Rels: "", - Snippet: "[Relative](f39c8) link", + SourceId: id, + TargetId: idPointer(4), + Title: "Relative", + Href: "f39c8", + Rels: "", + Snippet: "[Relative](f39c8) link", + SnippetStart: 50, + SnippetEnd: 100, }, { SourceId: id, @@ -248,6 +254,7 @@ func TestNoteDAOUpdate(t *testing.T) { Body: "Updated body", RawContent: "Updated raw content", Checksum: "updated checksum", + Metadata: map[string]interface{}{"updated-key": "updated-value"}, WordCount: 42, Created: time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC), Modified: time.Date(2020, 11, 22, 16, 49, 47, 0, time.UTC), @@ -267,6 +274,7 @@ func TestNoteDAOUpdate(t *testing.T) { WordCount: 42, Created: time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC), Modified: time.Date(2020, 11, 22, 16, 49, 47, 0, time.UTC), + Metadata: `{"updated-key":"updated-value"}`, }) }) } @@ -439,6 +447,7 @@ func TestNoteDAOFindMatch(t *testing.T) { WordCount: 4, Links: []note.Link{}, Tags: []string{}, + Metadata: map[string]interface{}{}, Created: time.Date(2019, 12, 4, 11, 59, 11, 0, time.UTC), Modified: time.Date(2019, 12, 4, 12, 17, 21, 0, time.UTC), Checksum: "iaefhv", @@ -455,6 +464,7 @@ func TestNoteDAOFindMatch(t *testing.T) { WordCount: 4, Links: []note.Link{}, Tags: []string{}, + Metadata: map[string]interface{}{}, Created: time.Date(2020, 11, 29, 8, 20, 18, 0, time.UTC), Modified: time.Date(2020, 11, 10, 8, 20, 18, 0, time.UTC), Checksum: "earkte", @@ -471,6 +481,7 @@ func TestNoteDAOFindMatch(t *testing.T) { WordCount: 4, Links: []note.Link{}, Tags: []string{}, + Metadata: map[string]interface{}{}, Created: time.Date(2020, 11, 29, 8, 20, 18, 0, time.UTC), Modified: time.Date(2020, 11, 29, 8, 20, 18, 0, time.UTC), Checksum: "arstde", @@ -487,9 +498,12 @@ func TestNoteDAOFindMatch(t *testing.T) { WordCount: 3, Links: []note.Link{}, Tags: []string{"fiction", "adventure"}, - Created: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC), - Modified: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC), - Checksum: "qwfpgj", + Metadata: map[string]interface{}{ + "author": "Dom", + }, + Created: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC), + Modified: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC), + Checksum: "qwfpgj", }, Snippets: []string{"A daily note\n\nWith lot of content"}, }, @@ -633,9 +647,12 @@ func TestNoteDAOFindLinkedByWithSnippets(t *testing.T) { WordCount: 5, Links: []note.Link{}, Tags: []string{}, - Created: time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC), - Modified: time.Date(2019, 11, 20, 20, 34, 6, 0, time.UTC), - Checksum: "iecywst", + Metadata: map[string]interface{}{ + "alias": "a.md", + }, + Created: time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC), + Modified: time.Date(2019, 11, 20, 20, 34, 6, 0, time.UTC), + Checksum: "iecywst", }, Snippets: []string{ "[[Link from 4 to 6]]", @@ -652,9 +669,12 @@ func TestNoteDAOFindLinkedByWithSnippets(t *testing.T) { WordCount: 3, Links: []note.Link{}, Tags: []string{"fiction", "adventure"}, - Created: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC), - Modified: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC), - Checksum: "qwfpgj", + Metadata: map[string]interface{}{ + "author": "Dom", + }, + Created: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC), + Modified: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC), + Checksum: "qwfpgj", }, Snippets: []string{ "[[Another link]]", @@ -939,18 +959,18 @@ func testNoteDAOWithoutFixtures(t *testing.T, callback func(tx Transaction, dao } type noteRow struct { - Path, Title, Lead, Body, RawContent, Checksum string - WordCount int - Created, Modified time.Time + Path, Title, Lead, Body, RawContent, Checksum, Metadata string + WordCount int + Created, Modified time.Time } func queryNoteRow(tx Transaction, where string) (noteRow, error) { var row noteRow err := tx.QueryRow(fmt.Sprintf(` - SELECT path, title, lead, body, raw_content, word_count, checksum, created, modified + SELECT path, title, lead, body, raw_content, word_count, checksum, created, modified, metadata FROM notes WHERE %v - `, where)).Scan(&row.Path, &row.Title, &row.Lead, &row.Body, &row.RawContent, &row.WordCount, &row.Checksum, &row.Created, &row.Modified) + `, where)).Scan(&row.Path, &row.Title, &row.Lead, &row.Body, &row.RawContent, &row.WordCount, &row.Checksum, &row.Created, &row.Modified, &row.Metadata) return row, err } @@ -958,6 +978,7 @@ type linkRow struct { SourceId core.NoteId TargetId *core.NoteId Href, Title, Rels, Snippet string + SnippetStart, SnippetEnd int External bool } @@ -965,7 +986,7 @@ func queryLinkRows(t *testing.T, tx Transaction, where string) []linkRow { links := make([]linkRow, 0) rows, err := tx.Query(fmt.Sprintf(` - SELECT source_id, target_id, title, href, external, rels, snippet + SELECT source_id, target_id, title, href, external, rels, snippet, snippet_start, snippet_end FROM links WHERE %v ORDER BY id @@ -976,7 +997,7 @@ func queryLinkRows(t *testing.T, tx Transaction, where string) []linkRow { var row linkRow var sourceId int64 var targetId *int64 - err = rows.Scan(&sourceId, &targetId, &row.Title, &row.Href, &row.External, &row.Rels, &row.Snippet) + err = rows.Scan(&sourceId, &targetId, &row.Title, &row.Href, &row.External, &row.Rels, &row.Snippet, &row.SnippetStart, &row.SnippetEnd) assert.Nil(t, err) row.SourceId = core.NoteId(sourceId) if targetId != nil { diff --git a/adapter/sqlite/transaction_test.go b/adapter/sqlite/transaction_test.go index 8b8b27c..340107c 100644 --- a/adapter/sqlite/transaction_test.go +++ b/adapter/sqlite/transaction_test.go @@ -25,7 +25,7 @@ func testTransactionWithoutFixtures(t *testing.T, test func(tx Transaction)) { func testTransactionWithFixtures(t *testing.T, fixturesDir opt.String, test func(tx Transaction)) { db, err := OpenInMemory() assert.Nil(t, err) - err = db.Migrate() + _, err = db.Migrate() assert.Nil(t, err) if !fixturesDir.IsNull() { diff --git a/cmd/container.go b/cmd/container.go index bece518..3168007 100644 --- a/cmd/container.go +++ b/cmd/container.go @@ -4,16 +4,21 @@ import ( "io" "os" "sync" + "time" "github.com/mickael-menu/zk/adapter/fzf" "github.com/mickael-menu/zk/adapter/handlebars" "github.com/mickael-menu/zk/adapter/markdown" "github.com/mickael-menu/zk/adapter/sqlite" "github.com/mickael-menu/zk/adapter/term" + "github.com/mickael-menu/zk/core/note" "github.com/mickael-menu/zk/core/zk" "github.com/mickael-menu/zk/util" "github.com/mickael-menu/zk/util/date" + "github.com/mickael-menu/zk/util/errors" "github.com/mickael-menu/zk/util/pager" + "github.com/mickael-menu/zk/util/paths" + "github.com/schollz/progressbar/v3" ) type Container struct { @@ -77,14 +82,53 @@ func (c *Container) NoteIndexer(tx sqlite.Transaction) *sqlite.NoteIndexer { } // Database returns the DB instance for the given notebook, after executing any -// pending migration. -func (c *Container) Database(path string) (*sqlite.DB, error) { - db, err := sqlite.Open(path) +// pending migration and indexing the notes if needed. +func (c *Container) Database(zk *zk.Zk, forceIndexing bool) (*sqlite.DB, note.IndexingStats, error) { + var stats note.IndexingStats + + db, err := sqlite.Open(zk.DBPath()) + if err != nil { + return nil, stats, err + } + needsReindexing, err := db.Migrate() if err != nil { - return nil, err + return nil, stats, errors.Wrap(err, "failed to migrate the database") } - err = db.Migrate() - return db, err + + stats, err = c.index(zk, db, forceIndexing || needsReindexing) + if err != nil { + return nil, stats, err + } + + return db, stats, err +} + +func (c *Container) index(zk *zk.Zk, db *sqlite.DB, force bool) (note.IndexingStats, error) { + var bar = progressbar.NewOptions(-1, + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionThrottle(100*time.Millisecond), + progressbar.OptionSpinnerType(14), + ) + + var err error + var stats note.IndexingStats + err = db.WithTransaction(func(tx sqlite.Transaction) error { + stats, err = note.Index( + zk, + force, + c.Parser(), + c.NoteIndexer(tx), + c.Logger, + func(change paths.DiffChange) { + bar.Add(1) + bar.Describe(change.String()) + }, + ) + return err + }) + bar.Clear() + + return stats, err } // Paginate creates an auto-closing io.Writer which will be automatically diff --git a/cmd/edit.go b/cmd/edit.go index 56e5da4..9d417d6 100644 --- a/cmd/edit.go +++ b/cmd/edit.go @@ -36,7 +36,7 @@ func (cmd *Edit) Run(container *Container) error { return errors.Wrapf(err, "incorrect criteria") } - db, err := container.Database(zk.DBPath()) + db, _, err := container.Database(zk, false) if err != nil { return err } diff --git a/cmd/index.go b/cmd/index.go index 11dbe55..203bb1f 100644 --- a/cmd/index.go +++ b/cmd/index.go @@ -2,13 +2,6 @@ package cmd import ( "fmt" - "os" - "time" - - "github.com/mickael-menu/zk/adapter/sqlite" - "github.com/mickael-menu/zk/core/note" - "github.com/mickael-menu/zk/util/paths" - "github.com/schollz/progressbar/v3" ) // Index indexes the content of all the notes in the notebook. @@ -18,7 +11,7 @@ type Index struct { } func (cmd *Index) Help() string { - return "You usually don't need to run `zk index` manually, as notes are indexed automatically before each zk invocation." + return "You usually do not need to run `zk index` manually, as notes are indexed automatically when needed." } func (cmd *Index) Run(container *Container) error { @@ -27,34 +20,11 @@ func (cmd *Index) Run(container *Container) error { return err } - db, err := container.Database(zk.DBPath()) + _, stats, err := container.Database(zk, cmd.Force) if err != nil { return err } - var bar = progressbar.NewOptions(-1, - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionThrottle(100*time.Millisecond), - progressbar.OptionSpinnerType(14), - ) - - var stats note.IndexingStats - err = db.WithTransaction(func(tx sqlite.Transaction) error { - stats, err = note.Index( - zk, - cmd.Force, - container.Parser(), - container.NoteIndexer(tx), - container.Logger, - func(change paths.DiffChange) { - bar.Add(1) - bar.Describe(change.String()) - }, - ) - return err - }) - bar.Clear() - if err == nil && !cmd.Quiet { fmt.Println(stats) } diff --git a/cmd/list.go b/cmd/list.go index 840f02c..0193e35 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -39,7 +39,7 @@ func (cmd *List) Run(container *Container) error { return err } - db, err := container.Database(zk.DBPath()) + db, _, err := container.Database(zk, false) if err != nil { return err } diff --git a/core/note/format.go b/core/note/format.go index 723a56f..c94a738 100644 --- a/core/note/format.go +++ b/core/note/format.go @@ -108,6 +108,7 @@ func (f *Formatter) Format(match Match) (string, error) { Tags: match.Tags, RawContent: match.RawContent, WordCount: match.WordCount, + Metadata: match.Metadata.Metadata, Created: match.Created, Modified: match.Modified, Checksum: match.Checksum, @@ -123,6 +124,7 @@ type formatRenderContext struct { RawContent string `handlebars:"raw-content"` WordCount int `handlebars:"word-count"` Tags []string + Metadata map[string]interface{} Created time.Time Modified time.Time Checksum string diff --git a/core/note/index.go b/core/note/index.go index 1b049f1..ff42acc 100644 --- a/core/note/index.go +++ b/core/note/index.go @@ -27,6 +27,7 @@ type Metadata struct { WordCount int Links []Link Tags []string + Metadata map[string]interface{} Created time.Time Modified time.Time Checksum string @@ -139,6 +140,7 @@ func metadata(path string, zk *zk.Zk, parser Parser) (Metadata, error) { metadata.WordCount = len(strings.Fields(contentStr)) metadata.Links = make([]Link, 0) metadata.Tags = contentParts.Tags + metadata.Metadata = contentParts.Metadata metadata.Checksum = fmt.Sprintf("%x", sha256.Sum256(content)) for _, link := range contentParts.Links { diff --git a/core/note/parse.go b/core/note/parse.go index da9513f..15433d4 100644 --- a/core/note/parse.go +++ b/core/note/parse.go @@ -15,15 +15,19 @@ type Content struct { Tags []string // Links is the list of outbound links found in the note. Links []Link + // Additional metadata. For example, extracted from a YAML frontmatter. + Metadata map[string]interface{} } // Link links a note to another note or an external resource. type Link struct { - Title string - Href string - External bool - Rels []string - Snippet string + Title string + Href string + External bool + Rels []string + Snippet string + SnippetStart int + SnippetEnd int } // LinkRelation defines the relationship between a link's source and target. diff --git a/docs/template-format.md b/docs/template-format.md index 763c0a5..8357cf3 100644 --- a/docs/template-format.md +++ b/docs/template-format.md @@ -2,17 +2,19 @@ The following variables are available in the templates used when formatting notes, for example with `zk list --format