Save additional metadata in the database (#7)

* YAML frontmatter as a JSON object in `notes.metadata` * Print metadata from the YAML frontmatter in `list` output using `{{metadata.<key>}}`, e.g. `{{metadata.description}}`. Keys are normalized to lower case. * Start and end offsets for link snippets. * This could be useful to expand backlinks contexts.
3 years ago · f5b3102deb
parent bae1fab567
commit f5b3102deb
18 changed files with 360 additions and 200 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -9,8 +9,10 @@ All notable changes to this project will be documented in this file.
 * Support for tags.
    * Filter notes by their tags using `--tag "history, europe"`.
        * To match notes associated with either tags, use a pipe `|` or `OR` (all caps), e.g. `--tag "inbox OR todo"`.
-        * If you want to exclude notes having a particular tag, prefix it with `-` or `NOT` (all caps), e.g. `--tag "NOT done"`
+        * If you want to exclude notes having a particular tag, prefix it with `-` or `NOT` (all caps), e.g. `--tag "NOT done"`.
+        * Use glob patterns to match multiple tags, e.g. `--tag "book-*"`.
    * Many tag flavors are supported: `#hashtags`, `:colon:separated:tags:` and even Bear's [`#multi-word tags#`](https://blog.bear.app/2017/11/bear-tips-how-to-create-multi-word-tags/). If you prefer to use a YAML frontmatter, list your tags with the key `tags` or `keywords`.
+* Print metadata from the YAML frontmatter in `list` output using `{{metadata.<key>}}`, e.g. `{{metadata.description}}`. Keys are normalized to lower case.

 ### Changed

--- a/adapter/markdown/markdown.go
+++ b/adapter/markdown/markdown.go
@ -91,11 +91,12 @@ func (p *Parser) Parse(source string) (*note.Content, error) {
 	}

 	return &note.Content{
-		Title: title,
-		Body:  body,
-		Lead:  parseLead(body),
-		Links: links,
-		Tags:  tags,
+		Title:    title,
+		Body:     body,
+		Lead:     parseLead(body),
+		Links:    links,
+		Tags:     tags,
+		Metadata: frontmatter.values,
 	}, nil
 }

@ -215,23 +216,29 @@ func parseLinks(root ast.Node, source []byte) ([]note.Link, error) {
 			case *ast.Link:
 				href := string(link.Destination)
 				if href != "" {
+					snippet, snStart, snEnd := extractLines(n.Parent(), source)
 					links = append(links, note.Link{
-						Title:    string(link.Text(source)),
-						Href:     href,
-						Rels:     strings.Fields(string(link.Title)),
-						External: strutil.IsURL(href),
-						Snippet:  extractLines(n.Parent(), source),
+						Title:        string(link.Text(source)),
+						Href:         href,
+						Rels:         strings.Fields(string(link.Title)),
+						External:     strutil.IsURL(href),
+						Snippet:      snippet,
+						SnippetStart: snStart,
+						SnippetEnd:   snEnd,
 					})
 				}

 			case *ast.AutoLink:
 				if href := string(link.URL(source)); href != "" && link.AutoLinkType == ast.AutoLinkURL {
+					snippet, snStart, snEnd := extractLines(n.Parent(), source)
 					links = append(links, note.Link{
-						Title:    string(link.Label(source)),
-						Href:     href,
-						Rels:     []string{},
-						External: true,
-						Snippet:  extractLines(n.Parent(), source),
+						Title:        string(link.Label(source)),
+						Href:         href,
+						Rels:         []string{},
+						External:     true,
+						Snippet:      snippet,
+						SnippetStart: snStart,
+						SnippetEnd:   snEnd,
 					})
 				}
 			}
@ -241,17 +248,18 @@ func parseLinks(root ast.Node, source []byte) ([]note.Link, error) {
 	return links, err
 }

-func extractLines(n ast.Node, source []byte) string {
+func extractLines(n ast.Node, source []byte) (content string, start, end int) {
 	if n == nil {
-		return ""
+		return
 	}
 	segs := n.Lines()
 	if segs.Len() == 0 {
-		return ""
+		return
 	}
-	start := segs.At(0).Start
-	end := segs.At(segs.Len() - 1).Stop
-	return string(source[start:end])
+	start = segs.At(0).Start
+	end = segs.At(segs.Len() - 1).Stop
+	content = string(source[start:end])
+	return
 }

 // frontmatter contains metadata parsed from a YAML frontmatter.
@ -265,6 +273,7 @@ var frontmatterRegex = regexp.MustCompile(`(?ms)^\s*-+\s*$.*?^\s*-+\s*$`)

 func parseFrontmatter(context parser.Context, source []byte) (frontmatter, error) {
 	var front frontmatter
+	front.values = map[string]interface{}{}

 	index := frontmatterRegex.FindIndex(source)
 	if index == nil {
@ -273,7 +282,6 @@ func parseFrontmatter(context parser.Context, source []byte) (frontmatter, error

 	front.start = index[0]
 	front.end = index[1]
-	front.values = map[string]interface{}{}

 	values, err := meta.TryGet(context)
 	if err != nil {
--- a/adapter/markdown/markdown_test.go
+++ b/adapter/markdown/markdown_test.go
@ -377,11 +377,13 @@ Neuron links with titles: [[trailing|Trailing link]]# #[[leading |  Leading link
 [External links](http://example.com) are marked [as such](ftp://domain).
 `, []note.Link{
 		{
-			Title:    "link",
-			Href:     "heading",
-			Rels:     []string{},
-			External: false,
-			Snippet:  "Heading with a [link](heading)",
+			Title:        "link",
+			Href:         "heading",
+			Rels:         []string{},
+			External:     false,
+			Snippet:      "Heading with a [link](heading)",
+			SnippetStart: 3,
+			SnippetEnd:   33,
 		},
 		{
 			Title:    "multiple links",
@ -390,6 +392,8 @@ Neuron links with titles: [[trailing|Trailing link]]# #[[leading |  Leading link
 			External: false,
 			Snippet: `Paragraph containing [multiple **links**](stripped-formatting), here's one [relative](../other).
 A link can have [one relation](one "rel-1") or [several relations](several "rel-1 rel-2").`,
+			SnippetStart: 35,
+			SnippetEnd:   222,
 		},
 		{
 			Title:    "relative",
@ -398,6 +402,8 @@ A link can have [one relation](one "rel-1") or [several relations](several "rel-
 			External: false,
 			Snippet: `Paragraph containing [multiple **links**](stripped-formatting), here's one [relative](../other).
 A link can have [one relation](one "rel-1") or [several relations](several "rel-1 rel-2").`,
+			SnippetStart: 35,
+			SnippetEnd:   222,
 		},
 		{
 			Title:    "one relation",
@ -406,6 +412,8 @@ A link can have [one relation](one "rel-1") or [several relations](several "rel-
 			External: false,
 			Snippet: `Paragraph containing [multiple **links**](stripped-formatting), here's one [relative](../other).
 A link can have [one relation](one "rel-1") or [several relations](several "rel-1 rel-2").`,
+			SnippetStart: 35,
+			SnippetEnd:   222,
 		},
 		{
 			Title:    "several relations",
@ -414,94 +422,143 @@ A link can have [one relation](one "rel-1") or [several relations](several "rel-
 			External: false,
 			Snippet: `Paragraph containing [multiple **links**](stripped-formatting), here's one [relative](../other).
 A link can have [one relation](one "rel-1") or [several relations](several "rel-1 rel-2").`,
+			SnippetStart: 35,
+			SnippetEnd:   222,
 		},
 		{
-			Title:    "https://inline-link.com",
-			Href:     "https://inline-link.com",
-			External: true,
-			Rels:     []string{},
-			Snippet:  "An https://inline-link.com and http://another-inline-link.com.",
+			Title:        "https://inline-link.com",
+			Href:         "https://inline-link.com",
+			External:     true,
+			Rels:         []string{},
+			Snippet:      "An https://inline-link.com and http://another-inline-link.com.",
+			SnippetStart: 224,
+			SnippetEnd:   286,
 		},
 		{
-			Title:    "http://another-inline-link.com",
-			Href:     "http://another-inline-link.com",
-			External: true,
-			Rels:     []string{},
-			Snippet:  "An https://inline-link.com and http://another-inline-link.com.",
+			Title:        "http://another-inline-link.com",
+			Href:         "http://another-inline-link.com",
+			External:     true,
+			Rels:         []string{},
+			Snippet:      "An https://inline-link.com and http://another-inline-link.com.",
+			SnippetStart: 224,
+			SnippetEnd:   286,
 		},
 		{
-			Title:    "Wiki link",
-			Href:     "Wiki link",
-			External: false,
-			Rels:     []string{},
-			Snippet:  "A [[Wiki link]] is surrounded by [[2-brackets | two brackets]].",
+			Title:        "Wiki link",
+			Href:         "Wiki link",
+			External:     false,
+			Rels:         []string{},
+			Snippet:      "A [[Wiki link]] is surrounded by [[2-brackets | two brackets]].",
+			SnippetStart: 288,
+			SnippetEnd:   351,
 		},
 		{
-			Title:    "two brackets",
-			Href:     "2-brackets",
-			External: false,
-			Rels:     []string{},
-			Snippet:  "A [[Wiki link]] is surrounded by [[2-brackets | two brackets]].",
+			Title:        "two brackets",
+			Href:         "2-brackets",
+			External:     false,
+			Rels:         []string{},
+			Snippet:      "A [[Wiki link]] is surrounded by [[2-brackets | two brackets]].",
+			SnippetStart: 288,
+			SnippetEnd:   351,
 		},
 		{
-			Title:    `esca]]ped [chara\cters`,
-			Href:     `esca]]ped [chara\cters`,
-			External: false,
-			Rels:     []string{},
-			Snippet:  `It can contain [[esca]\]ped \[chara\\cters]].`,
+			Title:        `esca]]ped [chara\cters`,
+			Href:         `esca]]ped [chara\cters`,
+			External:     false,
+			Rels:         []string{},
+			Snippet:      `It can contain [[esca]\]ped \[chara\\cters]].`,
+			SnippetStart: 353,
+			SnippetEnd:   398,
 		},
 		{
-			Title:    "Folgezettel link",
-			Href:     "Folgezettel link",
-			External: false,
-			Rels:     []string{"down"},
-			Snippet:  "A [[[Folgezettel link]]] is surrounded by three brackets.",
+			Title:        "Folgezettel link",
+			Href:         "Folgezettel link",
+			External:     false,
+			Rels:         []string{"down"},
+			Snippet:      "A [[[Folgezettel link]]] is surrounded by three brackets.",
+			SnippetStart: 400,
+			SnippetEnd:   457,
 		},
 		{
-			Title:    "trailing hash",
-			Href:     "trailing hash",
-			External: false,
-			Rels:     []string{"down"},
-			Snippet:  "Neuron also supports a [[trailing hash]]# for Folgezettel links.",
+			Title:        "trailing hash",
+			Href:         "trailing hash",
+			External:     false,
+			Rels:         []string{"down"},
+			Snippet:      "Neuron also supports a [[trailing hash]]# for Folgezettel links.",
+			SnippetStart: 459,
+			SnippetEnd:   523,
 		},
 		{
-			Title:    "leading hash",
-			Href:     "leading hash",
-			External: false,
-			Rels:     []string{"up"},
-			Snippet:  "A #[[leading hash]] is used for #uplinks.",
+			Title:        "leading hash",
+			Href:         "leading hash",
+			External:     false,
+			Rels:         []string{"up"},
+			Snippet:      "A #[[leading hash]] is used for #uplinks.",
+			SnippetStart: 525,
+			SnippetEnd:   566,
 		},
 		{
-			Title:    "Trailing link",
-			Href:     "trailing",
-			External: false,
-			Rels:     []string{"down"},
-			Snippet:  "Neuron links with titles: [[trailing|Trailing link]]# #[[leading |  Leading link]]",
+			Title:        "Trailing link",
+			Href:         "trailing",
+			External:     false,
+			Rels:         []string{"down"},
+			Snippet:      "Neuron links with titles: [[trailing|Trailing link]]# #[[leading |  Leading link]]",
+			SnippetStart: 568,
+			SnippetEnd:   650,
 		},
 		{
-			Title:    "Leading link",
-			Href:     "leading",
-			External: false,
-			Rels:     []string{"up"},
-			Snippet:  "Neuron links with titles: [[trailing|Trailing link]]# #[[leading |  Leading link]]",
+			Title:        "Leading link",
+			Href:         "leading",
+			External:     false,
+			Rels:         []string{"up"},
+			Snippet:      "Neuron links with titles: [[trailing|Trailing link]]# #[[leading |  Leading link]]",
+			SnippetStart: 568,
+			SnippetEnd:   650,
 		},
 		{
-			Title:    "External links",
-			Href:     "http://example.com",
-			Rels:     []string{},
-			External: true,
-			Snippet:  `[External links](http://example.com) are marked [as such](ftp://domain).`,
+			Title:        "External links",
+			Href:         "http://example.com",
+			Rels:         []string{},
+			External:     true,
+			Snippet:      `[External links](http://example.com) are marked [as such](ftp://domain).`,
+			SnippetStart: 652,
+			SnippetEnd:   724,
 		},
 		{
-			Title:    "as such",
-			Href:     "ftp://domain",
-			Rels:     []string{},
-			External: true,
-			Snippet:  `[External links](http://example.com) are marked [as such](ftp://domain).`,
+			Title:        "as such",
+			Href:         "ftp://domain",
+			Rels:         []string{},
+			External:     true,
+			Snippet:      `[External links](http://example.com) are marked [as such](ftp://domain).`,
+			SnippetStart: 652,
+			SnippetEnd:   724,
 		},
 	})
 }

+func TestParseMetadataFromFrontmatter(t *testing.T) {
+	test := func(source string, expectedMetadata map[string]interface{}) {
+		content := parse(t, source)
+		assert.Equal(t, content.Metadata, expectedMetadata)
+	}
+
+	test("", map[string]interface{}{})
+	test("# A title", map[string]interface{}{})
+	test("---\n---\n# A title", map[string]interface{}{})
+	test(`---
+title: A title
+tags:
+  - tag1
+  - "tag 2"
+---
+
+Paragraph
+`, map[string]interface{}{
+		"title": "A title",
+		"tags":  []interface{}{"tag1", "tag 2"},
+	})
+}
+
 func parse(t *testing.T, source string) note.Content {
 	return parseWithOptions(t, source, ParserOpts{
 		HashtagEnabled:      true,
--- a/adapter/sqlite/db.go
+++ b/adapter/sqlite/db.go
@ -48,8 +48,8 @@ func (db *DB) Close() error {
 }

 // Migrate upgrades the SQL schema of the database.
-func (db *DB) Migrate() error {
-	err := db.WithTransaction(func(tx Transaction) error {
+func (db *DB) Migrate() (needsReindexing bool, err error) {
+	err = db.WithTransaction(func(tx Transaction) error {
 		var version int
 		err := tx.QueryRow("PRAGMA user_version").Scan(&version)
 		if err != nil {
@ -154,8 +154,28 @@ func (db *DB) Migrate() error {
 			}
 		}

+		if version <= 2 {
+			err = tx.ExecStmts([]string{
+				// Add a `metadata` column to `notes`
+				`ALTER TABLE notes ADD COLUMN metadata TEXT DEFAULT('{}') NOT NULL`,
+
+				// Add snippet's start and end offsets to `links`
+				`ALTER TABLE links ADD COLUMN snippet_start INTEGER DEFAULT(0) NOT NULL`,
+				`ALTER TABLE links ADD COLUMN snippet_end INTEGER DEFAULT(0) NOT NULL`,
+
+				`PRAGMA user_version = 3`,
+			})
+
+			if err != nil {
+				return err
+			}
+
+			needsReindexing = true
+		}
+
 		return nil
 	})

-	return errors.Wrap(err, "database migration failed")
+	err = errors.Wrap(err, "database migration failed")
+	return
 }
--- a/adapter/sqlite/db_test.go
+++ b/adapter/sqlite/db_test.go
@ -23,17 +23,17 @@ func TestMigrateFrom0(t *testing.T) {
 	db, err := OpenInMemory()
 	assert.Nil(t, err)

-	err = db.Migrate()
+	_, err = db.Migrate()
 	assert.Nil(t, err)
 	// Should be able to migrate twice in a row
-	err = db.Migrate()
+	_, err = db.Migrate()
 	assert.Nil(t, err)

 	err = db.WithTransaction(func(tx Transaction) error {
 		var version int
 		err := tx.QueryRow("PRAGMA user_version").Scan(&version)
 		assert.Nil(t, err)
-		assert.Equal(t, version, 2)
+		assert.Equal(t, version, 3)

 		_, err = tx.Exec(`
 			INSERT INTO notes (path, sortable_path, title, body, word_count, checksum)
--- a/adapter/sqlite/fixtures/default/notes.yml
+++ b/adapter/sqlite/fixtures/default/notes.yml
@ -9,6 +9,7 @@
  checksum: "qwfpgj"
  created: "2020-11-22T16:27:45Z"
  modified: "2020-11-22T16:27:45Z"
+  metadata: '{"author":"Dom"}'

 - id: 2
  path: "log/2021-01-04.md"
@ -21,6 +22,7 @@
  checksum: "arstde"
  created: "2020-11-29T08:20:18Z"
  modified: "2020-11-29T08:20:18Z"
+  metadata: "{}"

 - id: 3
  path: "index.md"
@ -33,6 +35,7 @@
  checksum: "iaefhv"
  created: "2019-12-04T11:59:11Z"
  modified: "2019-12-04T12:17:21Z"
+  metadata: "{}"

 - id: 4
  path: "f39c8.md"
@ -45,6 +48,7 @@
  checksum: "irkwyc"
  created: "2020-01-19T10:58:41Z"
  modified: "2020-01-20T08:52:42Z"
+  metadata: "{}"

 - id: 5
  path: "ref/test/b.md"
@ -57,6 +61,7 @@
  checksum: "yvwbae"
  created: "2019-11-20T20:32:56Z"
  modified: "2019-11-20T20:34:06Z"
+  metadata: "{}"

 - id: 6
  path: "ref/test/a.md"
@ -69,6 +74,7 @@
  checksum: "iecywst"
  created: "2019-11-20T20:32:56Z"
  modified: "2019-11-20T20:34:06Z"
+  metadata: '{"alias":"a.md"}'

 - id: 7
  path: "log/2021-02-04.md"
@ -81,3 +87,4 @@
  checksum: "earkte"
  created: "2020-11-29T08:20:18Z"
  modified: "2020-11-10T08:20:18Z"
+  metadata: "{}"
--- a/adapter/sqlite/note_dao.go
+++ b/adapter/sqlite/note_dao.go
@ -2,6 +2,7 @@ package sqlite

 import (
 	"database/sql"
+	"encoding/json"
 	"fmt"
 	"regexp"
 	"strconv"
@ -51,14 +52,14 @@ func NewNoteDAO(tx Transaction, logger util.Logger) *NoteDAO {

 		// Add a new note to the index.
 		addStmt: tx.PrepareLazy(`
-			INSERT INTO notes (path, sortable_path, title, lead, body, raw_content, word_count, checksum, created, modified)
-			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+			INSERT INTO notes (path, sortable_path, title, lead, body, raw_content, word_count, metadata, checksum, created, modified)
+			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 		`),

 		// Update the content of a note.
 		updateStmt: tx.PrepareLazy(`
 			UPDATE notes
-			   SET title = ?, lead = ?, body = ?, raw_content = ?, word_count = ?, checksum = ?, modified = ?
+			   SET title = ?, lead = ?, body = ?, raw_content = ?, word_count = ?, metadata = ?, checksum = ?, modified = ?
 			 WHERE path = ?
 		`),

@ -82,8 +83,8 @@ func NewNoteDAO(tx Transaction, logger util.Logger) *NoteDAO {

 		// Add a new link.
 		addLinkStmt: tx.PrepareLazy(`
-			INSERT INTO links (source_id, target_id, title, href, external, rels, snippet)
-			VALUES (?, ?, ?, ?, ?, ?, ?)
+			INSERT INTO links (source_id, target_id, title, href, external, rels, snippet, snippet_start, snippet_end)
+			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
 		`),

 		// Set links matching a given href and missing a target ID to the given
@ -149,9 +150,15 @@ func (d *NoteDAO) Add(note note.Metadata) (core.NoteId, error) {
 	// string.
 	sortablePath := strings.ReplaceAll(note.Path, "/", "\x01")

+	metadata, err := d.metadataToJson(note)
+	if err != nil {
+		return 0, err
+	}
+
 	res, err := d.addStmt.Exec(
-		note.Path, sortablePath, note.Title, note.Lead, note.Body, note.RawContent, note.WordCount, note.Checksum,
-		note.Created, note.Modified,
+		note.Path, sortablePath, note.Title, note.Lead, note.Body,
+		note.RawContent, note.WordCount, metadata, note.Checksum, note.Created,
+		note.Modified,
 	)
 	if err != nil {
 		return 0, err
@ -177,9 +184,14 @@ func (d *NoteDAO) Update(note note.Metadata) (core.NoteId, error) {
 		return 0, errors.New("note not found in the index")
 	}

+	metadata, err := d.metadataToJson(note)
+	if err != nil {
+		return 0, err
+	}
+
 	_, err = d.updateStmt.Exec(
-		note.Title, note.Lead, note.Body, note.RawContent, note.WordCount, note.Checksum, note.Modified,
-		note.Path,
+		note.Title, note.Lead, note.Body, note.RawContent, note.WordCount,
+		metadata, note.Checksum, note.Modified, note.Path,
 	)
 	if err != nil {
 		return id, err
@ -194,6 +206,14 @@ func (d *NoteDAO) Update(note note.Metadata) (core.NoteId, error) {
 	return id, err
 }

+func (d *NoteDAO) metadataToJson(note note.Metadata) (string, error) {
+	json, err := json.Marshal(note.Metadata)
+	if err != nil {
+		return "", errors.Wrapf(err, "cannot serialize note metadata to JSON: %s", note.Path)
+	}
+	return string(json), nil
+}
+
 // addLinks inserts all the outbound links of the given note.
 func (d *NoteDAO) addLinks(id core.NoteId, note note.Metadata) error {
 	for _, link := range note.Links {
@ -202,7 +222,7 @@ func (d *NoteDAO) addLinks(id core.NoteId, note note.Metadata) error {
 			return err
 		}

-		_, err = d.addLinkStmt.Exec(id, d.idToSql(targetId), link.Title, link.Href, link.External, joinLinkRels(link.Rels), link.Snippet)
+		_, err = d.addLinkStmt.Exec(id, d.idToSql(targetId), link.Title, link.Href, link.External, joinLinkRels(link.Rels), link.Snippet, link.SnippetStart, link.SnippetEnd)
 		if err != nil {
 			return err
 		}
@ -295,16 +315,25 @@ func (d *NoteDAO) Find(opts note.FinderOpts) ([]note.Match, error) {
 			id, wordCount                 int
 			title, lead, body, rawContent string
 			snippets, tags                sql.NullString
-			path, checksum                string
+			path, metadataJSON, checksum  string
 			created, modified             time.Time
 		)

-		err := rows.Scan(&id, &path, &title, &lead, &body, &rawContent, &wordCount, &created, &modified, &checksum, &tags, &snippets)
+		err := rows.Scan(
+			&id, &path, &title, &lead, &body, &rawContent, &wordCount,
+			&created, &modified, &metadataJSON, &checksum, &tags, &snippets,
+		)
 		if err != nil {
 			d.logger.Err(err)
 			continue
 		}

+		var metadata map[string]interface{}
+		err = json.Unmarshal([]byte(metadataJSON), &metadata)
+		if err != nil {
+			d.logger.Err(errors.Wrapf(err, "cannot parse note metadata from JSON: %s", path))
+		}
+
 		matches = append(matches, note.Match{
 			Snippets: parseListFromNullString(snippets),
 			Metadata: note.Metadata{
@ -316,6 +345,7 @@ func (d *NoteDAO) Find(opts note.FinderOpts) ([]note.Match, error) {
 				WordCount:  wordCount,
 				Links:      []note.Link{},
 				Tags:       parseListFromNullString(tags),
+				Metadata:   metadata,
 				Created:    created,
 				Modified:   modified,
 				Checksum:   checksum,
@ -585,7 +615,7 @@ SELECT note_id FROM notes_collections
 		query += "\n)\n"
 	}

-	query += fmt.Sprintf("SELECT n.id, n.path, n.title, n.lead, n.body, n.raw_content, n.word_count, n.created, n.modified, n.checksum, n.tags, %s AS snippet\n", snippetCol)
+	query += fmt.Sprintf("SELECT n.id, n.path, n.title, n.lead, n.body, n.raw_content, n.word_count, n.created, n.modified, n.metadata, n.checksum, n.tags, %s AS snippet\n", snippetCol)

 	query += "FROM notes_with_metadata n\n"

--- a/adapter/sqlite/note_dao_test.go
+++ b/adapter/sqlite/note_dao_test.go
@ -111,6 +111,7 @@ func TestNoteDAOAdd(t *testing.T) {
 			Body:       "Note body",
 			RawContent: "# Added note\nNote body",
 			WordCount:  2,
+			Metadata:   map[string]interface{}{"key": "value"},
 			Created:    time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC),
 			Modified:   time.Date(2020, 11, 22, 16, 49, 47, 0, time.UTC),
 			Checksum:   "check",
@ -129,6 +130,7 @@ func TestNoteDAOAdd(t *testing.T) {
 			Checksum:   "check",
 			Created:    time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC),
 			Modified:   time.Date(2020, 11, 22, 16, 49, 47, 0, time.UTC),
+			Metadata:   `{"key":"value"}`,
 		})
 	})
 }
@ -144,9 +146,11 @@ func TestNoteDAOAddWithLinks(t *testing.T) {
 					Rels:  []string{"rel-1", "rel-2"},
 				},
 				{
-					Title:   "Relative",
-					Href:    "f39c8",
-					Snippet: "[Relative](f39c8) link",
+					Title:        "Relative",
+					Href:         "f39c8",
+					Snippet:      "[Relative](f39c8) link",
+					SnippetStart: 50,
+					SnippetEnd:   100,
 				},
 				{
 					Title: "Second is added",
@ -177,12 +181,14 @@ func TestNoteDAOAddWithLinks(t *testing.T) {
 				Rels:     "\x01rel-1\x01rel-2\x01",
 			},
 			{
-				SourceId: id,
-				TargetId: idPointer(4),
-				Title:    "Relative",
-				Href:     "f39c8",
-				Rels:     "",
-				Snippet:  "[Relative](f39c8) link",
+				SourceId:     id,
+				TargetId:     idPointer(4),
+				Title:        "Relative",
+				Href:         "f39c8",
+				Rels:         "",
+				Snippet:      "[Relative](f39c8) link",
+				SnippetStart: 50,
+				SnippetEnd:   100,
 			},
 			{
 				SourceId: id,
@ -248,6 +254,7 @@ func TestNoteDAOUpdate(t *testing.T) {
 			Body:       "Updated body",
 			RawContent: "Updated raw content",
 			Checksum:   "updated checksum",
+			Metadata:   map[string]interface{}{"updated-key": "updated-value"},
 			WordCount:  42,
 			Created:    time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC),
 			Modified:   time.Date(2020, 11, 22, 16, 49, 47, 0, time.UTC),
@ -267,6 +274,7 @@ func TestNoteDAOUpdate(t *testing.T) {
 			WordCount:  42,
 			Created:    time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC),
 			Modified:   time.Date(2020, 11, 22, 16, 49, 47, 0, time.UTC),
+			Metadata:   `{"updated-key":"updated-value"}`,
 		})
 	})
 }
@ -439,6 +447,7 @@ func TestNoteDAOFindMatch(t *testing.T) {
 					WordCount:  4,
 					Links:      []note.Link{},
 					Tags:       []string{},
+					Metadata:   map[string]interface{}{},
 					Created:    time.Date(2019, 12, 4, 11, 59, 11, 0, time.UTC),
 					Modified:   time.Date(2019, 12, 4, 12, 17, 21, 0, time.UTC),
 					Checksum:   "iaefhv",
@ -455,6 +464,7 @@ func TestNoteDAOFindMatch(t *testing.T) {
 					WordCount:  4,
 					Links:      []note.Link{},
 					Tags:       []string{},
+					Metadata:   map[string]interface{}{},
 					Created:    time.Date(2020, 11, 29, 8, 20, 18, 0, time.UTC),
 					Modified:   time.Date(2020, 11, 10, 8, 20, 18, 0, time.UTC),
 					Checksum:   "earkte",
@ -471,6 +481,7 @@ func TestNoteDAOFindMatch(t *testing.T) {
 					WordCount:  4,
 					Links:      []note.Link{},
 					Tags:       []string{},
+					Metadata:   map[string]interface{}{},
 					Created:    time.Date(2020, 11, 29, 8, 20, 18, 0, time.UTC),
 					Modified:   time.Date(2020, 11, 29, 8, 20, 18, 0, time.UTC),
 					Checksum:   "arstde",
@ -487,9 +498,12 @@ func TestNoteDAOFindMatch(t *testing.T) {
 					WordCount:  3,
 					Links:      []note.Link{},
 					Tags:       []string{"fiction", "adventure"},
-					Created:    time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
-					Modified:   time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
-					Checksum:   "qwfpgj",
+					Metadata: map[string]interface{}{
+						"author": "Dom",
+					},
+					Created:  time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
+					Modified: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
+					Checksum: "qwfpgj",
 				},
 				Snippets: []string{"A <zk:match>daily</zk:match> note\n\nWith lot of content"},
 			},
@ -633,9 +647,12 @@ func TestNoteDAOFindLinkedByWithSnippets(t *testing.T) {
 					WordCount:  5,
 					Links:      []note.Link{},
 					Tags:       []string{},
-					Created:    time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC),
-					Modified:   time.Date(2019, 11, 20, 20, 34, 6, 0, time.UTC),
-					Checksum:   "iecywst",
+					Metadata: map[string]interface{}{
+						"alias": "a.md",
+					},
+					Created:  time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC),
+					Modified: time.Date(2019, 11, 20, 20, 34, 6, 0, time.UTC),
+					Checksum: "iecywst",
 				},
 				Snippets: []string{
 					"[[<zk:match>Link from 4 to 6</zk:match>]]",
@ -652,9 +669,12 @@ func TestNoteDAOFindLinkedByWithSnippets(t *testing.T) {
 					WordCount:  3,
 					Links:      []note.Link{},
 					Tags:       []string{"fiction", "adventure"},
-					Created:    time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
-					Modified:   time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
-					Checksum:   "qwfpgj",
+					Metadata: map[string]interface{}{
+						"author": "Dom",
+					},
+					Created:  time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
+					Modified: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
+					Checksum: "qwfpgj",
 				},
 				Snippets: []string{
 					"[[<zk:match>Another link</zk:match>]]",
@ -939,18 +959,18 @@ func testNoteDAOWithoutFixtures(t *testing.T, callback func(tx Transaction, dao
 }

 type noteRow struct {
-	Path, Title, Lead, Body, RawContent, Checksum string
-	WordCount                                     int
-	Created, Modified                             time.Time
+	Path, Title, Lead, Body, RawContent, Checksum, Metadata string
+	WordCount                                               int
+	Created, Modified                                       time.Time
 }

 func queryNoteRow(tx Transaction, where string) (noteRow, error) {
 	var row noteRow
 	err := tx.QueryRow(fmt.Sprintf(`
-		SELECT path, title, lead, body, raw_content, word_count, checksum, created, modified
+		SELECT path, title, lead, body, raw_content, word_count, checksum, created, modified, metadata
 		  FROM notes
 		 WHERE %v
-	`, where)).Scan(&row.Path, &row.Title, &row.Lead, &row.Body, &row.RawContent, &row.WordCount, &row.Checksum, &row.Created, &row.Modified)
+	`, where)).Scan(&row.Path, &row.Title, &row.Lead, &row.Body, &row.RawContent, &row.WordCount, &row.Checksum, &row.Created, &row.Modified, &row.Metadata)
 	return row, err
 }

@ -958,6 +978,7 @@ type linkRow struct {
 	SourceId                   core.NoteId
 	TargetId                   *core.NoteId
 	Href, Title, Rels, Snippet string
+	SnippetStart, SnippetEnd   int
 	External                   bool
 }

@ -965,7 +986,7 @@ func queryLinkRows(t *testing.T, tx Transaction, where string) []linkRow {
 	links := make([]linkRow, 0)

 	rows, err := tx.Query(fmt.Sprintf(`
-		SELECT source_id, target_id, title, href, external, rels, snippet
+		SELECT source_id, target_id, title, href, external, rels, snippet, snippet_start, snippet_end
 		  FROM links
 		 WHERE %v
 		 ORDER BY id
@ -976,7 +997,7 @@ func queryLinkRows(t *testing.T, tx Transaction, where string) []linkRow {
 		var row linkRow
 		var sourceId int64
 		var targetId *int64
-		err = rows.Scan(&sourceId, &targetId, &row.Title, &row.Href, &row.External, &row.Rels, &row.Snippet)
+		err = rows.Scan(&sourceId, &targetId, &row.Title, &row.Href, &row.External, &row.Rels, &row.Snippet, &row.SnippetStart, &row.SnippetEnd)
 		assert.Nil(t, err)
 		row.SourceId = core.NoteId(sourceId)
 		if targetId != nil {
--- a/adapter/sqlite/transaction_test.go
+++ b/adapter/sqlite/transaction_test.go
@ -25,7 +25,7 @@ func testTransactionWithoutFixtures(t *testing.T, test func(tx Transaction)) {
 func testTransactionWithFixtures(t *testing.T, fixturesDir opt.String, test func(tx Transaction)) {
 	db, err := OpenInMemory()
 	assert.Nil(t, err)
-	err = db.Migrate()
+	_, err = db.Migrate()
 	assert.Nil(t, err)

 	if !fixturesDir.IsNull() {
--- a/cmd/container.go
+++ b/cmd/container.go
@ -4,16 +4,21 @@ import (
 	"io"
 	"os"
 	"sync"
+	"time"

 	"github.com/mickael-menu/zk/adapter/fzf"
 	"github.com/mickael-menu/zk/adapter/handlebars"
 	"github.com/mickael-menu/zk/adapter/markdown"
 	"github.com/mickael-menu/zk/adapter/sqlite"
 	"github.com/mickael-menu/zk/adapter/term"
+	"github.com/mickael-menu/zk/core/note"
 	"github.com/mickael-menu/zk/core/zk"
 	"github.com/mickael-menu/zk/util"
 	"github.com/mickael-menu/zk/util/date"
+	"github.com/mickael-menu/zk/util/errors"
 	"github.com/mickael-menu/zk/util/pager"
+	"github.com/mickael-menu/zk/util/paths"
+	"github.com/schollz/progressbar/v3"
 )

 type Container struct {
@ -77,14 +82,53 @@ func (c *Container) NoteIndexer(tx sqlite.Transaction) *sqlite.NoteIndexer {
 }

 // Database returns the DB instance for the given notebook, after executing any
-// pending migration.
-func (c *Container) Database(path string) (*sqlite.DB, error) {
-	db, err := sqlite.Open(path)
+// pending migration and indexing the notes if needed.
+func (c *Container) Database(zk *zk.Zk, forceIndexing bool) (*sqlite.DB, note.IndexingStats, error) {
+	var stats note.IndexingStats
+
+	db, err := sqlite.Open(zk.DBPath())
+	if err != nil {
+		return nil, stats, err
+	}
+	needsReindexing, err := db.Migrate()
 	if err != nil {
-		return nil, err
+		return nil, stats, errors.Wrap(err, "failed to migrate the database")
 	}
-	err = db.Migrate()
-	return db, err
+
+	stats, err = c.index(zk, db, forceIndexing || needsReindexing)
+	if err != nil {
+		return nil, stats, err
+	}
+
+	return db, stats, err
+}
+
+func (c *Container) index(zk *zk.Zk, db *sqlite.DB, force bool) (note.IndexingStats, error) {
+	var bar = progressbar.NewOptions(-1,
+		progressbar.OptionSetWriter(os.Stderr),
+		progressbar.OptionThrottle(100*time.Millisecond),
+		progressbar.OptionSpinnerType(14),
+	)
+
+	var err error
+	var stats note.IndexingStats
+	err = db.WithTransaction(func(tx sqlite.Transaction) error {
+		stats, err = note.Index(
+			zk,
+			force,
+			c.Parser(),
+			c.NoteIndexer(tx),
+			c.Logger,
+			func(change paths.DiffChange) {
+				bar.Add(1)
+				bar.Describe(change.String())
+			},
+		)
+		return err
+	})
+	bar.Clear()
+
+	return stats, err
 }

 // Paginate creates an auto-closing io.Writer which will be automatically
--- a/cmd/edit.go
+++ b/cmd/edit.go
@ -36,7 +36,7 @@ func (cmd *Edit) Run(container *Container) error {
 		return errors.Wrapf(err, "incorrect criteria")
 	}

-	db, err := container.Database(zk.DBPath())
+	db, _, err := container.Database(zk, false)
 	if err != nil {
 		return err
 	}
--- a/cmd/index.go
+++ b/cmd/index.go
@ -2,13 +2,6 @@ package cmd

 import (
 	"fmt"
-	"os"
-	"time"
-
-	"github.com/mickael-menu/zk/adapter/sqlite"
-	"github.com/mickael-menu/zk/core/note"
-	"github.com/mickael-menu/zk/util/paths"
-	"github.com/schollz/progressbar/v3"
 )

 // Index indexes the content of all the notes in the notebook.
@ -18,7 +11,7 @@ type Index struct {
 }

 func (cmd *Index) Help() string {
-	return "You usually don't need to run `zk index` manually, as notes are indexed automatically before each zk invocation."
+	return "You usually do not need to run `zk index` manually, as notes are indexed automatically when needed."
 }

 func (cmd *Index) Run(container *Container) error {
@ -27,34 +20,11 @@ func (cmd *Index) Run(container *Container) error {
 		return err
 	}

-	db, err := container.Database(zk.DBPath())
+	_, stats, err := container.Database(zk, cmd.Force)
 	if err != nil {
 		return err
 	}

-	var bar = progressbar.NewOptions(-1,
-		progressbar.OptionSetWriter(os.Stderr),
-		progressbar.OptionThrottle(100*time.Millisecond),
-		progressbar.OptionSpinnerType(14),
-	)
-
-	var stats note.IndexingStats
-	err = db.WithTransaction(func(tx sqlite.Transaction) error {
-		stats, err = note.Index(
-			zk,
-			cmd.Force,
-			container.Parser(),
-			container.NoteIndexer(tx),
-			container.Logger,
-			func(change paths.DiffChange) {
-				bar.Add(1)
-				bar.Describe(change.String())
-			},
-		)
-		return err
-	})
-	bar.Clear()
-
 	if err == nil && !cmd.Quiet {
 		fmt.Println(stats)
 	}
--- a/cmd/list.go
+++ b/cmd/list.go
@ -39,7 +39,7 @@ func (cmd *List) Run(container *Container) error {
 		return err
 	}

-	db, err := container.Database(zk.DBPath())
+	db, _, err := container.Database(zk, false)
 	if err != nil {
 		return err
 	}
--- a/core/note/format.go
+++ b/core/note/format.go
@ -108,6 +108,7 @@ func (f *Formatter) Format(match Match) (string, error) {
 		Tags:       match.Tags,
 		RawContent: match.RawContent,
 		WordCount:  match.WordCount,
+		Metadata:   match.Metadata.Metadata,
 		Created:    match.Created,
 		Modified:   match.Modified,
 		Checksum:   match.Checksum,
@ -123,6 +124,7 @@ type formatRenderContext struct {
 	RawContent string `handlebars:"raw-content"`
 	WordCount  int    `handlebars:"word-count"`
 	Tags       []string
+	Metadata   map[string]interface{}
 	Created    time.Time
 	Modified   time.Time
 	Checksum   string
--- a/core/note/index.go
+++ b/core/note/index.go
@ -27,6 +27,7 @@ type Metadata struct {
 	WordCount  int
 	Links      []Link
 	Tags       []string
+	Metadata   map[string]interface{}
 	Created    time.Time
 	Modified   time.Time
 	Checksum   string
@ -139,6 +140,7 @@ func metadata(path string, zk *zk.Zk, parser Parser) (Metadata, error) {
 	metadata.WordCount = len(strings.Fields(contentStr))
 	metadata.Links = make([]Link, 0)
 	metadata.Tags = contentParts.Tags
+	metadata.Metadata = contentParts.Metadata
 	metadata.Checksum = fmt.Sprintf("%x", sha256.Sum256(content))

 	for _, link := range contentParts.Links {
--- a/core/note/parse.go
+++ b/core/note/parse.go
@ -15,15 +15,19 @@ type Content struct {
 	Tags []string
 	// Links is the list of outbound links found in the note.
 	Links []Link
+	// Additional metadata. For example, extracted from a YAML frontmatter.
+	Metadata map[string]interface{}
 }

 // Link links a note to another note or an external resource.
 type Link struct {
-	Title    string
-	Href     string
-	External bool
-	Rels     []string
-	Snippet  string
+	Title        string
+	Href         string
+	External     bool
+	Rels         []string
+	Snippet      string
+	SnippetStart int
+	SnippetEnd   int
 }

 // LinkRelation defines the relationship between a link's source and target.
--- a/docs/template-format.md
+++ b/docs/template-format.md
@ -2,17 +2,19 @@

 The following variables are available in the templates used when formatting notes, for example with `zk list --format <template>`.

-| Variable      | Type     | Description                                               |
-|---------------|----------|-----------------------------------------------------------|
-| `path`        | string   | File path to the note, relative to the current directory  |
-| `title`       | string   | Note title                                                |
-| `lead`        | string   | First paragraph extracted from the note content           |
-| `body`        | string   | All of the note content, minus the heading                |
-| `snippets`    | [string] | List of context-sensitive relevant excerpts from the note |
-| `raw-content` | string   | The full raw content of the note file                     |
-| `word-count`  | int      | Number of words in the note                               |
-| `tags`        | [string] | List of tags found in the note                            |
-| `created`     | date     | Date of creation of the note                              |
-| `modified`    | date     | Last date of modification of the note                     |
-| `checksum`    | string   | SHA-256 checksum of the note file                         |
+| Variable      | Type     | Description                                                         |
+|---------------|----------|---------------------------------------------------------------------|
+| `path`        | string   | File path to the note, relative to the current directory            |
+| `title`       | string   | Note title                                                          |
+| `lead`        | string   | First paragraph extracted from the note content                     |
+| `body`        | string   | All of the note content, minus the heading                          |
+| `snippets`    | [string] | List of context-sensitive relevant excerpts from the note           |
+| `raw-content` | string   | The full raw content of the note file                               |
+| `word-count`  | int      | Number of words in the note                                         |
+| `tags`        | [string] | List of tags found in the note                                      |
+| `metadata`    | map      | YAML frontmatter metadata, e.g. `metadata.description`<sup>1</sup>  |
+| `created`     | date     | Date of creation of the note                                        |
+| `modified`    | date     | Last date of modification of the note                               |
+| `checksum`    | string   | SHA-256 checksum of the note file                                   |

+1. YAML keys are normalized to lower case.
--- a/main.go
+++ b/main.go
@ -55,8 +55,6 @@ func main() {
 	// Create the dependency graph.
 	container := cmd.NewContainer()

-	indexZk(container)
-
 	if isAlias, err := runAlias(container, os.Args[1:]); isAlias {
 		fatalIfError(err)

@ -97,13 +95,6 @@ func fatalIfError(err error) {
 	}
 }

-// indexZk will index any notebook in the working directory.
-func indexZk(container *cmd.Container) {
-	if len(os.Args) > 1 && os.Args[1] != "index" {
-		(&cmd.Index{Quiet: true}).Run(container)
-	}
-}
-
 // runAlias will execute a user alias if the command is one of them.
 func runAlias(container *cmd.Container, args []string) (bool, error) {
 	runningAlias := os.Getenv("ZK_RUNNING_ALIAS")