Add a _lot_ of specification testing & fix some parse bugs

.envrc

@@ -1,4 +1,4 @@
 PATH_add bin
-PATH_add _out/bin
+PATH_add .out/bin
 PATH_add .env/bin
-export NIX_PATH="nixpkgs=$PWD/nix/nixpkgs.nix"
+export NIX_PATH="nixpkgs=$PWD/nix/nixpkgs.nix"

.gitattributes

@@ -1 +1,2 @@
-*.golden -text
+/data/golden/*/*.pro -text
+/data/golden/*/*.json -text

.gitignore

@@ -3,5 +3,6 @@
 dist/
 dist-newstyle/
 /cabal.project.local
-/_out/
+/.out/
+/.prosidy-manual/
 *~

.vscode/settings.json

@@ -1,18 +1,18 @@
 {
     "files.exclude": {
-        "_out/**": true,
+        ".out/**": true,
         "dist/**": true,
         "dist-newstyle/**": true,
         ".ghc.environment.*": true,
         ".gitattributes": true,
-        ".ghc/**": true
+        ".env/**": true
     },
     "files.watcherExclude": {
-        "_out/**": true,
+        "**/.out/**": true,
         "**/.git/objects/**": true,
         "**/.git/subtree-cache/**": true,
-        ".ghc/**": true,
-        "dist/**": true,
-        "dist-newstyle/**": true
+        "**/.env/**": true,
+        "**/dist/**": true,
+        "**/dist-newstyle/**": true
     }
-}
+}

.vscode/tasks.json

@@ -5,7 +5,7 @@
             "label": "Build everything",
             "type": "shell",
             "command": "./bin/m",
-            "args": ["all"],
+            "args": ["build", "all"],
             "group": {
                 "kind": "build",
                 "isDefault": true
@@ -15,7 +15,7 @@
             "label": "Run tests",
             "type": "shell",
             "command": "./bin/m",
-            "args": ["test"],
+            "args": ["test", "all"],
             "group": {
                 "kind": "test",
                 "isDefault": true

README.pro

@@ -34,10 +34,16 @@ although it may work without both.
 
 #-h{Changelog}
 #-h+{2020-01-24}
-Combined multiple Cabal projects into a super-Cabal file.
+#-list:
+  #-item{Combined multiple Cabal projects into a super-Cabal file.}
+#:
 
 #-h+{2020-01-19}
-Added source tags to all Prosidy elements meaning better errors!
+#-list:
+  #-item{Added source tags to all Prosidy elements meaning better errors!}
+#:
 
 #-h+{2020-01-03}
-Refactored the various Prosidy repositories into a single monorepo.
+#-list:
+  #-item{Refactored the various Prosidy repositories into a single monorepo.}
+#:

bin/compose

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -euo pipefail
+cd "$(git rev-parse --show-toplevel)"
+source ./bin/mkenv
+
+mkdir -p .out/doc
+warp --docroot .out/doc --host 127.0.0.1 &
+trap "kill $!" EXIT
+
+fswatch --recursive --one-per-batch --event Created --event Updated ./doc |\
+while read -r event
+do
+    echo "Triggering update (event: ${event})" >&2
+    m v2-run prosidy-manual || true
+done

bin/hie-bios-setup

@@ -1,9 +1,10 @@
 #!/bin/bash
 set -eu
 cd "$(git rev-parse --show-toplevel)"
-HIE_BIOS_OUTPUT="${HIE_BIOS_OUTPUT:-/dev/fd/1}"
 ./bin/mkenv
 
+HIE_BIOS_OUTPUT="${HIE_BIOS_OUTPUT:-/dev/fd/1}"
+
 function write
 {
     cat >> "$HIE_BIOS_OUTPUT"
@@ -11,9 +12,10 @@ function write
 
 if [ -L .env ] 2>&1
 then
+    dbpath="$(realpath "$(find -L .env/lib -type d -name package.conf.d)")"
     write <<EOF
 -clear-package-db
--package-db="$(realpath "$(find .env/lib -type d -name package.conf.d)")/"
+-package-db=$dbpath
 EOF
 else
     version="$(ghc --numeric-version)"

bin/m

@@ -1,14 +1,13 @@
 #!/bin/bash
 set -eu
 cd "$(git rev-parse --show-toplevel)"
-
-./bin/mkenv
+source ./bin/mkenv
 
 ghc_version="$(ghc --numeric-version)"
 
 cabal_opts=( 
     --offline
-    --builddir="$PWD/_out/cabal" 
+    --builddir="$PWD/.out/cabal" 
     --package-db=clear
     --package-db="$PWD/.env/lib/ghc-${ghc_version}/package.conf.d"
 )
@@ -18,7 +17,7 @@ then
     set - build all
 elif [[ "${1:-}" = *install ]]
 then
-    cabal_opts+=( --installdir="$PWD/_out/bin" )
+    cabal_opts+=( --installdir="$PWD/.out/bin" )
 fi
 
 if [ -n "${JENKINS_URL:-}" ]

bin/mkenv

@@ -1,16 +1,23 @@
 #!/bin/bash
-command -v nix >/dev/null 2>&1 || exit 0
+[ -z "${_MKENV:-}" ] || return 0
+command -v nix >/dev/null 2>&1 || return 0
+[ -n "${DIRENV_DIR:-}" ] || eval "$(direnv export bash)"
 
-export NIX_PATH="nixpkgs=$PWD/nix/nixpkgs.nix"
-nix_substituters=(
-    'https://cache.nixos.org' 
-    'https://static-haskell-nix.cachix.org'
-)
-nix_keys=(
-    'cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY='
-    'static-haskell-nix.cachix.org-1:Q17HawmAwaM1/BfIxaEDKAxwTOyRVhPG5Ji9K3+FvUU=' 
-)
-nix-build ./nix/default.nix -A env \
-    --substituters "$nix_substituters" \
-    --trusted-public-keys "$nix_keys" \
-    --out-link .env
+function setup_nix
+{
+    local nix_substituters=(
+        'https://cache.nixos.org' 
+        'https://static-haskell-nix.cachix.org'
+    )
+    local nix_keys=(
+        'cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY='
+        'static-haskell-nix.cachix.org-1:Q17HawmAwaM1/BfIxaEDKAxwTOyRVhPG5Ji9K3+FvUU=' 
+    )
+    nix-build ./nix/default.nix -A env \
+        --substituters "$nix_substituters" \
+        --trusted-public-keys "$nix_keys" \
+        --out-link .env
+}
+
+setup_nix
+export _MKENV=1

cabal.project

@@ -1,3 +1,4 @@
 packages: prosidy.cabal
 
 optimization: True
+split-sections: True

data/golden/comment/input.pro

@@ -0,0 +1,127 @@
+case: Comments
+
+## Empty line in header
+    ## Empty line after spaces in header
+
+prop            ## After a property
+empty =         ## After an empty setting
+nonempty = yes  ## After a non-empty setting
+
+---
+
+## Empty line in the body
+    ## Empty line after spaces in the body
+
+
+#-tag   ## Immediately after an empty block-tag
+
+## On the line preceeding an empty block tag
+#-tag
+
+#-tag
+## On the line proceeding an empty block tag
+
+
+#-tag[prop, setting='']  ## Immediately after an empty block-tag with metadata
+
+## On the line preceeding an empty block tag with metadata
+#-tag[prop, setting='']
+
+#-tag[prop, setting='']
+## On the line proceeding an empty block tag with metadata
+
+#-tag[    ## After the metadata open bracket
+prop      ## After a property in the metadata
+,         ## After a metadata separator
+setting   ## After a setting key in the metadata
+=         ## After the association operator
+"foo"     ## After a quoted value
+]         ## After a metadata close bracket
+
+#-tag[    ## After the metadata open bracket
+prop      ## After a property in the metadata
+,         ## After a metadata separator
+setting   ## After a setting key in the metadata
+=         ## After the association operator
+"foo"     ## After a quoted value
+]:        ## After the block open colon
+    ## Within a block
+    Inner contents
+
+    Another paragraph
+    ## Within a block
+#:        ## After a named block close
+
+#-tag[    ## After the metadata open bracket
+prop      ## After a property in the metadata
+,         ## After a metadata separator
+setting   ## After a setting key in the metadata
+=         ## After the association operator
+"foo"     ## After a quoted value
+]:end     ## After the block open colon
+    ## Within a block
+    Inner contents
+
+    Another paragraph
+    ## Within a block
+#:end     ## After a named block close
+
+#-tag[    ## After the metadata open bracket
+prop      ## After a property in the metadata
+,         ## After a metadata separator
+setting   ## After a setting key in the metadata
+=         ## After the association operator
+"foo"     ## After a quoted value
+]{        ## After the inline brace
+
+## Within the brace
+
+Inner contents
+
+## Within the brace
+
+}        ## After the inline brace
+
+## Before a paragraph
+Blah blah  ## After a line in a paragraph
+blah blah. ## After a line in a paragraph
+#inline[  ## After a dangling inline meta open
+prop      ## After a property in the metadata
+,         ## After a metadata separator
+setting   ## After a setting key in the metadata
+=         ## After the association operator
+"foo"     ## After a quoted value
+]         ## After the inline meta close
+#inline[  ## After a dangling inline meta open
+prop      ## After a property in the metadata
+,         ## After a metadata separator
+setting   ## After a setting key in the metadata
+=         ## After the association operator
+"foo"     ## After a quoted value
+]{        ## Within the brace
+          ## Within the brace
+look ma   ## After a paragraph in a brace
+          ## Within the brace
+          ## Within the brace
+}         ## After a brace closes
+#inline[  ## After a dangling inline meta open
+prop      ## After a property in the metadata
+,         ## After a metadata separator
+setting   ## After a setting key in the metadata
+=         ## After the association operator
+"foo"     ## After a quoted value
+]{        ## Within the brace
+          ## Within the brace
+no hands ## After a paragraph in a brace
+          ## Within the brace
+          ## Within the brace
+} content ## After a brace closes with trailing content
+## After a paragraph
+
+
+#=src:end ## After a literal tag's opening line
+          ## This is _not_ a comment; it's part of the tag around it
+#:end     ## After a literal tag's closing line
+
+## As the last line of a document, without a trailing newline
+## This is a direct remediation of a bug I found

data/golden/comment/output.json

@@ -0,0 +1,317 @@
+{
+    "content": [
+        {
+            "value": {
+                "content": [],
+                "name": "tag",
+                "metadata": {
+                    "settings": {},
+                    "properties": []
+                }
+            },
+            "subtype": "block",
+            "type": "tag"
+        },
+        {
+            "value": {
+                "content": [],
+                "name": "tag",
+                "metadata": {
+                    "settings": {},
+                    "properties": []
+                }
+            },
+            "subtype": "block",
+            "type": "tag"
+        },
+        {
+            "value": {
+                "content": [],
+                "name": "tag",
+                "metadata": {
+                    "settings": {},
+                    "properties": []
+                }
+            },
+            "subtype": "block",
+            "type": "tag"
+        },
+        {
+            "value": {
+                "content": [],
+                "name": "tag",
+                "metadata": {
+                    "settings": {
+                        "setting": ""
+                    },
+                    "properties": [
+                        "prop"
+                    ]
+                }
+            },
+            "subtype": "block",
+            "type": "tag"
+        },
+        {
+            "value": {
+                "content": [],
+                "name": "tag",
+                "metadata": {
+                    "settings": {
+                        "setting": ""
+                    },
+                    "properties": [
+                        "prop"
+                    ]
+                }
+            },
+            "subtype": "block",
+            "type": "tag"
+        },
+        {
+            "value": {
+                "content": [],
+                "name": "tag",
+                "metadata": {
+                    "settings": {
+                        "setting": ""
+                    },
+                    "properties": [
+                        "prop"
+                    ]
+                }
+            },
+            "subtype": "block",
+            "type": "tag"
+        },
+        {
+            "value": {
+                "content": [],
+                "name": "tag",
+                "metadata": {
+                    "settings": {
+                        "setting": "foo"
+                    },
+                    "properties": [
+                        "prop"
+                    ]
+                }
+            },
+            "subtype": "block",
+            "type": "tag"
+        },
+        {
+            "value": {
+                "content": [
+                    {
+                        "value": [
+                            {
+                                "value": "Inner contents",
+                                "type": "text"
+                            }
+                        ],
+                        "type": "paragraph"
+                    },
+                    {
+                        "value": [
+                            {
+                                "value": "Another paragraph",
+                                "type": "text"
+                            }
+                        ],
+                        "type": "paragraph"
+                    }
+                ],
+                "name": "tag",
+                "metadata": {
+                    "settings": {
+                        "setting": "foo"
+                    },
+                    "properties": [
+                        "prop"
+                    ]
+                }
+            },
+            "subtype": "block",
+            "type": "tag"
+        },
+        {
+            "value": {
+                "content": [
+                    {
+                        "value": [
+                            {
+                                "value": "Inner contents",
+                                "type": "text"
+                            }
+                        ],
+                        "type": "paragraph"
+                    },
+                    {
+                        "value": [
+                            {
+                                "value": "Another paragraph",
+                                "type": "text"
+                            }
+                        ],
+                        "type": "paragraph"
+                    }
+                ],
+                "name": "tag",
+                "metadata": {
+                    "settings": {
+                        "setting": "foo"
+                    },
+                    "properties": [
+                        "prop"
+                    ]
+                }
+            },
+            "subtype": "block",
+            "type": "tag"
+        },
+        {
+            "value": {
+                "content": [
+                    {
+                        "value": [
+                            {
+                                "value": "Inner contents",
+                                "type": "text"
+                            }
+                        ],
+                        "type": "paragraph"
+                    }
+                ],
+                "name": "tag",
+                "metadata": {
+                    "settings": {
+                        "setting": "foo"
+                    },
+                    "properties": [
+                        "prop"
+                    ]
+                }
+            },
+            "subtype": "block",
+            "type": "tag"
+        },
+        {
+            "value": [
+                {
+                    "value": "Blah blah",
+                    "type": "text"
+                },
+                {
+                    "value": [],
+                    "type": "break"
+                },
+                {
+                    "value": "blah blah.",
+                    "type": "text"
+                },
+                {
+                    "value": [],
+                    "type": "break"
+                },
+                {
+                    "value": {
+                        "content": [],
+                        "name": "inline",
+                        "metadata": {
+                            "settings": {
+                                "setting": "foo"
+                            },
+                            "properties": [
+                                "prop"
+                            ]
+                        }
+                    },
+                    "subtype": "inline",
+                    "type": "tag"
+                },
+                {
+                    "value": [],
+                    "type": "break"
+                },
+                {
+                    "value": {
+                        "content": [
+                            {
+                                "value": "look ma",
+                                "type": "text"
+                            }
+                        ],
+                        "name": "inline",
+                        "metadata": {
+                            "settings": {
+                                "setting": "foo"
+                            },
+                            "properties": [
+                                "prop"
+                            ]
+                        }
+                    },
+                    "subtype": "inline",
+                    "type": "tag"
+                },
+                {
+                    "value": [],
+                    "type": "break"
+                },
+                {
+                    "value": {
+                        "content": [
+                            {
+                                "value": "no hands",
+                                "type": "text"
+                            }
+                        ],
+                        "name": "inline",
+                        "metadata": {
+                            "settings": {
+                                "setting": "foo"
+                            },
+                            "properties": [
+                                "prop"
+                            ]
+                        }
+                    },
+                    "subtype": "inline",
+                    "type": "tag"
+                },
+                {
+                    "value": [],
+                    "type": "break"
+                },
+                {
+                    "value": "content",
+                    "type": "text"
+                }
+            ],
+            "type": "paragraph"
+        },
+        {
+            "value": {
+                "content": "          ## This is _not_ a comment; it's part of the tag around it",
+                "name": "src",
+                "metadata": {
+                    "settings": {},
+                    "properties": []
+                }
+            },
+            "subtype": "literal",
+            "type": "tag"
+        }
+    ],
+    "metadata": {
+        "settings": {
+            "empty": "",
+            "case": "Comments",
+            "nonempty": "yes"
+        },
+        "properties": [
+            "prop"
+        ]
+    }
+}

data/golden/empty-document/input.pro

@@ -0,0 +1 @@
+---

data/golden/empty-document/output.json

@@ -0,0 +1,7 @@
+{
+    "metadata": {
+        "properties": [],
+        "settings": {}
+    },
+    "content": []
+}

data/golden/gamut/input.pro

@@ -0,0 +1,164 @@
+## This is an example document which enumerates many valid syntaxes. It is
+## probably not exhaustive, but should serve as a good document to implement
+## against.
+
+## Properties
+prop
+prop-with-comment    ## this has a trailing comment
+    prop-with-leading-spaces ## arbitrary leading spaces are permitted
+
+## Settings
+set-a1:42
+set-a2=42
+set-a3: 42
+set-a4= 42
+set-a5 :42
+set-a6 =42
+set-a7 = 42
+set-a8 : 42
+
+    set-a9 = 42  ## arbitrary leading spaces are permitted
+
+## Settings with trailing comments
+set-b1:42            ## this has a trailing comment
+set-b2=42            ## this has a trailing comment
+set-b3: 42           ## this has a trailing comment
+set-b4= 42           ## this has a trailing comment
+set-b5 :42           ## this has a trailing comment
+set-b6 =42           ## this has a trailing comment
+set-b7 = 42          ## this has a trailing comment
+set-b8 : 42          ## this has a trailing comment
+
+## Settings with escaped values
+set-c1 = foo \# bar
+set-c2 = \#\# not a comment ## is a comment
+
+## Settings with null values
+set-d1 =
+set-d2 :
+set-d3 = ## Just a comment, not a value
+set-d4 : ## Just a comment, not a value
+---
+
+Paragraph with only one line.
+
+Paragraph trailed by comment. ## This is the comment
+
+Paragraph split
+over multiple lines.
+
+  Paragraph split ## with an indent!
+over multiple lines ## Another sneaky comment
+with a comment trailing some.
+
+#tag{This} paragraph starts with a simple tag.
+
+This paragraph ends with a simple #tag{tag.}
+
+This #tag{paragraph} has a few #tag{tags}.
+
+#a{}b#c{}#d{}
+
+This paragraph has a number of inline tag styles.
+#tag              ## no meta, no children
+#tag[]            ## no meta, no children
+#tag[]{}          ## no meta, no children
+#tag[prop]        ## 1 prop, no children
+#tag[prop1,prop2] ## 2 props, no children
+#tag[set = '42']  ## 1 setting (equal, single quote), no children
+#tag[set = "42"]  ## 1 setting (equal, double quote), no children
+#tag[set: '42']   ## 1 setting (colon, single quote), no children
+#tag[set: "42"]   ## 1 setting (colon, double quote), no children
+#tag[prop, foo: 'bar', baz = "quz"] ## complicated, no children
+#tag{content}     ## no meta, children
+#tag[]{content}   ## no meta, children
+#tag{#tag{ok}}    ## no meta, children containing tags
+#tag[prop, foo: 'bar', baz = "quz"]{#tag{ok}} ## ok wow
+#tag[
+] ## Meta blocks can span multiple lines
+#tag[ ## Comments can go here
+  ## Because items in the metadata block are relatively structured
+  ## compared to the rest of the document, you can go wild with
+  ## how properties are laid out
+
+  prop1, prop2 ## Comments can go here
+, set ## Comments can go here
+= '42', prop3 ## Comments can go here
+]{This one is pathological}
+
+This paragraph has some tags that are awfuly close to one another.
+#tag{}text#tag{}text#tag#tag{}.
+
+#tag{This is a paragraph in an #kind{inline} tag.}
+
+#tag{This is a paragraph in an #kind{inline} tag.
+     It spans many lines.
+     Content may follow.} This follows!
+
+#tag{This is a paragraph in an #kind{inline} tag. ## Comments can go here
+     It spans many lines .                        ## Comments can go here
+     Content may follow.} This follows!           ## Comments can go here
+
+#-tag{This is a paragraph in a #kind{block} tag.  ## Comments can go here
+      Content may not follow.}                    ## Comments can go here
+
+#-block: ## Comments can go here
+  This is the other kind of block tag.
+  It can have #italic{nested} block children.
+
+  #-block: ## Empty blocks are fine too
+  #:
+
+  #-block[prop]:label ## Comments can go here
+    Block starts/ends can have aribtrary labels attached.
+  #:label ## Comments can go here
+
+  #-block[set='42']:label ## Comments can go here
+    Block starts/ends can have aribtrary labels attached.
+  #:label ## Comments can go here
+
+  #-block[prop1, set: '42', prop2]:label ## Comments can go here
+    Block starts/ends can have aribtrary labels attached.
+  #:label ## Comments can go here
+#: ## Comments can go here
+
+#-block[ ## Comments can go here
+  ## Because items in the metadata block are relatively structured
+  ## compared to the rest of the document, you can go wild with
+  ## how properties are laid out
+
+  prop1, prop2 ## Comments can go here
+, set ## Comments can go here
+= '42', prop3 ## Comments can go here
+]:end ## Comments can go here
+  This is another pathological tag, just a block this time.
+  #:end
+
+## This is a literal tag
+## The label is once again optional like block tags,
+## but a label is _very_ helpful for quoting prosidy
+#=lit:eof ## Comments can go here, but not on the next line
+#-block: ## this is treated literally!
+  This is the other kind of block tag.
+  It can have #italic{nested} block children.
+
+  #-block: ## this is treated literally!
+  #:
+
+  #-block[prop]:label ## this is treated literally!
+    Block starts/ends can have aribtrary labels attached.
+  #:label ## this is treated literally!
+
+  #-block[set='42']:label ## this is treated literally!
+    Block starts/ends can have aribtrary labels attached.
+  #:label ## this is treated literally!
+
+  #-block[prop1, set: '42', prop2]:label ## this is treated literally!
+    Block starts/ends can have aribtrary labels attached.
+  #:label ## this is treated literally!
+#: ## this is treated literally!
+#:eof ## Comments can go here again
+
+Some unicode escapes:
+  "\u1F638" is a smiling cat face,
+  and "\uB" is a vertical tab.

data/golden/invalid-utf8/input.pro

@@ -0,0 +1,6 @@
+case: Invalid Utf-8
+---
+
+The next paragraph contains the character #lit{0xC0}, which is not valid UTF-8.
+
+Invalid:À

data/golden/invalid-utf8/output.json

@@ -0,0 +1,53 @@
+{
+    "content": [
+        {
+            "value": [
+                {
+                    "value": "The next paragraph contains the character",
+                    "type": "text"
+                },
+                {
+                    "value": [],
+                    "type": "break"
+                },
+                {
+                    "value": {
+                        "content": [
+                            {
+                                "value": "0xC0",
+                                "type": "text"
+                            }
+                        ],
+                        "name": "lit",
+                        "metadata": {
+                            "settings": {},
+                            "properties": []
+                        }
+                    },
+                    "subtype": "inline",
+                    "type": "tag"
+                },
+                {
+                    "value": ", which is not valid UTF-8.",
+                    "type": "text"
+                }
+            ],
+            "type": "paragraph"
+        },
+        {
+            "value": [
+                {
+                    "value": "Invalid:�",
+                    "type": "text"
+                }
+            ],
+            "type": "paragraph"
+        }
+    ],
+    "metadata": {
+        "settings": {
+            "case": "Invalid Utf-8"
+        },
+        "properties": []
+    }
+}

data/golden/line-break/input.pro

@@ -0,0 +1,7 @@
+case: Line breaks
+---
+This is a paragraph ending in a Unix break.
+
+This is a paragraph ending in a Windows break.
+
+This is a paragraph ending in a classic Mac OS break.

data/golden/line-break/output.json

@@ -0,0 +1,37 @@
+{
+    "content": [
+        {
+            "value": [
+                {
+                    "value": "This is a paragraph ending in a Unix break.",
+                    "type": "text"
+                }
+            ],
+            "type": "paragraph"
+        },
+        {
+            "value": [
+                {
+                    "value": "This is a paragraph ending in a Windows break.",
+                    "type": "text"
+                }
+            ],
+            "type": "paragraph"
+        },
+        {
+            "value": [
+                {
+                    "value": "This is a paragraph ending in a classic Mac OS break.",
+                    "type": "text"
+                }
+            ],
+            "type": "paragraph"
+        }
+    ],
+    "metadata": {
+        "settings": {
+            "case": "Line breaks"
+        },
+        "properties": []
+    }
+}

doc/index.pro

@@ -13,7 +13,6 @@ Prosidy is an "easy-to-read, easy to write" plain-text format.
 It doesn't get in the way of your text so the source document is legible,
 and has only a few rules to keep in mind when writing.
 
-
 Like #link[url='https://www.w3.org/XML/']{XML}, Prosidy is #i{extensible}:
 it doesn't have any preconceived ideas about the what you write,
 letting the author tailor the language presicely to their use case.

doc/res/info.svg

@@ -1,4 +1,56 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <!-- Generated by IcoMoon.io -->
-<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="768" height="768" viewBox="0 0 768 768">
-<path d="M736 384c0-97.184-39.424-185.248-103.104-248.896s-151.712-103.104-248.896-103.104-185.248 39.424-248.896 103.104-103.104 151.712-103.104 248.896 39.424 185.248 103.104 248.896 151.712 103.104 248.896 103.104 185.248-39.424 248.896-103.104 103.104-151.712 103.104-248.896zM672 384c0 79.552-32.192 151.488-84.352 203.648s-124.096 84.352-203.648 84.352-151.488-32.192-203.648-84.352-84.352-124.096-84.352-203.648 32.192-151.488 84.352-203.648 124.096-84.352 203.648-84.352 151.488 32.192 203.648 84.352 84.352 124.096 84.352 203.648zM416 512v-128c0-17.664-14.336-32-32-32s-32 14.336-32 32v128c0 17.664 14.336 32 32 32s32-14.336 32-32zM384 288c17.664 0 32-14.336 32-32s-14.336-32-32-32-32 14.336-32 32 14.336 32 32 32z"></path>
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   version="1.1"
+   width="768"
+   height="768"
+   viewBox="0 0 768 768"
+   id="svg4"
+   sodipodi:docname="info.svg"
+   inkscape:version="0.92.4 (unknown)">
+  <metadata
+     id="metadata10">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs8" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="841"
+     inkscape:window-height="480"
+     id="namedview6"
+     showgrid="false"
+     inkscape:zoom="0.30729167"
+     inkscape:cx="384"
+     inkscape:cy="384"
+     inkscape:window-x="0"
+     inkscape:window-y="48"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg4" />
+  <path
+     d="M736 384c0-97.184-39.424-185.248-103.104-248.896s-151.712-103.104-248.896-103.104-185.248 39.424-248.896 103.104-103.104 151.712-103.104 248.896 39.424 185.248 103.104 248.896 151.712 103.104 248.896 103.104 185.248-39.424 248.896-103.104 103.104-151.712 103.104-248.896zM672 384c0 79.552-32.192 151.488-84.352 203.648s-124.096 84.352-203.648 84.352-151.488-32.192-203.648-84.352-84.352-124.096-84.352-203.648 32.192-151.488 84.352-203.648 124.096-84.352 203.648-84.352 151.488 32.192 203.648 84.352 84.352 124.096 84.352 203.648zM416 512v-128c0-17.664-14.336-32-32-32s-32 14.336-32 32v128c0 17.664 14.336 32 32 32s32-14.336 32-32zM384 288c17.664 0 32-14.336 32-32s-14.336-32-32-32-32 14.336-32 32 14.336 32 32 32z"
+     id="path2"
+     style="fill:#23c6a8;fill-opacity:1" />
 </svg>

doc/res/manual.css

@@ -2,6 +2,7 @@
 
 html {
     font: 16px/1.4 'PT Serif', Georgia, Times, Times New Roman, serif;
+    height: 100%;
 }
 
 body {
@@ -101,9 +102,12 @@ nav li
     top: 0;
 }
 
-nav > ol > li {
+nav > ol > li.current {
     font-weight: bold;
-    margin-bottom: 1em;
+}
+
+nav > ol > li {
+    margin: 0.5em 0;
 }
 
 nav > ol > li > ol {
@@ -112,9 +116,9 @@ nav > ol > li > ol {
 
 nav > ol > li > ol > li > ol {
     display: none;
-    border-right: 1px solid #2454C7;
-    padding-right: 0.5ch;
-    font-style: italic;
+    border-left: 1px solid #2454C7;
+    margin-left: 1ch;
+    padding-left: 1ch;
 }
 
 nav > ol > li > ol > li.viewing-child > ol,
@@ -167,12 +171,12 @@ aside p:first-child {
     font-weight: bold;
 }
 
-aside.note {
-    background-color: #d7dff4;
-    border-color: #2454C7;
+aside.info {
+    background-color: #d7f4ef;
+    border-color: #23C6A8;
 }
 
-aside.note::before {
+aside.info::before {
     background-image: url('info.svg');
 }
 
@@ -194,7 +198,6 @@ aside.wip::before {
     background-image: url('wip.svg');
 }
 
-
 .char-sequence-literal > span,
 .char-sequence-utf-8 > span {
     margin-left: 0.25ch;
@@ -202,13 +205,37 @@ aside.wip::before {
 
 /* footer */
 footer {
-    color: #444;
+    background-color: #444;
+    color: #ddd;
     font-size: 0.8em;
+    margin: 2rem -1rem -1rem;
+    min-height: 8em;
 
     text-align: center;
     padding: 2rem 0;
 }
 
+pre.source-code {
+    background-color: #fdf6e3;
+    border: 1px solid #eee8d5;
+    color: #839496; 
+}
+
+pre.source-code > code {
+    display: block;
+    padding: 0 1ch;
+    background-color: #fdf6e3;
+}
+
+pre.source-code::before {
+    content: "Source code";
+    background-color: #eee8d5;
+    color: #93a1a1;
+    display: block;
+    font-size: 0.8em;
+    padding: 0 1ch;
+}
+
 /* Special rules for tiny views */
 @media screen and (max-width: 60ch) {
     nav {
@@ -240,26 +267,26 @@ footer {
 
     body {
         display: grid;
-        grid-template-columns: 1fr 15ch minmax(45ch, 70ch) 1fr;
-        grid-template-rows: max-content max-content min-content;
+        grid-template-columns: 1fr minmax(45ch, 70ch) 15ch 1fr;
+        grid-template-rows: max-content max-content auto;
         grid-column-gap: 4ch;
     }
 
     body > nav
     {
-        grid-column: 2;
+        grid-column: 3;
         grid-row: 2;
     }
     
     body > header,
     body > main
     {
-        grid-column: 3;
+        grid-column: 2;
     }
 
     body > footer {
         grid-row: 3;
-        grid-column: 2 / 4;
+        grid-column: 1 / 5;
     }
 
     aside {
@@ -280,7 +307,6 @@ footer {
     }
 
     nav {
-        text-align: right;
         opacity: 0.6;
         transition: 0.2s opacity;
     }

doc/res/manual.js

@@ -1,9 +1,9 @@
 class CalculateViewing {
     constructor() {
         this.main = document.querySelector('main');
-        this.sections = document.querySelectorAll('main > section, main > section > section');
         this.currentLinks = document.querySelectorAll('nav > ol > li.current a')
         this.parents = document.querySelectorAll('nav > ol > li, nav > ol > li > ol > li')
+        this.sections = document.querySelectorAll('main > section, main > section > section')
         this.update();
         window.currentLinks = this.currentLinks
     }
@@ -11,7 +11,7 @@ class CalculateViewing {
 
     update() {
         if (!this.currentLinks) return;
-        const currentTop = window.scrollY
+        const currentTop = window.scrollY + window.innerHeight / 2;
         let viewing;
         for (const section of this.sections) {
             if (section.offsetTop > currentTop) break;

doc/specification.pro

@@ -17,32 +17,51 @@ as well as straightforward to customize and specialize to the user's needs.
   #link[url='mailto:ask@prosidy.org']{ask@prosidy.org}.
 #:
 
-#-section[title='Basic Syntax']:
-  #-section[title='Character']:
+#-section[title='Base Syntax']:
+  This section defines the smallest parts of Prosidy syntax,
+  such as the syntax for #term[lemma='character']{characters}
+  and #term[lemma='escape sequence']{escapes}.
+
+  #-note[level='info']:
+    This section is low-level.
+
+    If you're writing a parser, it's a great place to start!
+    However, newer Prosidy users may wish to skip forward to 
+    #ref[section='Document Structure']{Document Structure}
+    to get a larger view of what Prosidy is.
+  #:
+
+  #-section[title='Characters']:
     A #def{character} is a single Unicode codepoint
     (regardless of whether or not that codepoint represents a single glyph
     or combines with other neighboring codepoints as a modifier).
 
-    Because Prosidy documents are #link[url='#encoding']{encoded in UTF-8},
-    some byte sequences may not represent a valid codepoint.
-    Invalid byte-sequences should be normalized to the
+    #-spec[test='test.prosidy.invalid-utf-8']:
+      Because Prosidy documents are #ref[section='Encoding']{encoded in UTF-8},
+      some byte sequences may not represent a valid codepoint.
+      Implementations should either fail on inputs not encoded in valid UTF-8,
+      or replace all invalid sequences with the #chars[rep='\uFFFD', no-escape]{replacement character}.
+    #:
   #:
 
-  #-section[title='Line break']:
-    A #def{line break} is a sequence of characters that marks the end of a #term{line}.
+  #-section[title='Line breaks']:
+    #-spec[test='test.prosidy.line-break']:
+      A #def{line break} is a sequence of characters that marks the end of a #term{line}.
 
-    All Prosidy parsers must treat the following sequences as a line break:
+      All Prosidy parsers must treat the following sequences as a line break:
 
-    #-list:
-      #-item{#chars[rep='\n']{Unix newline}}
-      #-item{#chars[rep='\r\n']{Windows newline}}
-      #-item{#chars[rep='\r']{Classic Mac OS newline}}
+      #-list:
+        #-item{#chars[rep='\n']{Unix newline}}
+        #-item{#chars[rep='\r\n']{Windows newline}}
+        #-item{#chars[rep='\r']{Classic Mac OS newline}}
+      #:
     #:
   #:
 
-  #-section[title='Line']:
+  #-section[title='Lines']:
     A #def{line} is a sequence of characters terminated by a #term{line break}.
-    A #term{line break} cannot appear in a line unless it is #term[lemma='escape sequence']{escaped}.
+    A #term{line break} cannot otherwise appear as plain-text 
+    unless it is #term[lemma='escape sequence']{escaped}.
   #:
 
   #-section[title='Escape sequences']:
@@ -84,27 +103,123 @@ as well as straightforward to customize and specialize to the user's needs.
       #-item{#chars[rep='u'] followed by 1-6 hexadecimal digits inserts a unicode codepoint}
     #:
   #:
+
+  #-section[title='Comments']:
+    A #def{comment} is text present in a Prosidy document that is not present in its output.
+    A comment begins with #chars[rep='##']{two hash signs} and 
+    continues until #ref[section='Line breaks']{the end of the line}.
+
+    #-spec[test='test.prosidy.comment']:
+      Comments can appoear in any #ref[section='Tags & contexts']{context}
+      #i{except} for #ref[section='Literal context']{literal contexts}
+      (where they are included as text)
+      and on the same line as the #term{document divider}.
+    #:
+
+    This document consists only of the text “hello, world”;
+    the comment is stripped while parsing.
+
+    #=src[lang='pro']:
+---
+Hello, world!   ## Maybe a bit cliché?
+    #:
+  #:
 #:
 
 #-section[title='Document structure']:
   Prosidy is organized into #def[lemma='document']{documents};
-  Each #term{document} is normally stored in a single file on a filesystem,
+  each #term{document} is normally stored in a single file on a filesystem,
   but ultimately the storage mechanism is application dependent.
 
   Each #term{document} contains exactly two parts: 
   a #term{header} followed by the #term{body}.
 
-  #-section[title='The Header']:
+  #-section[title='The header']:
     The #def{header} begins at the start of the document 
-    and continues until a #term{line} containing only 
-    #chars[rep='---']{three hyphens} is encountered.
+    and continues until the #term{document divider}.
+
+    The #term{header} contains #term{metadata} about the rest of the document.
+    Each non-empty #term{line} in the header represents a single #term{property} or #term{setting}.
+
+    ## TODO: Fill in with information about header syntax.
+
+    Its syntax is different from the #term{body}'s syntax to simplify scripting of the metadata.
+    For instance, you can build a table of contents from the #term[lemma='header']{headers}
+    of all of your documents with tools like #lit{sed}, 
+    while doing the same for the #term{body} would require a full Prosidy parser.
   #:
 
-  #-section[title='The Body']:
+  #-section[title='The body']:
     The #def{body} begins on the first line proceeding the #term{header},
     and continues until the end of the document.
   #:
+
+  #-section[title='The divide']:
+    The #term{header} and the #term{body} are separated by the #def{document divider}—
+    a #term{line} containing #chars[rep='---']{three consecutive hyphens} and nothing else.
+    
+    #-spec[test='test.prosidy.empty-document']:
+      This delimiter is the #i{only} required element in a #term{document}.
+      A #term{document} with a completely empty #term{header} and #term{body}
+      looks like this:
+
+      #=src[lang='pro']:
+---
+      #:
+    #:
+  #:
+#:
+
+#-section[title='Metadata']:
+  Metadata is composed of #term[lemma='property']{properties} and #term[lemma='setting']{settings},
+  applied to a region of a #term{document}.
+  #def[lemma='metadata']{Metadata} can be attached to both #term[lemma='tag']{tags} 
+  and #term[lemma='document']{documents}.
+
+  #-section[title='Properties']:
+    A #def{property} is a #term{key} which may either be specified or unspecified.
+    #term[lemma='property']{Properties} are #i{boolean} attributes— they're either #i{on} or #i{off}.
+    For instance, a hypothetical #term{dialect} may use attributes to style text.
+
+    #=src[lang='pro']:
+---
+This isn't a #style[italic]{real} dialect, 
+but it #style[bold]{does} work as an example!
+    #:
+
+    Here, the #term[lemma='property']{properties} #i{bold} and #i{italic}
+    annotate how the text in the #i{style} #term{tag} should be displayed.
+  #:
+
+  #-section[title='Settings']:
+    A #def{setting} is a #term{key} which is either unspecified or assigned a value.
+    The same hypothetical #term{dialect} from #ref[section='Properties']{before}
+    may allow #i{color} to be specified as a setting.
+
+    #=src[lang='pro']:
+---
+#style[color='#F00']{This text is red};
+but #style[bold, color='#0F0']{this text is blue and bold}.
+    #:
+  #:
+#:
+
+#-section[title='Keys']:
+  A #def{key} is a sequence of characters used as the names of 
+  #term[lemma='property']{properties}, #term[lemma='setting']{settings}, and #term[lemma='tag']{tags}.
+#:
+
+#-section[title='Tags & contexts']:
+  A #def{tag} is a #term{key} used to annotate a region of a document.
+
+  #-section[title='Literal context']:
+  #:
 #:
 
 #-section[title='Encoding']:
+  Prosidy #b{only} supports UTF-8 encoding.
+#:
+
+#-section[title='Dialects']:
+  A #def{dialect} is a convention for writing documents with a standardized structure.
 #:

nix/default.nix

@@ -23,6 +23,7 @@ let
 	        name = "prosidy-env";
 	        paths = [
                 pkgs.binutils.bintools
+                pkgs.fswatch
                 self.ghc
 	        ];
 	    };
@@ -39,9 +40,11 @@ let
             mmorph
             megaparsec
             optparse-applicative
+            shake
             tasty
             tasty-golden
             tasty-hunit
+            wai-app-static
         ]);
     };
 in

prosidy.cabal

@@ -1,6 +1,6 @@
 cabal-version: 3.0
 name:          prosidy
-version:       1.2.0.0
+version:       1.3.0.0
 synopsis:      A simple language for writing documents.
 license:       MPL-2.0
 license-file:  LICENSE
@@ -9,14 +9,16 @@ maintainer:    alex@fldcr.com
 copyright:     ©2020 to James Alexander Feldman-Crough
 category:      Language
 tested-with:   GHC == 8.8.1
+data-dir:      data
+data-files:    golden/**/*.pro, golden/**/*.json
 
 description:   
-  Prosidy is a small language for writing documents.
-  More details are on its site at https://prosidy.org.
+    Prosidy is a small language for writing documents.
+    More details are on its site at https://prosidy.org.
 
 source-repository head
-  type:     git
-  location: https://git.fldcr.com/prosidy
+   type:     git
+   location: https://git.fldcr.com/prosidy
 
 flag fatal-warnings
   description: Turns all warnings into errors. Used in CI.
@@ -28,12 +30,24 @@ common prosidy-shared
   default-language: Haskell2010
   build-depends:    base >=4.12 && <5
   ghc-options:
-    -Wall 
-    -Wno-name-shadowing
+      -Wall 
+      -Wno-name-shadowing
 
   if flag(fatal-warnings)
-    ghc-options:
-      -Werror
+      ghc-options:
+          -Werror
+
+
+common prosidy-shared-test
+  default-language: Haskell2010
+  ghc-options:
+      -Wall 
+      -Wno-name-shadowing
+  build-depends:
+      base         >=4.12 && <5
+    , tasty        >=1.2  && <1.3
+    , tasty-hunit  >=0.10 && <0.11
+    , tasty-golden >=2.3  && <2.4
 
 -------------------------------------------------------------------------------
 library
@@ -58,7 +72,7 @@ library
     , template-haskell  >= 2.14 && < 2.16
     , text             ^>= 1.2
     , transformers     ^>= 0.5
-
+  
 
 -------------------------------------------------------------------------------
 library prosidyc
@@ -103,6 +117,7 @@ executable prosidy-manual
   other-modules:
       Prosidy.Manual.Compile
     , Prosidy.Manual.Monad
+    , Prosidy.Manual.Opts
     , Prosidy.Manual.Slug
     , Prosidy.Manual.TableOfContents
 
@@ -118,6 +133,7 @@ executable prosidy-manual
     , hashable              >= 1.2  && < 1.4
     , lens                 ^>= 4.18
     , optparse-applicative ^>= 0.15
+    , shake                ^>= 0.18
     , text                 ^>= 1.2
     , transformers         ^>= 0.5
     , unordered-containers ^>= 0.2
@@ -142,6 +158,33 @@ executable prosidy-markup
     , text                 ^>= 1.2
     , transformers         ^>= 0.5
 
+  
+-------------------------------------------------------------------------------
+test-suite prosidy-test
+  import:          prosidy-shared-test
+  type:            exitcode-stdio-1.0
+  hs-source-dirs:  test
+  ghc-options:     -main-is Prosidy.Test.main
+
+  main-is:
+      Prosidy/Test.hs
+
+  other-modules:
+      Prosidy.Test.Prosidy
+    , Paths_prosidy
+
+  autogen-modules:
+      Paths_prosidy
+
+  build-depends:
+      prosidy
+    , aeson            ^>= 1.4
+    , aeson-pretty     ^>= 0.8
+    , aeson-diff       ^>= 1.1
+    , bytestring       ^>= 0.10
+    , directory        ^>= 1.3
+    , filepath         ^>= 1.4
+    , text             ^>= 1.2
 
 -------------------------------------------------------------------------------
 library prosidy-internal

src/prosidy-manual/Prosidy/Manual.hs

@@ -4,58 +4,120 @@
 {-# LANGUAGE TupleSections #-}
 {-# LANGUAGE TypeApplications #-}
 {-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE GeneralisedNewtypeDeriving #-}
+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE TypeFamilies #-}
 module Prosidy.Manual where
 
-import Control.Applicative (Alternative(..))
-import qualified Options.Applicative as O
-import Prosidy (readDocument)
-import Data.Foldable (foldr', for_)
-import Prosidy.Manual.TableOfContents (documentEntry, insertEntry)
-import Prosidy.Manual.Compile (compile, document)
-import Data.Bifunctor (second)
-import Control.Exception (throwIO, displayException, handle, fromException)
-import System.Exit (ExitCode, exitFailure)
+import Development.Shake.Classes
+import Development.Shake ((%>), (<//>), (~>), (|%>))
+import qualified Development.Shake as Shake
+import Development.Shake.FilePath ((</>), (-<.>), makeRelative)
+import GHC.Generics (Generic)
+import qualified Data.Map.Strict as Map
+import Data.Maybe (fromMaybe)
+import Control.Exception (throwIO)
+
+import Control.Lens.Operators
+import Data.Text.Lens (packed)
+import qualified Control.Lens as L
 import qualified Data.ByteString.Lazy as LBS
-import System.FilePath ((</>), (-<.>))
-import qualified System.IO as IO
+
+import Prosidy
+import Prosidy.Manual.Compile (compile, document)
+import Prosidy.Manual.Opts
+import Prosidy.Manual.Slug
+import Prosidy.Manual.TableOfContents
 
 main :: IO ()
-main = handleException $ do
-    Opts{inputs, outputDir} <- O.execParser optOpts
-    documents <- zip inputs <$> traverse readDocument inputs
-    let entries = fmap (second documentEntry) documents
-        toc     = foldr' (uncurry insertEntry) mempty entries
-    for_ documents $ \(path, doc) ->
-        case compile document toc path doc of
-            Left e -> throwIO e
-            Right ok -> LBS.writeFile (outputDir </> path -<.> "html") ok
-
-handleException :: IO a -> IO a
-handleException =
-    handle (\e -> case fromException @ExitCode e of
-                    Just code -> throwIO code
-                    Nothing -> IO.hPutStrLn IO.stderr (displayException e) *> exitFailure)
-
-data Opts = Opts
-    { outputDir :: FilePath
-    , inputs    :: [FilePath]
-    }
-  deriving Show
-
-optOpts :: O.ParserInfo Opts
-optOpts = O.info (Opts <$> optOutputDir <*> optInputs) $ mconcat
-    [
-    ]
-
-optInputs :: O.Parser [FilePath]
-optInputs = some . O.strArgument $ mconcat
-    [ O.help "A document to process."
-    , O.metavar "PRO"
-    ]
-
-optOutputDir :: O.Parser FilePath
-optOutputDir = O.strOption $ mconcat
-    [ O.short 'o', O.long "output-root"
-    , O.help "The root directory to write resulting HTML in to."
-    , O.metavar "HTML"
-    ]
+main = do
+    opts <- getOpts
+    Shake.shake (mainOpts opts) $ do
+        installOracles opts 
+        mainBuild opts
+
+mainOpts :: Opts -> Shake.ShakeOptions
+mainOpts Opts{dbDir, outDir} = Shake.shakeOptions