fix: ignore punctuation at end of URL

Problem:
URLs in help docs may be followed by "." or ",", but it's usually not
intended as part of the URL.

Examples from neovim/neovim#36597:

    https://luarocks.org,
    https://neovim.io/doc/,

Solution:
- Treat "." as a word.
- Assume that `)].,` at the end of a URL is not part of the URL.

Now NESTED parens work:

    (https://neovim.io/doc/user/vimfn.html#get()-blob)

but it's not possible to support a trailing closing paren ")":

    (https://neovim.io/doc/user/api.html#nvim_input())

workaround: URL-encode the trailing paren:

    (https://neovim.io/doc/user/api.html#nvim_input%28%29)

URL cannot contain a closing bracket `]` anywhere in the URL.
(Workaround: URL-encode the bracket.) This is a tradeoff so that
markdown hyperlinks work:

    [https://example.com](https://example.com)

Bonus(?): now the inline code in this example is recognized:

    `foo`.bar

Author: justinmk

.editorconfig

@@ -4,7 +4,10 @@ root = true
 charset = utf-8
 end_of_line = lf
 insert_final_newline = true
-trim_trailing_whitespace = true
+
+[*.txt]
+# Some test files have intentional whitespace at EOL.
+trim_trailing_whitespace = false
 
 [*.{json,toml,yml,gyp}]
 indent_style = space

README.md

@@ -32,6 +32,7 @@ Overview
     nature; parsing the contents would require loading a "child" language
     (injection). See [#2](https://github.com/neovim/tree-sitter-vimdoc/issues/2).
   - the terminating `<` (and any following whitespace) is discarded (anonymous).
+- `url` intentionally does not capture `.,)` at the end of the URL. See also [Known issues](#known-issues).
 - `h1` = "Heading 1": `======` followed by text and optional `*tags*`.
 - `h2` = "Heading 2": `------` followed by text and optional `*tags*`.
 - `h3` = "Heading 3": UPPERCASE WORDS, followed by optional `*tags*`, followed
@@ -45,8 +46,11 @@ Known issues
 - Spec requires that `codeblock` delimiter ">" must be preceded by a space
   (" >"), not a tab. But currently the grammar doesn't enforce this. Example:
   `:help lcs-tab`.
-- `url` doesn't handle _surrounding_ parens. E.g. `(https://example.com/#yay)` yields `word`
-- `url` doesn't handle _nested_ parens. E.g. `(https://example.com/(foo)#yay)`
+- `url` cannot contain a closing bracket `]` anywhere in the URL. (Workaround:
+  URL-encode the bracket.) This is a tradeoff so that markdown hyperlinks work:
+  ```
+  [https://example.com](https://example.com)
+  ```
 - `column_heading` currently only recognizes tilde `~` preceded by space (i.e.
   `foo ~` not `foo~`). This covers 99% of :help files.
 - `column_heading` children should be plaintext, but currently are parsed as `$._atom`.
@@ -55,8 +59,8 @@ Known issues
 TODO
 ----
 
-- `tag_heading` : line(s) containing only tags, typically implies a "heading"
-  before a block.
+- `h4` ("tag heading") : a line containing only tags, or ending with a tag, is
+  a "h4" heading.
 
 Release
 -------

grammar.js

@@ -41,7 +41,7 @@ module.exports = grammar({
       $._atom_common,
     ),
     word: ($) => choice(
-      token(prec(-1, /[^,(\[\n\t ]+/)),
+      token(prec(-1, /[^.,(\[\n\t ]+/)),
       $._word_common,
     ),
 
@@ -89,11 +89,14 @@ module.exports = grammar({
       /\{\{+[0-9]*/,
 
       '(',
+      ')',
       '[',
+      ']',
       '~',
       // NOT codeblock: random ">" in middle of the motherflippin text.
       '>',
       ',',
+      '.',
     ),
 
     note: () => choice(
@@ -223,7 +226,7 @@ module.exports = grammar({
       '*', '*'),
 
     // URL without surrounding (), [], etc.
-    url_word: () => /https?:[^\n\t)\] ]+/,
+    url_word: () => /https?:\/\/[^\n\t\] ]*[^\n\t )\].,]/,
     url: ($) => choice(
       // seq('(', field('text', prec.left(alias($.url_word, $.word))), token.immediate(')')),
       // seq('[', field('text', prec.left(alias($.url_word, $.word))), token.immediate(']')),

test/corpus/arguments.txt

@@ -207,6 +207,7 @@ EXTERNAL *netrw-externapp* {{{2
       (h1
         (delimiter)
         (heading
+          (word)
           (word)
           (word))
         (tag

test/corpus/codeblock.txt

@@ -82,6 +82,8 @@ text
           (line)))))
   (block
     (line
+      (word)
+      (word)
       (word)
       (word)))
   (block
@@ -90,6 +92,7 @@ text
       (word)
       (taglink
         (word))
+      (word)
       (word))
     (line
       (codeblock
@@ -336,6 +339,7 @@ To test for a non-empty string, use empty(): >
       (word)
       (word)
       (word)
+      (word)
       (word))
     (line
       (word)
@@ -359,6 +363,7 @@ To test for a non-empty string, use empty(): >
       (word)
       (word)
       (word)
+      (word)
       (codeblock
         (code
           (line))))))

test/corpus/codespan.txt

@@ -44,6 +44,7 @@ an error`.
         (word)
         (MISSING "`")))
     (line
+      (word)
       (word)
       (word))))
 
@@ -79,4 +80,4 @@ g'{mark}  g`{mark}
         (word)))
     (line
       (word)
-        (word))))
+      (word))))

test/corpus/heading3-column_heading.txt

@@ -229,17 +229,27 @@ ABC not-h3
         (word))
       (codespan
         (word))
+      (word)
       (word))
     (line
       (codespan
         (word))
+      (word)
       (word))
     (line
       (word)
       (word)
       (word)
+      (word)
       (word))
     (line
+      (word)
+      (word)
+      (word)
+      (codespan
+        (word))
+      (word)
+      (word)
       (word))))
 
 ================================================================================

test/corpus/line_block.txt

@@ -103,6 +103,8 @@ li continues
   (block
     (line_li
       (line
+        (word)
+        (word)
         (word)
         (word))
       (line))
@@ -121,6 +123,7 @@ li continues
         (word)
         (word)
         (word)
+        (word)
         (word))
       (line
         (optionlink
@@ -130,6 +133,9 @@ li continues
           (word))))
     (line_li
       (line
+        (word)
+        (word)
+        (word)
         (word)
         (word)
         (word)
@@ -199,13 +205,17 @@ listitem with codeblock
   (block
     (line_li
       (line
+        (word)
+        (word)
         (word)
         (word))
       (codeblock
         (code
           (line))))
     (line_li
       (line
+        (word)
+        (word)
         (word)
         (word))
       (line))

test/corpus/optionlink.txt

@@ -75,6 +75,7 @@ Regular	  /	:help /[
 (help_file
   (block
     (line
+      (word)
       (word)
       (word))
     (line
@@ -96,6 +97,8 @@ Regular	  /	:help /[
       (word)
       (word)
       (word)
+      (word)
+      (word)
       (ERROR
         (word))
       (word)
@@ -162,6 +165,7 @@ foo '"\ '.  Notice
       (word)
       (word)
       (word)
+      (word)
       (word))
     (line
       (taglink

test/corpus/taglink.txt

@@ -148,6 +148,9 @@ Note: ":autocmd" can...
     (line
       (note)
       (word)
+      (word)
+      (word)
+      (word)
       (word))
     (line
       (word)