Ensure that the parse tool produces trees with the same structure as the generator

.pylintrc

@@ -33,6 +33,7 @@ disable=
     redefined-builtin,
     too-few-public-methods,
     too-many-arguments,
+    too-many-boolean-expressions,
     too-many-branches,
     too-many-instance-attributes,
     too-many-lines,

grammarinator/parse.py

@@ -56,6 +56,8 @@ def execute():
                         help='directory to save the trees (default: %(default)s).')
     parser.add_argument('--parser-dir', metavar='DIR',
                         help='directory to save the parser grammars (default: <OUTDIR>/grammars).')
+    parser.add_argument('--lib', metavar='DIR',
+                        help='alternative location of import grammars.')
     add_tree_format_argument(parser)
     add_encoding_argument(parser, help='input file encoding (default: %(default)s).')
     add_encoding_errors_argument(parser)
@@ -80,7 +82,7 @@ def execute():
         parser.error(e)
 
     with ParserTool(grammars=args.grammar, hidden=args.hidden, transformers=args.transformer, parser_dir=args.parser_dir, antlr=args.antlr, rule=args.rule,
-                    population=DefaultPopulation(args.out, args.tree_extension, codec=args.tree_codec), max_depth=args.max_depth, cleanup=args.cleanup, encoding=args.encoding, errors=args.encoding_errors) as parser:
+                    population=DefaultPopulation(args.out, args.tree_extension, codec=args.tree_codec), max_depth=args.max_depth, lib_dir=args.lib, cleanup=args.cleanup, encoding=args.encoding, errors=args.encoding_errors) as parser:
         if args.jobs > 1:
             with Pool(args.jobs) as pool:
                 for _ in pool.imap_unordered(parser.parse, iter_files(args)):

tests/parser/Parse.g4

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2024 Renata Hodovan, Akos Kiss.
+ *
+ * Licensed under the BSD 3-Clause License
+ * <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
+ * This file may not be copied, modified, or distributed except
+ * according to those terms.
+ */
+
+/*
+ * This test checks whether the parser utility of Grammarinator creates
+ * the same tree structures as generator would do.
+ */
+
+grammar Parse;
+
+start
+  : element (' ' element)*    # Quantifiers_test
+  | element (' | ' element)+  # Quantifiers_test
+  | list_with_recursion       # Recursion_test
+  ;
+
+element
+  : 'pass' ('?' | '!')?
+  ;
+
+list_with_recursion
+  : list_with_recursion (', ' element)
+  | element
+  ;

tests/parser/exp1.grtj

@@ -0,0 +1 @@
+{"t": "p", "n": "start", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "p", "n": "start_Quantifiers_test", "c": [{"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": []}]}, {"t": "q", "i": 0, "b": 0, "e": Infinity, "c": []}]}]}]}

tests/parser/exp2.grtj

@@ -0,0 +1 @@
+{"t": "p", "n": "start", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "p", "n": "start_Quantifiers_test", "c": [{"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": [{"t": "qd", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "l", "n": "<INVALID>", "s": "?", "z": [1, 1]}]}]}]}]}, {"t": "q", "i": 0, "b": 0, "e": Infinity, "c": []}]}]}]}

tests/parser/exp3.grtj

@@ -0,0 +1 @@
+{"t": "p", "n": "start", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "p", "n": "start_Quantifiers_test", "c": [{"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": [{"t": "qd", "c": [{"t": "a", "ai": 0, "i": 1, "c": [{"t": "l", "n": "<INVALID>", "s": "!", "z": [1, 1]}]}]}]}]}, {"t": "q", "i": 0, "b": 0, "e": Infinity, "c": [{"t": "qd", "c": [{"t": "l", "n": "<INVALID>", "s": " ", "z": [1, 1]}, {"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": [{"t": "qd", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "l", "n": "<INVALID>", "s": "?", "z": [1, 1]}]}]}]}]}]}]}]}]}]}

tests/parser/exp4.grtj

@@ -0,0 +1 @@
+{"t": "p", "n": "start", "c": [{"t": "a", "ai": 0, "i": 1, "c": [{"t": "p", "n": "start_Quantifiers_test", "c": [{"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": [{"t": "qd", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "l", "n": "<INVALID>", "s": "?", "z": [1, 1]}]}]}]}]}, {"t": "q", "i": 0, "b": 1, "e": Infinity, "c": [{"t": "qd", "c": [{"t": "l", "n": "<INVALID>", "s": " | ", "z": [1, 1]}, {"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": []}]}]}]}]}]}]}

tests/parser/exp5.grtj

@@ -0,0 +1 @@
+{"t": "p", "n": "start", "c": [{"t": "a", "ai": 0, "i": 2, "c": [{"t": "p", "n": "start_Recursion_test", "c": [{"t": "p", "n": "list_with_recursion", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "p", "n": "list_with_recursion", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "p", "n": "list_with_recursion", "c": [{"t": "a", "ai": 0, "i": 1, "c": [{"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": [{"t": "qd", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "l", "n": "<INVALID>", "s": "?", "z": [1, 1]}]}]}]}]}]}]}, {"t": "l", "n": "<INVALID>", "s": ", ", "z": [1, 1]}, {"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": [{"t": "qd", "c": [{"t": "a", "ai": 0, "i": 1, "c": [{"t": "l", "n": "<INVALID>", "s": "!", "z": [1, 1]}]}]}]}]}]}]}, {"t": "l", "n": "<INVALID>", "s": ", ", "z": [1, 1]}, {"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": []}]}]}]}]}]}]}

tests/parser/inp1.txt

@@ -0,0 +1 @@
+pass

tests/parser/inp2.txt

@@ -0,0 +1 @@
+pass?

tests/parser/inp3.txt

@@ -0,0 +1 @@
+pass! pass?

tests/parser/inp4.txt

@@ -0,0 +1 @@
+pass? | pass

tests/parser/inp5.txt

@@ -0,0 +1 @@
+pass?, pass!, pass

tests/test_parser.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2024 Renata Hodovan, Akos Kiss.
+#
+# Licensed under the BSD 3-Clause License
+# <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
+# This file may not be copied, modified, or distributed except
+# according to those terms.
+
+import os
+
+import pytest
+
+from antlerinator import default_antlr_jar_path
+from antlr4 import InputStream
+
+from grammarinator.parse import ParserTool
+from grammarinator.tool import JsonTreeCodec
+
+parser_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'parser')
+
+
+@pytest.mark.parametrize('inp, expected', [
+    (os.path.join(parser_dir, 'inp1.txt'), os.path.join(parser_dir, 'exp1.grtj')),
+    (os.path.join(parser_dir, 'inp2.txt'), os.path.join(parser_dir, 'exp2.grtj')),
+    (os.path.join(parser_dir, 'inp3.txt'), os.path.join(parser_dir, 'exp3.grtj')),
+    (os.path.join(parser_dir, 'inp4.txt'), os.path.join(parser_dir, 'exp4.grtj')),
+    (os.path.join(parser_dir, 'inp5.txt'), os.path.join(parser_dir, 'exp5.grtj')),
+])
+def test_parser(inp, expected, tmpdir):
+    with open(inp, 'r') as f:
+        src = f.read()
+
+    tool = ParserTool(grammars=[os.path.join(parser_dir, 'Parse.g4')], rule='start', parser_dir=str(tmpdir), antlr=default_antlr_jar_path(), population=None)
+    root = tool._create_tree(InputStream(src), None)
+
+    with open(expected, 'rb') as f:
+        expected_root = JsonTreeCodec().decode(f.read())
+
+    assert root.equals(expected_root), f'{root:|} != {expected_root:|}'