Added support for comments between arguments

Author: aluttik

crossplane/parser.py

@@ -76,6 +76,8 @@ def _parse(parsing, tokens, ctx=(), consume=False):
 
         # parse recursively by pulling from a flat stream of tokens
         for token, lineno, quoted in tokens:
+            comments_in_args = []
+
             # we are parsing a block, so break if it's closing
             if token == '}' and not quoted:
                 break
@@ -118,7 +120,11 @@ def _parse(parsing, tokens, ctx=(), consume=False):
             args = stmt['args']
             token, __, quoted = next(tokens)  # disregard line numbers of args
             while token not in ('{', ';', '}') or quoted:
-                stmt['args'].append(token)
+                if token.startswith('#') and not quoted:
+                    comments_in_args.append(token[1:])
+                else:
+                    stmt['args'].append(token)
+
                 token, __, quoted = next(tokens)
 
             # consume the directive if it is ignored and move on
@@ -196,6 +202,16 @@ def _parse(parsing, tokens, ctx=(), consume=False):
 
             parsed.append(stmt)
 
+            # add all comments found inside args after stmt is added
+            for comment in comments_in_args:
+                comment_stmt = {
+                    'directive': '#',
+                    'line': stmt['line'],
+                    'args': [],
+                    'comment': comment
+                }
+                parsed.append(comment_stmt)
+
         return parsed
 
     # the includes list grows as "include" directives are found in _parse

tests/configs/comments-between-args/nginx.conf

@@ -0,0 +1,7 @@
+http { #comment 1
+    log_format #comment 2
+        \#arg\ 1 #comment 3
+        '#arg 2' #comment 4
+        #comment 5
+        ;
+}

tests/test_build.py

@@ -196,6 +196,37 @@ def test_build_with_quoted_unicode():
     assert built == u"env 'русский текст';"
 
 
+def test_build_multiple_comments_on_one_line():
+    payload = [
+        {
+            "directive": "#",
+            "line": 1,
+            "args": [],
+            "comment": "comment1"
+        },
+        {
+            "directive": "user",
+            "line": 2,
+            "args": ["root"]
+        },
+        {
+            "directive": "#",
+            "line": 2,
+            "args": [],
+            "comment": "comment2"
+        },
+        {
+            "directive": "#",
+            "line": 2,
+            "args": [],
+            "comment": "comment3"
+        }
+    ]
+    built = crossplane.build(payload, indent=4, tabs=False)
+    assert built == '#comment1\nuser root; #comment2 #comment3'
+
+
+
 def test_build_files_with_missing_status_and_errors(tmpdir):
     assert len(tmpdir.listdir()) == 0
     payload = {

tests/test_parse.py

@@ -582,7 +582,7 @@ def test_config_with_comments():
                 }
              ],
              "status" : "ok",
-             "file" : os.path.join(dirname, 'nginx.conf')
+             "file" : config
           }
        ]
     }
@@ -914,3 +914,64 @@ def test_combine_parsed_missing_values():
             }
         ]
     }
+
+
+def test_comments_between_args():
+    dirname = os.path.join(here, 'configs', 'comments-between-args')
+    config = os.path.join(dirname, 'nginx.conf')
+    payload = crossplane.parse(config, comments=True)
+    assert payload == {
+        'status': 'ok',
+        'errors': [],
+        'config': [
+            {
+                'file': config,
+                'status': 'ok',
+                'errors': [],
+                'parsed': [
+                    {
+                        'directive': 'http',
+                        'line': 1,
+                        'args': [],
+                        'block': [
+                            {
+                                'directive': '#',
+                                'line': 1,
+                                'args': [],
+                                'comment': 'comment 1'
+                            },
+                            {
+                                'directive': 'log_format',
+                                'line': 2,
+                                'args': ['\\#arg\\ 1', '#arg 2']
+                            },
+                            {
+                                'directive': '#',
+                                'line': 2,
+                                'args': [],
+                                'comment': 'comment 2'
+                            },
+                            {
+                                'directive': '#',
+                                'line': 2,
+                                'args': [],
+                                'comment': 'comment 3'
+                            },
+                            {
+                                'directive': '#',
+                                'line': 2,
+                                'args': [],
+                                'comment': 'comment 4'
+                            },
+                            {
+                                'directive': '#',
+                                'line': 2,
+                                'args': [],
+                                'comment': 'comment 5'
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
+    }