Separate flatdata-py and flatdata-generator at the root of the repository (#123)

* Separate flatdata-py and flatdata-generator in their own directories

Signed-off-by: Gabriel Féron <[email protected]>

* Add instructions on how to run the generator and inspector with the new package structure

Signed-off-by: Gabriel Féron <[email protected]>

* pip3 install without --user

Signed-off-by: Gabriel Féron <[email protected]>

* Fix CI and CMake path to generator

Signed-off-by: Gabriel Féron <[email protected]>

* Add generator requirements.txt link to flatdata-py requirements.txt

Signed-off-by: Gabriel Féron <[email protected]>

* Move flatdata-py to flatdata.lib

Signed-off-by: Gabriel Féron <[email protected]>

* Install flatdata-generator in Travis

Signed-off-by: Gabriel Féron <[email protected]>

* Mention PYTHONPATH for local development

Signed-off-by: Gabriel Féron <[email protected]>

* Fix generator test

Signed-off-by: Gabriel Féron <[email protected]>

* Trying to fix Travis CI

Signed-off-by: Gabriel Féron <[email protected]>

* Trying to fix pip3 install in CI

Signed-off-by: Gabriel Féron <[email protected]>

* Use find_package since find_namespace_packages is not widely available

Signed-off-by: Gabriel Féron <[email protected]>

* Various fixes for older setuptools

Signed-off-by: Gabriel Féron <[email protected]>

Author: gferon

.travis.yml

@@ -5,17 +5,34 @@ matrix:
     - rust: nightly
 
   include:
-    # flatdata-py and generator
+    # flatdata-generator
     - language: python
+      name: "flatdata-generator"
       dist: xenial
       python: 3.5
       script:
+        - cd flatdata-generator
+        - pip3 install -r requirements.txt  # runtime requirements
+        - pip3 install nose pylint  # CI requirements
+        - python3 -m nose
+        # try to install
+        - pip3 install .
+        - flatdata-generator --help
+
+    # flatdata-py
+    - language: python
+      name: "flatdata-py"
+      dist: xenial
+      python: 3.5
+      script:
+        - pip3 install ./flatdata-generator
         - cd flatdata-py
-        - pip3 install -r requirements.txt
-        - pip3 install -r generator/requirements.txt
-        - pip3 install pylint
+        - pip3 install -r requirements.txt  # runtime requirements
+        - pip3 install nose pylint  # CI requirements
         - python3 -m nose
-        - pylint flatdata generator || true
+        # try to install
+        - pip3 install .
+        - flatdata-inspector --help
 
     # cpp
     - language: cpp
@@ -25,9 +42,10 @@ matrix:
         apt:
           packages:
             - python3-pip
+            - python3-setuptools
             - libboost-filesystem-dev
       before_script:
-        - pip3 install -r flatdata-py/generator/requirements.txt
+        - pip3 install ./flatdata-generator
       script:
         - flatdata-cpp/ci/build-and-test-cpp.sh
 
@@ -39,9 +57,10 @@ matrix:
         apt:
           packages:
             - python3-pip
+            - python3-setuptools
             - libboost-filesystem-dev
       before_script:
-        - pip3 install -r flatdata-py/generator/requirements.txt
+        - pip3 install ./flatdata-generator
       script:
         - flatdata-cpp/ci/build-and-test-cpp.sh
 
@@ -68,9 +87,10 @@ matrix:
         apt:
           packages:
             - python3-pip
+            - python3-setuptools
             - graphviz
       before_script:
-        - pip3 install -r flatdata-py/generator/requirements.txt
+        - pip3 install ./flatdata-generator
       script:
         - ci/dot_test_cases.sh
 
@@ -84,8 +104,9 @@ matrix:
         apt:
           packages:
             - python3-pip
+            - python3-setuptools
       before_script:
-        - pip3 install -r flatdata-py/generator/requirements.txt
+        - pip3 install ./flatdata-generator
         - cd flatdata-rs
       script:
         - cargo build
@@ -103,8 +124,9 @@ matrix:
         apt:
           packages:
             - python3-pip
+            - python3-setuptools
       before_script:
-        - pip3 install -r flatdata-py/generator/requirements.txt
+        - pip3 install ./flatdata-generator
         - cd flatdata-rs
       script:
         - cargo build
@@ -120,9 +142,10 @@ matrix:
         apt:
           packages:
             - python3-pip
+            - python3-setuptools
       before_script:
-        - pip3 install -r flatdata-py/generator/requirements.txt
+        - pip3 install ./flatdata-generator
         - cd flatdata-rs
       script:
         - cargo build
-        - cargo test
+        - cargo test

flatdata-cpp/cmake/flatdata/GenerateSource.cmake

@@ -1,4 +1,4 @@
-set(FLATDATA_GENERATOR_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../flatdata-py)
+set(FLATDATA_GENERATOR_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../flatdata-generator)
 
 # Generates sources from flatdata schema
 #
@@ -17,7 +17,7 @@ function(flatdata_generate_source TARGET_NAME SCHEMA_FILENAME OUTPUT_FILENAME)
 
     add_custom_command(
         OUTPUT ${OUTPUT_FILENAME}
-        COMMAND ${PYTHON3_EXECUTABLE} ${FLATDATA_GENERATOR_PATH}/generate.py
+        COMMAND ${PYTHON3_EXECUTABLE} ${FLATDATA_GENERATOR_PATH}/generator.py
         --gen cpp
         --schema ${SCHEMA_FILENAME}
         --output-file ${OUTPUT_FILENAME}

flatdata-generator/.pylintrc

@@ -0,0 +1,5 @@
+[MASTER]
+max-line-length=120
+
+[MESSAGES CONTROL]
+disable=missing-docstring,line-too-long

flatdata-py/generator/README.md renamed to flatdata-generator/README.md

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 '''
  Copyright (c) 2017 HERE Europe B.V.
  See the LICENSE file in the root of this project for license details.
@@ -11,15 +10,15 @@
 
 # check that requirements are installed here
 try:
-    #pylint: disable=unused-import
+    # pylint: disable=unused-import
     import pyparsing
     import jinja2
 except ModuleNotFoundError as exc:
     print("Cannot import `%s`, you probably need to install it. See `generator/requirements.txt` or `README.md`." % exc.name, file=sys.stderr)
     sys.exit(2)
 
-from generator.engine import Engine
-from generator.tree.errors import FlatdataSyntaxError
+from flatdata.generator.engine import Engine
+from flatdata.generator.tree.errors import FlatdataSyntaxError
 
 
 def _parse_command_line():
@@ -89,8 +88,4 @@ def _run(args):
 
 def main():
     """Entrypoint"""
-    _run(_parse_command_line())
-
-
-if __name__ == "__main__":
-    main()
+    _run(_parse_command_line())

flatdata-py/generator/__init__.py renamed to flatdata-generator/flatdata/generator/__init__.py

@@ -5,8 +5,8 @@
 
 import imp
 
-from generator.tree.builder import build_ast
-from generator.tree.nodes.trivial.namespace import Namespace
+from flatdata.generator.tree.builder import build_ast
+from flatdata.generator.tree.nodes.trivial.namespace import Namespace
 
 from .generators.cpp import CppGenerator
 from .generators.dot import DotGenerator

flatdata-py/generator/app.py renamed to flatdata-generator/flatdata/generator/app.py

@@ -9,12 +9,12 @@
 from jinja2.ext import Extension
 from jinja2.exceptions import TemplateRuntimeError
 
-from generator.tree.nodes.archive import Archive
-from generator.tree.nodes.trivial import Structure, Enumeration, Constant, Namespace
-from generator.tree.nodes.resources import ResourceBase, BoundResource, Archive as \
+from flatdata.generator.tree.nodes.archive import Archive
+from flatdata.generator.tree.nodes.trivial import Structure, Enumeration, Constant, Namespace
+from flatdata.generator.tree.nodes.resources import ResourceBase, BoundResource, Archive as \
     ArchiveResource, Vector, Multivector, Instance, RawData
-from generator.tree.syntax_tree import SyntaxTree
-from generator.tree.traversal import DfsTraversal
+from flatdata.generator.tree.syntax_tree import SyntaxTree
+from flatdata.generator.tree.traversal import DfsTraversal
 
 
 class BaseGenerator(metaclass=ABCMeta):
@@ -36,7 +36,7 @@ def _populate_environment(self, env):
 
     def render(self, tree):
         """Generate the language implementation from the AST"""
-        env = Environment(loader=PackageLoader('generator', 'templates'), lstrip_blocks=True,
+        env = Environment(loader=PackageLoader('flatdata.generator', 'templates'), lstrip_blocks=True,
                           trim_blocks=True, autoescape=False, extensions=[RaiseExtension])
         env.filters['is_archive'] = lambda n: isinstance(n, Archive)
         env.filters['is_instance'] = lambda n: isinstance(n, Instance)

flatdata-py/generator/engine.py renamed to flatdata-generator/flatdata/generator/engine.py

@@ -3,10 +3,10 @@
  See the LICENSE file in the root of this project for license details.
 '''
 
-from generator.tree.nodes.resources import Vector, Multivector, Instance, RawData, BoundResource, \
+from flatdata.generator.tree.nodes.resources import Vector, Multivector, Instance, RawData, BoundResource, \
     ResourceBase, Archive as ArchiveResource
-from generator.tree.nodes.trivial import Structure, Enumeration, Constant, Field
-from generator.tree.nodes.archive import Archive
+from flatdata.generator.tree.nodes.trivial import Structure, Enumeration, Constant, Field
+from flatdata.generator.tree.nodes.archive import Archive
 from . import BaseGenerator
 
 

flatdata-py/generator/generators/__init__.py renamed to flatdata-generator/flatdata/generator/generators/__init__.py

@@ -3,7 +3,7 @@
  See the LICENSE file in the root of this project for license details.
 '''
 
-from generator.tree.nodes.archive import Archive
+from flatdata.generator.tree.nodes.archive import Archive
 from . import BaseGenerator
 
 SCOPE_SEPARATOR = "__"

flatdata-py/generator/generators/cpp.py renamed to flatdata-generator/flatdata/generator/generators/cpp.py

@@ -3,10 +3,10 @@
  See the LICENSE file in the root of this project for license details.
 '''
 
-from generator.tree.nodes.resources import BoundResource
-from generator.tree.nodes.trivial import Structure, Enumeration, Constant
-from generator.tree.nodes.archive import Archive
-from generator.tree.syntax_tree import SyntaxTree
+from flatdata.generator.tree.nodes.resources import BoundResource
+from flatdata.generator.tree.nodes.trivial import Structure, Enumeration, Constant
+from flatdata.generator.tree.nodes.archive import Archive
+from flatdata.generator.tree.syntax_tree import SyntaxTree
 from . import BaseGenerator
 
 

flatdata-py/generator/generators/dot.py renamed to flatdata-generator/flatdata/generator/generators/dot.py

@@ -2,11 +2,11 @@
  Copyright (c) 2017 HERE Europe B.V.
  See the LICENSE file in the root of this project for license details.
 '''
-from generator.tree.nodes.archive import Archive
-from generator.tree.nodes.node import Node
-from generator.tree.nodes.resources import Instance, Vector, Multivector, RawData
-from generator.tree.nodes.resources.archive import Archive as ArchiveResource
-from generator.tree.nodes.trivial import Structure, Constant
+from flatdata.generator.tree.nodes.archive import Archive
+from flatdata.generator.tree.nodes.node import Node
+from flatdata.generator.tree.nodes.resources import Instance, Vector, Multivector, RawData
+from flatdata.generator.tree.nodes.resources.archive import Archive as ArchiveResource
+from flatdata.generator.tree.nodes.trivial import Structure, Constant
 
 from . import BaseGenerator
 

flatdata-py/generator/generators/flatdata.py renamed to flatdata-generator/flatdata/generator/generators/flatdata.py

@@ -2,11 +2,11 @@
  Copyright (c) 2017 HERE Europe B.V.
  See the LICENSE file in the root of this project for license details.
 '''
-from generator.tree.nodes.resources import Instance, Vector, Multivector, RawData
-from generator.tree.nodes.resources.archive import Archive as ArchiveResource
-from generator.tree.nodes.trivial import Structure
-from generator.tree.nodes.node import Node
-from generator.tree.nodes.archive import Archive
+from flatdata.generator.tree.nodes.resources import Instance, Vector, Multivector, RawData
+from flatdata.generator.tree.nodes.resources.archive import Archive as ArchiveResource
+from flatdata.generator.tree.nodes.trivial import Structure
+from flatdata.generator.tree.nodes.node import Node
+from flatdata.generator.tree.nodes.archive import Archive
 from . import BaseGenerator
 
 

flatdata-py/generator/generators/go.py renamed to flatdata-generator/flatdata/generator/generators/go.py

@@ -4,12 +4,12 @@
 '''
 import re
 
-from generator.tree.nodes.resources import (Vector, Multivector, Instance, RawData, BoundResource,
+from flatdata.generator.tree.nodes.resources import (Vector, Multivector, Instance, RawData, BoundResource,
                                             Archive as ArchiveResource)
-from generator.tree.nodes.trivial import Structure, Constant, Enumeration
-from generator.tree.helpers.enumtype import EnumType
-from generator.tree.nodes.archive import Archive
-from generator.tree.syntax_tree import SyntaxTree
+from flatdata.generator.tree.nodes.trivial import Structure, Constant, Enumeration
+from flatdata.generator.tree.helpers.enumtype import EnumType
+from flatdata.generator.tree.nodes.archive import Archive
+from flatdata.generator.tree.syntax_tree import SyntaxTree
 from . import BaseGenerator
 
 

flatdata-py/generator/generators/python.py renamed to flatdata-generator/flatdata/generator/generators/python.py

@@ -4,7 +4,7 @@
 #    Any modifications to this file will be lost upon next regeneration.    #
 #############################################################################
 
-import flatdata
+import flatdata.lib as flatdata
 
 {% macro structure_definition(tree, struct) %}
 {{ struct.doc|to_python_doc}}

flatdata-py/generator/generators/rust.py renamed to flatdata-generator/flatdata/generator/generators/rust.py

@@ -5,19 +5,19 @@
 
 from pyparsing import ParseException, ParseSyntaxException
 
-import generator.tree.nodes.trivial as nodes
-from generator.grammar import flatdata_grammar
-from generator.tree.errors import InvalidEnumWidthError
-from generator.tree.nodes.archive import Archive
-from generator.tree.nodes.node import Node
-from generator.tree.syntax_tree import SyntaxTree
-from generator.tree.nodes.resources import Multivector
-from generator.tree.nodes.references import BuiltinStructureReference, ConstantReference, EnumerationReference
-from generator.tree.nodes.root import Root
-from generator.tree.errors import ParsingError
-from generator.tree.traversal import DfsTraversal
-from generator.tree.helpers.basictype import BasicType
-from generator.tree.helpers.enumtype import EnumType
+import flatdata.generator.tree.nodes.trivial as nodes
+from flatdata.generator.grammar import flatdata_grammar
+from flatdata.generator.tree.errors import InvalidEnumWidthError
+from flatdata.generator.tree.nodes.archive import Archive
+from flatdata.generator.tree.nodes.node import Node
+from flatdata.generator.tree.syntax_tree import SyntaxTree
+from flatdata.generator.tree.nodes.resources import Multivector
+from flatdata.generator.tree.nodes.references import BuiltinStructureReference, ConstantReference, EnumerationReference
+from flatdata.generator.tree.nodes.root import Root
+from flatdata.generator.tree.errors import ParsingError
+from flatdata.generator.tree.traversal import DfsTraversal
+from flatdata.generator.tree.helpers.basictype import BasicType
+from flatdata.generator.tree.helpers.enumtype import EnumType
 
 from .resolver import resolve_references
 

flatdata-py/generator/grammar.py renamed to flatdata-generator/flatdata/generator/grammar.py

@@ -1,6 +1,6 @@
-from generator import grammar
-from generator.tree.errors import InvalidWidthError
-from generator.tree.errors import InvalidSignError
+from flatdata.generator import grammar
+from flatdata.generator.tree.errors import InvalidWidthError
+from flatdata.generator.tree.errors import InvalidSignError
 
 
 class BasicType:

flatdata-py/generator/tests/generators/__init__.py renamed to flatdata-generator/flatdata/generator/templates/__init__.py

@@ -1,6 +1,6 @@
-from generator.tree.errors import UnexpectedResourceType
-from generator.tree.nodes.node import Node
-import generator.tree.nodes.resources as resources
+from flatdata.generator.tree.errors import UnexpectedResourceType
+from flatdata.generator.tree.nodes.node import Node
+import flatdata.generator.tree.nodes.resources as resources
 
 
 def _create_resource(properties):

flatdata-py/generator/tests/tree/__init__.py renamed to flatdata-generator/flatdata/generator/templates/cpp/__init__.py

@@ -1,5 +1,5 @@
-from generator.tree.nodes.node import Node
-from generator.tree.nodes.references import ResourceReference, FieldReference, StructureReference
+from flatdata.generator.tree.nodes.node import Node
+from flatdata.generator.tree.nodes.references import ResourceReference, FieldReference, StructureReference
 
 
 class ExplicitReference(Node):