From d71cab7935a5c4a8b51b245f6fba39eca12a8602 Mon Sep 17 00:00:00 2001
From: isikl <isabell_klipper@web.de>
Date: Wed, 23 Jan 2019 16:10:17 +0100
Subject: [PATCH 1/6] [WP] Add geopy geocoding library

---
 openpoiservice/server/api/query_builder.py   |  4 +++
 openpoiservice/server/db_import/models.py    |  2 +-
 openpoiservice/server/db_import/objects.py   | 26 +++++++++++++++++++-
 openpoiservice/server/db_import/parse_osm.py |  3 ++-
 4 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/openpoiservice/server/api/query_builder.py b/openpoiservice/server/api/query_builder.py
index 06b3053..90536d5 100644
--- a/openpoiservice/server/api/query_builder.py
+++ b/openpoiservice/server/api/query_builder.py
@@ -270,6 +270,10 @@ def generate_geojson_features(cls, query, limit):
                     key_values[key] = q[5][idx]
                 properties["osm_tags"] = key_values
 
+            # if q[7] is not None:
+            properties["address"] = q[7]
+            print(properties['address'])
+
             geojson_feature = geojson.Feature(geometry=trimmed_point,
                                               properties=properties)
             geojson_features.append(geojson_feature)
diff --git a/openpoiservice/server/db_import/models.py b/openpoiservice/server/db_import/models.py
index be79faf..ec16147 100755
--- a/openpoiservice/server/db_import/models.py
+++ b/openpoiservice/server/db_import/models.py
@@ -14,7 +14,7 @@ class Pois(db.Model):
     uuid = db.Column(db.LargeBinary, primary_key=True)
     osm_id = db.Column(db.BigInteger, nullable=False, index=True)
     osm_type = db.Column(db.Integer, nullable=False)
-    # address = db.Column(db.Text, nullable=True)
+    address = db.Column(db.Text, nullable=True)
     geom = db.Column(Geography(geometry_type="POINT", srid=4326, spatial_index=True), nullable=False)
 
     tags = db.relationship("Tags", backref='{}'.format(ops_settings['provider_parameters']['table_name']),
diff --git a/openpoiservice/server/db_import/objects.py b/openpoiservice/server/db_import/objects.py
index de96d1e..c02c6bc 100644
--- a/openpoiservice/server/db_import/objects.py
+++ b/openpoiservice/server/db_import/objects.py
@@ -1,5 +1,9 @@
 # openpoiservice/server/poi_entity.py
 
+from openpoiservice.server import ops_settings
+from geopy.geocoders import * # get_geocoder_for_service
+import json
+
 
 class PoiObject(object):
 
@@ -16,7 +20,7 @@ def __init__(self, uuid, categories, osmid, lat_lng, osm_type):
                                                   float(lat_lng[1]))
 
         # add geocoder connector here...
-        self.address = None
+        self.address = AddressObject(lat_lng).address_request()
 
 
 class TagsObject(object):
@@ -26,3 +30,23 @@ def __init__(self, uuid, osmid, key, value):
         self.osmid = int(osmid)
         self.key = key
         self.value = value
+
+
+class AddressObject(object):
+
+    def __init__(self, lat_lng):
+        self.lat_lng = lat_lng[::-1]
+
+    def address_request(self):
+
+        for geocoder, settings in ops_settings['geocoder'].items():
+            if geocoder == 'pelias':
+                domain = ops_settings['geocoder']['pelias']['domain']
+                api_key = ops_settings['geocoder']['pelias']['api_key']
+                geolocator = Pelias(domain=domain, api_key=api_key)
+                response = geolocator.reverse(query=self.lat_lng)
+                return json.dumps(response.raw['properties'], sort_keys=True)
+            else:
+                geolocator = get_geocoder_for_service(geocoder)
+                response = geolocator().reverse(query=self.lat_lng)
+                return json.dumps(response.raw['address'], sort_keys=True)
diff --git a/openpoiservice/server/db_import/parse_osm.py b/openpoiservice/server/db_import/parse_osm.py
index 0c2b969..be6ce5b 100644
--- a/openpoiservice/server/db_import/parse_osm.py
+++ b/openpoiservice/server/db_import/parse_osm.py
@@ -183,7 +183,8 @@ def store_poi(self, poi_object):
             uuid=poi_object.uuid,
             osm_id=poi_object.osmid,
             osm_type=poi_object.type,
-            geom=poi_object.geom
+            geom=poi_object.geom,
+            address=poi_object.address
         ))
 
         if self.pois_cnt % 1000 == 0:

From ea3a42bab709936e552afc5b7f75436ba1888124 Mon Sep 17 00:00:00 2001
From: Isabell Klipper <isabell_klipper@web.de>
Date: Tue, 29 Jan 2019 12:47:47 +0100
Subject: [PATCH 2/6] [WP] Modified JSON column

---
 openpoiservice/server/api/pois_post.yaml     |  9 +++++++
 openpoiservice/server/api/query_builder.py   | 25 +++++++++++++-------
 openpoiservice/server/db_import/models.py    |  3 ++-
 openpoiservice/server/db_import/objects.py   | 15 ++++++++----
 openpoiservice/server/db_import/parse_osm.py |  8 +++++--
 5 files changed, 44 insertions(+), 16 deletions(-)

diff --git a/openpoiservice/server/api/pois_post.yaml b/openpoiservice/server/api/pois_post.yaml
index cd81c72..232fe37 100644
--- a/openpoiservice/server/api/pois_post.yaml
+++ b/openpoiservice/server/api/pois_post.yaml
@@ -220,6 +220,8 @@ definitions:
       $ref: "#/definitions/limit"
     sortby:
       $ref: "#/definitions/sortby"
+    address:
+      $ref: "#/definitions/geocoder"
     title:  "Openpoiservice poi request"
 
   LocationsBodyStats:
@@ -293,6 +295,11 @@ definitions:
     enum:
     - category
     - distance
+  address:
+    type:
+    example: pelias
+    enum:
+      - pelias
   bbox:
     type: array
     #items:
@@ -433,6 +440,8 @@ definitions:
       osm_tags:
         type: "object"
         $ref: "#/definitions/location_features_properties_osm_tags"
+      address:
+        type: "json string" # "object"
 
   location_features_properties_category_ids:
     properties:
diff --git a/openpoiservice/server/api/query_builder.py b/openpoiservice/server/api/query_builder.py
index 90536d5..edac7d4 100644
--- a/openpoiservice/server/api/query_builder.py
+++ b/openpoiservice/server/api/query_builder.py
@@ -13,6 +13,7 @@
 from sqlalchemy import dialects
 import geojson as geojson
 import logging
+import json
 from timeit import default_timer as timer
 
 logger = logging.getLogger(__name__)
@@ -103,6 +104,7 @@ def request_pois(self):
                        bbox_query.c.osm_type,
                        bbox_query.c.geom.ST_Distance(type_coerce(geom, Geography)),
                        bbox_query.c.geom,
+                       bbox_query.c.address.op('->')('label'),
                        keys_agg,
                        values_agg,
                        categories_agg) \
@@ -115,12 +117,13 @@ def request_pois(self):
                 .group_by(bbox_query.c.osm_id) \
                 .group_by(bbox_query.c.osm_type) \
                 .group_by(bbox_query.c.geom) \
+                .group_by(bbox_query.c.address) \
                 # .all()
 
             # end = timer()
             # print(end - start)
 
-            # print(str(pois_query))
+            # print(str(pois_query))q
             # for dude in pois_query:
             # print(dude)
 
@@ -240,6 +243,7 @@ def generate_geojson_features(cls, query, limit):
         lat_lngs = []
 
         for q_idx, q in enumerate(query):
+            print(q)
 
             geometry = wkb.loads(str(q[3]), hex=True)
             x = float(format(geometry.x, ".6f"))
@@ -254,8 +258,15 @@ def generate_geojson_features(cls, query, limit):
                 distance=float(q[2])
             )
 
+            if q[4] is not None:
+                address_data = json.loads(q[4])
+                address_dict = {}
+                for k_add, v_add in address_data.items():
+                    address_dict[k_add] = v_add
+                properties['address'] = address_dict
+
             category_ids_obj = {}
-            for c_id in set(q[6]):
+            for c_id in set(q[7]):
                 category_name = categories_tools.category_ids_index[c_id]['poi_name']
                 category_group = categories_tools.category_ids_index[c_id]['poi_group']
                 category_ids_obj[c_id] = {
@@ -264,15 +275,13 @@ def generate_geojson_features(cls, query, limit):
                 }
             properties["category_ids"] = category_ids_obj
 
-            if q[5][0] is not None:
+            if q[6][0] is not None:
                 key_values = {}
-                for idx, key in enumerate(q[4]):
-                    key_values[key] = q[5][idx]
+                for idx, key in enumerate(q[5]):
+                    key_values[key] = q[6][idx]
                 properties["osm_tags"] = key_values
 
-            # if q[7] is not None:
-            properties["address"] = q[7]
-            print(properties['address'])
+            print(properties)
 
             geojson_feature = geojson.Feature(geometry=trimmed_point,
                                               properties=properties)
diff --git a/openpoiservice/server/db_import/models.py b/openpoiservice/server/db_import/models.py
index ec16147..9cdde4d 100755
--- a/openpoiservice/server/db_import/models.py
+++ b/openpoiservice/server/db_import/models.py
@@ -2,6 +2,7 @@
 
 from openpoiservice.server import db, ops_settings
 from geoalchemy2 import Geography
+from sqlalchemy.dialects.postgresql import JSONB
 import logging
 
 logger = logging.getLogger(__name__)
@@ -14,7 +15,6 @@ class Pois(db.Model):
     uuid = db.Column(db.LargeBinary, primary_key=True)
     osm_id = db.Column(db.BigInteger, nullable=False, index=True)
     osm_type = db.Column(db.Integer, nullable=False)
-    address = db.Column(db.Text, nullable=True)
     geom = db.Column(Geography(geometry_type="POINT", srid=4326, spatial_index=True), nullable=False)
 
     tags = db.relationship("Tags", backref='{}'.format(ops_settings['provider_parameters']['table_name']),
@@ -22,6 +22,7 @@ class Pois(db.Model):
 
     categories = db.relationship("Categories", backref='{}'.format(ops_settings['provider_parameters']['table_name']),
                                  lazy='dynamic')
+    address = db.Column(JSONB) # ,  nullable=True)
 
     def __repr__(self):
         return '<osm id %r>' % self.osm_id
diff --git a/openpoiservice/server/db_import/objects.py b/openpoiservice/server/db_import/objects.py
index c02c6bc..4a07cf9 100644
--- a/openpoiservice/server/db_import/objects.py
+++ b/openpoiservice/server/db_import/objects.py
@@ -7,7 +7,7 @@
 
 class PoiObject(object):
 
-    def __init__(self, uuid, categories, osmid, lat_lng, osm_type):
+    def __init__(self, uuid, categories, osmid, lat_lng, osm_type, address):
         self.uuid = uuid
         self.osmid = int(osmid)
         self.type = int(osm_type)
@@ -18,9 +18,9 @@ def __init__(self, uuid, categories, osmid, lat_lng, osm_type):
 
         self.geom = 'SRID={};POINT({} {})'.format(4326, float(lat_lng[0]),
                                                   float(lat_lng[1]))
+        self.address = address
 
-        # add geocoder connector here...
-        self.address = AddressObject(lat_lng).address_request()
+        # self.address = AddressObject(lat_lng).address_request()
 
 
 class TagsObject(object):
@@ -45,8 +45,13 @@ def address_request(self):
                 api_key = ops_settings['geocoder']['pelias']['api_key']
                 geolocator = Pelias(domain=domain, api_key=api_key)
                 response = geolocator.reverse(query=self.lat_lng)
-                return json.dumps(response.raw['properties'], sort_keys=True)
+                return json.dumps(response.raw['properties'])
+                # return json.dumps(response.raw['properties'], sort_keys=True, ensure_ascii=False)
+                # json_data = json.dumps(response.raw['properties'], ensure_ascii=False)
+                # return json.loads(json_data)
             else:
                 geolocator = get_geocoder_for_service(geocoder)
                 response = geolocator().reverse(query=self.lat_lng)
-                return json.dumps(response.raw['address'], sort_keys=True)
+                return response.raw['address']
+                # json_data = json.dumps(response.raw['address'], sort_keys=True)
+                # return json.loads(json_data)
diff --git a/openpoiservice/server/db_import/parse_osm.py b/openpoiservice/server/db_import/parse_osm.py
index be6ce5b..0d0faf9 100644
--- a/openpoiservice/server/db_import/parse_osm.py
+++ b/openpoiservice/server/db_import/parse_osm.py
@@ -3,7 +3,7 @@
 from openpoiservice.server import db
 from openpoiservice.server import categories_tools, ops_settings
 from openpoiservice.server.db_import.models import Pois, Tags, Categories
-from openpoiservice.server.db_import.objects import PoiObject, TagsObject
+from openpoiservice.server.db_import.objects import PoiObject, TagsObject, AddressObject
 from openpoiservice.server.utils.decorators import get_size
 import shapely as shapely
 from shapely.geometry import Point, Polygon, LineString, MultiPoint
@@ -14,6 +14,7 @@
 from timeit import Timer
 from bisect import bisect_left
 from collections import deque
+import json
 
 logger = logging.getLogger(__name__)
 
@@ -79,6 +80,7 @@ def __init__(self):
         self.tags_object = None
         self.poi_object = None
         self.process_ways_length = None
+        # self.address = None
 
     def parse_relations(self, relations):
         """
@@ -262,7 +264,9 @@ def create_poi(self, tags, osmid, lat_lng, osm_type, categories=[]):
                     self.tags_object = TagsObject(my_uuid, osmid, tag, value)
                     self.store_tags(self.tags_object)
 
-            self.poi_object = PoiObject(my_uuid, categories, osmid, lat_lng, osm_type)
+            address = AddressObject(lat_lng).address_request() # json.dumps()
+
+            self.poi_object = PoiObject(my_uuid, categories, osmid, lat_lng, osm_type, address)
             self.store_poi(self.poi_object)
 
             for category in categories:

From 513def3c0a0bc7b22f8fea2357e751a6d3b72254 Mon Sep 17 00:00:00 2001
From: Isabell Klipper <isabell_klipper@web.de>
Date: Mon, 11 Feb 2019 18:09:11 +0100
Subject: [PATCH 3/6] [WP] Modified class AddressObject and optional address
 feature

---
 openpoiservice/server/api/query_builder.py   | 44 ++++++++++----------
 openpoiservice/server/db_import/models.py    |  5 ++-
 openpoiservice/server/db_import/objects.py   | 40 +++++++++---------
 openpoiservice/server/db_import/parse_osm.py | 32 ++++++++++----
 openpoiservice/server/db_import/parser.py    |  4 ++
 5 files changed, 74 insertions(+), 51 deletions(-)

diff --git a/openpoiservice/server/api/query_builder.py b/openpoiservice/server/api/query_builder.py
index edac7d4..f1d3fc6 100644
--- a/openpoiservice/server/api/query_builder.py
+++ b/openpoiservice/server/api/query_builder.py
@@ -104,10 +104,11 @@ def request_pois(self):
                        bbox_query.c.osm_type,
                        bbox_query.c.geom.ST_Distance(type_coerce(geom, Geography)),
                        bbox_query.c.geom,
-                       bbox_query.c.address.op('->')('label'),
                        keys_agg,
                        values_agg,
-                       categories_agg) \
+                       categories_agg,
+                       bbox_query.c.address
+                       ) \
                 .order_by(*sortby_group) \
                 .filter(*category_filters) \
                 .filter(*custom_filters) \
@@ -117,16 +118,9 @@ def request_pois(self):
                 .group_by(bbox_query.c.osm_id) \
                 .group_by(bbox_query.c.osm_type) \
                 .group_by(bbox_query.c.geom) \
-                .group_by(bbox_query.c.address) \
+                .group_by(bbox_query.c.address)
                 # .all()
 
-            # end = timer()
-            # print(end - start)
-
-            # print(str(pois_query))q
-            # for dude in pois_query:
-            # print(dude)
-
             # response as geojson feature collection
             features = self.generate_geojson_features(pois_query, params['limit'])
 
@@ -242,6 +236,11 @@ def generate_geojson_features(cls, query, limit):
         geojson_features = []
         lat_lngs = []
 
+        # for v in query.statement.c:
+        #     print(v)
+        for v in query:
+            print(v)
+
         for q_idx, q in enumerate(query):
             print(q)
 
@@ -258,15 +257,8 @@ def generate_geojson_features(cls, query, limit):
                 distance=float(q[2])
             )
 
-            if q[4] is not None:
-                address_data = json.loads(q[4])
-                address_dict = {}
-                for k_add, v_add in address_data.items():
-                    address_dict[k_add] = v_add
-                properties['address'] = address_dict
-
             category_ids_obj = {}
-            for c_id in set(q[7]):
+            for c_id in set(q[6]):
                 category_name = categories_tools.category_ids_index[c_id]['poi_name']
                 category_group = categories_tools.category_ids_index[c_id]['poi_group']
                 category_ids_obj[c_id] = {
@@ -275,12 +267,22 @@ def generate_geojson_features(cls, query, limit):
                 }
             properties["category_ids"] = category_ids_obj
 
-            if q[6][0] is not None:
+            # Checks if Tags are available
+            if q[5][0] is not None:
                 key_values = {}
-                for idx, key in enumerate(q[5]):
-                    key_values[key] = q[6][idx]
+                for idx, key in enumerate(q[4]):
+                    key_values[key] = q[5][idx]
                 properties["osm_tags"] = key_values
 
+            # Checks if addresses are available
+            if q[7] is not None:
+                #####
+                address_data = json.loads(q[7])
+                address_dict = {}
+                for k_add, v_add in address_data.items():
+                    address_dict[k_add] = v_add
+                properties['address'] = address_dict
+
             print(properties)
 
             geojson_feature = geojson.Feature(geometry=trimmed_point,
diff --git a/openpoiservice/server/db_import/models.py b/openpoiservice/server/db_import/models.py
index 9cdde4d..1c91269 100755
--- a/openpoiservice/server/db_import/models.py
+++ b/openpoiservice/server/db_import/models.py
@@ -2,7 +2,6 @@
 
 from openpoiservice.server import db, ops_settings
 from geoalchemy2 import Geography
-from sqlalchemy.dialects.postgresql import JSONB
 import logging
 
 logger = logging.getLogger(__name__)
@@ -22,7 +21,9 @@ class Pois(db.Model):
 
     categories = db.relationship("Categories", backref='{}'.format(ops_settings['provider_parameters']['table_name']),
                                  lazy='dynamic')
-    address = db.Column(JSONB) # ,  nullable=True)
+
+    if ops_settings['geocoder'] is not None:
+        address = db.Column(db.String, nullable=True)
 
     def __repr__(self):
         return '<osm id %r>' % self.osm_id
diff --git a/openpoiservice/server/db_import/objects.py b/openpoiservice/server/db_import/objects.py
index 4a07cf9..c74ace9 100644
--- a/openpoiservice/server/db_import/objects.py
+++ b/openpoiservice/server/db_import/objects.py
@@ -3,11 +3,12 @@
 from openpoiservice.server import ops_settings
 from geopy.geocoders import * # get_geocoder_for_service
 import json
+from flask import jsonify, Response
 
 
 class PoiObject(object):
 
-    def __init__(self, uuid, categories, osmid, lat_lng, osm_type, address):
+    def __init__(self, uuid, categories, osmid, lat_lng, osm_type, address=None):
         self.uuid = uuid
         self.osmid = int(osmid)
         self.type = int(osm_type)
@@ -20,8 +21,6 @@ def __init__(self, uuid, categories, osmid, lat_lng, osm_type, address):
                                                   float(lat_lng[1]))
         self.address = address
 
-        # self.address = AddressObject(lat_lng).address_request()
-
 
 class TagsObject(object):
 
@@ -39,19 +38,22 @@ def __init__(self, lat_lng):
 
     def address_request(self):
 
-        for geocoder, settings in ops_settings['geocoder'].items():
-            if geocoder == 'pelias':
-                domain = ops_settings['geocoder']['pelias']['domain']
-                api_key = ops_settings['geocoder']['pelias']['api_key']
-                geolocator = Pelias(domain=domain, api_key=api_key)
-                response = geolocator.reverse(query=self.lat_lng)
-                return json.dumps(response.raw['properties'])
-                # return json.dumps(response.raw['properties'], sort_keys=True, ensure_ascii=False)
-                # json_data = json.dumps(response.raw['properties'], ensure_ascii=False)
-                # return json.loads(json_data)
-            else:
-                geolocator = get_geocoder_for_service(geocoder)
-                response = geolocator().reverse(query=self.lat_lng)
-                return response.raw['address']
-                # json_data = json.dumps(response.raw['address'], sort_keys=True)
-                # return json.loads(json_data)
+        # try:
+        geocoder_settings = list(ops_settings['geocoder'].items())[0]
+        geolocator = get_geocoder_for_service(geocoder_settings[0])
+
+        if list(ops_settings['geocoder'].values())[0] is not None:
+            setup_geolocator = geolocator(domain=geocoder_settings[1]['domain'],
+                                          api_key=geocoder_settings[1]['api_key'])
+        else:
+            setup_geolocator = geolocator()
+
+        response = setup_geolocator.reverse(query=self.lat_lng)
+        # Checks if address for location is available
+        if response is not None:
+            return json.dumps(response.raw['properties'])
+
+        # except AttributeError:
+        #     pass
+
+
diff --git a/openpoiservice/server/db_import/parse_osm.py b/openpoiservice/server/db_import/parse_osm.py
index 0d0faf9..96d8f3a 100644
--- a/openpoiservice/server/db_import/parse_osm.py
+++ b/openpoiservice/server/db_import/parse_osm.py
@@ -80,7 +80,6 @@ def __init__(self):
         self.tags_object = None
         self.poi_object = None
         self.process_ways_length = None
-        # self.address = None
 
     def parse_relations(self, relations):
         """
@@ -180,14 +179,24 @@ def store_poi(self, poi_object):
         :type poi_object: object
         """
 
+        ######
         self.pois_cnt += 1
-        self.poi_objects.append(Pois(
-            uuid=poi_object.uuid,
-            osm_id=poi_object.osmid,
-            osm_type=poi_object.type,
-            geom=poi_object.geom,
-            address=poi_object.address
-        ))
+        # print(self.pois_cnt)
+        if poi_object.address is not None:
+            self.poi_objects.append(Pois(
+                uuid=poi_object.uuid,
+                osm_id=poi_object.osmid,
+                osm_type=poi_object.type,
+                geom=poi_object.geom,
+                address=poi_object.address
+            ))
+        else:
+            self.poi_objects.append(Pois(
+                uuid=poi_object.uuid,
+                osm_id=poi_object.osmid,
+                osm_type=poi_object.type,
+                geom=poi_object.geom
+            ))
 
         if self.pois_cnt % 1000 == 0:
             logger.info('Pois: {}, tags: {}, categories: {}'.format(self.pois_cnt, self.tags_cnt, self.categories_cnt))
@@ -264,9 +273,14 @@ def create_poi(self, tags, osmid, lat_lng, osm_type, categories=[]):
                     self.tags_object = TagsObject(my_uuid, osmid, tag, value)
                     self.store_tags(self.tags_object)
 
-            address = AddressObject(lat_lng).address_request() # json.dumps()
+            if ops_settings['geocoder'] is not None:
+                address = AddressObject(lat_lng).address_request()
+            else:
+                address = None
 
             self.poi_object = PoiObject(my_uuid, categories, osmid, lat_lng, osm_type, address)
+            # address=poi_object.address
+
             self.store_poi(self.poi_object)
 
             for category in categories:
diff --git a/openpoiservice/server/db_import/parser.py b/openpoiservice/server/db_import/parser.py
index 2c5be40..d40df70 100644
--- a/openpoiservice/server/db_import/parser.py
+++ b/openpoiservice/server/db_import/parser.py
@@ -50,6 +50,10 @@ def parse_import(osm_file):
     coords = OSMParser(concurrency=1, coords_callback=osm_importer.parse_coords_for_ways)
     coords.parse(osm_file)
 
+    # Checks if geocoder is provided
+    if ops_settings['geocoder'] is not None:
+        logger.info('Importing addresses...')
+
     logger.info('Storing remaining pois')
     osm_importer.save_remainder()
 

From d155c9346577737e8876dc5d73240116083ec086 Mon Sep 17 00:00:00 2001
From: Isabell Klipper <isabell_klipper@web.de>
Date: Fri, 15 Mar 2019 17:09:34 +0100
Subject: [PATCH 4/6] [WP] Added class GeocoderSetup(), def
 generate_geocode_categories()

---
 openpoiservice/server/__init__.py             |  8 +++
 openpoiservice/server/api/pois_post.yaml      | 11 ----
 openpoiservice/server/api/query_builder.py    | 35 ++++--------
 .../server/categories/categories.py           | 13 +++++
 .../server/categories/categories.yml          |  1 +
 openpoiservice/server/db_import/models.py     |  5 +-
 openpoiservice/server/db_import/objects.py    | 57 ++++++++++++-------
 openpoiservice/server/db_import/parse_osm.py  | 16 ++----
 openpoiservice/server/db_import/parser.py     |  5 +-
 openpoiservice/server/ops_settings.yml        | 33 +++++++++--
 10 files changed, 110 insertions(+), 74 deletions(-)

diff --git a/openpoiservice/server/__init__.py b/openpoiservice/server/__init__.py
index 9580481..05036c6 100755
--- a/openpoiservice/server/__init__.py
+++ b/openpoiservice/server/__init__.py
@@ -6,6 +6,7 @@
 from flask_cors import CORS
 from openpoiservice.server.categories.categories import CategoryTools
 from openpoiservice.server.api import api_exceptions
+from openpoiservice.server.db_import.objects import GeocoderSetup
 import yaml
 import os
 import time
@@ -19,6 +20,13 @@
 basedir = os.path.abspath(os.path.dirname(__file__))
 ops_settings = yaml.safe_load(open(os.path.join(basedir, 'ops_settings.yml')))
 
+geocoder = None
+geocode_categories = None
+if ops_settings['geocoder'] is not None:
+    geocoder = GeocoderSetup(list(ops_settings['geocoder'].items())[0]).define_geocoder()
+    geocode_categories = CategoryTools('categories.yml').generate_geocode_categories()
+
+
 if "TESTING" in os.environ:
     ops_settings['provider_parameters']['table_name'] = ops_settings['provider_parameters']['table_name'] + '_test'
 
diff --git a/openpoiservice/server/api/pois_post.yaml b/openpoiservice/server/api/pois_post.yaml
index 232fe37..114c378 100644
--- a/openpoiservice/server/api/pois_post.yaml
+++ b/openpoiservice/server/api/pois_post.yaml
@@ -220,8 +220,6 @@ definitions:
       $ref: "#/definitions/limit"
     sortby:
       $ref: "#/definitions/sortby"
-    address:
-      $ref: "#/definitions/geocoder"
     title:  "Openpoiservice poi request"
 
   LocationsBodyStats:
@@ -295,11 +293,6 @@ definitions:
     enum:
     - category
     - distance
-  address:
-    type:
-    example: pelias
-    enum:
-      - pelias
   bbox:
     type: array
     #items:
@@ -440,8 +433,6 @@ definitions:
       osm_tags:
         type: "object"
         $ref: "#/definitions/location_features_properties_osm_tags"
-      address:
-        type: "json string" # "object"
 
   location_features_properties_category_ids:
     properties:
@@ -458,8 +449,6 @@ definitions:
     properties:
       name:
         type: "string"
-      address:
-        type: "string"
       website:
         type: "string"
       opening_hours:
diff --git a/openpoiservice/server/api/query_builder.py b/openpoiservice/server/api/query_builder.py
index f1d3fc6..8a18f32 100644
--- a/openpoiservice/server/api/query_builder.py
+++ b/openpoiservice/server/api/query_builder.py
@@ -1,7 +1,7 @@
 # openpoiservice/server/query_builder.py
 
 from openpoiservice.server import db
-from openpoiservice.server import categories_tools, ops_settings
+from openpoiservice.server import categories_tools, ops_settings, geocoder
 import geoalchemy2.functions as geo_func
 from geoalchemy2.types import Geography, Geometry
 from geoalchemy2.elements import WKBElement, WKTElement
@@ -107,8 +107,7 @@ def request_pois(self):
                        keys_agg,
                        values_agg,
                        categories_agg,
-                       bbox_query.c.address
-                       ) \
+                       bbox_query.c.address) \
                 .order_by(*sortby_group) \
                 .filter(*category_filters) \
                 .filter(*custom_filters) \
@@ -119,7 +118,6 @@ def request_pois(self):
                 .group_by(bbox_query.c.osm_type) \
                 .group_by(bbox_query.c.geom) \
                 .group_by(bbox_query.c.address)
-                # .all()
 
             # response as geojson feature collection
             features = self.generate_geojson_features(pois_query, params['limit'])
@@ -144,12 +142,9 @@ def generate_geom_filters(geometry, Pois):
                                              type_coerce(geom_bbox, Geography)), Pois.geom, 0))
 
         elif 'bbox' not in geometry and 'geom' in geometry:
-
             geom = geometry['geom'].wkt
-
             filters.append(  # buffer around geom
-                geo_func.ST_DWithin(geo_func.ST_Buffer(type_coerce(geom, Geography), geometry['buffer']), Pois.geom, 0)
-            )
+                geo_func.ST_DWithin(geo_func.ST_Buffer(type_coerce(geom, Geography), geometry['buffer']), Pois.geom, 0))
 
         return filters, geom
 
@@ -236,13 +231,7 @@ def generate_geojson_features(cls, query, limit):
         geojson_features = []
         lat_lngs = []
 
-        # for v in query.statement.c:
-        #     print(v)
-        for v in query:
-            print(v)
-
         for q_idx, q in enumerate(query):
-            print(q)
 
             geometry = wkb.loads(str(q[3]), hex=True)
             x = float(format(geometry.x, ".6f"))
@@ -275,15 +264,15 @@ def generate_geojson_features(cls, query, limit):
                 properties["osm_tags"] = key_values
 
             # Checks if addresses are available
-            if q[7] is not None:
-                #####
-                address_data = json.loads(q[7])
-                address_dict = {}
-                for k_add, v_add in address_data.items():
-                    address_dict[k_add] = v_add
-                properties['address'] = address_dict
-
-            print(properties)
+            try:
+                if q[7] is not None:
+                    address_dict = {}
+                    address_data = json.loads(q[7])
+                    for key, value in address_data.items():
+                        address_dict[key] = value
+                    properties['address'] = address_dict
+            except IndexError:
+                pass
 
             geojson_feature = geojson.Feature(geometry=trimmed_point,
                                               properties=properties)
diff --git a/openpoiservice/server/categories/categories.py b/openpoiservice/server/categories/categories.py
index e127713..a174251 100644
--- a/openpoiservice/server/categories/categories.py
+++ b/openpoiservice/server/categories/categories.py
@@ -94,3 +94,16 @@ def get_category(self, tags):
                                 categories.append(category_id)
 
         return categories
+
+    def generate_geocode_categories(self):
+
+        geocode_categories = {}
+
+        for category in self.categories_object.items():
+            if 'geocoder' not in category[1]:
+                for child in category[1]['children'].values():
+                    for id in child.values():
+                        geocode_categories[id] = {}
+
+        return geocode_categories
+
diff --git a/openpoiservice/server/categories/categories.yml b/openpoiservice/server/categories/categories.yml
index e832ab8..95d3473 100644
--- a/openpoiservice/server/categories/categories.yml
+++ b/openpoiservice/server/categories/categories.yml
@@ -49,6 +49,7 @@ education:
 
 facilities:
     id: 160
+    geocoder: False
     children:
         amenity:
             compressed_air: 161
diff --git a/openpoiservice/server/db_import/models.py b/openpoiservice/server/db_import/models.py
index 1c91269..3be5912 100755
--- a/openpoiservice/server/db_import/models.py
+++ b/openpoiservice/server/db_import/models.py
@@ -1,6 +1,6 @@
 # openpoiservice/server/models.py
 
-from openpoiservice.server import db, ops_settings
+from openpoiservice.server import db, ops_settings, geocoder
 from geoalchemy2 import Geography
 import logging
 
@@ -22,8 +22,7 @@ class Pois(db.Model):
     categories = db.relationship("Categories", backref='{}'.format(ops_settings['provider_parameters']['table_name']),
                                  lazy='dynamic')
 
-    if ops_settings['geocoder'] is not None:
-        address = db.Column(db.String, nullable=True)
+    address = db.Column(db.String, nullable=True)
 
     def __repr__(self):
         return '<osm id %r>' % self.osm_id
diff --git a/openpoiservice/server/db_import/objects.py b/openpoiservice/server/db_import/objects.py
index c74ace9..eb39398 100644
--- a/openpoiservice/server/db_import/objects.py
+++ b/openpoiservice/server/db_import/objects.py
@@ -1,10 +1,11 @@
 # openpoiservice/server/poi_entity.py
 
-from openpoiservice.server import ops_settings
-from geopy.geocoders import * # get_geocoder_for_service
 import json
-from flask import jsonify, Response
+import logging
+from geopy.geocoders import get_geocoder_for_service
+from geopy.extra.rate_limiter import RateLimiter
 
+logger = logging.getLogger(__name__)
 
 class PoiObject(object):
 
@@ -31,29 +32,47 @@ def __init__(self, uuid, osmid, key, value):
         self.value = value
 
 
+class GeocoderSetup(object):
+    """Initialises geocoder"""
+
+    def __init__(self, geocoder_name):
+        self.geocoder_name = geocoder_name
+        self.geocoder = None
+
+    def define_geocoder(self):
+
+        # returns error if no valid geocoder is provided
+        try:
+            self.geocoder_settings = get_geocoder_for_service(self.geocoder_name[0])
+
+            if self.geocoder_name[1] is not None:
+                self.geocoder = self.geocoder_settings(**self.geocoder_name[1])
+
+            else:
+                self.geocoder = self.geocoder_settings()
+
+        except Exception as err:
+            logger.error(err)
+
+        return self.geocoder
+
+
 class AddressObject(object):
 
-    def __init__(self, lat_lng):
+    def __init__(self, lat_lng, geocoder):
         self.lat_lng = lat_lng[::-1]
+        self.geocoder = geocoder
 
     def address_request(self):
 
-        # try:
-        geocoder_settings = list(ops_settings['geocoder'].items())[0]
-        geolocator = get_geocoder_for_service(geocoder_settings[0])
+        # address_delaied = RateLimiter(self.geocoder.reverse, min_delay_seconds=1)
+        response = self.geocoder.reverse(query=self.lat_lng)
 
-        if list(ops_settings['geocoder'].values())[0] is not None:
-            setup_geolocator = geolocator(domain=geocoder_settings[1]['domain'],
-                                          api_key=geocoder_settings[1]['api_key'])
-        else:
-            setup_geolocator = geolocator()
-
-        response = setup_geolocator.reverse(query=self.lat_lng)
         # Checks if address for location is available
         if response is not None:
-            return json.dumps(response.raw['properties'])
-
-        # except AttributeError:
-        #     pass
-
+            try:
+                return json.dumps(response.raw)
+            except AttributeError:
+                return json.dumps(response)
 
+        return None
diff --git a/openpoiservice/server/db_import/parse_osm.py b/openpoiservice/server/db_import/parse_osm.py
index 96d8f3a..6d01b51 100644
--- a/openpoiservice/server/db_import/parse_osm.py
+++ b/openpoiservice/server/db_import/parse_osm.py
@@ -1,7 +1,7 @@
 # openpoiservice/server/parse_osm.py
 
 from openpoiservice.server import db
-from openpoiservice.server import categories_tools, ops_settings
+from openpoiservice.server import categories_tools, ops_settings, geocoder, geocode_categories
 from openpoiservice.server.db_import.models import Pois, Tags, Categories
 from openpoiservice.server.db_import.objects import PoiObject, TagsObject, AddressObject
 from openpoiservice.server.utils.decorators import get_size
@@ -14,7 +14,6 @@
 from timeit import Timer
 from bisect import bisect_left
 from collections import deque
-import json
 
 logger = logging.getLogger(__name__)
 
@@ -179,10 +178,8 @@ def store_poi(self, poi_object):
         :type poi_object: object
         """
 
-        ######
         self.pois_cnt += 1
-        # print(self.pois_cnt)
-        if poi_object.address is not None:
+        if geocoder is not None:
             self.poi_objects.append(Pois(
                 uuid=poi_object.uuid,
                 osm_id=poi_object.osmid,
@@ -273,13 +270,12 @@ def create_poi(self, tags, osmid, lat_lng, osm_type, categories=[]):
                     self.tags_object = TagsObject(my_uuid, osmid, tag, value)
                     self.store_tags(self.tags_object)
 
-            if ops_settings['geocoder'] is not None:
-                address = AddressObject(lat_lng).address_request()
-            else:
-                address = None
+            address = None
+            if geocoder is not None and categories[0] in geocode_categories:
+                address = AddressObject(lat_lng, geocoder).address_request()
+                time.sleep(1)
 
             self.poi_object = PoiObject(my_uuid, categories, osmid, lat_lng, osm_type, address)
-            # address=poi_object.address
 
             self.store_poi(self.poi_object)
 
diff --git a/openpoiservice/server/db_import/parser.py b/openpoiservice/server/db_import/parser.py
index d40df70..e3a8207 100644
--- a/openpoiservice/server/db_import/parser.py
+++ b/openpoiservice/server/db_import/parser.py
@@ -2,10 +2,9 @@
 
 from openpoiservice.server.db_import.parse_osm import OsmImporter
 from openpoiservice.server.utils.decorators import timeit, processify
-from openpoiservice.server import ops_settings
+from openpoiservice.server import ops_settings, geocoder
 from imposm.parser import OSMParser
 import logging
-import time
 
 # from guppy import hpy
 from collections import deque
@@ -51,7 +50,7 @@ def parse_import(osm_file):
     coords.parse(osm_file)
 
     # Checks if geocoder is provided
-    if ops_settings['geocoder'] is not None:
+    if geocoder is not None:
         logger.info('Importing addresses...')
 
     logger.info('Storing remaining pois')
diff --git a/openpoiservice/server/ops_settings.yml b/openpoiservice/server/ops_settings.yml
index e20edd3..21904cb 100644
--- a/openpoiservice/server/ops_settings.yml
+++ b/openpoiservice/server/ops_settings.yml
@@ -16,13 +16,12 @@ concurrent_workers: 4
 # Database parameters
 provider_parameters:
   table_name: ops_planet_pois
-  db_name: gis
-  user_name: gis_admin
+  db_name: opsdb
+  user_name: postgres
   table_schema: public
-  #user_name: admin
-  password: admin
+  password: ops
   host: localhost
-  port: 5434
+  port: 5432
   port_tests: 5432
 # add any additional tags you want to save to the database
 column_mappings:
@@ -40,3 +39,27 @@ column_mappings:
   phone:
   # https://wiki.openstreetmap.org/wiki/Key:website
   website:
+# choose one geopy geocoder and provide all needed parameters - https://geopy.readthedocs.io/
+geocoder:
+#  ArcGIS:
+#    domain: 'geocode.arcgis.com'
+  pelias:
+    domain: 'api.geocode.earth'
+    api_key: ''
+#    timeout: 60
+#  nominatim:
+#    timeout: 60
+#   GeocodeFarm:
+#     timeout: 60
+#  AzureMaps:
+#    subscription_key: ''
+#    domain: 'atlas.microsoft.com'
+#  Baidu:
+#    api_key: ''
+#  BANFrance:
+#    domain: 'api-adresse.data.gouv.fr'
+#  Bing:
+#    api_key: ''
+#restrictions:
+#  quota: 40
+#  per: min

From 28e4d5ba8ebeccaa1487ce4edd77a266e513a9fd Mon Sep 17 00:00:00 2001
From: Isabell Klipper <isabell_klipper@web.de>
Date: Wed, 20 Mar 2019 12:23:56 +0100
Subject: [PATCH 5/6] [WP] Added test_geocoder.py

---
 openpoiservice/server/__init__.py          |  4 +-
 openpoiservice/server/db_import/objects.py | 27 ++++---
 openpoiservice/server/ops_settings.yml     | 15 ++--
 openpoiservice/tests/test_geocoder.py      | 89 ++++++++++++++++++++++
 4 files changed, 114 insertions(+), 21 deletions(-)
 create mode 100644 openpoiservice/tests/test_geocoder.py

diff --git a/openpoiservice/server/__init__.py b/openpoiservice/server/__init__.py
index 05036c6..0f0313d 100755
--- a/openpoiservice/server/__init__.py
+++ b/openpoiservice/server/__init__.py
@@ -23,8 +23,8 @@
 geocoder = None
 geocode_categories = None
 if ops_settings['geocoder'] is not None:
-    geocoder = GeocoderSetup(list(ops_settings['geocoder'].items())[0]).define_geocoder()
-    geocode_categories = CategoryTools('categories.yml').generate_geocode_categories()
+        geocoder = GeocoderSetup(list(ops_settings['geocoder'].items())[0]).define_geocoder()
+        geocode_categories = CategoryTools('categories.yml').generate_geocode_categories()
 
 
 if "TESTING" in os.environ:
diff --git a/openpoiservice/server/db_import/objects.py b/openpoiservice/server/db_import/objects.py
index eb39398..698b622 100644
--- a/openpoiservice/server/db_import/objects.py
+++ b/openpoiservice/server/db_import/objects.py
@@ -3,7 +3,7 @@
 import json
 import logging
 from geopy.geocoders import get_geocoder_for_service
-from geopy.extra.rate_limiter import RateLimiter
+# from geopy.extra.rate_limiter import RateLimiter
 
 logger = logging.getLogger(__name__)
 
@@ -41,20 +41,26 @@ def __init__(self, geocoder_name):
 
     def define_geocoder(self):
 
-        # returns error if no valid geocoder is provided
+        # returns warning if no valid geocoder is provided
         try:
             self.geocoder_settings = get_geocoder_for_service(self.geocoder_name[0])
 
+        except Exception as err_geocoder:
+            logger.warning(err_geocoder)
+            return err_geocoder
+
+        # returns warning if no valid geocoder settings are provided
+        try:
             if self.geocoder_name[1] is not None:
                 self.geocoder = self.geocoder_settings(**self.geocoder_name[1])
 
             else:
                 self.geocoder = self.geocoder_settings()
+            return self.geocoder
 
-        except Exception as err:
-            logger.error(err)
-
-        return self.geocoder
+        except Exception as err_parameter:
+            logger.warning(err_parameter)
+            return err_parameter
 
 
 class AddressObject(object):
@@ -70,9 +76,10 @@ def address_request(self):
 
         # Checks if address for location is available
         if response is not None:
-            try:
-                return json.dumps(response.raw)
-            except AttributeError:
-                return json.dumps(response)
+            return json.dumps(response.raw)
+            # try:
+            #     return json.dumps(response.raw)
+            # except AttributeError:
+            #     return json.dumps(response)
 
         return None
diff --git a/openpoiservice/server/ops_settings.yml b/openpoiservice/server/ops_settings.yml
index 21904cb..4f96d7c 100644
--- a/openpoiservice/server/ops_settings.yml
+++ b/openpoiservice/server/ops_settings.yml
@@ -43,15 +43,15 @@ column_mappings:
 geocoder:
 #  ArcGIS:
 #    domain: 'geocode.arcgis.com'
-  pelias:
-    domain: 'api.geocode.earth'
-    api_key: ''
+#  pelias:
+#    domain: 'api.geocode.earth'
+#    api_key: ''
 #    timeout: 60
 #  nominatim:
 #    timeout: 60
-#   GeocodeFarm:
-#     timeout: 60
-#  AzureMaps:
+#  GeocodeFarm:
+#    timeout: 60
+#  Azure:
 #    subscription_key: ''
 #    domain: 'atlas.microsoft.com'
 #  Baidu:
@@ -60,6 +60,3 @@ geocoder:
 #    domain: 'api-adresse.data.gouv.fr'
 #  Bing:
 #    api_key: ''
-#restrictions:
-#  quota: 40
-#  per: min
diff --git a/openpoiservice/tests/test_geocoder.py b/openpoiservice/tests/test_geocoder.py
new file mode 100644
index 0000000..44cf6be
--- /dev/null
+++ b/openpoiservice/tests/test_geocoder.py
@@ -0,0 +1,89 @@
+# openpoiservice/server/tests/test_main.py
+
+
+import unittest
+import json
+from openpoiservice.server.db_import.objects import AddressObject, GeocoderSetup
+from openpoiservice.server.categories.categories import CategoryTools
+
+# from base import BaseTestCase
+from openpoiservice.tests.base import BaseTestCase
+
+# VALID GEOCODER AND PARAMETER
+valid_geocoder_param = dict(
+    nominatim=dict(
+        timeout=60
+    )
+)
+
+# VALID GEOCODER FOR class TestGeocoderBlueprint
+valid_geocoder = GeocoderSetup(list(valid_geocoder_param.items())[0]).define_geocoder()
+
+# INVALID GEOCODER
+invalid_geocoder = dict(
+    peliaas=dict()
+)
+
+# INVALID GEOCODER LOGGER RESPONSE
+logger_message_geocoder = "Unknown geocoder 'peliaas'; " \
+                 "options are: dict_keys(['arcgis', 'azure', 'baidu', 'banfrance', 'bing', 'databc', 'geocodeearth', " \
+                 "'geocodefarm', 'geonames', 'google', 'googlev3', 'geolake', 'here', 'ignfrance', 'mapbox', " \
+                 "'opencage', 'openmapquest', 'pickpoint', 'nominatim', 'pelias', 'photon', 'liveaddress', 'tomtom', " \
+                 "'what3words', 'yandex'])"
+
+# MISSING PARAMETER
+missing_key = dict(
+    azure=dict(
+        domain='atlas.microsoft.com'
+    )
+)
+
+# MISSING PARAMETER LOGGER RESPONSE
+logger_message_parameter = "__init__() missing 1 required positional argument: 'subscription_key'"
+
+# VALID COORDINATES
+coordinates = [8.807527, 53.07620980000001]
+
+# VALID GEOCODE CATEGORY ID
+college_category_id = 151
+
+# INVALID GEOCODE CATEGORY ID
+bench_category_id = 162
+
+
+class TestGeocoderBlueprint(BaseTestCase):
+
+    def test_valid_geocoder(self):
+        response = GeocoderSetup(list(valid_geocoder_param.items())[0]).define_geocoder()
+        self.assertEqual('https://nominatim.openstreetmap.org/search', response.api)
+
+    def test_unvalid_geocoder(self):
+        response = GeocoderSetup(list(invalid_geocoder.items())[0]).define_geocoder()
+        self.assertEqual(logger_message_geocoder, response.args[0])
+
+    def test_missing_key(self):
+        response = GeocoderSetup(list(missing_key.items())[0]).define_geocoder()
+        self.assertEqual(logger_message_parameter, response.args[0])
+
+
+class TestAddressBlueprint(BaseTestCase):
+
+    def test_address_request(self):
+        response = json.loads(AddressObject(coordinates, valid_geocoder).address_request())
+        self.assertEqual('Stadtbezirk Bremen-Mitte', response['address']['city_district'])
+
+    def test_address_type(self):
+        response = AddressObject(coordinates, valid_geocoder).address_request()
+        self.assertTrue(json.loads(response))
+
+
+class TestGeocodeCategoriesBlueprint(BaseTestCase):
+
+    def test_geocode_categories(self):
+        response = CategoryTools('categories.yml').generate_geocode_categories()
+        self.assertTrue(college_category_id in response)
+        self.assertTrue(bench_category_id not in response)
+
+
+if __name__ == '__main__':
+    unittest.main()

From 44892c0fbaf651830ab2f1a5371b91acad5c32c4 Mon Sep 17 00:00:00 2001
From: isabell <36471321+isikl@users.noreply.github.com>
Date: Wed, 20 Mar 2019 12:27:48 +0100
Subject: [PATCH 6/6] Update __init__.py

Corrected space distance
---
 openpoiservice/server/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/openpoiservice/server/__init__.py b/openpoiservice/server/__init__.py
index 0f0313d..05036c6 100755
--- a/openpoiservice/server/__init__.py
+++ b/openpoiservice/server/__init__.py
@@ -23,8 +23,8 @@
 geocoder = None
 geocode_categories = None
 if ops_settings['geocoder'] is not None:
-        geocoder = GeocoderSetup(list(ops_settings['geocoder'].items())[0]).define_geocoder()
-        geocode_categories = CategoryTools('categories.yml').generate_geocode_categories()
+    geocoder = GeocoderSetup(list(ops_settings['geocoder'].items())[0]).define_geocoder()
+    geocode_categories = CategoryTools('categories.yml').generate_geocode_categories()
 
 
 if "TESTING" in os.environ: