diff --git a/openpoiservice/server/__init__.py b/openpoiservice/server/__init__.py index 9580481..05036c6 100755 --- a/openpoiservice/server/__init__.py +++ b/openpoiservice/server/__init__.py @@ -6,6 +6,7 @@ from flask_cors import CORS from openpoiservice.server.categories.categories import CategoryTools from openpoiservice.server.api import api_exceptions +from openpoiservice.server.db_import.objects import GeocoderSetup import yaml import os import time @@ -19,6 +20,13 @@ basedir = os.path.abspath(os.path.dirname(__file__)) ops_settings = yaml.safe_load(open(os.path.join(basedir, 'ops_settings.yml'))) +geocoder = None +geocode_categories = None +if ops_settings['geocoder'] is not None: + geocoder = GeocoderSetup(list(ops_settings['geocoder'].items())[0]).define_geocoder() + geocode_categories = CategoryTools('categories.yml').generate_geocode_categories() + + if "TESTING" in os.environ: ops_settings['provider_parameters']['table_name'] = ops_settings['provider_parameters']['table_name'] + '_test' diff --git a/openpoiservice/server/api/pois_post.yaml b/openpoiservice/server/api/pois_post.yaml index cd81c72..114c378 100644 --- a/openpoiservice/server/api/pois_post.yaml +++ b/openpoiservice/server/api/pois_post.yaml @@ -449,8 +449,6 @@ definitions: properties: name: type: "string" - address: - type: "string" website: type: "string" opening_hours: diff --git a/openpoiservice/server/api/query_builder.py b/openpoiservice/server/api/query_builder.py index 06b3053..8a18f32 100644 --- a/openpoiservice/server/api/query_builder.py +++ b/openpoiservice/server/api/query_builder.py @@ -1,7 +1,7 @@ # openpoiservice/server/query_builder.py from openpoiservice.server import db -from openpoiservice.server import categories_tools, ops_settings +from openpoiservice.server import categories_tools, ops_settings, geocoder import geoalchemy2.functions as geo_func from geoalchemy2.types import Geography, Geometry from geoalchemy2.elements import WKBElement, WKTElement @@ -13,6 +13,7 @@ from sqlalchemy import dialects import geojson as geojson import logging +import json from timeit import default_timer as timer logger = logging.getLogger(__name__) @@ -105,7 +106,8 @@ def request_pois(self): bbox_query.c.geom, keys_agg, values_agg, - categories_agg) \ + categories_agg, + bbox_query.c.address) \ .order_by(*sortby_group) \ .filter(*category_filters) \ .filter(*custom_filters) \ @@ -115,14 +117,7 @@ def request_pois(self): .group_by(bbox_query.c.osm_id) \ .group_by(bbox_query.c.osm_type) \ .group_by(bbox_query.c.geom) \ - # .all() - - # end = timer() - # print(end - start) - - # print(str(pois_query)) - # for dude in pois_query: - # print(dude) + .group_by(bbox_query.c.address) # response as geojson feature collection features = self.generate_geojson_features(pois_query, params['limit']) @@ -147,12 +142,9 @@ def generate_geom_filters(geometry, Pois): type_coerce(geom_bbox, Geography)), Pois.geom, 0)) elif 'bbox' not in geometry and 'geom' in geometry: - geom = geometry['geom'].wkt - filters.append( # buffer around geom - geo_func.ST_DWithin(geo_func.ST_Buffer(type_coerce(geom, Geography), geometry['buffer']), Pois.geom, 0) - ) + geo_func.ST_DWithin(geo_func.ST_Buffer(type_coerce(geom, Geography), geometry['buffer']), Pois.geom, 0)) return filters, geom @@ -264,12 +256,24 @@ def generate_geojson_features(cls, query, limit): } properties["category_ids"] = category_ids_obj + # Checks if Tags are available if q[5][0] is not None: key_values = {} for idx, key in enumerate(q[4]): key_values[key] = q[5][idx] properties["osm_tags"] = key_values + # Checks if addresses are available + try: + if q[7] is not None: + address_dict = {} + address_data = json.loads(q[7]) + for key, value in address_data.items(): + address_dict[key] = value + properties['address'] = address_dict + except IndexError: + pass + geojson_feature = geojson.Feature(geometry=trimmed_point, properties=properties) geojson_features.append(geojson_feature) diff --git a/openpoiservice/server/categories/categories.py b/openpoiservice/server/categories/categories.py index e127713..a174251 100644 --- a/openpoiservice/server/categories/categories.py +++ b/openpoiservice/server/categories/categories.py @@ -94,3 +94,16 @@ def get_category(self, tags): categories.append(category_id) return categories + + def generate_geocode_categories(self): + + geocode_categories = {} + + for category in self.categories_object.items(): + if 'geocoder' not in category[1]: + for child in category[1]['children'].values(): + for id in child.values(): + geocode_categories[id] = {} + + return geocode_categories + diff --git a/openpoiservice/server/categories/categories.yml b/openpoiservice/server/categories/categories.yml index e832ab8..95d3473 100644 --- a/openpoiservice/server/categories/categories.yml +++ b/openpoiservice/server/categories/categories.yml @@ -49,6 +49,7 @@ education: facilities: id: 160 + geocoder: False children: amenity: compressed_air: 161 diff --git a/openpoiservice/server/db_import/models.py b/openpoiservice/server/db_import/models.py index be79faf..3be5912 100755 --- a/openpoiservice/server/db_import/models.py +++ b/openpoiservice/server/db_import/models.py @@ -1,6 +1,6 @@ # openpoiservice/server/models.py -from openpoiservice.server import db, ops_settings +from openpoiservice.server import db, ops_settings, geocoder from geoalchemy2 import Geography import logging @@ -14,7 +14,6 @@ class Pois(db.Model): uuid = db.Column(db.LargeBinary, primary_key=True) osm_id = db.Column(db.BigInteger, nullable=False, index=True) osm_type = db.Column(db.Integer, nullable=False) - # address = db.Column(db.Text, nullable=True) geom = db.Column(Geography(geometry_type="POINT", srid=4326, spatial_index=True), nullable=False) tags = db.relationship("Tags", backref='{}'.format(ops_settings['provider_parameters']['table_name']), @@ -23,6 +22,8 @@ class Pois(db.Model): categories = db.relationship("Categories", backref='{}'.format(ops_settings['provider_parameters']['table_name']), lazy='dynamic') + address = db.Column(db.String, nullable=True) + def __repr__(self): return '' % self.osm_id diff --git a/openpoiservice/server/db_import/objects.py b/openpoiservice/server/db_import/objects.py index de96d1e..698b622 100644 --- a/openpoiservice/server/db_import/objects.py +++ b/openpoiservice/server/db_import/objects.py @@ -1,9 +1,15 @@ # openpoiservice/server/poi_entity.py +import json +import logging +from geopy.geocoders import get_geocoder_for_service +# from geopy.extra.rate_limiter import RateLimiter + +logger = logging.getLogger(__name__) class PoiObject(object): - def __init__(self, uuid, categories, osmid, lat_lng, osm_type): + def __init__(self, uuid, categories, osmid, lat_lng, osm_type, address=None): self.uuid = uuid self.osmid = int(osmid) self.type = int(osm_type) @@ -14,9 +20,7 @@ def __init__(self, uuid, categories, osmid, lat_lng, osm_type): self.geom = 'SRID={};POINT({} {})'.format(4326, float(lat_lng[0]), float(lat_lng[1])) - - # add geocoder connector here... - self.address = None + self.address = address class TagsObject(object): @@ -26,3 +30,56 @@ def __init__(self, uuid, osmid, key, value): self.osmid = int(osmid) self.key = key self.value = value + + +class GeocoderSetup(object): + """Initialises geocoder""" + + def __init__(self, geocoder_name): + self.geocoder_name = geocoder_name + self.geocoder = None + + def define_geocoder(self): + + # returns warning if no valid geocoder is provided + try: + self.geocoder_settings = get_geocoder_for_service(self.geocoder_name[0]) + + except Exception as err_geocoder: + logger.warning(err_geocoder) + return err_geocoder + + # returns warning if no valid geocoder settings are provided + try: + if self.geocoder_name[1] is not None: + self.geocoder = self.geocoder_settings(**self.geocoder_name[1]) + + else: + self.geocoder = self.geocoder_settings() + return self.geocoder + + except Exception as err_parameter: + logger.warning(err_parameter) + return err_parameter + + +class AddressObject(object): + + def __init__(self, lat_lng, geocoder): + self.lat_lng = lat_lng[::-1] + self.geocoder = geocoder + + def address_request(self): + + # address_delaied = RateLimiter(self.geocoder.reverse, min_delay_seconds=1) + response = self.geocoder.reverse(query=self.lat_lng) + + # Checks if address for location is available + if response is not None: + return json.dumps(response.raw) + # try: + # return json.dumps(response.raw) + # except AttributeError: + # return json.dumps(response) + + return None diff --git a/openpoiservice/server/db_import/parse_osm.py b/openpoiservice/server/db_import/parse_osm.py index 0c2b969..6d01b51 100644 --- a/openpoiservice/server/db_import/parse_osm.py +++ b/openpoiservice/server/db_import/parse_osm.py @@ -1,9 +1,9 @@ # openpoiservice/server/parse_osm.py from openpoiservice.server import db -from openpoiservice.server import categories_tools, ops_settings +from openpoiservice.server import categories_tools, ops_settings, geocoder, geocode_categories from openpoiservice.server.db_import.models import Pois, Tags, Categories -from openpoiservice.server.db_import.objects import PoiObject, TagsObject +from openpoiservice.server.db_import.objects import PoiObject, TagsObject, AddressObject from openpoiservice.server.utils.decorators import get_size import shapely as shapely from shapely.geometry import Point, Polygon, LineString, MultiPoint @@ -179,12 +179,21 @@ def store_poi(self, poi_object): """ self.pois_cnt += 1 - self.poi_objects.append(Pois( - uuid=poi_object.uuid, - osm_id=poi_object.osmid, - osm_type=poi_object.type, - geom=poi_object.geom - )) + if geocoder is not None: + self.poi_objects.append(Pois( + uuid=poi_object.uuid, + osm_id=poi_object.osmid, + osm_type=poi_object.type, + geom=poi_object.geom, + address=poi_object.address + )) + else: + self.poi_objects.append(Pois( + uuid=poi_object.uuid, + osm_id=poi_object.osmid, + osm_type=poi_object.type, + geom=poi_object.geom + )) if self.pois_cnt % 1000 == 0: logger.info('Pois: {}, tags: {}, categories: {}'.format(self.pois_cnt, self.tags_cnt, self.categories_cnt)) @@ -261,7 +270,13 @@ def create_poi(self, tags, osmid, lat_lng, osm_type, categories=[]): self.tags_object = TagsObject(my_uuid, osmid, tag, value) self.store_tags(self.tags_object) - self.poi_object = PoiObject(my_uuid, categories, osmid, lat_lng, osm_type) + address = None + if geocoder is not None and categories[0] in geocode_categories: + address = AddressObject(lat_lng, geocoder).address_request() + time.sleep(1) + + self.poi_object = PoiObject(my_uuid, categories, osmid, lat_lng, osm_type, address) + self.store_poi(self.poi_object) for category in categories: diff --git a/openpoiservice/server/db_import/parser.py b/openpoiservice/server/db_import/parser.py index 2c5be40..e3a8207 100644 --- a/openpoiservice/server/db_import/parser.py +++ b/openpoiservice/server/db_import/parser.py @@ -2,10 +2,9 @@ from openpoiservice.server.db_import.parse_osm import OsmImporter from openpoiservice.server.utils.decorators import timeit, processify -from openpoiservice.server import ops_settings +from openpoiservice.server import ops_settings, geocoder from imposm.parser import OSMParser import logging -import time # from guppy import hpy from collections import deque @@ -50,6 +49,10 @@ def parse_import(osm_file): coords = OSMParser(concurrency=1, coords_callback=osm_importer.parse_coords_for_ways) coords.parse(osm_file) + # Checks if geocoder is provided + if geocoder is not None: + logger.info('Importing addresses...') + logger.info('Storing remaining pois') osm_importer.save_remainder() diff --git a/openpoiservice/server/ops_settings.yml b/openpoiservice/server/ops_settings.yml index e20edd3..4f96d7c 100644 --- a/openpoiservice/server/ops_settings.yml +++ b/openpoiservice/server/ops_settings.yml @@ -16,13 +16,12 @@ concurrent_workers: 4 # Database parameters provider_parameters: table_name: ops_planet_pois - db_name: gis - user_name: gis_admin + db_name: opsdb + user_name: postgres table_schema: public - #user_name: admin - password: admin + password: ops host: localhost - port: 5434 + port: 5432 port_tests: 5432 # add any additional tags you want to save to the database column_mappings: @@ -40,3 +39,24 @@ column_mappings: phone: # https://wiki.openstreetmap.org/wiki/Key:website website: +# choose one geopy geocoder and provide all needed parameters - https://geopy.readthedocs.io/ +geocoder: +# ArcGIS: +# domain: 'geocode.arcgis.com' +# pelias: +# domain: 'api.geocode.earth' +# api_key: '' +# timeout: 60 +# nominatim: +# timeout: 60 +# GeocodeFarm: +# timeout: 60 +# Azure: +# subscription_key: '' +# domain: 'atlas.microsoft.com' +# Baidu: +# api_key: '' +# BANFrance: +# domain: 'api-adresse.data.gouv.fr' +# Bing: +# api_key: '' diff --git a/openpoiservice/tests/test_geocoder.py b/openpoiservice/tests/test_geocoder.py new file mode 100644 index 0000000..44cf6be --- /dev/null +++ b/openpoiservice/tests/test_geocoder.py @@ -0,0 +1,89 @@ +# openpoiservice/server/tests/test_main.py + + +import unittest +import json +from openpoiservice.server.db_import.objects import AddressObject, GeocoderSetup +from openpoiservice.server.categories.categories import CategoryTools + +# from base import BaseTestCase +from openpoiservice.tests.base import BaseTestCase + +# VALID GEOCODER AND PARAMETER +valid_geocoder_param = dict( + nominatim=dict( + timeout=60 + ) +) + +# VALID GEOCODER FOR class TestGeocoderBlueprint +valid_geocoder = GeocoderSetup(list(valid_geocoder_param.items())[0]).define_geocoder() + +# INVALID GEOCODER +invalid_geocoder = dict( + peliaas=dict() +) + +# INVALID GEOCODER LOGGER RESPONSE +logger_message_geocoder = "Unknown geocoder 'peliaas'; " \ + "options are: dict_keys(['arcgis', 'azure', 'baidu', 'banfrance', 'bing', 'databc', 'geocodeearth', " \ + "'geocodefarm', 'geonames', 'google', 'googlev3', 'geolake', 'here', 'ignfrance', 'mapbox', " \ + "'opencage', 'openmapquest', 'pickpoint', 'nominatim', 'pelias', 'photon', 'liveaddress', 'tomtom', " \ + "'what3words', 'yandex'])" + +# MISSING PARAMETER +missing_key = dict( + azure=dict( + domain='atlas.microsoft.com' + ) +) + +# MISSING PARAMETER LOGGER RESPONSE +logger_message_parameter = "__init__() missing 1 required positional argument: 'subscription_key'" + +# VALID COORDINATES +coordinates = [8.807527, 53.07620980000001] + +# VALID GEOCODE CATEGORY ID +college_category_id = 151 + +# INVALID GEOCODE CATEGORY ID +bench_category_id = 162 + + +class TestGeocoderBlueprint(BaseTestCase): + + def test_valid_geocoder(self): + response = GeocoderSetup(list(valid_geocoder_param.items())[0]).define_geocoder() + self.assertEqual('https://nominatim.openstreetmap.org/search', response.api) + + def test_unvalid_geocoder(self): + response = GeocoderSetup(list(invalid_geocoder.items())[0]).define_geocoder() + self.assertEqual(logger_message_geocoder, response.args[0]) + + def test_missing_key(self): + response = GeocoderSetup(list(missing_key.items())[0]).define_geocoder() + self.assertEqual(logger_message_parameter, response.args[0]) + + +class TestAddressBlueprint(BaseTestCase): + + def test_address_request(self): + response = json.loads(AddressObject(coordinates, valid_geocoder).address_request()) + self.assertEqual('Stadtbezirk Bremen-Mitte', response['address']['city_district']) + + def test_address_type(self): + response = AddressObject(coordinates, valid_geocoder).address_request() + self.assertTrue(json.loads(response)) + + +class TestGeocodeCategoriesBlueprint(BaseTestCase): + + def test_geocode_categories(self): + response = CategoryTools('categories.yml').generate_geocode_categories() + self.assertTrue(college_category_id in response) + self.assertTrue(bench_category_id not in response) + + +if __name__ == '__main__': + unittest.main()