Merge pull request osm2pgsql-dev#420 from lonvia/fix-job-enqueueing

avoid duplicates in job queues for pending ways and relations

Author: pnorman

id-tracker.cpp

@@ -171,6 +171,8 @@ osmid_t id_tracker::pop_mark() {
 
 size_t id_tracker::size() { return impl->count; }
 
+osmid_t id_tracker::last_returned() const { return impl->old_id; }
+
 bool id_tracker::is_valid(osmid_t id) { return id != max(); }
 osmid_t id_tracker::max() { return std::numeric_limits<osmid_t>::max(); }
 osmid_t id_tracker::min() { return std::numeric_limits<osmid_t>::min(); }

id-tracker.hpp

@@ -13,6 +13,7 @@ struct id_tracker : public boost::noncopyable {
     bool is_marked(osmid_t id);
     osmid_t pop_mark();
     size_t size();
+    osmid_t last_returned() const;
 
     static bool is_valid(osmid_t);
     static osmid_t max();

output-multi.cpp

@@ -61,6 +61,15 @@ size_t output_multi_t::pending_count() const {
 }
 
 void output_multi_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+    osmid_t const prev = ways_pending_tracker->last_returned();
+    if (id_tracker::is_valid(prev) && prev >= id) {
+        if (prev > id) {
+            job_queue.push(pending_job_t(id, output_id));
+        }
+        // already done the job
+        return;
+    }
+
     //make sure we get the one passed in
     if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) {
         job_queue.push(pending_job_t(id, output_id));
@@ -83,11 +92,10 @@ void output_multi_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t
 
     //make sure to get this one as well and move to the next
     if(popped == id) {
-        popped = ways_pending_tracker->pop_mark();
-    }
-    if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
-        job_queue.push(pending_job_t(popped, output_id));
-        added++;
+        if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
+            job_queue.push(pending_job_t(popped, output_id));
+            added++;
+        }
     }
 }
 
@@ -106,6 +114,15 @@ int output_multi_t::pending_way(osmid_t id, int exists) {
 }
 
 void output_multi_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+    osmid_t const prev = rels_pending_tracker->last_returned();
+    if (id_tracker::is_valid(prev) && prev >= id) {
+        if (prev > id) {
+            job_queue.push(pending_job_t(id, output_id));
+        }
+        // already done the job
+        return;
+    }
+
     //make sure we get the one passed in
     if(id_tracker::is_valid(id)) {
         job_queue.push(pending_job_t(id, output_id));
@@ -126,11 +143,10 @@ void output_multi_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, s
 
     //make sure to get this one as well and move to the next
     if(popped == id) {
-        popped = rels_pending_tracker->pop_mark();
-    }
-    if(id_tracker::is_valid(popped)) {
-        job_queue.push(pending_job_t(popped, output_id));
-        added++;
+        if(id_tracker::is_valid(popped)) {
+            job_queue.push(pending_job_t(popped, output_id));
+            added++;
+        }
     }
 }
 

output-pgsql.cpp

@@ -273,6 +273,15 @@ extern "C" void *pthread_output_pgsql_stop_one(void *arg) {
 } // anonymous namespace
 
 void output_pgsql_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+    osmid_t const prev = ways_pending_tracker->last_returned();
+    if (id_tracker::is_valid(prev) && prev >= id) {
+        if (prev > id) {
+            job_queue.push(pending_job_t(id, output_id));
+        }
+        // already done the job
+        return;
+    }
+
     //make sure we get the one passed in
     if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) {
         job_queue.push(pending_job_t(id, output_id));
@@ -294,12 +303,11 @@ void output_pgsql_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t
     }
 
     //make sure to get this one as well and move to the next
-    if(popped == id) {
-        popped = ways_pending_tracker->pop_mark();
-    }
-    if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
-        job_queue.push(pending_job_t(popped, output_id));
-        added++;
+    if(popped > id) {
+        if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
+            job_queue.push(pending_job_t(popped, output_id));
+            added++;
+        }
     }
 }
 
@@ -319,6 +327,15 @@ int output_pgsql_t::pending_way(osmid_t id, int exists) {
 }
 
 void output_pgsql_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+    osmid_t const prev = rels_pending_tracker->last_returned();
+    if (id_tracker::is_valid(prev) && prev >= id) {
+        if (prev > id) {
+            job_queue.push(pending_job_t(id, output_id));
+        }
+        // already done the job
+        return;
+    }
+
     //make sure we get the one passed in
     if(id_tracker::is_valid(id)) {
         job_queue.push(pending_job_t(id, output_id));
@@ -338,12 +355,11 @@ void output_pgsql_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, s
     }
 
     //make sure to get this one as well and move to the next
-    if(popped == id) {
-        popped = rels_pending_tracker->pop_mark();
-    }
-    if(id_tracker::is_valid(popped)) {
-        job_queue.push(pending_job_t(popped, output_id));
-        added++;
+    if(popped > id) {
+        if(id_tracker::is_valid(popped)) {
+            job_queue.push(pending_job_t(popped, output_id));
+            added++;
+        }
     }
 }