Skip to content

Commit a6ddaed

Browse files
committed
avoid duplicates in job queues for pending ways and relations
If the pending tracker of output was ahead of the pending tracker of middle when merging the two then an id coming from the middle tracker was unconditionally added even if it was present in the output tracker and therefore added from there. Fixes osm2pgsql-dev#419
1 parent 8b55475 commit a6ddaed

File tree

4 files changed

+57
-22
lines changed

4 files changed

+57
-22
lines changed

id-tracker.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,8 @@ osmid_t id_tracker::pop_mark() {
171171

172172
size_t id_tracker::size() { return impl->count; }
173173

174+
osmid_t id_tracker::last_returned() const { return impl->old_id; }
175+
174176
bool id_tracker::is_valid(osmid_t id) { return id != max(); }
175177
osmid_t id_tracker::max() { return std::numeric_limits<osmid_t>::max(); }
176178
osmid_t id_tracker::min() { return std::numeric_limits<osmid_t>::min(); }

id-tracker.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ struct id_tracker : public boost::noncopyable {
1313
bool is_marked(osmid_t id);
1414
osmid_t pop_mark();
1515
size_t size();
16+
osmid_t last_returned() const;
1617

1718
static bool is_valid(osmid_t);
1819
static osmid_t max();

output-multi.cpp

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,15 @@ size_t output_multi_t::pending_count() const {
6161
}
6262

6363
void output_multi_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
64+
osmid_t const prev = ways_pending_tracker->last_returned();
65+
if (id_tracker::is_valid(prev) && prev >= id) {
66+
if (prev > id) {
67+
job_queue.push(pending_job_t(id, output_id));
68+
}
69+
// already done the job
70+
return;
71+
}
72+
6473
//make sure we get the one passed in
6574
if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) {
6675
job_queue.push(pending_job_t(id, output_id));
@@ -83,11 +92,10 @@ void output_multi_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t
8392

8493
//make sure to get this one as well and move to the next
8594
if(popped == id) {
86-
popped = ways_pending_tracker->pop_mark();
87-
}
88-
if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
89-
job_queue.push(pending_job_t(popped, output_id));
90-
added++;
95+
if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
96+
job_queue.push(pending_job_t(popped, output_id));
97+
added++;
98+
}
9199
}
92100
}
93101

@@ -106,6 +114,15 @@ int output_multi_t::pending_way(osmid_t id, int exists) {
106114
}
107115

108116
void output_multi_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
117+
osmid_t const prev = rels_pending_tracker->last_returned();
118+
if (id_tracker::is_valid(prev) && prev >= id) {
119+
if (prev > id) {
120+
job_queue.push(pending_job_t(id, output_id));
121+
}
122+
// already done the job
123+
return;
124+
}
125+
109126
//make sure we get the one passed in
110127
if(id_tracker::is_valid(id)) {
111128
job_queue.push(pending_job_t(id, output_id));
@@ -126,11 +143,10 @@ void output_multi_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, s
126143

127144
//make sure to get this one as well and move to the next
128145
if(popped == id) {
129-
popped = rels_pending_tracker->pop_mark();
130-
}
131-
if(id_tracker::is_valid(popped)) {
132-
job_queue.push(pending_job_t(popped, output_id));
133-
added++;
146+
if(id_tracker::is_valid(popped)) {
147+
job_queue.push(pending_job_t(popped, output_id));
148+
added++;
149+
}
134150
}
135151
}
136152

output-pgsql.cpp

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,15 @@ extern "C" void *pthread_output_pgsql_stop_one(void *arg) {
273273
} // anonymous namespace
274274

275275
void output_pgsql_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
276+
osmid_t const prev = ways_pending_tracker->last_returned();
277+
if (id_tracker::is_valid(prev) && prev >= id) {
278+
if (prev > id) {
279+
job_queue.push(pending_job_t(id, output_id));
280+
}
281+
// already done the job
282+
return;
283+
}
284+
276285
//make sure we get the one passed in
277286
if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) {
278287
job_queue.push(pending_job_t(id, output_id));
@@ -294,12 +303,11 @@ void output_pgsql_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t
294303
}
295304

296305
//make sure to get this one as well and move to the next
297-
if(popped == id) {
298-
popped = ways_pending_tracker->pop_mark();
299-
}
300-
if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
301-
job_queue.push(pending_job_t(popped, output_id));
302-
added++;
306+
if(popped > id) {
307+
if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
308+
job_queue.push(pending_job_t(popped, output_id));
309+
added++;
310+
}
303311
}
304312
}
305313

@@ -319,6 +327,15 @@ int output_pgsql_t::pending_way(osmid_t id, int exists) {
319327
}
320328

321329
void output_pgsql_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
330+
osmid_t const prev = rels_pending_tracker->last_returned();
331+
if (id_tracker::is_valid(prev) && prev >= id) {
332+
if (prev > id) {
333+
job_queue.push(pending_job_t(id, output_id));
334+
}
335+
// already done the job
336+
return;
337+
}
338+
322339
//make sure we get the one passed in
323340
if(id_tracker::is_valid(id)) {
324341
job_queue.push(pending_job_t(id, output_id));
@@ -338,12 +355,11 @@ void output_pgsql_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, s
338355
}
339356

340357
//make sure to get this one as well and move to the next
341-
if(popped == id) {
342-
popped = rels_pending_tracker->pop_mark();
343-
}
344-
if(id_tracker::is_valid(popped)) {
345-
job_queue.push(pending_job_t(popped, output_id));
346-
added++;
358+
if(popped > id) {
359+
if(id_tracker::is_valid(popped)) {
360+
job_queue.push(pending_job_t(popped, output_id));
361+
added++;
362+
}
347363
}
348364
}
349365

0 commit comments

Comments
 (0)