Skip to content

Commit e671039

Browse files
anand870anand870
committed
osm parser in node js
0 parents  commit e671039

File tree

12 files changed

+45242
-0
lines changed

12 files changed

+45242
-0
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*.swp
2+
node_modules

‎db.js

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
const MongoClient = require("mongodb").MongoClient;
2+
class DB{
3+
constructor(config){
4+
this.url = config.connectionstr || '';
5+
this.getConnection();
6+
this.initializeCounter();
7+
}
8+
_createConnection(){
9+
return new Promise((resolve,reject)=>{
10+
MongoClient.connect(this.url,function(err,db){
11+
if(err){
12+
reject(err);
13+
return;
14+
}
15+
resolve(db);
16+
});
17+
});
18+
}
19+
getConnection(){
20+
if(!this.db){
21+
this.db=this._createConnection()
22+
.catch((err)=>{
23+
console.log("error creating connection:Err=>",err);
24+
});
25+
}
26+
return this.db;
27+
}
28+
initializeCounter(){
29+
this.counter={
30+
node:0,
31+
way:0,
32+
relation:0
33+
};
34+
}
35+
writeToMongo(dbname,data){
36+
this.counter[dbname] += data.length;
37+
return this.getConnection().then((db)=>{
38+
return db.collection(dbname).insertMany(data);
39+
})
40+
}
41+
clearDB(){
42+
return this.getConnection().then((db)=>{
43+
return Promise.all([
44+
db.collection('node').drop(),
45+
db.collection('way').drop(),
46+
db.collection('relation').drop()
47+
]);
48+
})
49+
.catch(()=>{
50+
51+
});
52+
}
53+
close(){
54+
if(this.db){
55+
this.db.then((db)=>{
56+
console.log("closing connection");
57+
this.db=null;
58+
db.close();
59+
});
60+
}
61+
}
62+
}
63+
exports.db = function(config){
64+
return new DB(config);
65+
}

‎index.js

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
"use strict";
2+
const filename='./input/sample-100.osm';
3+
const batchSize = 1000;
4+
const connectionstr = 'mongodb://localhost:27017/testdb';
5+
6+
const { db } = require('./db');
7+
8+
const dbConfig={
9+
connectionstr
10+
}
11+
const dbcon = db(dbConfig)
12+
13+
const streamConfig = {
14+
dbcon,
15+
batchSize,
16+
inputfile:filename
17+
}
18+
19+
const {
20+
inputStream,
21+
batchStream,
22+
transformStream,
23+
groupStream,
24+
dbStream,
25+
saxStream,
26+
progressStream
27+
} = require('./streams')(streamConfig)
28+
29+
function parse(dbcon){
30+
batchStream.pipe(progressStream).pipe(process.stdout);
31+
return inputStream
32+
.pipe(saxStream)
33+
.pipe(transformStream)
34+
.pipe(batchStream)
35+
.pipe(groupStream)
36+
.pipe(dbStream)
37+
}
38+
39+
dbcon.clearDB()
40+
.then(()=>{
41+
console.log("parsing started\n");
42+
let stream = parse();
43+
stream
44+
.on('finish',function(){
45+
dbcon.close();
46+
for(let dbname in dbcon.counter){
47+
console.log(`\ninserted ${dbcon.counter[dbname]} in ${dbname}`);
48+
}
49+
})
50+
.on('error',function(err){
51+
console.log(err)
52+
});
53+
})
54+
.catch(function(err){
55+
console.error(err);
56+
dbcon.close();
57+
});

‎input/sample-10-copy.osm

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<osm>
3+
<node changeset="10373979" id="25038483" lat="-22.9809491" lon="-43.2068586" timestamp="2012-01-12T21:51:07Z" uid="12293" user="Nighto" version="2">
4+
<tag k="source" v="Yahoo imaery" />
5+
</node>
6+
<node changeset="41021266" id="35574858" lat="-22.9252515" lon="-43.1802213" timestamp="2016-07-25T19:56:58Z" uid="4008694" user="smaprs_import" version="4" />
7+
<node changeset="12169922" id="38663056" lat="-22.9357383" lon="-43.1949616" timestamp="2012-07-10T07:15:18Z" uid="481662" user="Geaquinto" version="5" />
8+
<node changeset="33954340" id="50930714" lat="-22.9854283" lon="-43.2229035" timestamp="2015-09-11T10:52:10Z" uid="502691" user="ThiagoPv" version="14" />
9+
<node changeset="8393341" id="50935710" lat="-22.973438" lon="-43.2020022" timestamp="2011-06-09T23:47:41Z" uid="12293" user="Nighto" version="2" />
10+
<node changeset="10894496" id="97304223" lat="-23.005639" lon="-43.3107037" timestamp="2012-03-06T22:01:24Z" uid="481662" user="Geaquinto" version="3" />
11+
<node changeset="9791668" id="97338293" lat="-22.9113387" lon="-43.8390266" timestamp="2011-11-10T19:52:38Z" uid="179540" user="sushi_klaus" version="2" />
12+
</osm>

‎input/sample-10.osm

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<osm>
3+
<node changeset="10373979" id="25038483" lat="-22.9809491" lon="-43.2068586" timestamp="2012-01-12T21:51:07Z" uid="12293" user="Nighto" version="2">
4+
<tag k="source" v="Yahoo imaery" />
5+
</node>
6+
<node changeset="41021266" id="35574858" lat="-22.9252515" lon="-43.1802213" timestamp="2016-07-25T19:56:58Z" uid="4008694" user="smaprs_import" version="4" />
7+
<node changeset="12169922" id="38663056" lat="-22.9357383" lon="-43.1949616" timestamp="2012-07-10T07:15:18Z" uid="481662" user="Geaquinto" version="5" />
8+
<node changeset="33954340" id="50930714" lat="-22.9854283" lon="-43.2229035" timestamp="2015-09-11T10:52:10Z" uid="502691" user="ThiagoPv" version="14" />
9+
<node changeset="8393341" id="50935710" lat="-22.973438" lon="-43.2020022" timestamp="2011-06-09T23:47:41Z" uid="12293" user="Nighto" version="2" />
10+
<node changeset="10894496" id="97304223" lat="-23.005639" lon="-43.3107037" timestamp="2012-03-06T22:01:24Z" uid="481662" user="Geaquinto" version="3" />
11+
<node changeset="9791668" id="97338293" lat="-22.9113387" lon="-43.8390266" timestamp="2011-11-10T19:52:38Z" uid="179540" user="sushi_klaus" version="2" />
12+
<way changeset="42433432" id="444337539" timestamp="2016-09-25T22:22:39Z" uid="502691" user="ThiagoPv" version="1">
13+
<nd ref="2841027216" />
14+
<nd ref="2841027616" />
15+
<tag k="highway" v="secondary" />
16+
<tag k="lanes" v="2" />
17+
<tag k="lit" v="yes" />
18+
<tag k="name" v="Estrada do Pontal" />
19+
<tag k="oneway" v="yes" />
20+
<tag k="surface" v="asphalt" />
21+
</way>
22+
<relation changeset="34922564" id="121687" timestamp="2015-10-28T10:01:06Z" uid="502691" user="ThiagoPv" version="6">
23+
<member ref="30775792" role="outer" type="way" />
24+
<member ref="33747005" role="inner" type="way" />
25+
<member ref="33747063" role="inner" type="way" />
26+
<member ref="33747066" role="inner" type="way" />
27+
<member ref="33747078" role="inner" type="way" />
28+
<tag k="name" v="Pavilhão Reitor João Lyra Filho" />
29+
<tag k="type" v="multipolygon" />
30+
<tag k="building" v="university" />
31+
</relation>
32+
<relation changeset="28741164" id="1770055" timestamp="2015-02-10T00:18:58Z" uid="69210" user="Márcio Vínícius Pinheiro" version="16">
33+
<member ref="134836622" role="" type="way" />
34+
<member ref="327344046" role="" type="way" />
35+
<member ref="327344045" role="" type="way" />
36+
<member ref="125434685" role="" type="way" />
37+
<member ref="67557569" role="" type="way" />
38+
<member ref="126014854" role="" type="way" />
39+
<member ref="118690265" role="" type="way" />
40+
<member ref="142993737" role="" type="way" />
41+
<member ref="180213283" role="" type="way" />
42+
<member ref="67571046" role="" type="way" />
43+
<member ref="67551955" role="" type="way" />
44+
<member ref="131269851" role="" type="way" />
45+
<member ref="327344048" role="" type="way" />
46+
<member ref="131706984" role="" type="way" />
47+
<member ref="237330069" role="" type="way" />
48+
<member ref="327344052" role="" type="way" />
49+
<member ref="125275563" role="" type="way" />
50+
<member ref="131706982" role="" type="way" />
51+
<member ref="131706983" role="" type="way" />
52+
<member ref="53813561" role="" type="way" />
53+
<member ref="327344054" role="" type="way" />
54+
<member ref="327344050" role="" type="way" />
55+
<member ref="125774743" role="" type="way" />
56+
<member ref="1392011499" role="" type="node" />
57+
<member ref="1392011497" role="" type="node" />
58+
<member ref="1957464973" role="" type="node" />
59+
<member ref="2453741083" role="" type="node" />
60+
<tag k="ref" v="626" />
61+
<tag k="name" v="Muda - Praça Saens Peña" />
62+
<tag k="type" v="route" />
63+
<tag k="route" v="bus" />
64+
<tag k="old_ref" v="413A" />
65+
<tag k="operator" v="Alpha" />
66+
</relation>
67+
68+
69+
</osm>

0 commit comments

Comments
 (0)