#80 make data preprocessing more error-tolerant. Download incompleted ways

Author: deevroman

src/building.js

@@ -43,7 +43,8 @@ class Building {
     const nodelist = Building.buildNodeList(xmlData);
     const extents = Building.getExtents(id, xmlData, nodelist);
     const innerData = await Building.getInnerData(...extents);
-    return new Building(id, innerData);
+    const [augmentedNodelist, augmentedWays] = await Building.buildAugmentedData(innerData)
+    return new Building(id, innerData, augmentedNodelist, augmentedWays);
   }
 
   /**
@@ -52,9 +53,11 @@ class Building {
    * @param {string} id - the unique XML id of the object.
    * @param {string} FullXmlData - XML data.
    */
-  constructor(id, FullXmlData) {
+  constructor(id, FullXmlData, augmentedNodelist, augmentedWays) {
     this.id = id;
     this.fullXmlData = new window.DOMParser().parseFromString(FullXmlData, 'text/xml');
+    this.augmentedNodelist = augmentedNodelist;
+    this.augmentedWays = augmentedWays;
     const outerElementXml = this.fullXmlData.getElementById(id);
     if (outerElementXml.tagName.toLowerCase() === 'way') {
       this.type = 'way';
@@ -68,17 +71,17 @@ class Building {
       this.setHome();
       this.repositionNodes();
       if (this.type === 'way') {
-        this.outerElement = new BuildingPart(id, this.fullXmlData, this.nodelist);
+        this.outerElement = new BuildingPart(id, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays);
       } else if (this.type === 'multipolygon') {
-        this.outerElement = new MultiBuildingPart(id, this.fullXmlData, this.nodelist);
+        this.outerElement = new MultiBuildingPart(id, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays);
       } else {
         const outlineRef = outerElementXml.querySelector('member[role="outline"]').getAttribute('ref');
         const outline = this.fullXmlData.getElementById(outlineRef);
         const outlineType = outline.tagName.toLowerCase();
         if (outlineType === 'way') {
-          this.outerElement = new BuildingPart(id, this.fullXmlData, this.nodelist);
+          this.outerElement = new BuildingPart(id, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays);
         } else {
-          this.outerElement = new MultiBuildingPart(outlineRef, this.fullXmlData, this.nodelist);
+          this.outerElement = new MultiBuildingPart(outlineRef, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays);
         }
       }
       this.addParts();
@@ -121,6 +124,34 @@ class Building {
     return nodeList;
   }
 
+
+  /**
+   * @param {DOM.Element} fullXmlData - OSM XML with nodes
+   * @return {Promise<({}|*)[]>}
+   */
+  static async buildAugmentedData(fullXmlData) {
+    const xmlData = new DOMParser().parseFromString(fullXmlData, 'text/xml');
+    const completedWays = new Set(Array.from(xmlData.getElementsByTagName('way')).map(i => i.getAttribute('id')));
+    const memberWays = xmlData.querySelectorAll('member[type="way"]');
+    const nodeList = {};
+    const waysList = {};
+    await Promise.all(Array.from(memberWays).map(async currentWay => {
+      const wayID = currentWay.getAttribute('ref');
+      if (completedWays.has(wayID)) {
+        return
+      }
+      printError('Additional downloading way ' + wayID);
+      const wayData = new DOMParser().parseFromString(await Building.getWayData(wayID), 'text/xml');
+      printError(`Way ${wayID} was downloaded`);
+      waysList[wayID] = wayData.querySelector('way');
+      wayData.querySelectorAll('node').forEach(i => {
+        nodeList[i.getAttribute('id')] = [i.getAttribute('lon'), i.getAttribute('lat')];
+      });
+    }))
+    return [nodeList, waysList];
+  }
+
+
   /**
    * convert all the longitude latitude values
    * to meters from the home point.
@@ -129,6 +160,9 @@ class Building {
     for (const key in this.nodelist) {
       this.nodelist[key] = BuildingShapeUtils.repositionPoint(this.nodelist[key], this.home);
     }
+    for (const key in this.augmentedNodelist) {
+      this.augmentedNodelist[key] = BuildingShapeUtils.repositionPoint(this.augmentedNodelist[key], this.home);
+    }
   }
 
   /**
@@ -158,9 +192,9 @@ class Building {
         const ref = parts[i].getAttribute('ref');
         const part = this.fullXmlData.getElementById(ref);
         if (part.tagName.toLowerCase() === 'way') {
-          this.parts.push(new BuildingPart(ref, this.fullXmlData, this.nodelist, this.outerElement.options));
+          this.parts.push(new BuildingPart(ref, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays, this.outerElement.options));
         } else {
-          this.parts.push(new MultiBuildingPart(ref, this.fullXmlData, this.nodelist, this.outerElement.options));
+          this.parts.push(new MultiBuildingPart(ref, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays, this.outerElement.options));
         }
       }
     } else {
@@ -169,15 +203,15 @@ class Building {
       for (let j = 0; j < parts.length; j++) {
         if (parts[j].querySelector('[k="building:part"]')) {
           const id = parts[j].getAttribute('id');
-          this.parts.push(new BuildingPart(id, this.fullXmlData, this.nodelist, this.outerElement.options));
+          this.parts.push(new BuildingPart(id, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays, this.outerElement.options));
         }
       }
       // Filter all relations
       parts = this.fullXmlData.getElementsByTagName('relation');
       for (let i = 0; i < parts.length; i++) {
         if (parts[i].querySelector('[k="building:part"]')) {
           const id = parts[i].getAttribute('id');
-          this.parts.push(new MultiBuildingPart(id, this.fullXmlData, this.nodelist, this.outerElement.options));
+          this.parts.push(new MultiBuildingPart(id, this.fullXmlData, this.nodelist, this.augmentedNodelist, this.augmentedWays, this.outerElement.options));
         }
       }
     }

src/buildingpart.js

@@ -68,7 +68,7 @@ class BuildingPart {
    * @param {[[number, number],...]} nodelist - Cartesian coordinates of each node keyed by node refID
    * @param {object} options - default values for the building part.
    */
-  constructor(id, fullXmlData, nodelist, defaultOptions = {}) {
+  constructor(id, fullXmlData, nodelist, augmentedNodelist, augmentedWays, defaultOptions = {}) {
     this.options = this.blankOptions;
     if (Object.keys(defaultOptions).length === 0) {
       defaultOptions = this.blankOptions;
@@ -78,6 +78,8 @@ class BuildingPart {
     this.id = id;
     this.way = fullXmlData.getElementById(id);
     this.nodelist = nodelist;
+    this.augmentedNodelist = augmentedNodelist;
+    this.augmentedWays = augmentedWays;
     this.shape = this.buildShape();
     this.setOptions();
   }
@@ -89,7 +91,7 @@ class BuildingPart {
    */
   buildShape() {
     this.type = 'way';
-    return BuildingShapeUtils.createShape(this.way, this.nodelist);
+    return BuildingShapeUtils.createShape(this.way, this.nodelist, this.augmentedNodelist);
   }
 
   /**

src/extras/BuildingShapeUtils.js

@@ -13,7 +13,7 @@ class BuildingShapeUtils extends ShapeUtils {
    *
    * @return {THREE.Shape} shape - the shape
    */
-  static createShape(way, nodelist) {
+  static createShape(way, nodelist, augmentedNodelist = {}) {
     // Initialize objects
     const shape = new Shape();
     var ref;
@@ -25,7 +25,7 @@ class BuildingShapeUtils extends ShapeUtils {
     // Get the coordinates of all the nodes and add them to the shape outline.
     for (let i = 0; i < elements.length; i++) {
       ref = elements[i].getAttribute('ref');
-      node = nodelist[ref];
+      node = nodelist[ref] ?? augmentedNodelist[ref];
       // The first node requires a differnet function call.
       if (i === 0) {
         shape.moveTo(parseFloat(node[0]), parseFloat(node[1]));
@@ -58,10 +58,14 @@ class BuildingShapeUtils extends ShapeUtils {
    * @return {DOM.Element}
    */
   static combineWays(ways) {
-    var closedWays = [];
-    var openWays = [];
-    var changed = true;
+    const closedWays = [];
+    let openWays = [];
+    let changed = true;
+    let watchDogCounter = 0
     while (changed) {
+      if (watchDogCounter++ > 2_000_000) {
+        throw 'Infinite loop in combineWays';
+      }
       changed = false;
       for (let i = 0; i < ways.length - 1; i++) {
         if (BuildingShapeUtils.isClosed(ways[i])) {

src/multibuildingpart.js

@@ -17,19 +17,24 @@ class MultiBuildingPart extends BuildingPart {
     const innerMembers = this.way.querySelectorAll('member[role="inner"]');
     const outerMembers = this.way.querySelectorAll('member[role="outer"]');
     const innerShapes = [];
-    var shapes = [];
+    let shapes = [];
     for (let i = 0; i < innerMembers.length; i++) {
       const way = this.fullXmlData.getElementById(innerMembers[i].getAttribute('ref'));
-      innerShapes.push(BuildingShapeUtils.createShape(way, this.nodelist));
+      innerShapes.push(BuildingShapeUtils.createShape(way, this.nodelist, this.augmentedNodelist));
     }
     const ways = [];
     for (let j = 0; j < outerMembers.length; j++) {
       const way = this.fullXmlData.getElementById(outerMembers[j].getAttribute('ref'));
-      ways.push(way);
+      if (way) {
+        ways.push(way);
+      } else {
+        printError(`Missing way ${outerMembers[j].getAttribute('ref')} for relation ${this.id}`);
+        ways.push(this.augmentedWays[outerMembers[j].getAttribute('ref')])
+      }
     }
     const closedWays = BuildingShapeUtils.combineWays(ways);
     for (let k = 0; k < closedWays.length; k++) {
-      const shape = BuildingShapeUtils.createShape(closedWays[k], this.nodelist);
+      const shape = BuildingShapeUtils.createShape(closedWays[k], this.nodelist, this.augmentedNodelist);
       shape.holes.push(...innerShapes);
       shapes.push(shape);
     }