ignore whitespace only within a w-tag

Author: PrinsINT

server/src/main/kotlin/org/ivdnt/galahad/formats/xml/XmlReader.kt

@@ -30,6 +30,7 @@ abstract class XmlReader(stream: InputStream) : LayerReader() {
     private var ignoring: Boolean = false
     private var currentDepth: Int = 0
     private var ignoreDepth: Int? = null
+    private var insideWordTag: Boolean = false
 
     abstract val nerTags: Array<String>
     abstract val documentTags: Array<String>
@@ -66,7 +67,7 @@ abstract class XmlReader(stream: InputStream) : LayerReader() {
                         in documentTags -> docID = currentXmlID
                         in paragraphTags -> parID = currentXmlID
                         in sentenceTags -> sentID = currentXmlID
-                        in wordTags -> wordID = currentXmlID
+                        in wordTags -> { insideWordTag = true; wordID = currentXmlID }
                     }
                 }
 
@@ -170,14 +171,22 @@ abstract class XmlReader(stream: InputStream) : LayerReader() {
     }
 
     private fun parseChars() {
-        val words = reader.text.ifBlank { null }?.trim()?.split(whitespace) ?: emptyList()
-        for ((j, word) in words.withIndex()) {
-            if (j > 0) newWordform()
-            literal += word
+        // if we are within a word tag, simply add all text, given that there are no (valid) word boundaries
+        if (insideWordTag) {
+            reader.text.ifBlank { null }?.trim()?.let { literal += it}
+        }
+        // Outside a word tag, treat spaces as new words
+        else {
+            val words = reader.text.ifBlank { null }?.split(whitespace) ?: emptyList()
+            for ((j, word) in words.withIndex()) {
+                if (j > 0) newWordform()
+                literal += word
+            }
         }
     }
 
     override fun newWordform() {
+        insideWordTag = false
         if (literal.isBlank()) return
         val annotations = buildMap {
             lemma?.ifBlank { null }?.let { put(Annotation.LEMMA, it) }

server/src/test/kotlin/org/ivdnt/galahad/formats/tei/TeiReaderTest.kt

@@ -26,6 +26,11 @@ internal class TeiReaderTest : ReaderTest() {
             assertLayerAndText("formats/tei/reader/mixed-tags/")
         }
 
+        @Test
+        fun `Import TEI with whitespace in w tags`() {
+            assertLayerAndText("formats/tei/reader/space-in-tag/")
+        }
+
         @Test
         fun `Import TEI with w-tags without spaces in between`() {
             assertLayerAndText("formats/tei/reader/nospaces")

server/src/test/resources/formats/tei/reader/space-in-tag/input.tei.xml

@@ -0,0 +1,10 @@
+<TEI>
+    <text>
+        <p>
+            <w lemma="fraai"><join n="mw_fraai"/> fraaie</w>
+            <w lemma="historie">   historie   <join n="mw_historie"/></w>
+            <w lemma="en">   ende   <join n="mw_en"/></w>
+            <w lemma="alwaar"><join n="mw_alwaar"/> alwaer</w>
+        </p>
+    </text>
+</TEI>

server/src/test/resources/formats/tei/reader/space-in-tag/layer.json

@@ -0,0 +1,45 @@
+{
+  "documents" : [ {
+    "id" : "d1",
+    "paragraphs" : [ {
+      "id" : "d1.p1",
+      "sentences" : [ {
+        "id" : "d1.p1.s1",
+        "terms" : [ {
+          "annotations" : {
+            "token" : "fraaie",
+            "lemma" : "fraai",
+            "group" : "mw_fraai"
+          },
+          "id" : "d1.p1.s1.w1",
+          "offset" : 0
+        }, {
+          "annotations" : {
+            "token" : "historie",
+            "lemma" : "historie",
+            "group" : "mw_historie"
+          },
+          "id" : "d1.p1.s1.w2",
+          "offset" : 7
+        }, {
+          "annotations" : {
+            "token" : "ende",
+            "lemma" : "en",
+            "group" : "mw_en"
+          },
+          "id" : "d1.p1.s1.w3",
+          "offset" : 16
+        }, {
+          "annotations" : {
+            "token" : "alwaer",
+            "lemma" : "alwaar",
+            "group" : "mw_alwaar"
+          },
+          "id" : "d1.p1.s1.w4",
+          "offset" : 21
+        } ]
+      } ]
+    } ]
+  } ],
+  "id" : "UUID"
+}

server/src/test/resources/formats/tei/reader/space-in-tag/plaintext.txt

@@ -0,0 +1 @@
+fraaie historie ende alwaer