Fix XmlReader not ingnoring tags

Author: PrinsINT

server/src/main/kotlin/org/ivdnt/galahad/app/Galahad.kt

@@ -52,6 +52,7 @@ const val JOB_DOCUMENT_URL: String = "$JOB_URL/documents/{document}"
 const val EVALUATION_URL: String = "$JOB_URL/evaluation"
 const val DISTRIBUTION_URL: String = "$EVALUATION_URL/distribution"
 const val TOKEN_FREQUENCY_URL: String = "$EVALUATION_URL/frequency"
+const val ENTITIES_URL: String = "$JOB_DOCUMENT_URL/entities"
 const val METRICS_URL: String = "$EVALUATION_URL/metrics"
 const val METRICS_SAMPLES_URL: String = "$METRICS_URL/download"
 const val CONFUSION_URL: String = "$EVALUATION_URL/confusion"

server/src/main/kotlin/org/ivdnt/galahad/formats/folia/FoliaReader.kt

@@ -33,7 +33,7 @@ class FoliaReader(
         private val SENTENCE_TAGS = arrayOf("s", "utt")
         private val WORD_TAGS = arrayOf("w")
         private val WORD_DATA_TAGS = arrayOf("w", "lemma", "pos")
-        private val IGNORABLE_TAGS = arrayOf("morphology", "note", "figure", "comment", "original", "suggestion")
+        private val IGNORABLE_TAGS = arrayOf("morphology", "note", "figure", "comment", "original", "suggestion", "metadata")
         private val SPAN_TAGS = arrayOf("entity")
         private val SPAN_DATA_TAGS = arrayOf("wref", "entity")
     }

server/src/main/kotlin/org/ivdnt/galahad/formats/xml/XmlReader.kt

@@ -1,13 +1,10 @@
 package org.ivdnt.galahad.formats.xml
 
+import org.ivdnt.galahad.annotations.*
 import org.ivdnt.galahad.annotations.Annotation
-import org.ivdnt.galahad.annotations.AnnotationReader
-import org.ivdnt.galahad.annotations.Layer
-import org.ivdnt.galahad.annotations.Term
-import org.ivdnt.galahad.annotations.TermSpan
 import org.ivdnt.galahad.util.XmlUtil
 import java.io.InputStream
-import java.util.UUID
+import java.util.*
 import javax.xml.XMLConstants
 import javax.xml.stream.XMLStreamConstants
 import javax.xml.stream.XMLStreamReader
@@ -25,6 +22,8 @@ abstract class XmlReader(stream: InputStream) : AnnotationReader() {
     private val currentXmlID: String?
         get() = reader.getAttributeValue(XMLConstants.XML_NS_URI, "id")?.ifBlank { null }
     private var ignoring: Boolean = false
+    private var currentDepth: Int = 0
+    private var ignoreDepth: Int? = null
 
     abstract val spanTags: Array<String>
     abstract val spanDataTags: Array<String>
@@ -35,7 +34,6 @@ abstract class XmlReader(stream: InputStream) : AnnotationReader() {
     abstract val wordDataTags: Array<String>
     abstract val ignorableTags: Array<String>
 
-
     final override fun read(): Layer {
         // retrieve the XML ID of the document root
         var rootID: String = UUID.randomUUID().toString()
@@ -64,7 +62,7 @@ abstract class XmlReader(stream: InputStream) : AnnotationReader() {
                     }
                 }
                 XMLStreamConstants.CHARACTERS -> if (!ignoring) parseChars()
-                XMLStreamConstants.END_ELEMENT -> if (!ignoring) {
+                XMLStreamConstants.END_ELEMENT -> if (!shouldIgnore()) {
                     when (reader.localName) {
                         in documentTags -> newDocument()
                         in paragraphTags -> newParagraph()
@@ -100,7 +98,22 @@ abstract class XmlReader(stream: InputStream) : AnnotationReader() {
 
     protected abstract fun parseAttrs()
 
-    private fun shouldIgnore(): Boolean = ignoring.also { ignoring = reader.localName in ignorableTags }
+    private fun shouldIgnore(): Boolean {
+        if (reader.isStartElement) {
+            currentDepth++
+            if (!ignoring && reader.localName in ignorableTags) {
+                ignoring = true
+                ignoreDepth = currentDepth
+            }
+        } else if (reader.isEndElement) {
+            if (currentDepth == ignoreDepth) {
+                ignoring = false
+                ignoreDepth = null
+            }
+            currentDepth--
+        }
+        return ignoring
+    }
 
     private fun parseChars() {
         val words = reader.text.ifBlank { null }?.split(whitespace) ?: emptyList()

server/src/main/kotlin/org/ivdnt/galahad/web/controller/EvaluationController.kt

@@ -9,6 +9,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse
 import org.apache.logging.log4j.kotlin.Logging
 import org.ivdnt.galahad.annotations.Annotation
 import org.ivdnt.galahad.annotations.SOURCE_LAYER_NAME
+import org.ivdnt.galahad.annotations.Term
 import org.ivdnt.galahad.app.*
 import org.ivdnt.galahad.evaluation.comparison.TermComparison
 import org.ivdnt.galahad.evaluation.confusion.Confusion
@@ -191,4 +192,12 @@ class EvaluationController(
         @PathVariable @Parameter(description = "Tagger name or sourceLayer") job: String,
         @RequestParam(defaultValue = SOURCE_LAYER_NAME) @Parameter(description = "Tagger name or sourceLayer") reference: String? = SOURCE_LAYER_NAME,
     ): CorpusMetrics = evaluationService.getTokenFrequency(corpus, job, reference)
+
+    @CrossOrigin
+    @GetMapping(ENTITIES_URL)
+    fun getEntities(
+        @PathVariable @Parameter(description = "Corpus UUID") corpus: UUID,
+        @PathVariable @Parameter(description = "Document name") document: String,
+        @PathVariable @Parameter(description = "Tagger name or sourceLayer") job: String,
+    ): List<Pair<String, List<Term>>> = evaluationService.getEntities(corpus, document, job)
 }

server/src/main/kotlin/org/ivdnt/galahad/web/service/EvaluationService.kt

@@ -4,6 +4,7 @@ import jakarta.servlet.http.HttpServletRequest
 import jakarta.servlet.http.HttpServletResponse
 import org.ivdnt.galahad.annotations.Annotation
 import org.ivdnt.galahad.annotations.SOURCE_LAYER_NAME
+import org.ivdnt.galahad.annotations.Term
 import org.ivdnt.galahad.app.User
 import org.ivdnt.galahad.corpora.CorpusMetadata
 import org.ivdnt.galahad.evaluation.comparison.*
@@ -277,4 +278,9 @@ class EvaluationService(val corpora: CorporaService) {
         )
         return cm
     }
+
+    fun getEntities(corpus: UUID, document: String, job: String): List<Pair<String, List<Term>>>{
+        val layer = corpora.readAsReaderOrThrow(corpus, user).jobs.readOrThrow(job).getLayer(document)
+        return layer.documents.flatMap { it.paragraphs.flatMap { it.sentences.flatMap { sent -> sent.spans[Annotation.NER]?.map { span -> span.value to span.indices.map { sent.terms[it] } } ?: emptyList() } } }
+    }
 }

server/src/test/kotlin/org/ivdnt/galahad/formats/conllu/ConlluExportTest.kt

@@ -1,45 +1,40 @@
 package org.ivdnt.galahad.formats.folia
 
-import org.ivdnt.galahad.formats.Resource
-import org.ivdnt.galahad.formats.assertPlaintextAndSourcelayer
-import org.ivdnt.galahad.formats.tsv.TsvFile
-import org.junit.jupiter.api.Assertions.assertEquals
+import org.ivdnt.galahad.util.TestUtil
 import org.junit.jupiter.api.Test
 
-internal class FoliaImportTest {
+class FoliaImportTest {
 
     @Test
     fun `Import doc with correction tags`() {
-        val file = FoliaFile(Resource.get("folia/corrections/input.folia.xml"))
-        assertPlaintextAndSourcelayer("folia/corrections", file)
+        val file = FoliaFile(TestUtil.get("formats/folia/reader/corrections/input.folia.xml"))
+        TestUtil.assertPlaintextAndSourcelayer("formats/folia/reader/corrections", file)
     }
 
     @Test
     fun `Import doc with multiple pos & lemma per word, and morphology tags`() {
-        val foliaFile = FoliaFile(Resource.get("folia/hauraki/input.folia.xml"))
-        val expectedPlain = Resource.get("folia/hauraki/plaintext.txt").readText()
-
-        assertEquals(expectedPlain, foliaFile.plaintext.trim())
-
-        val sourceLayer = foliaFile.layer
-        assertEquals(97, sourceLayer.wordForms.size)
-        assertEquals(97, sourceLayer.terms.size)
-
-        val tsvFile = TsvFile(Resource.get("folia/hauraki/pie.tsv"))
-
-        val mergeLayer = tsvFile.mapOnPlainText(foliaFile.plaintext, "mappedLayer")
-        assertEquals(89, mergeLayer.wordForms.size)
-        assertEquals(89, mergeLayer.terms.size)
+        val file = FoliaFile(TestUtil.get("formats/folia/reader/morphology/input.folia.xml"))
+        TestUtil.assertPlaintextAndSourcelayer("formats/folia/reader/morphology", file)
+
+//        val sourceLayer = foliaFile.layer
+//        assertEquals(97, sourceLayer.wordForms.size)
+//        assertEquals(97, sourceLayer.terms.size)
+//
+//        val tsvFile = TsvFile(TestUtil.get("folia/hauraki/pie.tsv"))
+//
+//        val mergeLayer = tsvFile.mapOnPlainText(foliaFile.plaintext, "mappedLayer")
+//        assertEquals(89, mergeLayer.wordForms.size)
+//        assertEquals(89, mergeLayer.terms.size)
     }
 
     @Test
     fun `Import plaintext twined with many style tags`() {
-        val file = FoliaFile(Resource.get("folia/twine/twine.folia.xml"))
-        val plaintext = Resource.get("folia/twine/plaintext.txt").readText()
-        assertEquals(plaintext, file.plaintext)
-        // Source layer should be empty, there are no source annotations
-        val sourceLayer = file.layer
-        assertEquals(0, sourceLayer.wordForms.size)
-        assertEquals(0, sourceLayer.terms.size)
+        val file = FoliaFile(TestUtil.get("formats/folia/reader/twine/twine.folia.xml"))
+        TestUtil.assertPlainText("formats/folia/reader/twine", file)
+//        assertEquals(plaintext, file.plaintext)
+//        // Source layer should be empty, there are no source annotations
+//        val sourceLayer = file.layer
+//        assertEquals(0, sourceLayer.wordForms.size)
+//        assertEquals(0, sourceLayer.terms.size)
     }
 }