More NER support

Author: PrinsINT

server/build.gradle.kts

@@ -54,5 +54,6 @@ dependencies {
 
 tasks.withType<Test> {
     environment(mapOf("profile" to "dev"))
+    systemProperty("line.separator", "\n")
     useJUnitPlatform()
 }

server/src/main/kotlin/org/ivdnt/galahad/annotations/AnnotationReader.kt

@@ -1,5 +1,7 @@
 package org.ivdnt.galahad.annotations
 
+import kotlin.collections.plus
+
 abstract class AnnotationReader {
     val layer: Layer by lazy { read() }
 
@@ -16,9 +18,9 @@ abstract class AnnotationReader {
     protected var sentID: String? = null
     protected var wordID: String? = null
 
-    private fun docID(): String = docID ?: "d$dIndex"
-    private fun parID(): String = parID ?: "p$pIndex"
-    private fun sentID(): String = sentID ?: "s$sIndex"
+    protected fun docID(): String = docID ?: "d$dIndex"
+    protected fun parID(): String = parID ?: "p$pIndex"
+    protected fun sentID(): String = sentID ?: "s$sIndex"
     protected fun wordID(): String = wordID ?: "w$wIndex"
 
     private val wIndex: Int get() = terms.size + 1
@@ -46,6 +48,19 @@ abstract class AnnotationReader {
 
     protected open fun newSentence() {
         if (terms.isNotEmpty()) {
+            // loop through all the terms and turn the NER into a span
+            val indices = mutableListOf<Int>()
+            terms.forEachIndexed { i, t ->
+                t.ner?.let {
+                   indices += i
+                }
+            }
+            if (indices.isNotEmpty()) {
+                val ners = mutableListOf<TermSpan>()
+                ners += TermSpan(indices, terms[indices.first()].annotationHead(Annotation.NER)!!)
+                spans[Annotation.NER] = ners.toTypedArray()
+            }
+
             sentences.add(SentenceLayer(sentID(), terms.toTypedArray(), spans.toMap()))
             terms.clear()
             spans.clear()

server/src/main/kotlin/org/ivdnt/galahad/annotations/Layer.kt

@@ -1,6 +1,7 @@
 package org.ivdnt.galahad.annotations
 
 import com.fasterxml.jackson.annotation.JsonIgnore
+import java.util.UUID
 
 const val SOURCE_LAYER_NAME: String = "sourceLayer"
 
@@ -9,7 +10,8 @@ const val SOURCE_LAYER_NAME: String = "sourceLayer"
  * Those may be split into paragraphs, sentences, etc.
  */
 class Layer(
-    val documents: Array<DocumentLayer>
+    val documents: Array<DocumentLayer>,
+    val id: String = UUID.randomUUID().toString(),
 ) {
     @get:JsonIgnore
     val spans: Map<Annotation, Sequence<TermSpan>> by lazy {
@@ -23,10 +25,13 @@ class Layer(
             }
         }
     }
+
     @get:JsonIgnore
     val summary: LayerSummary by lazy { LayerSummary(tokens = terms.count()) }
+
     @get:JsonIgnore
     val preview: LayerPreview by lazy { LayerPreview(terms.take(LAYER_PREVIEW_LENGTH).toList()) }
+
     @get:JsonIgnore
     val terms: Sequence<Term> by lazy {
         documents.asSequence().flatMap { document ->
@@ -41,7 +46,7 @@ class Layer(
     override fun toString(): String = documents.joinToString("\n\n") + "\n" // Unix convention EOF
 
     companion object {
-        val EMPTY: Layer = Layer(emptyArray())
+        val EMPTY: Layer = Layer(emptyArray(), "")
     }
 }
 
@@ -64,5 +69,10 @@ class SentenceLayer(
     val terms: Array<Term>,
     val spans: Map<Annotation, Array<TermSpan>>,
 ) {
-    override fun toString(): String = terms.joinToString("") { it.token + (if (it.spaceAfter == false) "" else " ") }
+    override fun toString(): String = buildString {
+        terms.forEachIndexed { i, t ->
+            append(t.token)
+            if (i != terms.lastIndex) append(t.space)
+        }
+    }
 }

server/src/main/kotlin/org/ivdnt/galahad/annotations/Term.kt

@@ -11,6 +11,9 @@ class Term(
 ) {
     val spaceAfter: Boolean? = if (spaceAfter == false) false else null
 
+    @get:JsonIgnore
+    val space: String = if (spaceAfter == false) "" else " "
+
     @get:JsonIgnore
     val token: String = annotations[Annotation.TOKEN]!!
 

server/src/main/kotlin/org/ivdnt/galahad/app/User.kt

@@ -10,7 +10,7 @@ class User(
     companion object {
         private const val USERNAME: String = "user"
         private val ADMIN_FILE: File = File("data/admins/admins.txt")
-        private val DEFAULT_USER: User get() = User(id = USERNAME, isAdmin = isAdmin(USERNAME))
+        val DEFAULT_USER: User get() = User(id = USERNAME, isAdmin = isAdmin(USERNAME))
 
         private fun isAdmin(username: String): Boolean {
             if (!ADMIN_FILE.exists()) return false // When no admins are set, no one is admin by default

server/src/main/kotlin/org/ivdnt/galahad/corpora/documents/Document.kt

@@ -80,7 +80,7 @@ class Document(
             // First try to access the layer. If the file is invalid, this will throw.
             val sourceLayer = internalFile.layer
             // Set sourceLayer as job. Note that if we threw, we don't unnecessarily create a job folder, keeping the disk clean.
-            corpus.jobs.createOrThrow(SOURCE_LAYER_NAME).jobDocuments.createOrThrow(doc.name).layer = sourceLayer
+            corpus.jobs.createOrThrow(SOURCE_LAYER_NAME).setLayer(doc.name, sourceLayer)
             // plaintext
             ThreadPoolUtil.pool.execute {
                 doc.plaintextFile.writeText(internalFile.plaintext)

server/src/main/kotlin/org/ivdnt/galahad/corpora/documents/DocumentMetadata.kt

@@ -24,7 +24,7 @@ data class DocumentMetadata(
     /** Last modified timestamp in milliseconds. */
     val lastModified: Long,
     /** UUID of the document. */
-    val uuid: UUID,
+    val uuid: String,
     /** Annotation types in the source layer. */
     val annotations: Set<Annotation>,
 ) {
@@ -42,7 +42,7 @@ data class DocumentMetadata(
                 layerPreview = file.layer.preview,
                 layerSummary = file.layer.summary,
                 lastModified = System.currentTimeMillis(),
-                uuid = UUID.randomUUID(),
+                uuid = file.layer.id,
                 annotations = file.layer.terms.flatMap { it.annotations.keys }.toSet()
             )
         }

server/src/main/kotlin/org/ivdnt/galahad/corpora/jobs/Job.kt

@@ -113,11 +113,12 @@ class Job(
         override fun set() = JobMetadata.create(this@Job)
     }
 
-    fun layer(doc: Document): Layer = layer(doc.name)
-    fun layer(key: String): Layer = jobDocuments.readOrNull(key)?.layer ?: Layer.EMPTY
-    fun setLayerForKey(key: String, layer: Layer) {
+    fun getLayer(doc: Document): Layer = getLayer(doc.name)
+    fun getLayer(key: String): Layer = jobDocuments.readOrNull(key)?.layer ?: Layer.EMPTY
+    fun setLayer(key: String, layer: Layer) {
         jobDocuments.createOrThrow(key).layer = layer
     }
+    fun setLayer(doc: Document, layer: Layer): Unit = setLayer(doc.name, layer)
 
     //////////////////////////////////////////////////////
     // TODO: check everything below
@@ -256,7 +257,7 @@ class Job(
     }
 
     companion object {
-        private val mapper: ObjectMapper by lazy { ObjectMapper() }
+        private val mapper: ObjectMapper = ObjectMapper()
 
         private fun <T : Any> taggerRequest(
             job: Job, route: String, method: HttpMethod, type: Class<T>,

server/src/main/kotlin/org/ivdnt/galahad/evaluation/comparison/LayerComparison.kt

@@ -17,8 +17,8 @@ val PUNCTUATION: Array<Char> = arrayOf(',', '.', '?', '!', ':', ';', ')', '(', '
  * (Still, aggregating these matches is up to the (corpus/documents) evaluation classes)
  */
 open class LayerComparison(
-    private val hypothesisLayer: Layer,
-    private val referenceLayer: Layer,
+    hypothesisLayer: Layer,
+    referenceLayer: Layer,
     private val layerFilter: LayerFilter? = null,
 ) {
     constructor(export: DocumentExport) : this(export.layer, export.sourceLayer)

server/src/main/kotlin/org/ivdnt/galahad/evaluation/confusion/CorpusConfusion.kt

@@ -39,8 +39,8 @@ class CorpusConfusion(
         corpus.documents.readAll().forEach {
             add(
                 DocumentConfusion(
-                    hypothesisJob.layer(it),
-                    referenceJob.layer(it),
+                    hypothesisJob.getLayer(it),
+                    referenceJob.getLayer(it),
                     layerFilter,
                     annotation,
                 )