Write TEI as xml stream to zip

Author: PrinsINT

server/.run/Galahad.run.xml

@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Galahad" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
+  <configuration default="true" name="Galahad" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
     <option name="ALTERNATIVE_JRE_PATH" value="graalvm-ce-23" />
     <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="true" />
     <envs>

server/build.gradle.kts

@@ -49,7 +49,7 @@ dependencies {
     implementation("com.github.ben-manes.caffeine:caffeine:3.2.0")
 
     // immutable arrays
-    implementation("com.danrusu.pods4k:pods4k:0.7.0")
+    // implementation("com.danrusu.pods4k:pods4k:0.7.0")
 }
 
 tasks.withType<Test> {

server/src/main/kotlin/org/ivdnt/galahad/annotations/AnnotationReader.kt

@@ -11,7 +11,7 @@ abstract class AnnotationReader(
     protected val paragraphs: MutableList<ParagraphLayer> = mutableListOf()
     protected val sentences: MutableList<SentenceLayer> = mutableListOf()
     protected val terms: MutableList<Term> = mutableListOf()
-    protected val spans: MutableMap<Annotation, List<TermSpan>> = mutableMapOf()
+    protected val spans: MutableMap<Annotation, Array<TermSpan>> = mutableMapOf()
 
     protected var offset: Int = 0
 
@@ -35,22 +35,22 @@ abstract class AnnotationReader(
     protected open fun newDocument() {
         newParagraph()
         if (paragraphs.isNotEmpty()) {
-            documents.add(DocumentLayer(docID(), paragraphs.toList()))
+            documents.add(DocumentLayer(docID(), paragraphs.toTypedArray()))
             paragraphs.clear()
         }
     }
 
     protected open fun newParagraph() {
         newSentence()
         if (sentences.isNotEmpty()) {
-            paragraphs.add(ParagraphLayer(parID(), sentences.toList()))
+            paragraphs.add(ParagraphLayer(parID(), sentences.toTypedArray()))
             sentences.clear()
         }
     }
 
     protected open fun newSentence() {
         if (terms.isNotEmpty()) {
-            sentences.add(SentenceLayer(sentID(), terms.toList(), spans.toMap()))
+            sentences.add(SentenceLayer(sentID(), terms.toTypedArray(), spans.toMap()))
             terms.clear()
             spans.clear()
         }

server/src/main/kotlin/org/ivdnt/galahad/annotations/Layer.kt

@@ -9,7 +9,7 @@ const val SOURCE_LAYER_NAME: String = "sourceLayer"
  * Those may be split into paragraphs, sentences, etc.
  */
 class Layer(
-    val documents: List<DocumentLayer>
+    val documents: Array<DocumentLayer>
 ) {
     @get:JsonIgnore
     val spans: Map<Annotation, Sequence<TermSpan>> by lazy {
@@ -41,28 +41,28 @@ class Layer(
     override fun toString(): String = documents.joinToString("\n\n") + "\n" // Unix convention EOF
 
     companion object {
-        val EMPTY: Layer = Layer(emptyList())
+        val EMPTY: Layer = Layer(emptyArray())
     }
 }
 
 class DocumentLayer(
     val id: String,
-    val paragraphs: List<ParagraphLayer>,
+    val paragraphs: Array<ParagraphLayer>,
 ) {
     override fun toString(): String = paragraphs.joinToString("\n\n")
 }
 
 class ParagraphLayer(
     val id: String,
-    val sentences: List<SentenceLayer>,
+    val sentences: Array<SentenceLayer>,
 ) {
     override fun toString(): String = sentences.joinToString("\n")
 }
 
 class SentenceLayer(
     val id: String,
-    val terms: List<Term>,
-    val spans: Map<Annotation, List<TermSpan>>,
+    val terms: Array<Term>,
+    val spans: Map<Annotation, Array<TermSpan>>,
 ) {
     override fun toString(): String = terms.joinToString("") { it.token + (if (it.spaceAfter == false) "" else " ") }
 }

server/src/main/kotlin/org/ivdnt/galahad/corpora/documents/Document.kt

@@ -6,6 +6,7 @@ import org.ivdnt.galahad.corpora.Corpus
 import org.ivdnt.galahad.files.DiskValue
 import org.ivdnt.galahad.files.GalahadFolder
 import org.ivdnt.galahad.formats.InternalFile
+import org.ivdnt.galahad.util.ThreadPoolUtil
 import java.io.File
 
 /**
@@ -71,17 +72,19 @@ class Document(
             val doc = Document(dir)
 
             // uploaded file
-            file.copyTo(doc.uploadedFile, overwrite = true)
-
+            ThreadPoolUtil.pool.execute {
+                file.copyTo(doc.uploadedFile, overwrite = true)
+            }
             // plaintext & sourceLayer
             val internalFile = InternalFile.create(file)
             // First try to access the layer. If the file is invalid, this will throw.
             val sourceLayer = internalFile.layer
             // Set sourceLayer as job. Note that if we threw, we don't unnecessarily create a job folder, keeping the disk clean.
             corpus.jobs.createOrThrow(SOURCE_LAYER_NAME).jobDocuments.createOrThrow(doc.name).layer = sourceLayer
             // plaintext
-            doc.plaintextFile.writeText(internalFile.plaintext)
-
+            ThreadPoolUtil.pool.execute {
+                doc.plaintextFile.writeText(internalFile.plaintext)
+            }
             // metadata; needs to be serialized as well
             DiskValue<DocumentMetadata>(doc.metadataFile).write(DocumentMetadata.create(internalFile))
 

server/src/main/kotlin/org/ivdnt/galahad/corpora/documents/DocumentFormat.kt

@@ -44,18 +44,14 @@ enum class DocumentFormat(val identifier: String, val extension: String) {
         /**
          * Induce the format of a document based on its file extension and content (e.g. root XML node).
          */
-        fun fromFile(file: File): DocumentFormat {
-            val format = when (file.extension) {
-                "tsv" -> Tsv
-                "folia" -> Folia
-                "conllu" -> Conllu
-                "xml", "tei" -> determineXmlFormat(file) // TEI can be either P4 or P5, so still check.
-                "txt" -> Txt
-                "naf" -> Naf
-                else -> Unknown
-            }
-            logger.debug { "Induced format $format for file ${file.name}" }
-            return format
+        fun fromFile(file: File): DocumentFormat = when (file.extension) {
+            "tsv" -> Tsv
+            "folia" -> Folia
+            "conllu" -> Conllu
+            "xml", "tei" -> determineXmlFormat(file) // TEI can be either P4 or P5, so still check.
+            "txt" -> Txt
+            "naf" -> Naf
+            else -> Unknown
         }
 
         /**
@@ -76,18 +72,5 @@ enum class DocumentFormat(val identifier: String, val extension: String) {
             }
             return Unknown // No root element found
         }
-
-        /** Differentiate between TeiP5 and TeiP5Legacy by the presence of pos as an XML attribute.
-         * - 1 or more pos are present, it's TeiP5
-         * - if no pos are present, but at least one type is present, it's TeiP5Legacy
-         * - if no pos or type are present, it's unannotated and we default to TeiP5
-         */
-        private fun determineTeiP5Format(xmlDoc: Document): DocumentFormat {
-            val xPath: XPath = XPathFactory.newInstance().newXPath()
-            val numPos = xPath.compile("count(.//w[@pos])").evaluate(xmlDoc, XPathConstants.NUMBER) as Double
-            val numTypes = xPath.compile("count(.//w[@type])").evaluate(xmlDoc, XPathConstants.NUMBER) as Double
-            if (numTypes == 0.0 || numPos > 0) return TeiP5
-            return TeiP5Legacy // No pos but at least one type: assume legacy mode
-        }
     }
 }

server/src/main/kotlin/org/ivdnt/galahad/export/CmdiMetadata.kt

@@ -8,6 +8,7 @@ import org.ivdnt.galahad.util.child
 import org.ivdnt.galahad.util.childElements
 import org.ivdnt.galahad.util.ifNullOrBlank
 import java.io.File
+import java.io.OutputStream
 import java.text.SimpleDateFormat
 import java.util.*
 import javax.xml.transform.dom.DOMSource
@@ -33,10 +34,7 @@ class CmdiMetadata(val export: DocumentExport) {
     private val sourceName = corpus.sourceName.ifNullOrBlank { "!No source name defined!" }
     private val sourceUrl = corpus.sourceURL?.toString().ifNullOrBlank { "!No source URL defined!" }
 
-    /** After initialization this file will contain the CMDI */
-    val file: File
-
-    init {
+    fun write(out: OutputStream) {
         // Header
         val header = root.child("cmd:Header")
         header.child("cmd:MdCreationDate").textContent = date
@@ -93,8 +91,7 @@ class CmdiMetadata(val export: DocumentExport) {
             child("cmdp:dayTo").textContent = "---$day"
         }
         // Write to disk
-        file = tmp_dir.resolve("CMDI-$docTitle.xml")
-        XmlUtil.transformer.transform(DOMSource(xml), StreamResult(file.outputStream()))
+        XmlUtil.transformer.transform(DOMSource(xml), StreamResult(out))
     }
 
     companion object {

server/src/main/kotlin/org/ivdnt/galahad/export/CorpusExport.kt

@@ -10,6 +10,7 @@ import org.ivdnt.galahad.corpora.documents.Documents
 import org.ivdnt.galahad.corpora.jobs.Job
 import org.ivdnt.galahad.exceptions.MergeNotImplementedException
 import org.ivdnt.galahad.taggers.Tagger
+import org.ivdnt.galahad.util.FileMapper
 import org.ivdnt.galahad.util.createZipFile
 import java.io.File
 import java.io.OutputStream
@@ -38,16 +39,16 @@ class CorpusExport private constructor(
     }
 
 
-    fun formatMapper(doc: Document): File {
+    private fun formatMapper(doc: Document, out: OutputStream) {
         try {
             // Document conversions.
             val docExport = DocumentExport.create(this, doc)
-            return if (shouldMerge && mergeFormatMatches(doc, format)) {
+            if (shouldMerge && mergeFormatMatches(doc, format)) {
                 logger.info("Merging ${doc.name} of format ${doc.metadata.format}")
-                docExport.merge()
+                docExport.merge(out)
             } else {
                 logger.info("Converting ${doc.name} of format ${doc.metadata.format} to $format")
-                docExport.convert()
+                docExport.convert(out)
             }
         } catch (e: MergeNotImplementedException) {
             throw e
@@ -63,13 +64,8 @@ class CorpusExport private constructor(
         out: OutputStream,
     ) {
         val documents = corpus.documents.readAll().filter { DocumentExport.create(this, it).layer != Layer.EMPTY }
-        val convertedDocs = documents.asSequence().map(::formatMapper)
-        val docsToCmdi = documents.asSequence().map { CmdiMetadata(DocumentExport.create(this, it)).file }
-        val cmdiZip = createZipFile(docsToCmdi, includeCMDI = true)
-        // rename the cmdiZip to "metadata"
-        val dest = createTempDirectory("metadata").toFile().resolve("metadata.zip")
-        Files.move(cmdiZip.toPath(), dest.toPath())
-        createZipFile(convertedDocs + dest, out)
+        val seq: Sequence<FileMapper> = documents.asSequence().map { doc -> doc.name to { out -> formatMapper(doc, out) } }
+        createZipFile(seq, out)
     }
 
     companion object {

server/src/main/kotlin/org/ivdnt/galahad/export/DocumentExport.kt

@@ -9,6 +9,7 @@ import org.ivdnt.galahad.corpora.documents.DocumentFormat
 import org.ivdnt.galahad.corpora.jobs.Job
 import org.ivdnt.galahad.taggers.Tagger
 import java.io.File
+import java.io.OutputStream
 import kotlin.io.path.createTempDirectory
 
 class DocumentExport private constructor(
@@ -29,6 +30,10 @@ class DocumentExport private constructor(
     fun convert(): File = file.also { LayerConverter.create(this).convert(file.outputStream()) }
     fun merge(): File = file.also { LayerMerger.create(this).merge(file.outputStream()) }
 
+    fun convert(out: OutputStream): Unit = LayerConverter.create(this).convert(out)
+    fun merge(out: OutputStream): Unit = LayerMerger.create(this).merge(out)
+    fun cmdi(): CmdiMetadata = CmdiMetadata(this)
+
     companion object {
         fun create(export: CorpusExport, doc: Document): DocumentExport = create(export, doc.name)
 

server/src/main/kotlin/org/ivdnt/galahad/export/LayerConverter.kt

@@ -5,6 +5,7 @@ import org.ivdnt.galahad.exceptions.InvalidDocumentFormatException
 import org.ivdnt.galahad.formats.conllu.ConlluConverter
 import org.ivdnt.galahad.formats.folia.FoliaConverter
 import org.ivdnt.galahad.formats.naf.NafConverter
+import org.ivdnt.galahad.formats.tei.TeiAaltoConverter
 import org.ivdnt.galahad.formats.tei.TeiConverter
 import org.ivdnt.galahad.formats.tsv.TsvConverter
 import org.ivdnt.galahad.formats.txt.TxtConverter
@@ -20,7 +21,7 @@ abstract class LayerConverter(protected val export: DocumentExport) {
             DocumentFormat.Naf -> NafConverter(export)
             DocumentFormat.Txt -> TxtConverter(export)
             DocumentFormat.Conllu -> ConlluConverter(export)
-            DocumentFormat.TeiP5 -> TeiConverter(export)
+            DocumentFormat.TeiP5 -> TeiAaltoConverter(export)
             else -> throw InvalidDocumentFormatException("Unsupported export conversion format: ${export.format}")
         }
     }