Start work on synchronous job tagging

Author: Vincent Prins

server/src/main/kotlin/org/ivdnt/galahad/corpora/CorpusMetadata.kt

@@ -47,15 +47,12 @@ class CorpusMetadata(
 ) {
     companion object {
         fun create(corpus: Corpus): CorpusMetadata {
-            val allJobs = corpus.jobs.readAll()
-            val allDocs = corpus.documents.readAll()
             val meta = CorpusMetadata(
                 // Immutable/calculated fields
                 // sourceAnnotationTypes = uniqueAnnotations,
                 uuid = UUID.fromString(corpus.name),
-                activeJobs = allJobs.count { it.isActive },
-                numResults = allJobs.count { it.hasResult },
-                numDocs = allDocs.size,
+                numResults = corpus.jobs.readAll().count { it.hasResult },
+                numDocs = corpus.documents.readAll().size,
                 sizeInBytes = corpus.sizeInBytes,
                 lastModified = System.currentTimeMillis(),
             )

server/src/main/kotlin/org/ivdnt/galahad/documents/Document.kt

@@ -23,25 +23,11 @@ class Document(
     dir: File,
 ) : GalahadFolder(dir), Logging {
     // Files in the document folder.
-    val plaintextFile: File = dir.resolve(PLAINTEXT_FILE)
     private val metadataFile = dir.resolve(METADATA_FILE)
     val uploadedFile: File = dir.resolve("uploaded").resolve(name)
 
     // Values in those files.
 
-    /** The plaintext content of the document. */
-    val plaintext: String by lazy {
-        try {
-            plaintextFile.readText()
-        } catch (e: Exception) {
-            logger.warn("Error reading plaintext file, creating new plaintext", e)
-            val internalFile = InternalFile.create(uploadedFile)
-            val text = internalFile.plaintext
-            plaintextFile.writeText(text)
-            text
-        }
-    }
-
     /**
      * The UUID is only used as a metadata pid when converting a layer to TEI (for now).
      * When merging TEI, it is only used if the document itself defines no pid.
@@ -62,7 +48,6 @@ class Document(
 
     internal companion object {
         private const val METADATA_FILE = "metadata.json"
-        private const val PLAINTEXT_FILE = "plaintext.txt"
 
         /**
          * Create a new document folder from an uploaded file and fill it with the necessary data.
@@ -81,10 +66,6 @@ class Document(
             val sourceLayer = internalFile.layer
             // Set sourceLayer as job. Note that if we threw, we don't unnecessarily create a job folder, keeping the disk clean.
             corpus.jobs.createOrThrow(SOURCE_LAYER_NAME).setLayer(doc.name, sourceLayer)
-            // plaintext
-            ThreadPoolUtil.pool.execute {
-                doc.plaintextFile.writeText(internalFile.plaintext)
-            }
             // metadata; needs to be serialized as well
             DiskValue<DocumentMetadata>(doc.metadataFile).write(DocumentMetadata.create(internalFile))
 

server/src/main/kotlin/org/ivdnt/galahad/documents/DocumentMetadata.kt

@@ -29,7 +29,7 @@ data class DocumentMetadata(
         private const val PREVIEW_LENGTH: Int = 100
 
         fun create(file: InternalFile): DocumentMetadata {
-            val text = file.plaintext
+            val text = file.layer.toString()
             return DocumentMetadata(
                 name = file.file.name,
                 format = file.format,

server/src/main/kotlin/org/ivdnt/galahad/evaluation/confusion/CorpusConfusion.kt

@@ -50,7 +50,7 @@ class CorpusConfusion(
                 )
             }
         }
-
+        // TODO: is Sequence.size the best way? There is also [corpus.documents.size]
         for (i in 0..<allDocs.size) add(completionService.take().get())
     }
 

server/src/main/kotlin/org/ivdnt/galahad/files/GalahadFolderManager.kt

@@ -30,15 +30,16 @@ abstract class GalahadFolderManager<ReadType : GalahadFolder, CreateType : Any>(
 
     protected abstract fun throwNotFound(key: String): Nothing
 
-    abstract fun createOrThrow(key: CreateType): ReadType
+    open fun createOrThrow(key: CreateType): ReadType = ctor(key.toString())
 
-    open fun readAll(): Set<ReadType> = dir.list()?.map { readOrThrow(it) }?.toSet() ?: setOf()
+    open fun readAll(): List<ReadType> = dir.list()?.map { readOrThrow(it) } ?: emptyList()
+    open fun readAllSequence(): Sequence<ReadType> = dir.list()?.asSequence()?.map { readOrThrow(it) } ?: emptySequence()
 
     open fun readOrNull(key: String): ReadType? = if (dir.resolve(key).exists()) ctor(key) else null
 
     fun readOrThrow(key: String): ReadType = readOrNull(key) ?: throwNotFound(key)
 
-    fun deleteOrThrow(key: String) {
+    open fun deleteOrThrow(key: String) {
         readOrThrow(key) // does it exist?
         if (!dir.resolve(key).deleteRecursively()) {
             logger.warn("Partial deletion of $key")

server/src/main/kotlin/org/ivdnt/galahad/formats/InternalFile.kt

@@ -18,7 +18,6 @@ import java.io.File
 abstract class InternalFile protected constructor() {
     abstract val file: File
     abstract val format: DocumentFormat
-    val plaintext: String by lazy { reader.layer.toString() }
     val layer: Layer by lazy { reader.layer }
     protected abstract val reader: AnnotationReader