Added pdf and docx support

Author: Vincent Prins

client/src/components/input/UploadDocuments.vue

@@ -13,7 +13,7 @@
         </label>
         <!-- Actual input -->
         <input type="file" ref="uploadInput" name="filefield" multiple id="file-upload" style="display: none;"
-            accept=".xml, .tsv, .txt, .zip, .conllu, .naf"
+            accept=".xml, .tsv, .txt, .zip, .conllu, .naf, .pdf, .docx"
             @change="e => filesToUpload = Object.values(e.target.files as FileList)" />
 
 

server/build.gradle.kts

@@ -46,6 +46,12 @@ dependencies {
     // cache
     implementation("com.github.ben-manes.caffeine:caffeine:3.2.0")
 
+    // reading microsoft word docx
+    implementation("org.apache.poi:poi-ooxml:5.4.1")
+
+    // reading pdf
+    implementation("com.itextpdf:itextpdf:5.5.13.4")
+
     // immutable arrays
     // implementation("com.danrusu.pods4k:pods4k:0.7.0")
 }

server/src/main/kotlin/org/ivdnt/galahad/documents/DocumentFormat.kt

@@ -26,6 +26,8 @@ enum class DocumentFormat(val identifier: String, val extension: String) {
     Conllu("conllu", "conllu"),
     Folia("folia", "folia.xml"),
     Txt("txt", "txt"),
+    Docx("docx", "docx"),
+    Pdf("pdf", "pdf"),
     Unknown("unknown", "unknown");
 
     @JsonValue
@@ -48,9 +50,11 @@ enum class DocumentFormat(val identifier: String, val extension: String) {
             "tsv" -> Tsv
             "folia" -> Folia
             "conllu" -> Conllu
+            "docx" -> Docx
             "xml", "tei" -> determineXmlFormat(file) // TEI can be either P4 or P5, so still check.
             "txt" -> Txt
             "naf" -> Naf
+            "pdf" -> Pdf
             else -> Unknown
         }
 

server/src/main/kotlin/org/ivdnt/galahad/formats/InternalFile.kt

@@ -5,8 +5,10 @@ import org.ivdnt.galahad.annotations.Layer
 import org.ivdnt.galahad.documents.DocumentFormat
 import org.ivdnt.galahad.exceptions.DocumentInvalidException
 import org.ivdnt.galahad.formats.conllu.ConlluFile
+import org.ivdnt.galahad.formats.docx.DocxFile
 import org.ivdnt.galahad.formats.folia.FoliaFile
 import org.ivdnt.galahad.formats.naf.NafFile
+import org.ivdnt.galahad.formats.pdf.PdfFile
 import org.ivdnt.galahad.formats.tei.TeiFile
 import org.ivdnt.galahad.formats.tsv.TsvFile
 import org.ivdnt.galahad.formats.txt.TxtFile
@@ -28,6 +30,8 @@ abstract class InternalFile protected constructor() {
                 DocumentFormat.Naf -> NafFile(file)
                 DocumentFormat.Txt -> TxtFile(file)
                 DocumentFormat.Conllu -> ConlluFile(file)
+                DocumentFormat.Docx -> DocxFile(file)
+                DocumentFormat.Pdf -> PdfFile(file)
                 // Multiple TEI formats
                 DocumentFormat.TeiP4Legacy,
                 DocumentFormat.TeiP5Legacy,

server/src/main/kotlin/org/ivdnt/galahad/formats/docx/DocxFile.kt

@@ -0,0 +1,13 @@
+package org.ivdnt.galahad.formats.docx
+
+import org.ivdnt.galahad.documents.DocumentFormat
+import org.ivdnt.galahad.formats.InternalFile
+import java.io.BufferedInputStream
+import java.io.File
+
+class DocxFile(
+    override val file: File
+) : InternalFile() {
+    override val format: DocumentFormat = DocumentFormat.Docx
+    override val reader: DocxReader by lazy { DocxReader(BufferedInputStream(file.inputStream())) }
+}

server/src/main/kotlin/org/ivdnt/galahad/formats/docx/DocxReader.kt

@@ -0,0 +1,34 @@
+package org.ivdnt.galahad.formats.docx
+
+import org.apache.poi.xwpf.usermodel.XWPFDocument
+import org.ivdnt.galahad.annotations.Annotation
+import org.ivdnt.galahad.annotations.AnnotationReader
+import org.ivdnt.galahad.annotations.Layer
+import org.ivdnt.galahad.annotations.Term
+import java.io.InputStream
+
+class DocxReader(
+    stream: InputStream
+) : AnnotationReader() {
+    val doc: XWPFDocument = XWPFDocument(stream)
+
+    override fun read(): Layer {
+        doc.paragraphs.forEach { paragraph ->
+            paragraph.text.ifBlank { null }?.split(whitespace)?.forEach { word ->
+                terms += Term(
+                    wordID(), offset, mapOf(
+                        Annotation.TOKEN to word
+                    )
+                )
+                offset += word.length + 1
+            }
+            newParagraph()
+        }
+        newDocument()
+        return Layer(documents.toTypedArray())
+    }
+
+    companion object {
+        val whitespace: Regex = Regex("""\s+""")
+    }
+}

server/src/main/kotlin/org/ivdnt/galahad/formats/pdf/PdfFile.kt

@@ -0,0 +1,14 @@
+package org.ivdnt.galahad.formats.pdf
+
+import org.ivdnt.galahad.documents.DocumentFormat
+import org.ivdnt.galahad.formats.InternalFile
+import org.ivdnt.galahad.formats.docx.DocxReader
+import java.io.BufferedInputStream
+import java.io.File
+
+class PdfFile(
+    override val file: File
+) : InternalFile() {
+    override val format: DocumentFormat = DocumentFormat.Pdf
+    override val reader: PdfReader by lazy { PdfReader(BufferedInputStream(file.inputStream())) }
+}

server/src/main/kotlin/org/ivdnt/galahad/formats/pdf/PdfReader.kt

@@ -0,0 +1,36 @@
+package org.ivdnt.galahad.formats.pdf
+
+import com.itextpdf.text.pdf.parser.PdfTextExtractor
+import org.ivdnt.galahad.annotations.Annotation
+import com.itextpdf.text.pdf.PdfReader as PdfReaderIText
+import org.ivdnt.galahad.annotations.AnnotationReader
+import org.ivdnt.galahad.annotations.Layer
+import org.ivdnt.galahad.annotations.Term
+import java.io.InputStream
+
+class PdfReader(
+    stream: InputStream
+) : AnnotationReader() {
+    val reader = PdfReaderIText(stream)
+
+    override fun read(): Layer {
+        for (i in 1 .. reader.numberOfPages) {
+            val text = PdfTextExtractor.getTextFromPage(reader, i)
+            text.ifBlank { null }?.split(whitespace)?.filter { it.isNotBlank()}?.forEach {word ->
+                terms += Term(
+                    wordID(), offset, mapOf(
+                        Annotation.TOKEN to word
+                    )
+                )
+                offset += word.length + 1
+            }
+        }
+        newDocument()
+        return Layer(documents.toTypedArray())
+    }
+
+    companion object {
+        val whitespace: Regex = Regex("""\s+""")
+    }
+
+}