Added metrics.csv

Author: PrinsINT

server/src/main/kotlin/org/ivdnt/galahad/evaluation/EvaluationEntry.kt

@@ -9,10 +9,10 @@ const val MAX_SAMPLE_LENGTH: Int = 10
 
 data class EvaluationEntry(
     val count: Int = 0,
-    @JsonIgnore val samples: MutableList<TermComparison> = mutableListOf(),
+    val samples: MutableList<TermComparison> = mutableListOf(),
 ) {
-    @get:JsonProperty("samples")
-    val jsonSamples: List<TermComparison> get() = samples.asSequence().shuffled().take(MAX_SAMPLE_LENGTH).toList()
+//    @get:JsonProperty("samples")
+//    val jsonSamples: List<TermComparison> get() = samples.asSequence().shuffled().take(MAX_SAMPLE_LENGTH).toList()
 
     companion object {
         /**

server/src/main/kotlin/org/ivdnt/galahad/evaluation/metrics/DocumentMetric.kt

@@ -29,8 +29,16 @@ class ClsMetrics(
     )
 }
 
+class FlatMetricsSettings(
+    val id: String,
+    val annotation: String,
+    val group: String,
+) {
+    constructor(settings: MetricsSettings) : this(settings.id, settings.annotation, settings.group)
+}
+
 class NewMetric(
-    val settings: MetricsSettings,
+    val settings: FlatMetricsSettings,
     val grouped: MutableMap<String, ClsClasses> = mutableMapOf(),
     val classes: ClsClasses = grouped.filter { it.key != TermComparison.MISSING_MATCH }.values.reduce { a, b -> a + b },
     val accuracy: Float = classes.truePositive.count / classes.count.toFloat(),
@@ -126,7 +134,7 @@ class DocumentMetric(
                         }
                     }
                 }
-            }.mapValues { NewMetric(METRIC_TYPES.first{mt -> mt.id == it.key}, it.value)}.toMutableMap()
+            }.mapValues { NewMetric(FlatMetricsSettings(METRIC_TYPES.first{mt -> mt.id == it.key}), it.value)}.toMutableMap()
         )
 
         private fun truesFalses(

server/src/main/kotlin/org/ivdnt/galahad/evaluation/metrics/JobMetric.kt

@@ -3,18 +3,58 @@ package org.ivdnt.galahad.evaluation.metrics
 import com.fasterxml.jackson.annotation.JsonValue
 import org.ivdnt.galahad.corpora.Corpus
 import org.ivdnt.galahad.evaluation.DocumentEvaluations
+import org.ivdnt.galahad.export.csv.CsvFile
+import org.ivdnt.galahad.export.csv.CsvString
 import org.ivdnt.galahad.util.merge
 
 class JobMetric(
     @JsonValue
     val classesByGroup: Map<String, NewMetric>
 ) {
+
+    fun toGlobalCsv(): CsvString = buildString {
+        append(CsvFile.toCsvString(listOf("annotation", "grouped by", "macro precision", "macro recall", "macro f1", "micro accuracy", "count", "true positive", "false negative", "no match")))
+        for ((key, value) in classesByGroup) {
+            append(CsvFile.toCsvString(listOf(
+                value.settings.annotation,
+                value.settings.group,
+                value.macro.precision,
+                value.macro.recall,
+                value.macro.f1,
+                value.accuracy,
+                value.classes.count,
+                value.classes.truePositive.count,
+                value.classes.falseNegative.count,
+                value.classes.noMatch.count,
+            )))
+        }
+    }
+
     companion object {
         fun create(corpus: Corpus, docEvals: DocumentEvaluations): JobMetric = JobMetric(
             corpus.documents.readAllSequence().map { docEvals.createOrThrow(it.name).metrics.classesByGroup }
                 .reduce { map1, map2 ->
                     map1.merge(map2) { a, b -> a.apply { grouped.merge(b.grouped) { x, y -> x.add(y) } } } as MutableMap<String, NewMetric>
                 }.mapValues { NewMetric(it.value.settings, it.value.grouped) }
         )
+
+        fun toCsv(metric: NewMetric): CsvString = buildString {
+            append(CsvFile.toCsvString(listOf("group", "precision", "recall", "f1", "count", "true positive", "false positive", "false negative", "no match")))
+            for ((key, value) in metric.grouped) {
+                append(
+                    CsvFile.toCsvString(listOf(
+                        key,
+                        value.metrics.precision,
+                        value.metrics.recall,
+                        value.metrics.f1,
+                        value.count,
+                        value.truePositive.count,
+                        value.falsePositive.count,
+                        value.falseNegative.count,
+                        value.noMatch.count,
+                    ))
+                )
+            }
+        }
     }
 }

server/src/main/kotlin/org/ivdnt/galahad/web/service/EvaluationService.kt

@@ -150,7 +150,7 @@ class EvaluationService(val corpora: CorporaService) {
         createDistributionCsv(dir.resolve("distribution"), corpus, job)
         createEntitiesCsv(dir.resolve("entities"), corpus)
         if (reference != null) {
-//            createMetricsCsv(dir, corpus, job, reference)
+            createMetricsCsv(dir.resolve("metrics"), corpus, job, reference)
             createConfusionCsv(dir.resolve("confusion"), corpus, job, reference)
         }
         val metadata = writeMetadataToDir(corpus, job, reference, dir)
@@ -187,14 +187,15 @@ class EvaluationService(val corpora: CorporaService) {
         }
     }
 
-    private fun createMetricsCsv(dir: File, corpus: UUID, job: String, reference: String?) {
-        val metrics = getMetrics(corpus, job = job, reference = reference)
+    private fun createMetricsCsv(dir: File, corpus: UUID, hypothesis: String, reference: String) {
+        dir.mkdirs()
+        val metrics = getJobMetric(corpus, hypothesis, reference)
         val globFile = CsvFile(dir.resolve("metrics-global.csv"))
         globFile.append(metrics.toGlobalCsv())
 
-        metrics.metricTypes.values.forEach { mt ->
-            val file = CsvFile(dir.resolve("metrics-${mt.setting.id}.csv"))
-            file.append(mt.toGroupedCsv())
+        metrics.classesByGroup.values.forEach { mt ->
+            val file = CsvFile(dir.resolve("metrics-${mt.settings.id}.csv"))
+            file.append(JobMetric.toCsv(mt))
         }
     }