Metrics: fix serializability

PrinsINT · PrinsINT · commit 071ab8af67c3 · 2025-10-22T16:58:02.000+02:00
diff --git a/client/src/components/tables/MetricsTable.vue b/client/src/components/tables/MetricsTable.vue
@@ -31,7 +31,7 @@
             ]"
             #[cell]="data"
         >
-            <div :key="cell">
+            <div :key="cell" style="text-align: right">
                 {{ `${data.value ? parseFloat(data.value).toString().slice(0, 4) : 0}` }}
             </div>
         </template>
@@ -41,7 +41,7 @@
             #[cell]="data"
         >
             <div :key="cell">
-                <GButton :disabled="data.value?.count === 0" @click="openModal(data)">
+                <GButton :disabled="data.value?.count === 0" @click="openModal(data)" style="text-align: right">
                     {{ data.value?.count }}
                 </GButton>
             </div>
diff --git a/client/src/stores/evaluation/metrics.ts b/client/src/stores/evaluation/metrics.ts
@@ -21,18 +21,18 @@ export const metricsPerPosColumns = [
     { key: "recall", sortOn: (x: MetricsRow) => x.recall },
     { key: "f1", sortOn: (x: MetricsRow) => x.f1 },
     { key: "count", label: "count", sortOn: (x: MetricsRow) => x.count },
-    { key: "truePositive", label: "true positive", sortOn: (x: MetricsRow): number => x.truePositive.count / x.count },
+    { key: "truePositive", label: "true positive", sortOn: (x: MetricsRow): number => x.truePositive.count },
     {
         key: "falsePositive",
         label: "false positive",
-        sortOn: (x: MetricsRow): number => x.falsePositive.count / x.count,
+        sortOn: (x: MetricsRow): number => x.falsePositive.count,
     },
     {
         key: "falseNegative",
         label: "false negative",
-        sortOn: (x: MetricsRow): number => x.falseNegative.count / x.count,
+        sortOn: (x: MetricsRow): number => x.falseNegative.count,
     },
-    { key: "noMatch", label: "no match", sortOn: (x: MetricsRow): number => x.noMatch.count / x.count },
+    { key: "noMatch", label: "no match", sortOn: (x: MetricsRow): number => x.noMatch.count },
 ]
 
 /**
diff --git a/client/src/views/annotate/subviews/evaluate/subviews/GlobalMetricsView.vue b/client/src/views/annotate/subviews/evaluate/subviews/GlobalMetricsView.vue
@@ -87,12 +87,13 @@ const items = computed(() => {
             return {
                 id: i.name,
                 column: i.name,
-                name: i.name,
-                group: i.name,
+                name: i.settings.annotation,
+                group: i.settings.group,
                 count: i.classes.count,
                 truePositive: i.classes.truePositive,
                 falseNegative: i.classes.falseNegative,
                 noMatch: i.classes.noMatch,
+                microAccuracy: i.accuracy,
                 macroPrecision: i.macro.precision,
                 macroRecall: i.macro.recall,
                 macroF1: i.macro.f1,
diff --git a/server/src/main/kotlin/org/ivdnt/galahad/evaluation/DocumentEvaluation.kt b/server/src/main/kotlin/org/ivdnt/galahad/evaluation/DocumentEvaluation.kt
@@ -1,13 +1,13 @@
 package org.ivdnt.galahad.evaluation
 
+import org.ivdnt.galahad.annotations.Annotation
 import org.ivdnt.galahad.annotations.Layer
 import org.ivdnt.galahad.corpora.Corpus
 import org.ivdnt.galahad.evaluation.comparison.LayerComparison
 import org.ivdnt.galahad.evaluation.confusion.DocumentConfusion
 import org.ivdnt.galahad.evaluation.distribution.DocumentDistribution
 import org.ivdnt.galahad.evaluation.entities.DocumentEntities
 import org.ivdnt.galahad.evaluation.metrics.DocumentMetric
-import org.ivdnt.galahad.evaluation.metrics.DocumentMetrics
 import org.ivdnt.galahad.files.GalahadFolder
 import org.ivdnt.galahad.files.ValidatedDiskValue
 import org.ivdnt.galahad.jobs.Job
@@ -30,6 +30,7 @@ class DocumentEvaluation(
     private val refModified: Long get() = refJob.results.readOrThrow(name).modified
     private val hypModified: Long get() = hypJob.results.readOrThrow(name).modified
     private val lastModified: Long get() = hypModified.coerceAtLeast(refModified)
+    private val availableAnnotations: Set<Annotation> get() = refJob.metadata.annotations.annotations.keys.intersect(hypJob.metadata.annotations.annotations.keys).filter{  it != Annotation.TOKEN }.toSet()
 
     val entities: DocumentEntities
         get() = object : ValidatedDiskValue<DocumentEntities>(dir.resolve(ENTITIES_FILE)) {
@@ -46,13 +47,13 @@ class DocumentEvaluation(
     val confusion: DocumentConfusion
         get() = object : ValidatedDiskValue<DocumentConfusion>(dir.resolve(CONFUSION_FILE)) {
             override fun isValid(modified: Long) = modified >= lastModified
-            override fun set(): DocumentConfusion = DocumentConfusion.create(LayerComparison(hypLayer, refLayer))
+            override fun set(): DocumentConfusion = DocumentConfusion.create(LayerComparison(hypLayer, refLayer), availableAnnotations)
         }.readOrCreate<DocumentConfusion>()
 
     val metrics: DocumentMetric
         get() = object : ValidatedDiskValue<DocumentMetric>(dir.resolve(METRICS_FILE)) {
             override fun isValid(modified: Long) = modified >= lastModified
-            override fun set(): DocumentMetric = DocumentMetric.create(LayerComparison(hypLayer, refLayer))
+            override fun set(): DocumentMetric = DocumentMetric.create(LayerComparison(hypLayer, refLayer), availableAnnotations)
         }.readOrCreate<DocumentMetric>()
 
     companion object {
diff --git a/server/src/main/kotlin/org/ivdnt/galahad/evaluation/confusion/DocumentConfusion.kt b/server/src/main/kotlin/org/ivdnt/galahad/evaluation/confusion/DocumentConfusion.kt
@@ -12,12 +12,12 @@ class DocumentConfusion(
     @JsonValue val confusion: Map<Annotation, Map<String, Map<String, EvaluationEntry>>>
 ) {
     companion object {
-        private val ANNOTATIONS = arrayOf(Annotation.POS, Annotation.UPOS, Annotation.NER, Annotation.DEPREL)
+        private val ANNOTATIONS = setOf(Annotation.POS, Annotation.UPOS, Annotation.NER, Annotation.DEPREL)
 
-        fun create(layerComparison: LayerComparison): DocumentConfusion =
+        fun create(layerComparison: LayerComparison, annotations: Set<Annotation>): DocumentConfusion =
             DocumentConfusion(buildMap<Annotation, MutableMap<String, MutableMap<String, EvaluationEntry>>> {
                 layerComparison.matches.forEach { comparison ->
-                    ANNOTATIONS.forEach { annotation ->
+                    annotations.intersect(ANNOTATIONS).forEach { annotation ->
                         val reference = comparison.ref.format(annotation)
                         val hypothesis = comparison.hyp.format(annotation)
                         val entry = EvaluationEntry(1, mutableListOf(comparison))
diff --git a/server/src/main/kotlin/org/ivdnt/galahad/evaluation/metrics/DocumentMetric.kt b/server/src/main/kotlin/org/ivdnt/galahad/evaluation/metrics/DocumentMetric.kt
@@ -1,10 +1,12 @@
 package org.ivdnt.galahad.evaluation.metrics
 
 import com.fasterxml.jackson.annotation.JsonValue
+import org.ivdnt.galahad.annotations.Annotation
 import org.ivdnt.galahad.annotations.Term
 import org.ivdnt.galahad.evaluation.EvaluationEntry
 import org.ivdnt.galahad.evaluation.comparison.LayerComparison
 import org.ivdnt.galahad.evaluation.comparison.TermComparison
+import org.ivdnt.galahad.evaluation.metrics.ClsClasses.Companion.toMetrics
 
 class ClsMetrics(
     val precision: Float = 0f,
@@ -28,76 +30,74 @@ class ClsMetrics(
 }
 
 class NewMetric(
+    val settings: MetricsSettings,
     val grouped: MutableMap<String, ClsClasses> = mutableMapOf(),
-) {
-    val classes: ClsClasses
-        get() = grouped.filter { it.key != TermComparison.MISSING_MATCH }.values.reduce { a, b -> a + b }
-
-    val micro: ClsMetrics get() = classes.metrics
-
-    val macro: ClsMetrics
-        get() {
-            val validClasses = grouped.filter { it.key != TermComparison.MISSING_MATCH }
-            return validClasses.values.map { it.metrics }
-                .reduce { a, b -> a + b } / validClasses.size
-        }
-}
+    val classes: ClsClasses = grouped.filter { it.key != TermComparison.MISSING_MATCH }.values.reduce { a, b -> a + b },
+    val accuracy: Float = classes.truePositive.count / classes.count.toFloat(),
+    val macro: ClsMetrics = grouped.values.map { it.metrics }.reduce { a, b -> a + b } / grouped.size,
+)
 
 class ClsClasses(
     var truePositive: EvaluationEntry = EvaluationEntry(),
     var falsePositive: EvaluationEntry = EvaluationEntry(),
     var falseNegative: EvaluationEntry = EvaluationEntry(),
     var noMatch: EvaluationEntry = EvaluationEntry(),
+    var count: Int = truePositive.count + falseNegative.count + noMatch.count,
+    var metrics: ClsMetrics = toMetrics(truePositive, falsePositive, falseNegative, noMatch)
 ) {
     fun add(other: ClsClasses, truncate: Boolean = true): ClsClasses {
         truePositive = EvaluationEntry.add(truePositive, other.truePositive, truncate)
         falsePositive = EvaluationEntry.add(falsePositive, other.falsePositive, truncate)
         falseNegative = EvaluationEntry.add(falseNegative, other.falseNegative, truncate)
         noMatch = EvaluationEntry.add(noMatch, other.noMatch, truncate)
+        count = truePositive.count + falseNegative.count + noMatch.count
+        metrics = toMetrics(truePositive, falsePositive, falseNegative, noMatch)
         return this
     }
 
     operator fun plus(other: ClsClasses): ClsClasses = ClsClasses(
-        truePositive = EvaluationEntry.from(other.truePositive,truePositive),
-        falsePositive = EvaluationEntry.from(other.falsePositive,falsePositive),
-        falseNegative = EvaluationEntry.from(other.falseNegative,falseNegative),
-        noMatch = EvaluationEntry.from(other.noMatch,noMatch),
+        truePositive = EvaluationEntry.from(other.truePositive, truePositive),
+        falsePositive = EvaluationEntry.from(other.falsePositive, falsePositive),
+        falseNegative = EvaluationEntry.from(other.falseNegative, falseNegative),
+        noMatch = EvaluationEntry.from(other.noMatch, noMatch),
+        count = truePositive.count + falseNegative.count + noMatch.count,
+        metrics = toMetrics(truePositive, falsePositive, falseNegative, noMatch)
     )
 
-    val metrics: ClsMetrics
-        get() {
-            val precision = notNaN(truePositive.count / (truePositive.count + falsePositive.count).toFloat())
-            val recall = notNaN(truePositive.count / (truePositive.count + falsePositive.count + noMatch.count).toFloat())
+    companion object {
+        fun notNaN(value: Float): Float = if (value.isNaN()) 0.0f else value
+        fun toMetrics(
+            truePositive: EvaluationEntry,
+            falsePositive: EvaluationEntry,
+            falseNegative: EvaluationEntry,
+            noMatch: EvaluationEntry
+        ): ClsMetrics {
+            val tp = truePositive.count.toFloat()
+            val fp = falsePositive.count.toFloat()
+            val fn = falseNegative.count.toFloat()
+            val mm = noMatch.count.toFloat()
+            val precision = notNaN(tp / (tp + fp))
+            val recall = notNaN(tp / (tp + fn + mm))
             val f1 = notNaN(2.0f * (precision * recall) / (precision + recall))
             return ClsMetrics(precision, recall, f1)
         }
-
-    val count: Int
-        get() = truePositive.count + falseNegative.count + noMatch.count
-
-    companion object {
-        fun notNaN(value: Float): Float = if (value.isNaN()) 0.0f else value
     }
 }
 
-
 class DocumentMetric(
-    @JsonValue
-    val classesByGroup: MutableMap<String, NewMetric>
+    @JsonValue val classesByGroup: MutableMap<String, NewMetric>
 ) {
     companion object {
-        fun create(layerComparison: LayerComparison): DocumentMetric = DocumentMetric(
-            buildMap<String, NewMetric> {
+        fun create(layerComparison: LayerComparison, annotations: Set<Annotation>): DocumentMetric = DocumentMetric(
+            buildMap<String, MutableMap<String, ClsClasses>> {
                 layerComparison.matches.forEach { tc ->
-                    METRIC_TYPES.forEach { metricType ->
+                    METRIC_TYPES.filter { annotations.containsAll(it.requiredAnnotations) }.forEach { metricType ->
                         if (!metricType.filterBy(tc)) return@forEach
                         val mapsToAdd = mutableListOf<MutableMap<String, ClsClasses>>()
                         if (tc.hyp == Term.EMPTY) {
                             // handle missing match
                             val cls = ClsClasses(noMatch = EvaluationEntry(1, mutableListOf(tc)))
-                            val group = TermComparison.MISSING_MATCH
-                            val classesMap = mutableMapOf(group to cls)
-                            mapsToAdd.add(classesMap)
+                            mapsToAdd.add(mutableMapOf(metricType.groupBy(tc.ref) to cls))
                         } else {
                             // handle true positive & false negative
                             val (trueEntry, falseEntry) = truesFalses(tc, metricType::termsEqual)
@@ -115,16 +115,18 @@ class DocumentMetric(
                         }
                         for (map in mapsToAdd) {
 
-                            merge(metricType.id, NewMetric(grouped=map)) { oldMetricMap, newMetricMap -> oldMetricMap.apply {
-                                this.grouped.merge(newMetricMap.grouped.keys.first(), newMetricMap.grouped.values.first()) { oldCls, newCls ->
-                                    oldCls.add(newCls)
-                                }
-                            }
+                            merge(
+                                metricType.id,
+                                map
+                            ) {
+                                a, b ->
+                                a.apply { this.merge(b.keys.first(), b.values.first()) { x, y -> x.add(y) } }
                             }
+
                         }
                     }
                 }
-            }.toMutableMap()
+            }.mapValues { NewMetric(METRIC_TYPES.first{mt -> mt.id == it.key}, it.value)}.toMutableMap()
         )
 
         private fun truesFalses(
diff --git a/server/src/main/kotlin/org/ivdnt/galahad/evaluation/metrics/JobMetric.kt b/server/src/main/kotlin/org/ivdnt/galahad/evaluation/metrics/JobMetric.kt
@@ -7,14 +7,14 @@ import org.ivdnt.galahad.util.merge
 
 class JobMetric(
     @JsonValue
-    val classesByGroup: MutableMap<String, NewMetric>
+    val classesByGroup: Map<String, NewMetric>
 ) {
     companion object {
         fun create(corpus: Corpus, docEvals: DocumentEvaluations): JobMetric = JobMetric(
             corpus.documents.readAllSequence().map { docEvals.createOrThrow(it.name).metrics.classesByGroup }
                 .reduce { map1, map2 ->
                     map1.merge(map2) { a, b -> a.apply { grouped.merge(b.grouped) { x, y -> x.add(y) } } } as MutableMap<String, NewMetric>
-                }
+                }.mapValues { NewMetric(it.value.settings, it.value.grouped) }
         )
     }
 }
diff --git a/server/src/main/kotlin/org/ivdnt/galahad/evaluation/metrics/Metrics.kt b/server/src/main/kotlin/org/ivdnt/galahad/evaluation/metrics/Metrics.kt
@@ -4,7 +4,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore
 import com.fasterxml.jackson.annotation.JsonProperty
 import org.ivdnt.galahad.evaluation.comparison.TermComparison
 import org.ivdnt.galahad.export.csv.CsvFile
-import org.ivdnt.galahad.export.csv.CSVHeader
 import org.ivdnt.galahad.export.csv.CsvString
 import org.ivdnt.galahad.jobs.Job
 import org.ivdnt.galahad.taggers.Tagger

Original file line number	Diff line number	Diff line change
`@@ -7,14 +7,14 @@ import org.ivdnt.galahad.util.merge`
`7`	`7`
`8`	`8`	`class JobMetric(`
`9`	`9`	`@JsonValue`
`10`		`- val classesByGroup: MutableMap<String, NewMetric>`
	`10`	`+ val classesByGroup: Map<String, NewMetric>`
`11`	`11`	`) {`
`12`	`12`	`companion object {`
`13`	`13`	`fun create(corpus: Corpus, docEvals: DocumentEvaluations): JobMetric = JobMetric(`
`14`	`14`	`corpus.documents.readAllSequence().map { docEvals.createOrThrow(it.name).metrics.classesByGroup }`
`15`	`15`	`.reduce { map1, map2 ->`
`16`	`16`	`map1.merge(map2) { a, b -> a.apply { grouped.merge(b.grouped) { x, y -> x.add(y) } } } as MutableMap<String, NewMetric>`
`17`		`- }`
	`17`	`+ }.mapValues { NewMetric(it.value.settings, it.value.grouped) }`
`18`	`18`	`)`
`19`	`19`	`}`
`20`	`20`	`}`