Skip to content

Commit 734cbf3

Browse files
PrinsINTPrinsINT
committed
Basic conversion of Metrics Eval. to new structure
1 parent f438d48 commit 734cbf3

16 files changed

Lines changed: 257 additions & 35 deletions

File tree

‎client/src/components/tables/MetricsTable.vue‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,8 @@
4141
#[cell]="data"
4242
>
4343
<div :key="cell">
44-
<GButton :disabled="data.value.count === 0" @click="openModal(data)">
45-
{{ `${((data.value.count / data.item.count) * 100).toFixed(1)}%` }}
46-
<i>({{ data.value.count.toString() }})</i>
44+
<GButton :disabled="data.value?.count === 0" @click="openModal(data)">
45+
{{ data.value?.count }}
4746
</GButton>
4847
</div>
4948
</template>

‎client/src/views/annotate/subviews/evaluate/subviews/GlobalMetricsView.vue‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -74,29 +74,28 @@ const columns = computed(() => {
7474
return addColumns.concat(withoutName)
7575
})
7676
const items = computed(() => {
77-
if (metrics.value?.metrics == null) return []
77+
if (metrics.value == null) return []
7878
7979
// metrics has the form
8080
// { pos: { f1, recall, ... }, lemma: { f1, recall, ... }, lemmaPos: { f1, recall, ... } }
8181
// We want to transform this to
8282
// [ { name: "PoS", f1, recall, ... }, { name: "Lemma", f1, recall, ... }, { name: "Lemma & PoS", f1, recall, ... } ]
83-
const ret = Object.keys(metrics.value.metrics)
84-
.map((key) => ({ name: key, ...metrics.value.metrics[key] }))
83+
const ret = Object.keys(metrics.value)
84+
.map((key) => ({ name: key, ...metrics.value[key] }))
8585
.map((i) => {
8686
const annoAndGroup = annotationAndGroupFromName(i.name)
8787
return {
88-
id: i.setting.id,
89-
column: i.setting.annotation,
90-
name: i.setting.annotation,
91-
group: i.setting.group,
92-
count: i.classes.classCount,
88+
id: i.name,
89+
column: i.name,
90+
name: i.name,
91+
group: i.name,
92+
count: i.classes.count,
9393
truePositive: i.classes.truePositive,
9494
falseNegative: i.classes.falseNegative,
9595
noMatch: i.classes.noMatch,
9696
macroPrecision: i.macro.precision,
9797
macroRecall: i.macro.recall,
9898
macroF1: i.macro.f1,
99-
microAccuracy: i.micro.accuracy,
10099
}
101100
})
102101
return ret

‎client/src/views/annotate/subviews/evaluate/subviews/GroupedMetricsView.vue‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
on a percentage, a data sample is shown.
1616
</p>
1717
</template>
18-
<template #header v-if="metrics.metrics != null">
18+
<template #header>
1919
<GSelect :options="groupingOptions" v-model="selectedGrouping" />
2020
<p>
2121
<b> Only the 100 most frequent groups are shown. </b>
@@ -44,9 +44,7 @@ const jobSelection = stores.useJobSelection()
4444
// Fields
4545
const downloading = ref<boolean>()
4646
const selectedGrouping = ref<string>("lemmaByLemma")
47-
const groupingOptions = computed(() =>
48-
Object.keys(metrics.value?.metrics || {}).map((key) => ({ value: key, text: key })),
49-
)
47+
const groupingOptions = computed(() => Object.keys(metrics.value || {}).map((key) => ({ value: key, text: key })))
5048
5149
const columns = computed(() => metricsPerPosColumns)
5250
const metricsFilter = useTemplateRef<InstanceType<typeof MetricsFilter>>("metricsFilter")
@@ -55,19 +53,20 @@ const metricName = computed(() => {
5553
})
5654
5755
const posMetrics = computed(() => {
58-
if (metrics.value?.metrics?.[selectedGrouping.value] == null) return []
56+
if (metrics.value?.[selectedGrouping.value] == null) return []
57+
console.log(metrics.value[selectedGrouping.value])
5958
// Copy over the metrics (depending on selectedMetric.value) from:
6059
// { ADJ: { ADJ: { pos : { f1, recall, ... }, lemma : { f1, recall, ... } } } } }
6160
// to:
6261
// { ADJ: { ADJ: { f1, recall, ..., } } }
63-
const ret = metrics.value.metrics[selectedGrouping.value].grouped.map((i) => ({
62+
const ret = Object.entries(metrics.value[selectedGrouping.value]?.grouped || {}).map(([name, i]) => ({
6463
column: selectedGrouping.value.split("By")[1].toLowerCase(),
65-
name: i.name,
66-
count: i.classes.classCount,
67-
truePositive: i.classes.truePositive,
68-
falsePositive: i.classes.falsePositive,
69-
falseNegative: i.classes.falseNegative,
70-
noMatch: i.classes.noMatch,
64+
name: name,
65+
count: i.count,
66+
truePositive: i.truePositive,
67+
falsePositive: i.falsePositive,
68+
falseNegative: i.falseNegative,
69+
noMatch: i.noMatch,
7170
precision: i.metrics.precision,
7271
recall: i.metrics.recall,
7372
f1: i.metrics.f1,

‎server/src/main/kotlin/org/ivdnt/galahad/annotations/Term.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ class Term(
3232
val token: String = annotations[Annotation.TOKEN]!!
3333

3434
@get:JsonIgnore
35-
val lemma: String? get() = annotations[Annotation.LEMMA]
35+
val lemma: String? = annotations[Annotation.LEMMA]
3636

3737
@get:JsonIgnore
3838
val pos: String? = annotations[Annotation.POS]

‎server/src/main/kotlin/org/ivdnt/galahad/evaluation/DocumentEvaluation.kt‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import org.ivdnt.galahad.evaluation.comparison.LayerComparison
66
import org.ivdnt.galahad.evaluation.confusion.DocumentConfusion
77
import org.ivdnt.galahad.evaluation.distribution.DocumentDistribution
88
import org.ivdnt.galahad.evaluation.entities.DocumentEntities
9+
import org.ivdnt.galahad.evaluation.metrics.DocumentMetric
10+
import org.ivdnt.galahad.evaluation.metrics.DocumentMetrics
911
import org.ivdnt.galahad.files.GalahadFolder
1012
import org.ivdnt.galahad.files.ValidatedDiskValue
1113
import org.ivdnt.galahad.jobs.Job
@@ -44,12 +46,19 @@ class DocumentEvaluation(
4446
val confusion: DocumentConfusion
4547
get() = object : ValidatedDiskValue<DocumentConfusion>(dir.resolve(CONFUSION_FILE)) {
4648
override fun isValid(modified: Long) = modified >= lastModified
47-
override fun set(): DocumentConfusion = DocumentConfusion.create(LayerComparison(refLayer, hypLayer))
49+
override fun set(): DocumentConfusion = DocumentConfusion.create(LayerComparison(hypLayer, refLayer))
4850
}.readOrCreate<DocumentConfusion>()
4951

52+
val metrics: DocumentMetric
53+
get() = object : ValidatedDiskValue<DocumentMetric>(dir.resolve(METRICS_FILE)) {
54+
override fun isValid(modified: Long) = modified >= lastModified
55+
override fun set(): DocumentMetric = DocumentMetric.create(LayerComparison(hypLayer, refLayer))
56+
}.readOrCreate<DocumentMetric>()
57+
5058
companion object {
5159
private const val ENTITIES_FILE = "entities.json"
5260
private const val DISTRIBUTION_FILE = "distribution.json"
5361
private const val CONFUSION_FILE = "confusion.json"
62+
private const val METRICS_FILE = "metrics.json"
5463
}
5564
}

‎server/src/main/kotlin/org/ivdnt/galahad/evaluation/EvaluationEntry.kt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,10 @@ data class EvaluationEntry(
3232
return EvaluationEntry(a.count + b.count, a.samples)
3333

3434
}
35+
36+
fun from(a: EvaluationEntry, b: EvaluationEntry): EvaluationEntry {
37+
val aCopy = EvaluationEntry(a.count, a.samples.toMutableList())
38+
return EvaluationEntry.add(aCopy, b)
39+
}
3540
}
3641
}

‎server/src/main/kotlin/org/ivdnt/galahad/evaluation/JobEvaluation.kt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import org.ivdnt.galahad.corpora.Corpus
44
import org.ivdnt.galahad.evaluation.confusion.JobConfusion
55
import org.ivdnt.galahad.evaluation.distribution.JobDistribution
66
import org.ivdnt.galahad.evaluation.entities.JobEntities
7+
import org.ivdnt.galahad.evaluation.metrics.JobMetric
78
import org.ivdnt.galahad.files.GalahadFolder
89
import org.ivdnt.galahad.files.ValidatedDiskValue
910
import org.ivdnt.galahad.jobs.Job
@@ -39,10 +40,17 @@ class JobEvaluation(
3940
override fun set(): JobConfusion = JobConfusion.create(corpus, documents)
4041
}.readOrCreate()
4142

43+
val metrics: JobMetric
44+
get() = object : ValidatedDiskValue<JobMetric>(dir.resolve(METRICS_FILE)) {
45+
override fun isValid(modified: Long) = modified >= Math.max(refJob.modified, hypJob.modified)
46+
override fun set(): JobMetric = JobMetric.create(corpus, documents)
47+
}.readOrCreate()
48+
4249
companion object {
4350
private const val DISTRIBUTION_FILE = "distribution.json"
4451
private const val ENTITIES_FILE = "entities.json"
4552
private const val CONFUSION_FILE = "confusion.json"
53+
private const val METRICS_FILE = "metrics.json"
4654
private const val DOCUMENTS_FOLDER = "documents"
4755
}
4856
}

‎server/src/main/kotlin/org/ivdnt/galahad/evaluation/JobPair.kt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
package org.ivdnt.galahad.evaluation
22

33
class JobPair(
4-
val reference: String,
5-
val hypothesis: String = reference,
4+
val hypothesis: String,
5+
val reference: String = hypothesis,
66
) {
7-
override fun toString(): String = "$reference/$hypothesis"
7+
override fun toString(): String = "$hypothesis/$reference"
88

99
companion object {
1010
fun fromString(pair: String): JobPair = pair.split('/').let { JobPair(it[0], it[1]) }

‎server/src/main/kotlin/org/ivdnt/galahad/evaluation/confusion/DocumentConfusion.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@ class DocumentConfusion(
2727

2828
merge(annotation, groupedMap) { oldAnnotationMap, _ ->
2929
oldAnnotationMap.apply {
30-
merge(reference, entryMap) { oldEntry, _ ->
30+
this.merge(reference, entryMap) { oldEntry, _ ->
3131
oldEntry.apply {
32-
merge(hypothesis, entry) { e1, e2 -> EvaluationEntry.add(e1, e2) }
32+
this.merge(hypothesis, entry) { e1, e2 -> EvaluationEntry.add(e1, e2) }
3333
}
3434
}
3535
}

‎server/src/main/kotlin/org/ivdnt/galahad/evaluation/distribution/DocumentDistribution.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,10 @@ class DocumentDistribution(
2727
merge(annotation, mutableMapOf(pair to tokens)) { oldTypeTokens, newTypeTokens ->
2828
// merge this pair-tokencount mapping if needed
2929
oldTypeTokens.apply {
30-
merge(pair, tokens) { oldTokens, newTokens ->
30+
this.merge(pair, tokens) { oldTokens, newTokens ->
3131
// In doing so, we only need to sum the token counts
3232
oldTokens.apply {
33-
merge(t.token, 1, Integer::sum)
33+
this.merge(t.token, 1, Integer::sum)
3434
}
3535
}
3636
}

0 commit comments

Comments
 (0)
⚡