Maintenance for Poseidon v3.0.0

.github/workflows/main.yml

@@ -10,14 +10,14 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - uses: freckle/stack-action@v5
         with:
           stack-arguments: --coverage
       # prepare and upload test coverage report
       - uses: 8c6794b6/hpc-codecov-action@v4
         with:
           target: stack:spec
-      - uses: codecov/codecov-action@v4
+      - uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # set in organization settings

.github/workflows/release.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Create Release
         id: create_release
@@ -89,7 +89,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Build Docker image
         run: docker build -t linux -f .github/workflows/Dockerfile.centos .

.github/workflows/stylishHaskell.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
 
     - name: Check out code
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Install stylish-haskell
       run: |

CHANGELOG.md

@@ -1,6 +1,10 @@
+- V 1.0.1.0: Update for Poseidon v3.0.0
+	- Switched to new stackage resolver version v22.43.
+	- Replaced the .janno column order list with a new one for Poseidon v3.0.0.
+	- Introduced a mechanism to automatically position `_Note` columns in .janno files.
 - V 1.0.0.1:
-	- Switched to new stackage resolver version v21.21
-	- Update of the release pipeline according to the template developed for trident
+	- Switched to new stackage resolver version v21.21.
+	- Update of the release pipeline according to the template developed for trident.
 - V 1.0.0.0:
 	- Added more data input options for .janno files (d(), da(), j(), .janno).
 	- Reorganized golden tests and added new ones.

CHANGELOGRELEASE.md

@@ -1,3 +1,11 @@
+### V 1.0.1.0
+
+This is a maintenance release to bring `qjanno` in sync with Poseidon v3.0.0.
+
+It comes with a new column order template for `.janno` files, featuring the new columns introduced with the v3.0.0 schema release, and a mechanism to automatically position arbitrary `_Note` columns in this template (as in `trident`).
+
+As usual we also switched to a new GHC version (9.6.6) and a new Stackage resolver version (22.43).
+
 ### V 1.0.0.1
 
 This minor release replaces the pipeline to produce static `qjanno` executables for every release.

qjanno-hs.cabal

@@ -1,5 +1,5 @@
 name:                   qjanno-hs
-version:                1.0.0.1
+version:                1.0.1.0
 author:                 itchyny <https://github.com/itchyny>, Clemens Schmid
 maintainer:             Clemens Schmid <clemens@nevrome.de>
 license:                MIT

src/Qjanno/Janno.hs

@@ -8,8 +8,8 @@ import           Control.Monad    (filterM)
 import           Data.Aeson       (FromJSON, parseJSON, withObject, (.:), (.:?))
 import           Data.Either      (lefts, rights)
 import           Data.Foldable    (foldl')
-import           Data.List        (elemIndices, groupBy, sortBy, sortOn,
-                                   transpose)
+import           Data.List        (elemIndices, groupBy, insertBy, sortBy,
+                                   sortOn, transpose)
 import qualified Data.Map.Strict  as M
 import qualified Data.Set         as Set
 import           Data.Version     (Version)
@@ -111,17 +111,40 @@ mergeJannos xs =
 
 reorderJannoColumns :: ([String], [[String]]) -> ([String], [[String]])
 reorderJannoColumns (oldCols, oldRowsData) =
-    let orderedCols = sortOn getOrder oldCols
-        orderingIndices = concatMap (`elemIndices` oldCols) orderedCols
+    let baseOrderedCols = sortOn getOrder oldCols
+        finalCols       = applyNoteWeaving baseOrderedCols
+        orderingIndices = concatMap (`elemIndices` oldCols) finalCols
         orderedRowsData = map (\row -> map (row !!) orderingIndices) oldRowsData
-    in (orderedCols, orderedRowsData)
+    in (finalCols, orderedRowsData)
     where
         -- https://stackoverflow.com/a/26260968/3216883
         getOrder :: String -> Int
         getOrder k = M.findWithDefault (length jannoOrder) k ordermap
         ordermap :: M.Map String Int
         ordermap = M.fromList (zip jannoOrder [0..])
 
+-- _Note column weaving as in trident's Janno.hs module
+applyNoteWeaving :: [String] -> [String]
+applyNoteWeaving cols =
+    let noteCols    = filter isNote cols
+        nonNoteCols = filter (not . isNote) cols
+    in weave noteCols nonNoteCols
+  where
+    isNote x = reverse (takeWhile (/= '_') (reverse x)) == "Note"
+    weave :: [String] -> [String] -> [String]
+    weave inserts = reverse . insertByMulti findSpot inserts . reverse
+    -- reverse, because Note columns should be at the end of column groups (e.g. Date_*)
+    insertByMulti :: (a -> a -> Ordering) -> [a] -> [a] -> [a]
+    insertByMulti _ [] xs       = xs
+    insertByMulti f (i:rest) xs = insertBy f i (insertByMulti f rest xs)
+    findSpot :: String -> String -> Ordering
+    findSpot i x
+        | removeSuffix i == x = LT
+        | removeSuffix x == x = GT
+        | otherwise           = findSpot i (removeSuffix x)
+    removeSuffix :: String -> String
+    removeSuffix = reverse . drop 1 . dropWhile (/= '_') . reverse
+
 jannoOrder :: [String]
 jannoOrder = "package_title" : "package_version" : "source_file" : jannoHeader
 
@@ -130,44 +153,25 @@ jannoHeader = [
       "Poseidon_ID"
     , "Genetic_Sex"
     , "Group_Name"
-    , "Alternative_IDs"
-    , "Relation_To"
-    , "Relation_Degree"
-    , "Relation_Type"
-    , "Relation_Note"
-    , "Collection_ID"
-    , "Country"
-    , "Country_ISO"
-    , "Location"
-    , "Site"
-    , "Latitude"
-    , "Longitude"
+    , "Individual_ID"
+    , "Species"
+    , "Alternative_IDs", "Alternative_IDs_Context"
+    , "Relation_To", "Relation_Degree", "Relation_Type"
+    , "Collection_ID", "Custodian_Institution"
+    , "Cultural_Era", "Cultural_Era_URL", "Archaeological_Culture", "Archaeological_Culture_URL"
+    , "Country", "Country_ISO"
+    , "Location", "Site", "Latitude", "Longitude"
     , "Date_Type"
-    , "Date_C14_Labnr"
-    , "Date_C14_Uncal_BP"
-    , "Date_C14_Uncal_BP_Err"
-    , "Date_BC_AD_Start"
-    , "Date_BC_AD_Median"
-    , "Date_BC_AD_Stop"
-    , "Date_Note"
-    , "MT_Haplogroup"
-    , "Y_Haplogroup"
-    , "Source_Tissue"
-    , "Nr_Libraries"
-    , "Library_Names"
-    , "Capture_Type"
-    , "UDG"
-    , "Library_Built"
-    , "Genotype_Ploidy"
+    , "Date_C14_Labnr", "Date_C14_Uncal_BP", "Date_C14_Uncal_BP_Err"
+    , "Date_BC_AD_Start", "Date_BC_AD_Median", "Date_BC_AD_Stop"
+    , "Chromosomal_Anomalies"
+    , "MT_Haplogroup", "Y_Haplogroup"
+    , "Source_Material"
+    , "Nr_Libraries", "Library_Names"
+    , "Capture_Type", "UDG", "Library_Built", "Genotype_Ploidy"
     , "Data_Preparation_Pipeline_URL"
-    , "Endogenous"
-    , "Nr_SNPs"
-    , "Coverage_on_Target_SNPs"
-    , "Damage"
-    , "Contamination"
-    , "Contamination_Err"
-    , "Contamination_Meas"
-    , "Contamination_Note"
+    , "Endogenous", "Nr_SNPs", "Coverage_on_Target_SNPs", "Damage"
+    , "Contamination", "Contamination_Err", "Contamination_Meas"
     , "Genetic_Source_Accession_IDs"
     , "Primary_Contact"
     , "Publication"

stack.yaml

@@ -3,9 +3,7 @@ packages:
   - '.'
 extra-deps:
   - sqlite-simple-0.4.18.2
-  - direct-sqlite-2.3.28
   - simple-sql-parser-0.6.0
   - table-layout-0.9.1.0
-  - data-default-instances-base-0.1.0.1
 allow-newer: true
-resolver: lts-21.21
+resolver: lts-22.43

test/MainSpec.hs

@@ -1,10 +1,9 @@
 module MainSpec (spec) where
 
-import           Control.Applicative
 import           Control.Monad
 import           System.IO
 import           System.Process
-import           Test.Hspec          (Spec, describe, it, shouldReturn)
+import           Test.Hspec     (Spec, describe, it, shouldReturn)
 
 spec :: Spec
 spec = qjannoSpec
@@ -27,7 +26,8 @@ qjannoSpec =
 
     let poseidon_tests = [
             "janno_d", "janno_da", "janno_j", "janno_d_da_j", "janno_ext",
-            "janno_d_ext_ext", "show_columns", "janno_d_da_j_full"
+            "janno_d_ext_ext", "show_columns", "janno_d_da_j_full",
+            "janno_column_order"
             ]
 
     runTestScripts "poseidon" poseidon_tests

test/tests/poseidon/data/column_order_test.janno

@@ -0,0 +1,2 @@
+Nr_Libraries	Genetic_Sex	Source_Tissue	Poseidon_ID	Source_Material	Species	Location	Primary_Contact	Relation_Degree_Note	Nr_Libraries_Note	Relation_To	Relation_Note	Relation_Degree	Relation_Type	Group_Name
+10	F	bone	A	bone	Homo Sapiens	Chiayi City	Bob	test rel deg note	test nr lib note	C	test rel note	identical	identical twin	B

test/tests/poseidon/janno_column_order.out

@@ -0,0 +1,2 @@
+source_file	Poseidon_ID	Genetic_Sex	Group_Name	Species	Relation_To	Relation_Degree	Relation_Degree_Note	Relation_Type	Relation_Note	Location	Source_Material	Nr_Libraries	Nr_Libraries_Note	Primary_Contact	Source_Tissue
+data/column_order_test.janno	A	F	B	Homo Sapiens	C	identical	test rel deg note	identical twin	test rel note	Chiayi City	bone	10	test nr lib note	Bob	bone

test/tests/poseidon/janno_column_order.sh

@@ -0,0 +1 @@
+qjanno "SELECT * FROM data/column_order_test.janno" --raw

test/tests/poseidon/janno_d_da_j.out

@@ -1 +1 @@
-86
+87

test/tests/poseidon/janno_j.out

@@ -1 +1 @@
-32
+33