bird-wave/src/clj/bird_wave/pigpen.clj at 954995665abe1b2a0617d349b379a39d4f34e9c5 · edgecase/bird-wave · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
(ns bird-wave.pigpen
  (:require [pigpen.core :as pig]
            [pigpen.fold :as fold]
            [clojure.string :as cs]
            [clojure.instant :as inst]))

(defn birds-file
  [filename]
  (pig/load-tsv filename))

(def field-positions
  [:sighting/guid                   ;; "GLOBAL UNIQUE IDENTIFIER"
   :taxon/order                     ;; "TAXONOMIC ORDER"
   nil                              ;; "CATEGORY"
   :taxon/common-name               ;; "COMMON NAME"
   :taxon/scientific-name           ;; "SCIENTIFIC NAME"
   :taxon/subspecies-common-name    ;; "SUBSPECIES COMMON NAME"
   :taxon/subspecies-scientific-name;; "SUBSPECIES SCIENTIFIC NAME"
   :sighting/count                  ;; "OBSERVATION COUNT" ;; x indicates uncounted
   nil                              ;; "BREEDING BIRD ATLAS CODE"
   nil                              ;; "AGE/SEX"
   nil                              ;; "COUNTRY"
   nil                              ;; "COUNTRY_CODE"
   :sighting/state                  ;; "STATE_PROVINCE"
   :sighting/state-code             ;; "SUBNATIONAL1_CODE"
   :sighting/county                 ;; "COUNTY"
   :sighting/county-code            ;; "SUBNATIONAL2_CODE"
   nil                              ;; "IBA CODE"
   :sighting/locality               ;; "LOCALITY"
   nil                              ;; "LOCALITY ID"
   nil                              ;; "LOCALITY TYPE"
   :sighting/latitude               ;; "LATITUDE"
   :sighting/longitude              ;; "LONGITUDE"
   :sighting/date                   ;; "OBSERVATION DATE"
   nil                              ;; "TIME OBSERVATIONS STARTED"
   nil                              ;; "TRIP COMMENTS"
   nil                              ;; "SPECIES COMMENTS"
   nil                              ;; "OBSERVER ID"
   nil                              ;; "FIRST NAME"
   nil                              ;; "LAST NAME"
   nil                              ;; "SAMPLING EVENT IDENTIFIER"
   nil                              ;; "PROTOCOL TYPE"
   nil                              ;; "PROJECT CODE"
   nil                              ;; "DURATION MINUTES"
   nil                              ;; "EFFORT DISTANCE KM"
   nil                              ;; "EFFORT AREA HA"
   nil                              ;; "NUMBER OBSERVERS"
   nil                              ;; "ALL SPECIES REPORTED"
   nil                              ;; "GROUP IDENTIFIER"
   nil                              ;; "APPROVED"
   nil                              ;; "REVIEWED"
   nil                              ;; "REASON"
   ])

(def fields (concat (remove nil? field-positions)
                    [:sighting/month-yr]))

(defn sighting
  "given a line of text, split on tabs and return the fields we care about
   (indicated by non-nil presence in fields vector)"
  [row]
  (->> row
       (map #(if % [% %2]) field-positions)
       (remove nil?)
       (into {})))

(defn coerce [m f & [key & keys]]
  (if key
    (recur (assoc m key (f (get m key))) f keys)
    m))

(defn parse-count [s]
  (if (= "X" s) 1 (Long/parseLong s)))

(defn coerce-values [sighting]
  (let [date (inst/read-instant-date (:sighting/date sighting)) ]
    (-> sighting
        (coerce bigdec
                :sighting/latitude
                :sighting/longitude)
        (coerce parse-count
                :sighting/count)
        (assoc :sighting/date date)
        (assoc :sighting/month-yr (format "%tY/%tm" date date))
        )))

(defn birds-by-us-county
  [data]
  (->> data
       (pig/map
        (fn [row]
          (-> row
              (sighting)
              (coerce-values))))
       (pig/group-by (juxt :taxon/order :sighting/month-yr :sighting/state :sighting/county)
                     {:fold (fold/juxt (fold/count)
                                       (->> (fold/map :sighting/count) (fold/sum))
                                       (->> (fold/map :sighting/count) (fold/avg)))})
       (pig/map flatten)))

(defn species
  [data]
  (->> data
       (pig/map
        (fn [row]
          (-> row
              (sighting)
              (coerce-values))))
       (pig/group-by (juxt :taxon/order :taxon/common-name)
                     {:fold (fold/count)})
       (pig/map flatten)
       ;; (pig/filter (fn [x] (< 1 (last x))))
       ))

(defn transform-birds
  [input-file output-file]
  (->> (birds-file input-file)
       (birds-by-us-county)
       (pig/store-clj output-file)))

(comment

  (pig/dump (->> "sample_data/birds.txt"
                 (birds-file)
                 (birds-by-us-county)))

  (pig/dump (->> "sample_data/birds.txt"
                 (birds-file)
                 (species)))

  )