-
Notifications
You must be signed in to change notification settings - Fork 18
Expand file tree
/
Copy pathdecoding.sh
More file actions
executable file
·2924 lines (2624 loc) · 191 KB
/
decoding.sh
File metadata and controls
executable file
·2924 lines (2624 loc) · 191 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
############## Decoding ################################################
### For each real receiver/band there is one decode daemon and one recording daemon
### Waits for a new wav file then decodes and posts it to all of the posting client
declare -r DECODING_CLIENTS_SUBDIR="decoding_clients.d" ### Each decoding daemon will create its own subdir where it will copy YYMMDD_HHMM_wspr_spots.txt
declare MAX_ALL_WSPR_SIZE=200000 ### Delete the ALL_WSPR.TXT file once it reaches this size.. Stops wsprdaemon from filling ${WSPRDAEMON_TMP_DIR}/..
declare FFT_WINDOW_CMD=${WSPRDAEMON_ROOT_DIR}/wav_window.py
declare C2_FFT_ENABLED="yes" ### If "yes", then use the c2 file produced by wsprd to calculate FFT noise levels
declare C2_FFT_CMD=${WSPRDAEMON_ROOT_DIR}/c2_noise.py
function get_decode_mode_list() {
local modes_variable_to_return=$1
local receiver_modes_arg=$2
local receiver_band=$3
local temp_receiver_modes
local rc
temp_receiver_modes=${receiver_modes_arg}
if [[ ${receiver_modes_arg} == "DEFAULT" ]]; then
### Translate DEFAULT mode to a list of modes for this band
local default_modes=""
get_default_modes_for_band default_modes ${receiver_band}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: 'get_default_modes_for_band default_modes ${receiver_band}' => ${rc}"
sleep 1
return ${rc}
fi
wd_logger 1 "Translated decode mode '${receiver_modes_arg}' to '${default_modes}'"
temp_receiver_modes=${default_modes}
fi
### Validate the mode list
is_valid_mode_list ${temp_receiver_modes}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: 'is_valid_mode_list ${temp_receiver_modes}' => ${rc}"
return 1
fi
wd_logger 2 "Returning modes ${temp_receiver_modes}"
eval ${modes_variable_to_return}=${temp_receiver_modes}
return 0
}
##########
function get_af_db() {
local return_variable_name=$1
local local real_receiver_name=$2 ### 'real' as opposed to 'merged' receiver
local real_receiver_rx_band=$3
local default_value
local af_info_field="$(get_receiver_af_list_from_name ${real_receiver_name})"
if [[ -z "${af_info_field}" ]]; then
wd_logger 2 "Found no AF field for receiver ${real_receiver_name}, so return AF=0"
eval ${return_variable_name}=0
return 0
fi
local af_info_list=(${af_info_field//,/ })
wd_logger 1 "af_info_list= '${af_info_list[*]}'"
for element in ${af_info_list[@]}; do
local fields=(${element//:/ })
if [[ ${fields[0]} == "DEFAULT" ]]; then
default_value=${fields[1]}
wd_logger 1 "Found default value ${default_value}"
elif [[ ${fields[0]} == ${real_receiver_rx_band} ]]; then
if (( ${#fields[@]} < 2 )); then
wd_logger 1 "ERROR: The the AF gain for receiver ${real_receiver_name}, band ${real_receiver_rx_band} is missing on the receive config file line, so default to 0"
eval ${return_variable_name}=0
else
wd_logger 1 "Found AF value ${fields[1]} for receiver ${real_receiver_name}, band ${real_receiver_rx_band}"
eval ${return_variable_name}=${fields[1]}
fi
return 0
fi
done
if [[ -z "${default_value-}" ]]; then
wd_logger 1 "ERROR: can't find af value for receiver ${real_receiver_name}, band ${real_receiver_rx_band}, AND there is no DEFAULT. So return 0"
default_value=0
else
wd_logger 1 "Returning default value ${default_value} for receiver ${real_receiver_name}, band ${real_receiver_rx_band}"
fi
eval ${return_variable_name}=${default_value}
return 0
}
function calculate_nl_adjustments() {
local return_rms_corrections_variable_name=$1
local return_fft_corrections_variable_name=$2
local receiver_band=$3
wd_logger 1 "ARGS: return_rms_corrections_variable_name=${return_rms_corrections_variable_name}, return_fft_corrections_variable_name=${return_fft_corrections_variable_name}, receiver_band=${receiver_band}"
local rc
local wspr_band_freq_khz=$(get_wspr_band_freq_khz ${receiver_band})
local wspr_band_freq_mhz=$(awk "BEGIN {printf \"%.6f\", $wspr_band_freq_khz / 1000}")
local wspr_band_freq_hz=$(awk "BEGIN {printf \"%.0f\", $wspr_band_freq_khz * 1000}")
wd_logger 1 "CALCS: wspr_band_freq_khz=${wspr_band_freq_khz}, wspr_band_freq_mhz=${wspr_band_freq_mhz}, wspr_band_freq_hz=${wspr_band_freq_hz}"
if [[ -f ${WSPRDAEMON_ROOT_DIR}/noise_plot/noise_ca_vals.csv ]]; then
wd_logger 1 "Loading cal_vals[] from '${WSPRDAEMON_ROOT_DIR}/noise_plot/noise_ca_vals.csv'"
local cal_vals=($(sed -n '/^[0-9]/s/,/ /gp' ${WSPRDAEMON_ROOT_DIR}/noise_plot/noise_ca_vals.csv))
else
wd_logger 1 "No ${WSPRDAEMON_ROOT_DIR}/noise_plot/noise_ca_vals.csv, so use defaults"
fi
### In each of these assignments, if cal_vals[] was not defined above from the file 'noise_ca_vals.csv', then use the default value. e.g. cal_c2_correction will get the default value '-187.7
local cal_nom_bw=${cal_vals[0]-320} ### In this code I assume this is 320 hertz
local cal_ne_bw=${cal_vals[1]-246}
if (( cal_ne_bw == 0 )); then
wd_logger 1 "ERROR: cal_ne_bw = 0, which is an invalid value. So assign the default value of '246'"
cal_ne_bw=246
fi
local cal_rms_offset=${cal_vals[2]--50.4}
local cal_fft_offset=${cal_vals[3]--41.0}
local cal_fft_band=${cal_vals[4]--13.9}
local cal_threshold=${cal_vals[5]-13.1}
local cal_c2_correction=${cal_vals[6]--187.7}
local kiwi_amplitude_versus_frequency_correction=$(
gawk -v f=$wspr_band_freq_mhz}{ -f - <<'EOF'
BEGIN {
correction = -1 * ( \
(2.2474 * (10 ^ -7) * (f ^ 6)) - \
(2.1079 * (10 ^ -5) * (f ^ 5)) + \
(7.1058 * (10 ^ -4) * (f ^ 4)) - \
(1.1324 * (10 ^ -2) * (f ^ 3)) + \
(1.0013 * (10 ^ -1) * (f ^ 2)) - \
(3.7796 * (10 ^ -1) * f) - \
(9.1509 * (10 ^ -1)) \
)
print correction
}
EOF
)
wd_logger 1 "Calculated kiwi_amplitude_versus_frequency_correction=${kiwi_amplitude_versus_frequency_correction}"
if awk -v f="$wspr_band_freq_mhz" 'BEGIN { exit !(f > 30) }'; then
# Don't adjust Kiwi's amplitude frequency correction when fed by transverter
kiwi_amplitude_versus_frequency_correction=0
wd_logger 1 "Don't make a correcction since wspr_band_freq_mhz=${wspr_band_freq_mhz} > 30"
fi
local antenna_factor_adjust
get_af_db antenna_factor_adjust ${receiver_name} ${receiver_band}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: can't find AF for ${receiver_name} ${receiver_band}"
exit 1
fi
wd_logger 1 "Got AF = ${antenna_factor_adjust} for ${receiver_name} ${receiver_band}"
local rx_khz_offset=$(get_receiver_khz_offset_list_from_name ${receiver_name})
wd_logger 1 "local total_correction_db=\$(bc <<< 'scale = 10; ${kiwi_amplitude_versus_frequency_correction} + ${antenna_factor_adjust})'"
local total_correction_db=$(awk -v a="$kiwi_amplitude_versus_frequency_correction" -v b="$antenna_factor_adjust" 'BEGIN { printf "%.10f\n", a + b }')
local calculated_rms_nl_adjust=$(awk -v r="$cal_rms_offset" -v ne="$cal_ne_bw" -v t="$total_correction_db" 'BEGIN { var = (r + (10 * (log(1/ne) / log(10))) + t); printf "%.2f\n", var/1.0 }')
wd_logger 1 "calculated_rms_nl_adjust=\$(bc -l <<< \"var=(${cal_rms_offset} + (10 * (l( 1 / ${cal_ne_bw}) / l(10) ) ) + ${total_correction_db}); scale=2; var/1.0\" )"
eval ${return_rms_corrections_variable_name}=${calculated_rms_nl_adjust}
## G3ZIL implementation of algorithm using the c2 file by Christoph Mayer
local calculated_fft_nl_adjust=$(awk -v c="$cal_c2_correction" -v t="$total_correction_db" 'BEGIN { var = c + t; printf "%.2f\n", (var * 100)/100 }')
wd_logger 1 "calculated_fft_nl_adjust = ${calculated_fft_nl_adjust} from calculated_fft_nl_adjust=\$(bc <<< \"scale = 2;var=${cal_c2_correction};var+=${total_correction_db}; (var * 100)/100\")"
eval ${return_fft_corrections_variable_name}="'${calculated_fft_nl_adjust}'"
}
declare WAV_SAMPLES_LIST=(
"${SIGNAL_LEVEL_PRE_TX_SEC} ${SIGNAL_LEVEL_PRE_TX_LEN}"
"${SIGNAL_LEVEL_TX_SEC} ${SIGNAL_LEVEL_TX_LEN}"
"${SIGNAL_LEVEL_POST_TX_SEC} ${SIGNAL_LEVEL_POST_TX_LEN}"
)
### Record an error line to the log file if the wav file contains audio samples which exceed these levels
declare WAV_MIN_LEVEL=${WAV_MIN_LEVEL--1.0}
declare WAV_MAX_LEVEL=${WAV_MAX_LEVEL-1.0}
function get_wav_levels()
{
local __return_levels_var=$1
local wav_filename=$2
local sample_start_sec=$3
local sample_length_secs=$4
local rms_adjust=$5
if [[ ${sample_start_sec} == ${SIGNAL_LEVEL_PRE_TX_SEC} ]]; then
### This function is called three times for each wav file. We only need to check the whole wav file once to determine the min/max values
### So execute this check only the first time
### To see if the AGC might need to change from its default 60, check to see if any samples in the whole wav file closely approach the MAX or MIN sample values
### 'sox -n stats' output this information on seperate line:
### DC offset Min level Max level Pk lev dB RMS lev dB RMS Pk dB RMS Tr dB Crest factor Flat factor Pk count Bit-depth Num samples Length s Scale max Window s
### Field #: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14
### Run 'man sox' and search for 'stats' to find a description of those statistic fields
local full_wav_stats=$(sox ${wav_filename} -n stats 2>&1) ### sox -n stats prints those to stderr
local full_wav_stats_list=( $(echo "${full_wav_stats}" | awk '{printf "%s\t", $NF }') ) ### store them in an array
if [[ ${#full_wav_stats_list[@]} -ne ${EXPECTED_SOX_STATS_FIELDS_COUNT-15} ]]; then
wd_logger 1 "ERROR: Got ${#full_wav_stats_list[@]} stats from 'sox -n stats', not the expected ${EXPECTED_SOX_STATS_FIELDS_COUNT-15} fields:\n${full_wav_stats}"
else
local full_wav_min_level=${full_wav_stats_list[1]}
local full_wav_max_level=${full_wav_stats_list[2]}
local full_wav_peak_level_count=${full_wav_stats_list[9]}
local full_wav_bit_depth=${full_wav_stats_list[10]}
local full_wav_len_secs=${full_wav_stats_list[12]}
### Min and Max level are floating point numbers and their absolute values are less than or equal to 1.0000
if [[ $( echo "${full_wav_min_level} <= ${WAV_MIN_LEVEL}" | bc ) == "1" || $( echo "${full_wav_max_level} >= ${WAV_MAX_LEVEL}" | bc ) == "1" ]] ; then
wd_logger 1 "ERROR: ${full_wav_peak_level_count} full level (+/-1.0) samples detected in file ${wav_filename} of length=${full_wav_len_secs} seconds and with Bit-depth=${full_wav_bit_depth}: the min/max levels are: min=${full_wav_min_level}, max=${full_wav_max_level}"
else
wd_logger 2 "In file ${wav_filename} of length=${full_wav_len_secs} seconds and with Bit-depth=${full_wav_bit_depth}: the min/max levels are: min=${full_wav_min_level}, max=${full_wav_max_level}"
fi
### Create a status file associated with this indsividual wav file from which the decoding daemon will extract wav overload information for the spots decoded from this wav file
echo "WAV_stats: ${full_wav_min_level} ${full_wav_max_level} ${full_wav_peak_level_count}" > ${wav_filename}.stats
### Append these stats to a log file which can be searched by a yet-to-be-implemented 'wd-...' command
local wav_status_file="${WAV_STATUS_LOG_FILE-wav_status.log}"
touch ${wav_status_file} ### In case it doesn't yet exist
if grep -q "${wav_filename}" ${wav_status_file} ; then
wd_logger 1 "ERROR: unexpectly found log line for wav file ${wav_filename} in ${wav_status_file}"
else
wd_logger 2 "Appending '${wav_filename}: ${full_wav_min_level} ${full_wav_max_level} ${full_wav_peak_level_count}' to the log file '${wav_status_file}'"
echo "${wav_filename}: ${full_wav_min_level} ${full_wav_max_level} ${full_wav_peak_level_count}" >> ${wav_status_file}
truncate_file ${wav_status_file} 100000 ### Limit the size of this log file to 100 Kb
fi
fi
fi
local wav_levels_list=( $(sox ${wav_filename} -t wav - trim ${sample_start_sec} ${sample_length_secs} 2>/dev/null | sox - -n stats 2>&1 | awk '/dB/{print $(NF)}'))
if [[ ${#wav_levels_list[@]} -ne 4 ]]; then
wd_logger 1 "ERROR: found only ${#wav_levels_list[@]} dB lines, not the four expected dB lines from 'sox ${wav_filename} -t wav - trim ${sample_start_sec} ${sample_length_secs}'"
return 1
fi
wd_logger 2 "Got sox dB values: '${wav_levels_list[*]}'"
local return_line=""
for db_val in ${wav_levels_list[@]}; do
local adjusted_val=$(bc <<< "scale = 2; (${db_val} + ${rms_adjust})/1") ### '/1' forces bc to use the scale = 2 setting
return_line="${return_line} ${adjusted_val}"
done
wd_logger 2 "Returning adjusted dB values: '${return_line}'"
eval ${__return_levels_var}=\"${return_line}\"
return 0
}
declare WAV_SECOND_RANGE=${WAV_SECOND_RANGE-10} ### wav files of +/- this number of seconds are deemed OK for wsprd to decode
declare TARGET_RAW_WAV_SECONDS=60
declare MIN_VALID_RAW_WAV_SECONDS=${MIN_VALID_RAW_WAV_SECONDS-$(( ${TARGET_RAW_WAV_SECONDS} - ${WAV_SECOND_RANGE} )) }
declare MAX_VALID_RAW_WAV_SECONDS=${MAX_VALID_RAW_WAV_SECONDS-$(( ${TARGET_RAW_WAV_SECONDS} + ${WAV_SECOND_RANGE} )) }
declare TARGET_WSPR_WAV_SECONDS=120
declare MIN_VALID_WSPR_WAV_SECONDS=${MIN_VALID_WSPR_WAV_SECONDS-$(( ${TARGET_WSPR_WAV_SECONDS} - ${WAV_SECOND_RANGE} )) }
declare MAX_VALID_WSPR_WAV_SECONDS=${MAX_VALID_WSPR_WAV_SECONDS-$(( ${TARGET_WSPR_WAV_SECONDS} + ${WAV_SECOND_RANGE} )) }
function is_valid_wav_file()
{
local wav_filename=$1
local min_valid_secs=$2
local max_valid_secs=$3
local rc
if [[ ! -f ${wav_filename} ]]; then
wd_logger 1 "ERROR: no wav file ${wav_filename}"
return 1
fi
if [[ ! -s ${wav_filename} ]]; then
wd_logger 1 "ERROR: zero length wav file ${wav_filename}"
return 1
fi
local wav_stats=$(sox ${wav_filename} -n stats 2>&1 ) ### Don't add ' --keep-foreign-metadata"
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' => ${rc}"
return 1
fi
wd_logger 2 "'sox ${wav_filename} -n stats 2>&1' =>\n${wav_stats}"
local wav_length_line_list=( $(grep '^Length' <<< "${wav_stats}") )
if (( ! ${#wav_length_line_list[@]} )); then
wd_logger 1 "ERROR: can't find wav file 'Length' line in output of 'sox ${wav_filename} -n stats'"
return 1
fi
if (( ${#wav_length_line_list[@]} != 3 )); then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' ouput 'Length' line has ${#wav_length_line_list[@]} fields in it instead of the expected 3 fields"
return 1
fi
local wav_length_secs=${wav_length_line_list[2]/.*}
if [[ -z "${wav_length_secs}" ]]; then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' reports invalid wav file length '${wav_length_line_list[2]}'"
return 1
fi
if [[ ! ${wav_length_secs} =~ ^[0-9]+$ ]]; then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' reports wav file length ${wav_length_line_list[2]} which doesn't contain an integer number"
return 1
fi
if (( ( wav_length_secs < min_valid_secs ) || ( wav_length_secs > max_valid_secs) )); then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' reports invalid wav file length of ${wav_length_secs} seconds. valid min=${min_valid_secs}, valid max=${max_valid_secs}"
return 1
fi
return 0
}
function get_rms_levels()
{
local __return_var_name=$1
local __return_string_name=$2
local wav_filename=$3
local rms_adjust=$4
local rc
if ! is_valid_wav_file ${wav_filename} ${MIN_VALID_WSPR_WAV_SECONDS} ${MAX_VALID_WSPR_WAV_SECONDS} ; then
rc=$?
wd_logger 1 "ERROR: 'valid_wav_file ${wav_filename}' => ${rc}"
return 1
fi
local output_line=""
local sample_info
for sample_info in "${WAV_SAMPLES_LIST[@]}"; do
local sample_line_list=( ${sample_info} )
local sample_start_sec=${sample_line_list[0]}
local sample_length_secs=${sample_line_list[1]}
local sample_vals
get_wav_levels sample_vals ${wav_filename} ${sample_start_sec} ${sample_length_secs} ${rms_adjust}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: 'get_wav_levels sample_vals ${wav_filename} ${sample_start_sec} ${sample_length_secs}' => ${rc}"
return 1
fi
output_line="${output_line} ${sample_vals}"
done
local output_line_list=( ${output_line} )
if [[ ${#output_line_list[@]} -ne 12 ]]; then
wd_logger 1 "ERROR: expected 12 fields of dB info, but got only ${#output_line_list[@]} fields from calls to get_wav_levels()"
return 1
fi
local return_rms_value
local pre_rms_value=${output_line_list[3]} # RMS level is the minimum of the Pre and Post 'RMS Tr dB'
local post_rms_value=${output_line_list[11]} # RMS level is the minimum of the Pre and Post 'RMS Tr dB'
if [[ $(bc --mathlib <<< "${pre_rms_value} < ${post_rms_value}") -eq "1" ]]; then
return_rms_value=${pre_rms_value}
wd_logger 2 "So returning rms_level ${return_rms_value} which is from pre_tx"
else
return_rms_value=${post_rms_value}
wd_logger 2 "So returning rms_level ${return_rms_value} which is from post_tx"
fi
local signal_level_line=" ${output_line} ${return_rms_value}"
eval ${__return_var_name}=${return_rms_value}
eval ${__return_string_name}=\"${signal_level_line}\"
wd_logger 2 "Returning rms_value=${return_rms_value} and signal_level_line='${signal_level_line}'"
return 0
}
### Runs wsprd and outputs new spots to ALL_WSPR.TXT.new
function decode_wspr_wav_file() {
local wav_file_name=$1
local wspr_decode_capture_freq_hz=$2
local rx_khz_offset=$3
local stdout_file=$4
local wsprd_cmd_flags="$5" ### ${WSPRD_CMD_FLAGS}
local wsprd_spreading_cmd_flags="$6" ### ${WSPRD_CMD_FLAGS}
local rc
wd_logger 2 "Decode file ${wav_file_name} for frequency ${wspr_decode_capture_freq_hz} and send stdout to ${stdout_file}. rx_khz_offset=${rx_khz_offset}, wsprd_cmd_flags='${wsprd_cmd_flags}'"
local wspr_decode_capture_freq_hzx=${wav_file_name#*_} ### Remove the year/date/time
wspr_decode_capture_freq_hzx=${wspr_decode_capture_freq_hz%_*} ### Remove the _usb.wav
local wspr_decode_capture_freq_hzx=$( bc <<< "${wspr_decode_capture_freq_hz} + (${rx_khz_offset} * 1000)" )
local wspr_decode_capture_freq_mhz=$( printf "%2.4f\n" $(bc <<< "scale = 5; ${wspr_decode_capture_freq_hz}/1000000.0" ) )
if ! [[ -f ALL_WSPR.TXT ]]; then
touch ALL_WSPR.TXT
fi
sort -k 1,2 -k 5,5 ALL_WSPR.TXT > ALL_WSPR.TXT.save
cp -p ALL_WSPR.TXT.save ALL_WSPR.TXT
timeout ${WSPRD_TIMEOUT_SECS-110} nice -n ${WSPR_CMD_NICE_LEVEL} ${WSPRD_CMD} -c ${wsprd_cmd_flags} -f ${wspr_decode_capture_freq_mhz} ${wav_file_name} > ${stdout_file}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: Command 'timeout ${WSPRD_TIMEOUT_SECS-110} nice -n ${WSPR_CMD_NICE_LEVEL} ${WSPRD_CMD} -c ${wsprd_cmd_flags} -f ${wspr_decode_capture_freq_mhz} ${wav_file_name} > ${stdout_file}' returned error ${rc}"
return ${rc}
fi
sort -k 1,2 -k 5,5 ALL_WSPR.TXT > sort.tmp
mv sort.tmp ALL_WSPR.TXT
comm --nocheck-order -13 ALL_WSPR.TXT.save ALL_WSPR.TXT | sort -k 1,2 -k 5,5 > ALL_WSPR.TXT.new.tmp
wd_logger 1 "wsprd added $(wc -l < ALL_WSPR.TXT.new.tmp) spots to ALL_WSPR.txt and we saved those new spots in ALL_WSPR.TXT.new.tmp:\n$(< ALL_WSPR.TXT.new.tmp)"
### Start with the original ALL_WSPR.TXT and see what spots are reported by wsprd.spreading
wd_logger 2 "Decoding WSPR a second time to obtain spreading information"
cp -p ALL_WSPR.TXT.save ALL_WSPR.TXT
local n_arg="-n"
if [[ ${VERSION_ID} =~ 20.04 ]]; then
n_arg="" ## until we get a wsprd.spreading for U 20.04
fi
if [[ ${WSPRD_TWO_PASS-no} == "no" ]]; then
wd_logger 2 "Skipping wsprd second pass because WSPRD_TWO_PASS == 'no'"
> ${stdout_file}.spreading
else
timeout ${WSPRD_TIMEOUT_SECS-110} nice -n ${WSPR_CMD_NICE_LEVEL} ${WSPRD_SPREADING_CMD} ${n_arg} -c ${wsprd_spreading_cmd_flags} -f ${wspr_decode_capture_freq_mhz} ${wav_file_name} > ${stdout_file}.spreading
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: Command 'timeout ${WSPRD_TIMEOUT_SECS-110} nice -n ${WSPR_CMD_NICE_LEVEL} ${WSPRD_SPREADING_CMD} -n -c ${wsprd_spreading_cmd_flags} -f ${wspr_decode_capture_freq_mhz} ${wav_file_name} > ${stdout_file}.spreading' returned error ${rc}"
# return ${rc}
fi
fi
sort -k 1,2 -k 5,5 ALL_WSPR.TXT > sort.tmp
mv sort.tmp ALL_WSPR.TXT
comm --nocheck-order -13 ALL_WSPR.TXT.save ALL_WSPR.TXT | sort -k 1,2 -k 5,5 > ALL_WSPR.TXT.new.tmp.spreading
wd_logger 1 "wsprd.spreading added $(wc -l < ALL_WSPR.TXT.new.tmp.spreading) spots to ALL_WSPR.txt and added those new spots in ALL_WSPR.TXT.nspreading_ew.tmp:\n$(< ALL_WSPR.TXT.new.tmp.spreading)"
cat ALL_WSPR.TXT.new.tmp.spreading >> ALL_WSPR.TXT.new.tmp
### Restore ALL_WSPR.TXT to its state before either of the decodes added spots
mv ALL_WSPR.TXT.save ALL_WSPR.TXT
### Find the best set of spots from the two passes, giving preference to spots with WSPR-2 spreading information, and append them to ALL_WSPR.TXT so it can use them in the next decoding
awk -f ${AWK_FIND_BEST_SPOT_LINES} ALL_WSPR.TXT.new.tmp | sort -k 1,2 -k 5,5 > ALL_WSPR.TXT.new
cat ALL_WSPR.TXT.new >> ALL_WSPR.TXT
wd_logger 1 "Added the $(wc -l < ALL_WSPR.TXT.new) spots which are the union of the standard and spreading decodes:\n$(< ALL_WSPR.TXT.new)"
truncate_file ALL_WSPR.TXT ${MAX_ALL_WSPR_SIZE}
return ${rc}
}
declare WSPRD_BIN_DIR=${WSPRDAEMON_ROOT_DIR}/bin
declare WSPRD_X86_SPREADING_CMD=${WSPRD_BIN_DIR}/wsprd.spread_nodrift.x86
declare WSPRD_ARM_SPREADING_CMD=${WSPRD_BIN_DIR}/wsprd.spread_nodrift.arm
declare AWK_FIND_BEST_SPOT_LINES=${WSPRDAEMON_ROOT_DIR}/best_spots.awk
declare WSPR_CMD_NICE_LEVEL="${WSPR_CMD_NICE_LEVEL-19}"
declare JT9_CMD_NICE_LEVEL="${JT9_CMD_NICE_LEVEL-19}"
declare WSPRD_STDOUT_FILE=wsprd_stdout.txt ### wsprd stdout goes into this file, but we use wspr_spots.txt
declare MAX_ALL_WSPR_SIZE=200000 ### Truncate the ALL_WSPR.TXT file once it reaches this size.. Stops wsprdaemon from filling ${WSPRDAEMON_TMP_DIR}/..
declare RAW_FILE_FULL_SIZE=1440000 ### Approximate number of bytes in a full size one minute long raw or wav file
### We use 'soxi' to check the length of the 1 minute long wav files created by kiwirecorder.py in a field with the form HOURS:MINUTES:SECONDS.MILLISECONDS
### Because bash can only do integer comparisons, we strip the ':'s and '.' from that field
### As a result, valid wav files will bein the ranges from 6000 - (${MIN_VALID_RAW_WAV_SECONDS} * 100) to 5999
### or in the range from 10000 to (10000 + ${MIN_VALID_RAW_WAV_SECONDS})
### So this code gets the time duration of the wave file into an integer which has the form HHMMSSUU and thus can be compared by a bash expression
### Because the field rolls over from second 59 to minute 1, There can be no fields which have the values 6000 through 9999
declare WAV_FILE_MIN_HHMMSSUU=$(( ${MIN_VALID_RAW_WAV_SECONDS} * 100 )) ### by default this = 55 seconds == 5500
declare WAV_FILE_MAX_HHMMSSUU=$(( 10000 + ( ${WAV_SECOND_RANGE} * 100) )) ### by default this = 65 seconds == 10500
### If the wav recording daemon is running, we can calculate how many seconds until it starts to fill the raw file (if 0 length first file) or fills the 2nd raw file. Sleep until then
function flush_wav_files_older_than()
{
local reference_file=$1
local rc
if [[ ! -f ${reference_file} ]]; then
wd_logger 1 "ERROR: can't find expected reference file '${reference_file}"
return 1
fi
wd_logger 1 "Delete any files older than ${reference_file}"
find -name '*wav' >& find.log
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: 'find -name '*wav' > find.log'=> ${rc}:\n$(< find.log)"
return ${rc}
fi
local olders=0
local newers=0
local wav_file
for wav_file in $(<find.log); do
if [[ ${wav_file} -ot ${reference_file} ]]; then
(( ++olders ))
wd_logger 1 "Deleting older wav file '${wav_file}'"
wd_rm ${wav_file}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: Deleting older wav file '${wav_file}', 'wd_rm ${wav_file}' => ${rc}"
fi
elif [[ ${wav_file} -nt ${reference_file} ]]; then
(( ++newers ))
wd_logger 2 "Found wav file '${wav_file}' is newer than ${reference_file}"
else
### 'find' prepends './' to the filenames it returns, so we can't compare flenames. But if two wav file timestamps in the same directory match each other, then they must be the same wav file
wd_logger 1 "Found expected reference file ${reference_file}"
fi
done
if [[ ${olders} -gt 0 || ${newers} -gt 0 ]]; then
wd_logger 1 "Deleted ${olders} older wav files and/or found ${newers} new wav files"
fi
return 0
}
declare WD_RECORD_HDR_SIZE_BYTES=44 ## wd-record writes a wav file header and then waits until the first sample of the next minute before starting to write samples to the file
declare WAV_FILE_SIZE_POLL_SECS=${WAV_FILE_SIZE_POLL_SECS-2} ## Check that the wav file is growing every NN seconds, 2 seconds by default
function sleep_until_raw_file_is_full() {
local filename=$1
if [[ ! -f ${filename} ]]; then
wd_logger 1 "ERROR: ${filename} doesn't exist"
return 1
fi
local old_file_size=$( ${GET_FILE_SIZE_CMD} ${filename} )
local new_file_size
local start_seconds=${SECONDS}
local rc
sleep ${WAV_FILE_SIZE_POLL_SECS}
while [[ -f ${filename} ]] && new_file_size=$( ${GET_FILE_SIZE_CMD} ${filename}) && [[ ${new_file_size} -eq ${WD_RECORD_HDR_SIZE_BYTES} || ${new_file_size} -gt ${old_file_size} ]]; do
wd_logger 3 "Waiting for file ${filename} to stop growing in size. old_file_size=${old_file_size}, new_file_size=${new_file_size}"
old_file_size=${new_file_size}
sleep ${WAV_FILE_SIZE_POLL_SECS}
done
local loop_seconds=$(( SECONDS - start_seconds ))
if [[ ! -f ${filename} ]]; then
wd_logger 1 "ERROR: file ${filename} disappeared after ${loop_seconds} seconds"
return 1
fi
wd_logger 2 "'${filename}' stopped growing after ${loop_seconds} seconds"
local file_start_minute=${filename:11:2}
local file_start_second=${filename:13:2}
if [[ ${file_start_second} != "00" ]]; then
wd_logger 2 "'${filename} starts at second ${file_start_second}, not at the required second '00', so delete this file which should be the first file created after startup AND any older wav files"
flush_wav_files_older_than ${filename}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: Deleting non 00 second wav file'${filename}', 'flush_wav_files_older_than ${filename}' => ${rc}"
fi
wd_rm ${filename}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: Deleting non 00 second wav file'${filename}', 'wd_rm ${filename}' => ${rc}"
fi
return 2
fi
### Previously, I had just checked the size of the wav file to validate the duration of the recording
### My guesess of the min and max valid wav file size in bytes were too narrow and useful wav files were being thrown away
local wav_file_duration_hh_mm_sec_msec=$(soxi ${filename} | awk '/Duration/{print $3}')
local wav_file_duration_integer=$(sed 's/[\.:]//g' <<< "${wav_file_duration_hh_mm_sec_msec}")
wd_logger 1 "Got wav file ${filename} header which reports duration = ${wav_file_duration_hh_mm_sec_msec} => wav_file_duration_integer = ${wav_file_duration_integer}. WAV_FILE_MIN_HHMMSSUU=${WAV_FILE_MIN_HHMMSSUU}, WAV_FILE_MAX_HHMMSSUU=${WAV_FILE_MAX_HHMMSSUU}"
if [[ 10#${wav_file_duration_integer} -lt ${WAV_FILE_MIN_HHMMSSUU} ]]; then ### The 10#... forces bash to treat wav_file_duration_integer as a decimal, since its leading zeros would otherwise identify it at an octal number
wd_logger 2 "The wav file stabilized at invalid too short duration ${wav_file_duration_hh_mm_sec_msec} which almost always occurs at startup. Flush this file since it can't be used as part of a WSPR wav file"
flush_wav_files_older_than ${filename}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: While flushing too short wav file'${filename}', 'flush_wav_files_older_than ${filename}' => ${rc}"
fi
wd_rm ${filename}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: While flushing too shortwav file'${filename}', 'wd_rm ${filename}' => ${rc}"
fi
return 2
fi
if [[ 10#${wav_file_duration_integer} -gt ${WAV_FILE_MAX_HHMMSSUU} ]]; then
### If the wav file has grown to longer than one minute, then it is likely there are two kiwirecorder jobs running
### We really need to know the IP address of the Kiwi recording this band, since this freq may be recorded by other other Kiwis in a Merged group
local this_dir_path_list=( ${PWD//\// } )
local kiwi_name=${this_dir_path_list[-2]}
local kiwi_freq=${filename#*_}
kiwi_freq=${kiwi_freq::3}
local ps_output=$(ps aux | grep "${KIWI_RECORD_COMMAND}.*${kiwi_freq}.*${receiver_ip_address/:*}" | grep -v grep)
local kiwirecorder_pids=( $(awk '{print $2}' <<< "${ps_output}" ) )
if [[ ${#kiwirecorder_pids[@]} -eq 0 ]]; then
wd_logger 1 "ERROR: wav file stabilized at invalid too long duration ${wav_file_duration_hh_mm_sec_msec}, but can't find any kiwirecorder processes which would be creating it;\n$(soxi ${filename})"
else
wd_kill ${kiwirecorder_pids[@]}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: 'wd_kill ${kiwirecorder_pids[*]}' => ${rc}"
fi
wd_logger 1 "ERROR: wav file stabilized at invalid too long duration ${wav_file_duration_hh_mm_sec_msec}, so there appear to be more than one instance of the KWR running. 'ps' output was:\n${ps_output}\nSo executed 'wd_kill ${kiwirecorder_pids[*]}'"
echo ${force_abort}
fi
flush_wav_files_older_than ${filename}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: Deleting non 00 second wav file'${filename}', 'flush_wav_files_older_than ${filename}' => ${rc}"
fi
wd_rm ${filename}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: Deleting non 00 second wav file'${filename}', 'wd_rm ${filename}' => ${rc}"
fi
return 3
fi
wd_logger 2 "File ${filename} for minute ${filename:11:2} stabilized at size ${new_file_size} after ${loop_seconds} seconds"
return 0
}
### Returns the minute and epoch of the first sample in 'filename'. Variations in CPU and OS make using the file's timestamp a poor choice for the time source.
### So use the time in the file's name
function get_file_start_time_info()
{
local __epoch_return_variable_name=$1
local __minute_return_variable_name=$2
local file_name=$3
local rc
local epoch_from_file_stat=$( ${GET_FILE_MOD_TIME_CMD} ${file_name})
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: '${GET_FILE_MOD_TIME_CMD} ${file_name}' => ${rc}"
return 1
fi
local minute_from_file_epoch=$( printf "%(%M)T" ${epoch_from_file_stat} )
local year_from_file_name="${file_name:0:4}"
local month_from_file_name=${file_name:4:2}
local day_from_file_name=${file_name:6:2}
local hour_from_file_name=${file_name:9:2}
local minute_from_file_name=${file_name:11:2}
local file_spec_for_date_cmd="${month_from_file_name}/${day_from_file_name}/${year_from_file_name} ${hour_from_file_name}:${minute_from_file_name}:00"
local epoch_from_file_name=$( date --date="${file_spec_for_date_cmd}" +%s )
if [[ ${minute_from_file_epoch} != ${minute_from_file_name} ]]; then
wd_logger 1 "INFO: minute_from_file_epoch=${minute_from_file_epoch} != minute_from_file_name=${minute_from_file_name}, but always use file_name times"
fi
wd_logger 1 "File '${file_name}' => epoch_from_file_stat=${epoch_from_file_stat}, epoch_from_file_name=${epoch_from_file_name}, minute_from_file_epoch=${minute_from_file_epoch}, minute_from_file_name=${minute_from_file_name}"
eval ${__epoch_return_variable_name}=${epoch_from_file_name}
eval ${__minute_return_variable_name}=${minute_from_file_name}
return 0
}
###
### Get the epoch from the wav filename
function epoch_from_filename()
{
local file_name=${1##*/} ## strip off the path leaving only the filename
local rc
local file_date_format="${file_name:0:8} ${file_name:9:2}:${file_name:11:2}:${file_name:13:2}"
local file_epoch=$(date -d "${file_date_format}" +%s)
rc=$? ; if (( rc )); then
wd_1ogger 1 "ERROR: 'date -d "${file_date_format}" +%s' => ${rc}"
return ${rc}
fi
echo "${file_epoch}"
return 0
}
### Get the minute from the wav filename
function minute_from_filename()
{
local file_name=${1##*/} ## strip off the path leaving only the filename
echo "${file_name:11:2}"
return 0
}
### If filename includes second 59, then rename it to next minute second 00
declare MIN_ACCEPTED_GAP=${MIN_ACCEPTED_GAP-50} ## pcmrecord uses wall clock times, so fkilenames can have '59' seconds as start time
declare MAX_ACCEPTED_GAP=${MAX_ACCEPTED_GAP-70}
declare FILE_NAME_REJECT_SECONDS_MIN=$(( 60 - MIN_ACCEPTED_GAP )) ### Filenames outside the range of 50-59 an 0-10 will be rejected
declare FILE_NAME_REJECT_SECONDS_MAX=$(( 60 - FILE_NAME_REJECT_SECONDS_MIN ))
function adjust_file_named_59_seconds_to_nearest_minute() {
local __return_new_file_path=$1
local adjust_current_file_path=$2
local adjust_current_file_dir=${adjust_current_file_path%/*}
local adjust_current_file_name=${adjust_current_file_path##*/}
local adjust_current_file_minutes=${adjust_current_file_name:11:2}
local adjust_current_file_seconds=${adjust_current_file_name:13:2}
local adjust_current_file_Z_to_end="Z_${adjust_current_file_name##*Z_}"
if [[ "${adjust_current_file_seconds}" == "00" ]]; then
wd_logger 2 "File ${adjust_current_file_path##*/} is named for second 0, so nothing to do"
eval ${__return_new_file_path}=${adjust_current_file_path} ## By default don't rename the file
return 0
fi
if [[ "${ADJUST_FILENAME_TO_NEAREST_SECOND_ZERO-no}" == "no" ]]; then
wd_logger 1 "ERROR: ADJUST_FILENAME_TO_NEAREST_SECOND_ZERO is 'no' but '${adjust_current_file_path##*/}' is named for second '${adjust_current_file_seconds}' not the expected '00'"
eval ${__return_new_file_path}=${adjust_current_file_path} ## By default don't rename the file
return 0
fi
local current_file_seconds_int=$(( 10#${adjust_current_file_seconds} ))
if (( current_file_seconds_int > FILE_NAME_REJECT_SECONDS_MIN && current_file_seconds_int < FILE_NAME_REJECT_SECONDS_MAX )); then
wd_logger 1 "ERROR: File ${adjust_current_file_path##*/} is named for second ${adjust_current_file_seconds}, which is not in the acceptable range of ${FILE_NAME_REJECT_SECONDS_MAX} to ${FILE_NAME_REJECT_SECONDS_MIN} seconds, so dump the file"
wd_rm ${adjust_current_file_path}
return 1
fi
### This file is named for second 50-70, so rename it to second 00 of the nearest minute
local adjust_current_file_epoch=$(epoch_from_filename ${adjust_current_file_path} )
local nearest_minute_epoch=$(( (adjust_current_file_epoch + 30) /60 * 60 ))
local nearest_minute_file_path=$(printf "%s/%(%Y%m%dT%H%M%S)T%s" "${adjust_current_file_dir}" "${nearest_minute_epoch}" "${adjust_current_file_Z_to_end}")
wd_logger 1 "Renaming the second ${adjust_current_file_seconds} file ${adjust_current_file_path##*/} to the nearest minute ${nearest_minute_file_path##*/}"
local rc
mv ${adjust_current_file_path} ${nearest_minute_file_path}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: failed ' mv ${adjust_current_file_path##*/} ${nearest_minute_file_path}' => ${rc}"
return 3
fi
wd_logger 1 "File ${adjust_current_file_path##*/} is named for second ${adjust_current_file_seconds}, so rename it to second 00 of the nearest minute ${nearest_minute_file_path##*/}"
eval ${__return_new_file_path}=${nearest_minute_file_path} ## By default don't rename the file
}
### Given a list of filenames, start from the newest file, the one at the end of the list (i.e. [-1]), and work towards the front of the list
### Make sure that each earlier filename is 1 minute earlier. If not, then flush all the older files from the list
function cleanup_wav_file_list()
{
local __return_clean_files_string_name=$1
local check_file_list=( $2 )
local rc
wd_logger 2 "Testing list of ${#check_file_list[@]} raw files: '${check_file_list[*]}'"
if (( ! ${#check_file_list[@]} )); then
wd_logger 1 "ERROR: check_file_list[] is empty"
return 1
fi
local epoch_of_newest_file=$( epoch_from_filename "${check_file_list[-1]}" )
wd_logger 2 "Checking for valid list of wav_raw files which end with file ${check_file_list[-1]} = epoch ${epoch_of_newest_file} = minute $(( ( ${epoch_of_newest_file} % 3600 ) / 60 ))"
local flush_files="no"
### Walk back from the end of the file list verifying that each preceeding file starts one minute earlier and is full sized.
### If a invalid file is found, flush it and all earlier files
local raw_file_index=$(( ${#check_file_list[@]} - 2 )) ### Start testing the second to last file in the list
local epoch_of_last_file=${epoch_of_newest_file} ### So the epoch of the last file is the last
local return_clean_files_string="${check_file_list[-1]}" ### The last file is clean
### Now walk backwards through the check_file_list[] verifying that each file is full length and 60 seconds earlier than than its successor file
while (( raw_file_index )); do
local test_file_name
test_file_name=${check_file_list[${raw_file_index}]}
wd_logger 2 "Testing file ${test_file_name}"
if [[ ${flush_files} == "yes" ]]; then
wd_logger 1 "flush_files == 'yes', so flushing file ${test_file_name}"
wd_rm ${test_file_name}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: for flush_files == 'yes' ${test_file_name}', 'wd_rm ${test_file_name}' => ${rc}"
fi
(( --raw_file_index ))
continue
fi
is_valid_wav_file ${test_file_name} ${MIN_VALID_RAW_WAV_SECONDS} ${MAX_VALID_RAW_WAV_SECONDS}
rc=$? ; if (( rc )); then
### Found a wav file with invalid size
wd_logger 1 "ERROR: found wav file '${test_file_name}' has invalid size. Flush it and all earlier wav files"
wd_rm ${test_file_name}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: Failed to flush the first invalid file we found, ${test_file_name}', 'wd_rm ${test_file_name}' => ${rc}"
fi
flush_files="yes"
(( --raw_file_index ))
continue
fi
### wav file size is valid
local epoch_of_test_file=$( epoch_from_filename ${test_file_name} )
wd_logger 2 "test_file_name=${test_file_name} = ${epoch_of_test_file} = minute $(( ( ${epoch_of_test_file} % 3600 ) / 60 ))"
: <<'COMMENTED_OUT_LINES'
### see if it is one minute (60 second) earlier than the previous file
local file_epoch_gap=$(( ${epoch_of_last_file} - ${epoch_of_test_file} ))
if [[ ${file_epoch_gap} -ne 60 ]]; then
wd_logger 1 "ERROR: test_file_name=${test_file_name} is file_epoch_gap=${file_epoch_gap} seocnds, not 1 minute (60 seconds), earlier than the next file in the list. So delete it and all earlier files in the list"
local rc
wd_rm ${test_file_name}
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: Failed to flush ${test_file_name}' which is not one minute earlier than the next wav file in the list: 'wd_rm ${test_file_name}' => ${rc}"
fi
flush_files="yes"
wd_logger 1 "test_file_name=${test_file_name} is 1 minute (60 seconds) earlier than the next file in the list"
epoch_of_last_file=${epoch_of_test_file}
(( --raw_file_index ))
continue
fi
COMMENTED_OUT_LINES
wd_logger 2 "test_file_name='${test_file_name}' from index ${raw_file_index} is clean and 60 seconds earlier the the next file in the list. Proceed to check previous file on the list"
epoch_of_last_file=${epoch_of_test_file}
return_clean_files_string="${return_clean_files_string} ${test_file_name}"
(( --raw_file_index ))
done
local clean_files_list=( ${return_clean_files_string} )
wd_logger 2 "Given check_file_list[${#check_file_list[@]}] ='${check_file_list[*]}'"
wd_logger 2 "Returning clean_file_list[${#clean_files_list[*]}] ='${clean_files_list[*]}'"
if [[ ${#check_file_list[@]} -ne ${#clean_files_list[*]} ]]; then
wd_logger 1 "ERROR: Found errors in wav file list, so cleaned list check_file_list[${#check_file_list[@]}]='${check_file_list[*]}' => clean_file_list[${#clean_files_list[*]}]='${clean_files_list[*]}'"
fi
eval ${__return_clean_files_string_name}=\"${return_clean_files_string}\"
return 0
}
function file_is_open() {
local file_path="${1}"
local kiwirecorder_file_must_be_closed_seconds=${2} ### If the file is being written by kiwirecorder, then typically wait for it to be 2+ seconds old before report back that the wav file is filled
local file_directory="${file_path%/*}"
if [[ "${file_directory}" != "${PWD}" ]]; then
if lsof ${file_path} >& /dev/null; then
wd_logger 1 "Running in ${PWD}, ${file_path} is open"
return 0
else
wd_logger 1 "Running in ${PWD}, ${file_path} is closed"
return 1
fi
else
local file_age_seconds=$(( $(date +%s) - $(stat --format=%Y ${file_path}) ))
if (( file_age_seconds < kiwirecorder_file_must_be_closed_seconds )); then
wd_logger 1 "'${file_path}' has been closed only for ${file_age_seconds}, so we can assume kiwirecorder is still writing to it"
return 0
else
wd_logger 1 "'${file_path}' has been closed for ${file_age_seconds} which is more than the ${kiwirecorder_file_must_be_closed_seconds} seconds we expect will signal that kiwirecorder is finished writing sampled to it. So consider it a closed file"
return 1
fi
fi
}
declare last_wav_file_name=""
declare wav_file_create_time_log_file="./wav_file_create_time.log" ### each RX888 band has its own create lime log file
declare WWV_START_CMD="${WSPRDAEMON_ROOT_DIR}/wwv_start.py"
if ! [[ -x $WWV_START_CMD ]]; then
wd_logger 1 "ERROR: can't find expected program '$WWV_START_CMD'"
return 1
fi
function log_wav_file_create_times() {
local wav_file_name="$1"
wd_logger 1 "Logging wav file '$wav_file_name' create time line to $wav_file_create_time_log_file"
if [[ ! -f "$wav_file_name" ]]; then
wd_logger 1 "ERROR: can't find '$wav_file_name'"
return 1
fi
if [[ -z "$last_wav_file_name" ]]; then
echo "$(date): ================ starting decoding ===============" >> "$wav_file_create_time_log_file"
last_wav_file_name="$wav_file_name"
fi
local file_stat_list=($( stat "$wav_file_name" | awk '/File/{printf "%s:", $2} /Size/{printf " Size:%s", $2} /Birth/{printf " Birth:%s", $3} /Change/{printf " Change:%s", $3} END {printf "\n"}' ))
file_stat_list[4]="Tone_burst_offset:Not_measured"
if [[ ${WWV_TONE_BURST_LOGGING-no} == "yes" && "$wav_file_name" =~ WWV ]]; then
if ! [[ -x ${WSPRDAEMON_ROOT_DIR}/venv/bin/python3 ]]; then
wd_logger 1 "ERROR: WWV_TONE_BURST_LOGGING='yes' in WD.conf, but the needed 250 MB python virtual environment '${WSPRDAEMON_ROOT_DIR}/venv/bin/python3' is not installed. So skipping this feature"
return 0
fi
local wwv_burst_offset_msecs=0
wwv_burst_offset_msecs=$( $WWV_START_CMD "$wav_file_name" )
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: `$WWV_START_CMD $wav_file_name` => ${rc}"
else
file_stat_list[4]="Tone_burst_offset:${wwv_burst_offset_msecs// /_}" ### For ease of parsing, replace' 's with '_'s
wd_logger 1 "Found that the tone burst which starts each second is $wwv_burst_offset_msecs from the begining of '$wav_file_name'"
fi
fi
echo "${file_stat_list[0]} ${file_stat_list[1]} ${file_stat_list[3]} ${file_stat_list[2]} ${file_stat_list[4]}" >> "$wav_file_create_time_log_file"
truncate_file "$wav_file_create_time_log_file" ${MAX_CREATE_TIME_LOG-2000000} ### Default is to truncate this log file to 2 MB
return 0
}
function file_is_closed_or_last_write_was_seconds_ago() {
local file_path="${1}"
local wait_for_no_writes_seconds=${2} ### Tyypicaly we wait for 60+ seconds for a one minute file to be filled
local kiwirecorder_file_must_be_closed_seconds=${3} ### If the file is being written by kiwirecorder, then typically wait for it to be 2+ seconds old before report back that the wav file is filled
local file_directory="${file_path%/*}"
if [[ "${file_directory}" != "${PWD}" ]]; then
wd_logger 2 "Checking ${file_path} is not in ${PWD}, so assuming it is being written by pcmrecord and ths wait for it to be closed"
local start_second=${SECONDS}
if ! lsof ${file_path} >& /dev/null; then
wd_logger 2 "The file ${file_path} is closed"
return 0
else
wd_logger 2 "The file ${file_path} is open so run 'inotifywait --timeout ${INOTIFYWAIT_TIMEOUT_SECS-62} --event close_write,delete_self,move_self ${file_path}' so we wakeup when it is closed"
inotifywait --timeout ${INOTIFYWAIT_TIMEOUT_SECS-62} --event close_write,delete_self,move_self ${file_path} >& /dev/null
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: After $(( SECONDS - start_second )) seconds, 'inotifywait --timeout ${INOTIFYWAIT_TIMEOUT_SECS-62} --event close_write,delete_self,move_self ${file_path}' => ${rc}"
return ${rc}
else
log_wav_file_create_times "${file_path}"
local date_time_list=( $(stat ${file_path} | awk '/Modify/{printf "%s %s", $2, $3} ') )
local date=${date_time_list[0]}
local hhmmss=${date_time_list[1]%.*}
local microseconds=${date_time_list[1]##*.}
local seconds=$(( 10#${hhmmss##*:} )) ### converts second 00-59 to a decimal integer
local max_delay_secs=${RECORDING_MAX_DELAY_SECONDS-2} ### If 'stat' reports last write is greater than 2 seconds
local min_early_secs=${RECORDING_MIN_EARLY_SECONDS-59} ### and it is earlier than second 59, then log error
if (( (seconds > max_delay_secs) && (seconds < min_early_secs) )); then
wd_logger 1 "ERROR: File ${file_path##*/} last write time ${date_time_list[*]} is not in second 00"
else
wd_logger 1 "File ${file_path##*/} last write time ${date_time_list[*]} shows it has been closed in second $seconds after a wait of $(( SECONDS - start_second )) seconds"
fi
return 0
fi
fi
fi
wd_logger 2 "Check that file in ${PWD} has not been written for ${wait_for_no_writes_seconds} seconds since it is probably being written by kiwirecorders which opens and closes the file every second to write new samples"
local file_age_seconds=0
local stat_output
local timeout=${wait_for_no_writes_seconds}
while (( --timeout >= 0 )); do
if ! stat_output=$(stat --format=%Y "${file_path}" 2>/dev/null); then
wd_logger 1 "ERROR: stat failed on '${file_path}' - file may have disappeared"
sleep 2
return 2
fi
file_age_seconds=$(( $(date +%s) - stat_output ))
if (( file_age_seconds >= kiwirecorder_file_must_be_closed_seconds )); then
wd_logger 2 "File ${file_path} has been closed for ${file_age_seconds} seconds, so we can assume kiwirecorder is really done with it"
return 0
fi
wd_logger 2 "File ${file_path} is only ${file_age_seconds} seconds old, so sleep 1 second and check again"
sleep 1
done
wd_logger 1 "ERROR: timeout after ${wait_for_no_writes_seconds} seconds while waiting for ${file_path} to be at least ${kiwirecorder_file_must_be_closed_seconds} seconds old"
return 1
}
### Scott's pcmrecord creates the first wav file only at the next second 59 to second 00 transition
### so when pcmrecord is running but there is no wav file being recorded, wist up to 60+ seconds for the wav file to appear
declare MAX_SECONDS_TO_WAIT_FOR_FIRST_WAV=${MAX_SECONDS_TO_WAIT_FOR_FIRST_WAV-62}
function wait_for_pid_to_run_seconds()
{
local pid_to_wait_for=$1
local rc
wd_logger 1 "Wait for pid ${pid_to_wait_for} to be running for more than ${MAX_SECONDS_TO_WAIT_FOR_FIRST_WAV} seconds or until the next second 59->00 transition"
local pid_start_date=$(ps -o lstart= -p ${pid_to_wait_for})
rc=$? ; if (( ${rc} )); then
wd_logger 1 "ERROR: pcmrecording 'ps -o lstart= -p ${pcmrecord_pid}'=> $rc}"
return ${rc}
fi
local pid_start_epoch=$( date -d "${pid_start_date}" +%s )
rc=$? ; if (( rc )); then
wd_logger 1 "ERROR: 'date -d "${pid_start_date}" +%' => ${rc}"
echo ${force_abort}
fi
if [[ -z "${pid_start_epoch}" ]]; then
wd_logger 1 "ERROR: 'date -d "${pid_start_date}" +%' => ${rc}, but it returned an empty epoch"
echo ${force_abort}
fi
local seconds_since_pid_start=$(( EPOCHSECONDS - pid_start_epoch ))
local second_in_minute=$(( EPOCHSECONDS % 60 ))
local seconds_to_sleep=$(( 60 - second_in_minute + 2))
#wd_logger 1 "At current second ${second_in_minute} there are ${seconds_to_sleep} seconds before we look to see if the first wav file has appeared. So sleep for ${seconds_to_sleep} seconds"
wd_sleep ${seconds_to_sleep}
wd_logger 2 "Woke up after ${seconds_to_sleep} seconds"
return 0
}
### Waits for wav files needed to decode one or more of the WSPR packet length wav file have been fully recorded
### Then returns zero or more space-seperated strings each of which has the form 'WSPR_PKT_SECONDS:ONE_MINUTE_WAV_FILENAME_0,ONE_MINUTE_WAV_FILENAME_1[,ONE_MINUTE_WAV_FILENAME_2...]'
function wait_until_newest_tmp_file_is_closed()
{
local wav_file_dir_path=$1
local wav_file_regex=$2
local receiver_name=$3
local receiver_band=$4
wd_logger 2 "Starting with args: wav_file_dir_path=${wav_file_dir_path}, wav_file_regex=${wav_file_regex}, receiver_name=${receiver_name}, receiver_band=${receiver_band}"
local newest_tmp_wav_file
local rc
while true; do
wd_logger 1 "Looking for a '${wav_file_regex}' file in ${wav_file_dir_path}"
### WD's mutex library creates and almost immediately deletes transient ...mutex.lock directories. If find finds a mutex.lock directory and then it disappaears
### during the filtering stage of find, then find prints a 'No such file or directory' message to stderr
### Since I can't suppress those messages, direct stderr to /dev/null so those messages don't appear in the user's terminal window
find "${wav_file_dir_path}" -maxdepth 1 -type f \( -name "${wav_file_regex}" -o -name "${wav_file_regex}.tmp" \) >find.log 2>/dev/null
rc=$?; if (( rc )); then
wd_logger 1 "ERROR: 'find ${wav_file_dir_path} -maxdepth 1 -type f \( -name ${wav_file_regex} -o -name ${wav_file_regex}.tmp \) | sort | tail -1' "
echo ${force_abort}
fi
newest_tmp_wav_file=$(sort find.log | tail -1 )
if [[ -n "${newest_tmp_wav_file}" ]]; then
wd_logger 1 "Found the newest tmp file ${newest_tmp_wav_file}, so execute 'inotifywait --timeout ${INOTIFYWAIT_TIMEOUT_SECS-62} --event close_write,delete_self,move_self ${newest_tmp_wav_file} to wait for it to be closed"
local timeout
for (( timeout=0; timeout < 5; ++timeout )); do
if [[ -f "${newest_tmp_wav_file}" ]]; then
wd_logger 1 "Found ${newest_tmp_wav_file} exists after ${timeout} seconds"
break
fi
wd_logger 1 "WARNING: can't find expected file ${newest_tmp_wav_file}, so sleeping 1"
sleep 1
done
if ! [[ -f "${newest_tmp_wav_file}" ]]; then
wd_logger 1 "Timeout waiting for the expected file ${newest_tmp_wav_file} exists after ${timeout} seconds"
return 1
fi
inotifywait --timeout ${INOTIFYWAIT_TIMEOUT_SECS-62} --event close_write,delete_self,move_self ${newest_tmp_wav_file} >& inotifywait.log # /dev/null
rc=$? ; if (( rc == 0 )); then
### I expect this is the normal path through this function
wd_logger 1 "'inotifywait --timeout ${INOTIFYWAIT_TIMEOUT_SECS-62} --event close_write,delete_self,move_self ${newest_tmp_wav_file}' => ${rc}, so that file is closed or deleted and we can return"
return 0
else
wd_logger 1 "ERROR: unexpected that 'inotifywait --timeout ${INOTIFYWAIT_TIMEOUT_SECS-62} --event close_write,delete_self,move_self ${newest_tmp_wav_file}' => ${rc}':\n$(<inotifywait.log)"
# printf "$(date): ERROR: unexpected that 'inotifywait --timeout ${INOTIFYWAIT_TIMEOUT_SECS-62} --event close_write,delete_self,move_self ${newest_tmp_wav_file}' => ${rc}':\n$(<inotifywait.log)\n " >&2
return 1
# echo ${force_abort}
fi
else
### We found no '*wav*' files, so make sure pcmrecord is running
local receiver_ip=$(get_receiver_ip_from_name ${receiver_name})
local pcm_dns_regex
if [[ "${receiver_name}" =~ ^KA9Q ]]; then