Tidy up

[?]

AEj8dahVWy718uSSFPe9VSRJ5qX5G8pC2zvFzJJ8yzBd

Nov 10, 2024, 2:50 AM

7KO4BAOG5HYRSQOFMJTT3AD4K4OPOICXTTVO7K3UPPA24TACMLUAC

Dependencies

[2] QA2TJZRA ripped ML out into the SkraakML repo, compiles quicker now
[3] JYCKLP2E changes to clips and predict
[4] EDYR5C55 fixed ConstructPrimaryDataSet, licence date, Readme, re-arranged Predict.jl
[5] RQ56K2G4 minor work on train and predict
[6] L7G33K4C fixed a comment problem in Clips, and upgraded, now it compiles on Mac again
[7] PQ6OQCBQ work on construct primary dataset
[8] 2UBDFCJH new files tracked
[9] E3Y55MPR added perceptual colour maps and changed get_image_from_sample
[10] MMG2PLXK new functiion to make clips of downloaded bisd calls
[11] OFTU77S5 first commit
[12] FY7CEMM2 mac work on train.jl
[13] YODTMMPT added a whole bunch of functions to Labels.jl for working with raven and avians labels
[14] U46LDPL7 added model, CUDA works again now on ubuntu
[15] NV7FXZ5Q first commit
[16] PHGT4YMB extended Skraak.dddict, and work on all the functions that call it or make it
[17] NMQCXLNG catch up commit, about to do some work on prediction loop
[18] AQWXGGE3 mod to clips.jl
[19] X54TLSYE catch up commit, train now working with julia 1.10.5 but not 1.11
[20] BOPNWZL4 refactored Skraak.jl into sub files, tidy now
[21] 3FAESP6N catch up with changes, mainly construct primary dataset
[22] ROFI4OLA catch up, working on colour images
[23] WOSKMRI4 added module Base and module ML
[*] QPBH7QWC added MLBaste for confusion matrix, f1, roc, must still remove freq tables

Change contents

replacement in src/Utility.jl at line 3

[3.315018]→[3.340666:340745](∅→∅)

export check_png_wav_both_present, resize_image!, move_one_hour!, utc_to_nzdt!

[3.315018]

[3.315222]

export dawn_dusk_of_sunrise_sunset,
get_sunrise_sunset_utc,
make_spectro_from_file,
move_one_hour!,
resample_to_16000hz,
resample_to_8000hz,
resize_image!,
utc_to_nzdt!,
check_png_wav_both_present
edit in src/Utility.jl at line 90

[3.325555]→[3.325555:325601](∅→∅),[3.325601]→[3.340835:340898](∅→∅),[3.340898]→[3.325601:325864](∅→∅),[3.325601]→[3.325601:325864](∅→∅),[3.325864]→[2.329:382](∅→∅),[2.382]→[3.325906:325984](∅→∅),[3.325906]→[3.325906:325984](∅→∅),[3.325984]→[3.340899:340945](∅→∅),[3.340945]→[3.326031:326133](∅→∅),[3.326031]→[3.326031:326133](∅→∅),[3.326133]→[3.78:131](∅→∅),[3.131]→[2.383:451](∅→∅),[3.131]→[3.326185:326189](∅→∅),[2.451]→[3.326185:326189](∅→∅),[3.326185]→[3.326185:326189](∅→∅),[3.341009]→[3.326189:326406](∅→∅),[3.326189]→[3.326189:326406](∅→∅),[3.326406]→[3.132:238](∅→∅),[3.238]→[3.326458:327092](∅→∅),[3.326458]→[3.326458:327092](∅→∅)

end

"""
twilight_tuple_local_time(dt::Date)

#deprecated now have a big sunset_sunrise_utc.csv to work from
Takes a date and returns a tuple with local time twilight times. Use to make a Dataframe then csv.
Queries api.sunrise-sunset.org

was using civil_twilight_end, civil_twilight_begin, changed to sunrise, sunset

Use like this:
Using CSV, Dates, DataFrames, Skraak
df = DataFrames.DataFrame(Date=[], Dawn=[], Dusk=[])
dr = Dates.Date(2019,01,01):Dates.Day(1):Dates.Date(2024,12,31)
for day in dr
q = Skraak.twilight_tuple_local_time(day)
isempty(q) ? println("fail $day") : push!(df, q)
sleep(5)
end
CSV.write("dawn_dusk.csv", df)

using CSV, DataFrames, Dates, HTTP, JSON3, TimeZones

#####deprecated now have a big sunset_sunrise_utc.csv to work from
"""
function twilight_tuple_local_time(dt::Date)
# C05 co-ordinates hard coded into function
resp1 = HTTP.get(
"https://api.sunrise-sunset.org/json?lat=-45.50608&lng=167.47822&date=$dt&formatted=0",
)
#resp2 = String(resp1.body) |> JSON.Parser.parse
resp2 = String(resp1.body) |> x -> JSON3.read(x)
resp3 = get(resp2, "results", "missing")
dusk_utc = get(resp3, "sunset", "missing")
dusk_utc_zoned = ZonedDateTime(dusk_utc, "yyyy-mm-ddTHH:MM:SSzzzz")
dusk_local = astimezone(dusk_utc_zoned, tz"Pacific/Auckland")
dusk_string = Dates.format(dusk_local, "yyyy-mm-ddTHH:MM:SS")
dawn_utc = get(resp3, "sunrise", "missing")
dawn_utc_zoned = ZonedDateTime(dawn_utc, "yyyy-mm-ddTHH:MM:SSzzzz")
dawn_local = astimezone(dawn_utc_zoned, tz"Pacific/Auckland")
dawn_string = Dates.format(dawn_local, "yyyy-mm-ddTHH:MM:SS")
date = Dates.format(dt, "yyyy-mm-dd")
return (date, dawn_string, dusk_string)
edit in src/Utility.jl at line 273

[2.1015]

[2.1015]

end

# Convert mp3's with: for file in *.mp3; do ffmpeg -i "${file}" -ar 16000 "${file%.*}.wav"; done
# Requires 16000hz wav's, works in current folder, need ffmpeg to convert mp3's to wavs at 16000hz
#=
wavs = Glob.glob("*.wav")
for wav in wavs
Skraak.make_spectro_from_file(wav)
edit in src/Utility.jl at line 282

[2.1019]

=#
function make_spectro_from_file(file::String)
signal, freq = WAV.wavread("$file")
freq = freq |> Float32
partitioned_signal = Iterators.partition(signal, 80000) #5s clips

for (index, part) in enumerate(partitioned_signal)
length(part) > 50000 && begin
outfile = "$(chop(file, head=0, tail=4))__$(index)"
image = Skraak.get_image_from_sample(part, freq)
PNGFiles.save("$outfile.png", image)
end
end
end
replacement in src/Labels.jl at line 3

[3.15]→[3.0:87](∅→∅)

export aggregate_labels, audiodata_db, raven_of_avianz, avianz_of_raven, label_summary

[3.15]

[3.53]

export actual_from_folders,
aggregate_labels,
audiodata_db,
avianz_file_of_dict,
avianz_of_raven,
check_change_avianz_species!,
df_of_avianz_dict,
dict_of_avianz_file,
label_summary,
prepare_df_for_raven,
raven_of_avianz,
raven_of_avianz
edit in src/Labels.jl at line 56

[3.1071]→[3.1071:1072](∅→∅)
replacement in src/Labels.jl at line 89

[2.1389]→[2.1389:1563](∅→∅)

This function takes the csv output from my hand classification and ouputs a df, and csv for insertion into AudioData.duckdb using the duckdb cli or using DFto.audiodata_db()

[2.1389]

[2.1563]

This function takes the csv output from my hand classification and ouputs a df, and csv for insertion into AudioData.duckdb using the duckdb cli or using audiodata_db()
replacement in src/ConstructPrimaryDataset.jl at line 3

[3.514]→[3.2433:2530](∅→∅),[3.72]→[3.2433:2530](∅→∅)

using DataFrames, CSV, Glob
using DataFramesMeta: @transform!, @byrow #, @subset!, @passmissing

[3.514]

[3.115]

using DataFrames, CSV, Glob, DataFramesMeta
replacement in src/ConstructPrimaryDataset.jl at line 5

[3.116]→[3.116:224](∅→∅)

function make_dataset(input_file::String, output_path::String="/media/david/SSD2/PrimaryDataset/kiwi_set/")

[3.116]

[3.224]

function make_skraak_dataset(
input_file::String,
output_path::String = "/media/david/SSD2/PrimaryDataset/kiwi_set/",
)
replacement in src/ConstructPrimaryDataset.jl at line 10

[3.329]→[3.329:373](∅→∅),[3.373]→[3.467:502](∅→∅)

move_files(input_file, output_path) |>
x -> save_pngs(x, output_path)

[3.329]

[3.389]

move_files(input_file, output_path) |> x -> save_pngs(x, output_path)
replacement in src/ConstructPrimaryDataset.jl at line 23

[3.2951]→[3.394:415](∅→∅),[3.415]→[3.5082:5106](∅→∅),[3.5082]→[3.5082:5106](∅→∅),[3.5106]→[3.416:440](∅→∅),[3.440]→[3.5178:5180](∅→∅),[3.5178]→[3.5178:5180](∅→∅)

function move_files(
input_file::String,
output_path::String
)

[3.2951]

[2.2635]

function move_files(input_file::String, output_path::String)
replacement in src/ConstructPrimaryDataset.jl at line 54

[3.662]→[3.564:623](∅→∅)

function save_pngs(df::DataFrame,
output_path::String)

[3.662]

[3.6239]

function save_pngs(df::DataFrame, output_path::String)
replacement in src/ConstructPrimaryDataset.jl at line 67

[3.6787]→[3.765:807](∅→∅)

duration = length_signal ÷ freq

[3.6787]

[3.378849]

duration = length_signal ÷ freq
replacement in src/ConstructPrimaryDataset.jl at line 92

[3.7628]→[3.7628:7659](∅→∅),[3.7659]→[3.781:856](∅→∅),[3.856]→[3.7742:7786](∅→∅),[3.1126]→[3.7742:7786](∅→∅),[3.7742]→[3.7742:7786](∅→∅)

PNGFiles.save(
"$output_path$folder/K/$folder-$start-$(start+4).png",
plot,
)

[3.7534]

[3.7786]

PNGFiles.save("$output_path$folder/K/$folder-$start-$(start+4).png", plot)
replacement in src/ConstructPrimaryDataset.jl at line 96

[3.7958]→[3.7958:7989](∅→∅),[3.7989]→[3.857:932](∅→∅),[3.932]→[3.8072:8116](∅→∅),[3.1190]→[3.8072:8116](∅→∅),[3.8072]→[3.8072:8116](∅→∅)

PNGFiles.save(
"$output_path$folder/N/$folder-$start-$(start+4).png",
plot,
)

[3.7864]

[3.8116]

PNGFiles.save("$output_path$folder/N/$folder-$start-$(start+4).png", plot)
replacement in src/ConstructPrimaryDataset.jl at line 117

[3.1375]→[3.1375:1444](∅→∅)

s=(st*freq)-freq+1
en*freq <= len ? e=en*freq : e=len
return s, e

[3.1375]

[3.378850]

s = (st * freq) - freq + 1
en * freq <= len ? e = en * freq : e = len
return s, e
replacement in src/Clips.jl at line 3

[3.12369]→[3.12369:12410](∅→∅)

export make_clips, move_clips_to_folders

[3.12369]

[3.12410]

export make_clips, make_clips_generic, move_clips_to_folders
edit in src/Clips.jl at line 323

[3.22022]→[3.205:430](∅→∅),[3.430]→[2.5863:5889](∅→∅),[2.5889]→[3.451:559](∅→∅),[3.451]→[3.451:559](∅→∅),[3.559]→[2.5890:5930](∅→∅),[2.5930]→[3.595:850](∅→∅),[3.595]→[3.595:850](∅→∅),[3.850]→[2.5931:5992](∅→∅),[2.5992]→[3.904:953](∅→∅),[3.904]→[3.904:953](∅→∅)

end
end
end

# Convert mp3's with: for file in *.mp3; do ffmpeg -i "${file}" -ar 16000 "${file%.*}.wav"; done
# Requires 16000hz wav's, works in current folder, need ffmpeg to convert mp3's to wavs at 16000hz
#=
wavs = Glob.glob("*.wav")
for wav in wavs
Skraak.make_spectro_from_file(wav)
end
=#
function make_spectro_from_file(file::String)
signal, freq = WAV.wavread("$file")
freq = freq |> Float32
partitioned_signal = Iterators.partition(signal, 80000) #5s clips

for (index, part) in enumerate(partitioned_signal)
length(part) > 50000 && begin
outfile = "$(chop(file, head=0, tail=4))__$(index)"
image = Skraak.get_image_from_sample(part, freq)
PNGFiles.save("$outfile.png", image)
replacement in src/Clips.jl at line 328

[3.16]→[3.954:1061](∅→∅),[3.1061]→[3.72:104](∅→∅),[3.72]→[3.72:104](∅→∅)

For making colour images using GLMakie (moved on but may be useful one day), not wired up into skraak yet.
Using for 24/7 and 250kHZ data.

[3.16]

[3.104]

using Glob, Skraak, CSV, DataFrames, Dates, PNGFiles
replacement in src/Clips.jl at line 330

[3.106]→[3.106:478](∅→∅)

using DSP, GLMakie, PNGFiles
function get_colour_image_from_sample(sample, f)
dims = 224 #px
S = DSP.spectrogram(sample[:, 1], 400, 2; fs = f)
f = GLMakie.Figure(resolution = (dims, dims), figure_padding = 0)
ax = GLMakie.Axis(f[1, 1], spinewidth = 0)
GLMakie.hidedecorations!(ax)
GLMakie.heatmap!(ax, (DSP.pow2db.(S.power))', colormap = :inferno)

[3.105]

[3.478]

a=Glob.glob("*/*/preds-2024-08-29.csv")
b=Glob.glob("*/*/*/preds-2024-08-29.csv")
c=Glob.glob("*/*/*/*/preds-2024-08-29.csv")
predictions = [a ; b ; c]
## or
predictions = Glob.glob("*/*/preds-2024-10-21.csv")
replacement in src/Clips.jl at line 337

[3.479]→[3.479:541](∅→∅)

@assert size(f) == (dims, dims) "Wrong size"
return f

[3.479]

[3.541]

for file in predictions
try
make_clips_generic(file)
catch x
println(x)
edit in src/Clips.jl at line 343

[3.545]→[3.545:705](∅→∅)

function save_colour_image(f, outfile)
try
PNGFiles.save("$outfile.png", f)
catch err
@info "Saving $outfile.png failed\n$err"
end
replacement in src/Clips.jl at line 345

[3.2150]→[3.2150:2173](∅→∅),[3.2173]→[2.5993:6074](∅→∅),[2.6074]→[3.2239:2553](∅→∅),[3.2239]→[3.2239:2553](∅→∅)

#=
For Kahurangi Data
a=Glob.glob("*/*/*.csv")
b=Glob.glob("*/*/*/*.csv")
c=Glob.glob("*/*/*/*/*.csv")
list=[a ; b ; c]

# to delete empty preds.csv files
for file in list
size = stat(file).size
if size < 10
println("Deleting $file - $size")
rm(file)
end
end

## change date on preds below (no longer required at 25/10/24)
function make_clips_kahurangi(preds_path::String, label::Int = 1)

[3.2150]

[3.2553]

function make_clips_generic(preds_path::String, label::Int = 1)
replacement in src/Clips.jl at line 383

[3.4088]→[3.4088:4196](∅→∅)

p=chop(pth, tail=1) |> x -> replace(x, "/" => "---") #delete trailing / then replace / with ---

[3.4088]

[3.4196]

p = chop(pth, tail = 1) |> x -> replace(x, "/" => "---") #delete trailing / then replace / with ---
edit in src/Clips.jl at line 401

[3.4863]→[3.4863:4918](∅→∅),[3.4918]→[2.6272:6398](∅→∅),[2.6398]→[3.5029:5061](∅→∅),[3.5029]→[3.5029:5061](∅→∅),[3.5061]→[2.6399:6451](∅→∅),[2.6451]→[3.5108:5191](∅→∅),[3.5108]→[3.5108:5191](∅→∅),[3.5191]→[3.1062:1064](∅→∅),[3.709]→[3.1062:1064](∅→∅)

using Glob, Skraak, CSV, DataFrames, Dates, PNGFiles

a=Glob.glob("*/*/preds-2024-08-29.csv")
b=Glob.glob("*/*/*/preds-2024-08-29.csv")
c=Glob.glob("*/*/*/*/preds-2024-08-29.csv")
predictions = [a ; b ; c]
## or
predictions = Glob.glob("*/*/preds-2024-10-21.csv")

for file in predictions
try
make_clips_kahurangi(file)
catch x
println(x)
end
end
=#
replacement in Manifest.toml at line 3

[3.10259885]→[3.774:799](∅→∅)

julia_version = "1.10.6"

[3.10259885]

[3.10259914]

julia_version = "1.11.1"
edit in Manifest.toml at line 17

[3.1348]

[3.10259996]

[[deps.Adapt]]
deps = ["LinearAlgebra", "Requires"]
git-tree-sha1 = "cde29ddf7e5726c9fb511f340244ea3481267608"
uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
version = "3.7.2"
weakdeps = ["StaticArrays"]

[deps.Adapt.extensions]
AdaptStaticArraysExt = "StaticArrays"
replacement in Manifest.toml at line 30

[3.10260061]→[3.10260061:10260079](∅→∅)

version = "1.1.1"

[3.10260061]

[3.1766]

version = "1.1.2"
edit in Manifest.toml at line 40

[3.10260145]

[3.3087]

version = "1.11.0"
edit in Manifest.toml at line 56

[3.10260208]

[3.10260208]

version = "1.11.0"
edit in Manifest.toml at line 238

[3.10263999]

[3.12214]

version = "1.11.0"
edit in Manifest.toml at line 254

[3.13258]

[3.13258]

version = "1.11.0"
edit in Manifest.toml at line 345

[3.10266183]

[3.15849]

version = "1.11.0"
edit in Manifest.toml at line 362

[3.10266967]

[3.18101]

version = "1.11.0"
edit in Manifest.toml at line 533

[3.10267976]

[3.10267976]

version = "1.11.0"
replacement in Manifest.toml at line 536

[3.25949]→[3.7025:7261](∅→∅)

deps = ["AxisAlgorithms", "ChainRulesCore", "LinearAlgebra", "OffsetArrays", "Random", "Ratios", "Requires", "SharedArrays", "SparseArrays", "StaticArrays", "WoodburyMatrices"]
git-tree-sha1 = "00a19d6ab0cbdea2978fc23c5a6482e02c192501"

[3.25949]

[3.26194]

deps = ["Adapt", "AxisAlgorithms", "ChainRulesCore", "LinearAlgebra", "OffsetArrays", "Random", "Ratios", "Requires", "SharedArrays", "SparseArrays", "StaticArrays", "WoodburyMatrices"]
git-tree-sha1 = "721ec2cf720536ad005cb38f50dbba7b02419a15"
replacement in Manifest.toml at line 539

[3.26240]→[3.7262:7281](∅→∅)

version = "0.14.0"

[3.26240]

[3.26427]

version = "0.14.7"
edit in Manifest.toml at line 628

[3.10269707]

[3.10269707]

version = "1.11.0"
replacement in Manifest.toml at line 643

[3.10269989]→[3.10269989:10270009](∅→∅)

version = "8.4.0+0"

[3.10269989]

[3.10270009]

version = "8.6.0+0"
edit in Manifest.toml at line 648

[3.10270141]

[3.10270141]

version = "1.11.0"
replacement in Manifest.toml at line 653

[3.10270269]→[3.10270269:10270289](∅→∅)

version = "1.6.4+0"

[3.10270269]

[3.10270289]

version = "1.7.2+0"
edit in Manifest.toml at line 662

[3.10270485]

[3.29730]

version = "1.11.0"
edit in Manifest.toml at line 679

[3.10270613]

[25.3728]

version = "1.11.0"
edit in Manifest.toml at line 705

[3.10271742]

[3.10271742]

version = "1.11.0"
edit in Manifest.toml at line 733

[3.10272314]

[3.10272314]

version = "1.11.0"
replacement in Manifest.toml at line 744

[3.10272628]→[3.10272628:10272649](∅→∅)

version = "2.28.2+1"

[3.10272628]

[3.32084]

version = "2.28.6+0"
edit in Manifest.toml at line 760

[3.10272871]

[3.10272871]

version = "1.11.0"
replacement in Manifest.toml at line 776

[3.10273118]→[3.10273118:10273140](∅→∅)

version = "2023.1.10"

[3.10273118]

[3.34145]

version = "2023.12.12"
edit in Manifest.toml at line 804

[3.13404]

[3.35507]

weakdeps = ["Adapt"]
edit in Manifest.toml at line 808

[3.35578]→[2.6577:6662](∅→∅)

[deps.OffsetArrays.weakdeps]
Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
replacement in Manifest.toml at line 818

[3.10273658]→[3.1370:1391](∅→∅)

version = "0.3.23+4"

[3.10273658]

[3.36008]

version = "0.3.27+1"
replacement in Manifest.toml at line 897

[3.10274796]→[3.10274796:10274987](∅→∅)

deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]

[3.10274796]

[3.10274987]

deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"]
replacement in Manifest.toml at line 899

[3.10275033]→[3.10275033:10275052](∅→∅)

version = "1.10.0"

[3.10275033]

[3.37549]

version = "1.11.0"
weakdeps = ["REPL"]

[deps.Pkg.extensions]
REPLExt = "REPL"
edit in Manifest.toml at line 956

[3.10276585]

[3.10276585]

version = "1.11.0"
replacement in Manifest.toml at line 989

[3.10276792]→[3.10276792:10276854](∅→∅)

deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]

[3.10276792]

[3.10276854]

deps = ["InteractiveUtils", "Markdown", "Sockets", "StyledStrings", "Unicode"]
edit in Manifest.toml at line 991

[3.10276900]

[3.10276900]

version = "1.11.0"
edit in Manifest.toml at line 996

[3.10276978]

[3.39511]

version = "1.11.0"
edit in Manifest.toml at line 1076

[3.10277932]

[3.10277932]

version = "1.11.0"
edit in Manifest.toml at line 1087

[3.41320]

[3.10278146]

version = "1.11.0"
edit in Manifest.toml at line 1114

[3.10278362]

[3.10278362]

version = "1.11.0"
replacement in Manifest.toml at line 1125

[3.10278688]→[3.10278688:10278707](∅→∅)

version = "1.10.0"

[3.10278688]

[3.10278707]

version = "1.11.0"
replacement in Manifest.toml at line 1160

[3.10279308]→[3.10279308:10279349](∅→∅)

deps = ["LinearAlgebra", "SparseArrays"]

[3.10279308]

[3.10279349]

deps = ["LinearAlgebra"]
git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0"
replacement in Manifest.toml at line 1163

[3.10279395]→[3.10279395:10279414](∅→∅)

version = "1.10.0"

[3.10279395]

[3.43821]

version = "1.11.1"
weakdeps = ["SparseArrays"]

[deps.Statistics.extensions]
SparseArraysExt = ["SparseArrays"]
edit in Manifest.toml at line 1193

[3.44544]

[3.10279766]

[[deps.StyledStrings]]
uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b"
version = "1.11.0"
replacement in Manifest.toml at line 1200

[3.10279899]→[3.10279899:10279919](∅→∅)

version = "7.2.1+1"

[3.10279899]

[3.10279919]

version = "7.7.0+0"
edit in Manifest.toml at line 1245

[3.10281308]

[3.44905]

version = "1.11.0"
edit in Manifest.toml at line 1282

[3.10282591]

[3.47069]

version = "1.11.0"
edit in Manifest.toml at line 1291

[3.10282655]

[3.10282655]

version = "1.11.0"
replacement in Manifest.toml at line 1384

[3.10284428]→[3.10284428:10284449](∅→∅)

version = "1.52.0+1"

[3.10284428]

[3.17756]

version = "1.59.0+0"