changet train api to take a list of images instead of glob pattern

quietlight

Mar 8, 2024, 12:45 AM

27HKDBYTNASZMCIKAVZ4TMPZD35PAC7LMMIEHKY6MHJMZZ4AFUPAC

Dependencies

[2] RQ56K2G4 minor work on train and predict
[3] ZGLDIQ4G rolled back reduced and bifurcated training set used to test models before going away
[4] 3FAESP6N catch up with changes, mainly construct primary dataset
[5] RBXUHIO2 made somn small changes and reverted them mostly
[6] ETOIK7VE recording changes, but this does not work anymore
[7] E3Y55MPR added perceptual colour maps and changed get_image_from_sample
[8] QPBH7QWC added MLBaste for confusion matrix, f1, roc, must still remove freq tables
[9] FY7CEMM2 mac work on train.jl
[10] PQ6OQCBQ work on construct primary dataset
[11] NV7FXZ5Q first commit
[12] NMQCXLNG catch up commit, about to do some work on prediction loop
[13] 4BTZNCRM catch up commit before starting work

Change contents

replacement in src/Train.jl at line 15

[3.335928]→[3.335928:335964](∅→∅)

glob_pattern::String="*/*.png",

[3.335928]

[3.335964]

images::Vector{String},
replacement in src/Train.jl at line 27

[3.336185]→[3.336185:336198](∅→∅),[3.336198]→[2.6:80](∅→∅)

using Skraak
glob_pattern = "kiwi_set*/*/[N,K]/*.png" #11699814-element Vector{String}

[3.336185]

[2.80]

using Skraak, Glob
images = glob("kiwi_set*/*/[N,K]/*.png") #11699814-element Vector{String}
replacement in src/Train.jl at line 30

[2.182]→[2.182:259](∅→∅)

train("K1-10_total_set_no_augumentation", 2, glob_pattern, model, 0.97, 64)

[2.182]

[2.259]

train("K1-10_total_set_no_augumentation", 2, images, model, 0.97, 64)
replacement in src/Train.jl at line 32

[2.260]→[3.6:188](∅→∅),[3.336198]→[3.6:188](∅→∅),[3.61]→[3.0:1](∅→∅),[3.81]→[3.0:1](∅→∅),[3.174]→[3.0:1](∅→∅),[3.187]→[3.0:1](∅→∅),[3.188]→[3.0:1](∅→∅),[3.336312]→[3.0:1](∅→∅),[3.1]→[3.175:427](∅→∅),[3.427]→[3.225:278](∅→∅),[3.225]→[3.225:278](∅→∅)

glob_pattern = "original_set*/*/[N,K]/*.png" #from SSD2/PrimaryDataset 7758648 hard coded a random selection of not, 1:1
train("K1-9_original_set", 20, glob_pattern, true, 0.90, 64)

glob_pattern = "kiwi_set_2023-11-13/*/[N,K]/*.png" #from SSD2/PrimaryDataset
model = "/media/david/SSD2/PrimaryDataset/media/david/SSD2/PrimaryDataset/model_K1-6_CPU_epoch-3-0.9798-2024-02-01.jld2"
train("K1-6_ft1", 10, glob_pattern, model, 0.95, 64)

glob_pattern = "*/[D,F,M,N]/*.png" #from SSD2/Clips

[2.260]

[3.278]

images = glob("*/[D,F,M,N]/*.png") #from SSD2/Clips
replacement in src/Train.jl at line 34

[3.367]→[3.367:409](∅→∅)

train("DFMN1-5", 20, glob_pattern, model)

[3.367]

[3.336642]

train("DFMN1-5", 20, images, model)
replacement in src/Train.jl at line 43

[3.336787]→[2.261:323](∅→∅)

glob_pattern::String = "*/*.png", #images::Vector{String}

[3.336787]

[3.336825]

images::Vector{String} #glob_pattern::String = "*/*.png"
replacement in src/Train.jl at line 49

[3.336948]→[3.189:251](∅→∅)

images = glob(glob_pattern) #|> shuffle! |> x -> x[1:640]

[3.336948]

[3.337010]

#images = glob(glob_pattern) #|> shuffle! |> x -> x[1:640]
replacement in src/Train.jl at line 163

[3.340292]→[2.324:418](∅→∅)

#x -> Noise.add_gauss(x, (rand() * 0.2)) |>
#x -> apply_mask!(x, 3, 3, 12) |>

[3.340292]

[3.340384]

x -> Noise.add_gauss(x, (rand() * 0.2)) |>
x -> apply_mask!(x, 3, 3, 12) |>
replacement in src/Train.jl at line 323

[3.1071]→[3.1071:1162](∅→∅)

test_confusion_matrix[1,1] > a && begin
a = test_confusion_matrix[1,1]

[3.1071]

[3.345118]

#test_confusion_matrix[1,1] > a && begin
#a = test_confusion_matrix[1,1]
replacement in src/Train.jl at line 332

[3.345399]→[3.345399:345411](∅→∅)

end

[3.345399]

[3.345411]

#end