2C4FPBSQTF4FM4J45HZGWB6L3E56U7M26TLYIRUMXC6SULR6XSQAC P6OU2H3DSB5V53JKM2GZ3IXSNSUF23YRXQRWSEJ2U3ARCR5TP4IQC 2TDG53JBZHZA6ZPYONPINKVDV4UXLP4T4CI5C2MEZIIYO7DQE5RAC VZGXBNYYO3E7EPFQ4GOLNVMRXXTQDDQZUU2BZ6JHNBDY4B2QLDAAC IFVRAERTCCDICNTYTG3TX2WASB6RXQQEJWWXQMQZJSQDQ3HLE5OQC LLFYL4PV3LPUZNL2SPVF6FA7WMZSGHSGB4YWS3OUWPARECWI737QC RFSUR7ZEXTQNHH3IFJAL2NNOTGRPWOWB3PFIVH7VLI2JPTIBMW5AC EBCNGTNVY2YFFHKC4PHDEHNBOWJ4JDCEXTTJ3WKD5VWQZLLDZ65AC D4EL6RSTSZ3S3IDSETRNGLJHZKGZEE2V2OZIOKQK6LRLHQNS77JQC W3A2EECCD23SVHJZN6MXPH2PAVFHH5CNFD2XHPQRRW6M4GUTG3FAC func TestParseWAVHeaderMinimal(t *testing.T) {tmpDir := t.TempDir()t.Run("should parse basic WAV metadata", func(t *testing.T) {path := createTestWAVFile(t, tmpDir, "test_minimal.wav", struct {duration float64sampleRate intchannels intbitsPerSample intcomment stringartist string}{duration: 10.0,sampleRate: 44100,channels: 1,bitsPerSample: 16,comment: "",artist: "",})sampleRate, duration, err := ParseWAVHeaderMinimal(path)if err != nil {t.Fatalf("Failed to parse WAV header: %v", err)}if sampleRate != 44100 {t.Errorf("SampleRate incorrect: got %d, want 44100", sampleRate)}if duration < 9.9 || duration > 10.1 {t.Errorf("Duration incorrect: got %f, want ~10.0", duration)}})t.Run("should handle different sample rates", func(t *testing.T) {sampleRates := []int{8000, 22050, 44100, 48000, 96000}for _, sr := range sampleRates {t.Run(fmt.Sprintf("%dHz", sr), func(t *testing.T) {path := createTestWAVFile(t, tmpDir, fmt.Sprintf("test_sr_%d.wav", sr), struct {duration float64sampleRate intchannels intbitsPerSample intcomment stringartist string}{duration: 5.0,sampleRate: sr,channels: 1,bitsPerSample: 16,comment: "",artist: "",})sampleRate, duration, err := ParseWAVHeaderMinimal(path)if err != nil {t.Fatalf("Failed to parse WAV header: %v", err)}if sampleRate != sr {t.Errorf("SampleRate incorrect: got %d, want %d", sampleRate, sr)}if duration < 4.9 || duration > 5.1 {t.Errorf("Duration incorrect: got %f, want ~5.0", duration)}})}})t.Run("should handle stereo files", func(t *testing.T) {path := createTestWAVFile(t, tmpDir, "test_stereo.wav", struct {duration float64sampleRate intchannels intbitsPerSample intcomment stringartist string}{duration: 3.0,sampleRate: 44100,channels: 2,bitsPerSample: 16,comment: "",artist: "",})sampleRate, duration, err := ParseWAVHeaderMinimal(path)if err != nil {t.Fatalf("Failed to parse WAV header: %v", err)}if sampleRate != 44100 {t.Errorf("SampleRate incorrect: got %d, want 44100", sampleRate)}if duration < 2.9 || duration > 3.1 {t.Errorf("Duration incorrect: got %f, want ~3.0", duration)}})t.Run("should return error for non-existent file", func(t *testing.T) {_, _, err := ParseWAVHeaderMinimal("/nonexistent/file.wav")if err == nil {t.Error("Expected error for non-existent file")}})t.Run("should return error for non-WAV file", func(t *testing.T) {// Create a text filepath := filepath.Join(tmpDir, "notawav.wav")if err := os.WriteFile(path, []byte("Not a WAV file"), 0644); err != nil {t.Fatalf("Failed to create test file: %v", err)}_, _, err := ParseWAVHeaderMinimal(path)if err == nil {t.Error("Expected error for non-WAV file")}})}
}// ParseWAVHeaderMinimal reads only the first 4KB of a WAV file to extract essential metadata.// This is optimized for batch processing where INFO chunks (comment/artist) are not needed.// It's ~50x faster than ParseWAVHeader for large files due to reduced I/O.// Returns (sampleRate, duration, error) - the minimal data needed for .data file generation.func ParseWAVHeaderMinimal(filepath string) (sampleRate int, duration float64, err error) {file, err := os.Open(filepath)if err != nil {return 0, 0, fmt.Errorf("failed to open file: %w", err)}defer file.Close()// Get minimal header buffer from pool (4KB)headerBufPtr := getMinimalHeaderBuffer()defer putMinimalHeaderBuffer(headerBufPtr)headerBuf := (*headerBufPtr)[:cap(*headerBufPtr)]// Read first 4KB - sufficient for fmt + data chunk headers in 99% of filesn, err := file.Read(headerBuf)if err != nil && err != io.EOF {return 0, 0, fmt.Errorf("failed to read header: %w", err)}headerBuf = headerBuf[:n]// Parse minimal metadatasampleRate, duration, err = parseWAVMinimal(headerBuf)if err != nil {return 0, 0, err}return sampleRate, duration, nil}// parseWAVMinimal parses only essential WAV metadata from a byte buffer.// Returns (sampleRate, duration, error). Does not parse INFO chunks.func parseWAVMinimal(data []byte) (sampleRate int, duration float64, err error) {if len(data) < 44 {return 0, 0, fmt.Errorf("file too small to be valid WAV")}// Verify RIFF headerif string(data[0:4]) != "RIFF" {return 0, 0, fmt.Errorf("not a valid WAV file (missing RIFF header)")}// Verify WAVE formatif string(data[8:12]) != "WAVE" {return 0, 0, fmt.Errorf("not a valid WAV file (missing WAVE format)")}var channels, bitsPerSample int// Parse chunks - stop after finding data chunkoffset := 12for offset < len(data)-8 {chunkID := string(data[offset : offset+4])chunkSize := int(binary.LittleEndian.Uint32(data[offset+4 : offset+8]))offset += 8switch chunkID {case "fmt ":// Parse format chunkif chunkSize >= 16 && offset+16 <= len(data) {channels = int(binary.LittleEndian.Uint16(data[offset+2 : offset+4]))sampleRate = int(binary.LittleEndian.Uint32(data[offset+4 : offset+8]))bitsPerSample = int(binary.LittleEndian.Uint16(data[offset+14 : offset+16]))}case "data":// Found data chunk - calculate duration and returnif sampleRate > 0 && channels > 0 && bitsPerSample > 0 {bytesPerSample := bitsPerSample / 8bytesPerSecond := sampleRate * channels * bytesPerSampleif bytesPerSecond > 0 {duration = float64(chunkSize) / float64(bytesPerSecond)return sampleRate, duration, nil}}return 0, 0, fmt.Errorf("invalid WAV: fmt chunk missing or corrupt before data chunk")}// Move to next chunk (word-aligned)offset += chunkSizeif chunkSize%2 != 0 {offset++}}// Data chunk not found within 4KB - file may have large INFO chunksreturn 0, 0, fmt.Errorf("data chunk not found in first 4KB (try ParseWAVHeader for full parsing)")
CLUSTER_GAP_MULTIPLIER = 3 // Gap threshold = CLUSTER_GAP_MULTIPLIER * clip_durationMIN_DETECTIONS_PER_CLUSTER = 1 // Minimum detections per cluster (1 = filter single detections)
CLUSTER_GAP_MULTIPLIER = 3 // Gap threshold = CLUSTER_GAP_MULTIPLIER * clip_durationMIN_DETECTIONS_PER_CLUSTER = 1 // Minimum detections per cluster (1 = filter single detections)
CSVPath string `json:"csv_path" jsonschema:"required,Path to predictions CSV file"`Filter string `json:"filter" jsonschema:"Filter name for .data files"`WriteDotData bool `json:"write_dot_data" jsonschema:"Write .data files alongside audio files"`
CSVPath string `json:"csv_path" jsonschema:"required,Path to predictions CSV file"`Filter string `json:"filter" jsonschema:"Filter name for .data files"`WriteDotData bool `json:"write_dot_data" jsonschema:"Write .data files alongside audio files"`ProgressHandler ProgressHandler `json:"-"` // Optional progress callback (not serialized)
func writeDotFiles(csvPath, filter string, calls []ClusteredCall) (int, int, error) {
// Uses parallel workers for improved performance on large batchesfunc writeDotFiles(csvPath, filter string, calls []ClusteredCall, progress ProgressHandler) (int, int, error) {
// Report initial progressif progress != nil {progress(0, len(callsByFile), "Processing WAV files")}// If small batch, process sequentially (avoid goroutine overhead)if len(callsByFile) < 10 {return writeDotFilesSequential(csvDir, filter, callsByFile, progress)}// Parallel processing for larger batchesreturn writeDotFilesParallel(csvDir, filter, callsByFile, progress)}// dotDataJob represents a single file to processtype dotDataJob struct {filename stringfileCalls []ClusteredCall}
// dotDataResult represents the result of processing a single filetype dotDataResult struct {filename stringwritten boolerr error}// writeDotFilesSequential processes files one at a time (for small batches)func writeDotFilesSequential(csvDir, filter string, callsByFile map[string][]ClusteredCall, progress ProgressHandler) (int, int, error) {
// writeDotFilesParallel processes files concurrently using a worker poolfunc writeDotFilesParallel(csvDir, filter string, callsByFile map[string][]ClusteredCall, progress ProgressHandler) (int, int, error) {total := len(callsByFile)var processed atomic.Int32// Create job channeljobs := make(chan dotDataJob, len(callsByFile))results := make(chan dotDataResult, len(callsByFile))// Start workersvar wg sync.WaitGroupfor i := 0; i < DOT_DATA_WORKERS; i++ {wg.Add(1)go dotDataWorker(csvDir, filter, jobs, results, &wg)}// Send jobsfor filename, fileCalls := range callsByFile {jobs <- dotDataJob{filename: filename, fileCalls: fileCalls}}close(jobs)// Wait for workers to finishgo func() {wg.Wait()close(results)}()// Collect results with progress reportingdataFilesWritten := 0dataFilesSkipped := 0var firstErr errorfor result := range results {if result.err != nil && firstErr == nil {firstErr = result.err}if result.written {dataFilesWritten++} else {dataFilesSkipped++}// Report progressif progress != nil {current := int(processed.Add(1))progress(current, total, "")}}return dataFilesWritten, dataFilesSkipped, firstErr}// dotDataWorker processes files from the jobs channelfunc dotDataWorker(csvDir, filter string, jobs <-chan dotDataJob, results chan<- dotDataResult, wg *sync.WaitGroup) {defer wg.Done()for job := range jobs {wavPath := filepath.Join(csvDir, job.filename)dataPath := wavPath + ".data"sampleRate, duration, err := utils.ParseWAVHeaderMinimal(wavPath)if err != nil {results <- dotDataResult{filename: job.filename, written: false, err: nil}continue}dataFile := buildAviaNZDataFile(job.fileCalls, filter, duration, sampleRate)if err := writeAviaNZDataFile(dataPath, dataFile); err != nil {results <- dotDataResult{filename: job.filename, written: false, err: fmt.Errorf("failed to write %s: %w", dataPath, err)}continue}results <- dotDataResult{filename: job.filename, written: true, err: nil}}}
## [2026-03-04] Performance Optimizations for calls-from-preds**Problem:** Processing 7617 WAV files took 16 minutes due to excessive I/O and sequential processing.**Changes:**- `utils/wav_metadata.go` — Added `ParseWAVHeaderMinimal()` that reads only 4KB instead of 200KB per file (50× less I/O). Added separate buffer pool for minimal headers.- `tools/calls_from_preds.go` — Added parallel processing with 8 workers for .data file generation. Small batches (<10 files) use sequential processing to avoid goroutine overhead.- `tools/calls_from_preds.go` — Added `ProgressHandler` callback type for progress reporting during long operations.- `cmd/calls.go` — Added progress indicator showing "Processing WAV files: X/Y (Z%)" during .data file writing.