"skraak/tools"

	"skraak/tools/calls"

	state        *tools.ClassifyState

	state        *calls.ClassifyState

func New(state *tools.ClassifyState) Model {

func New(state *calls.ClassifyState) Model {

	sorted := make([]tools.KeyBinding, len(state.Config.Bindings))

	sorted := make([]calls.KeyBinding, len(state.Config.Bindings))

					m.state.ApplyBinding(&tools.BindingResult{Species: result.Species})

					m.state.ApplyBinding(&calls.BindingResult{Species: result.Species})

func saveClip(state *tools.ClassifyState, prefix string) error {

func saveClip(state *calls.ClassifyState, prefix string) error {

func playCurrentSegmentAtSpeed(state *tools.ClassifyState, speed float64) string {

func playCurrentSegmentAtSpeed(state *calls.ClassifyState, speed float64) string {

			fmt.Fprintf(b, "  • %s\n", tools.FormatLabels([]*utils.Label{l}, m.state.Config.Filter))

			fmt.Fprintf(b, "  • %s\n", calls.FormatLabels([]*utils.Label{l}, m.state.Config.Filter))

func generateSpectrogramImage(state *tools.ClassifyState, dataPath string, seg *utils.Segment) image.Image {

func generateSpectrogramImage(state *calls.ClassifyState, dataPath string, seg *utils.Segment) image.Image {

func inlineImageCmd(state *tools.ClassifyState, protocol utils.ImageProtocol, gen uint64, currentGen *uint64) tea.Cmd {

func inlineImageCmd(state *calls.ClassifyState, protocol utils.ImageProtocol, gen uint64, currentGen *uint64) tea.Cmd {

package tools
// AviaNZMeta is the metadata element in a .data file
type AviaNZMeta struct {
	Operator string  `json:"Operator"`
	Reviewer *string `json:"Reviewer,omitempty"`
	Duration float64 `json:"Duration"`
}
// AviaNZLabel represents a species label in a segment
type AviaNZLabel struct {
	Species   string `json:"species"`
	Certainty int    `json:"certainty"`
	Filter    string `json:"filter"`
}
// AviaNZSegment represents a detection segment [start, end, freq_low, freq_high, labels]
type AviaNZSegment [5]any

package tools
import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"sync/atomic"
)
// parallelResult is the common interface for birda/raven worker results.
type parallelResult interface {
	filePath() string
	getCalls() []ClusteredCall
	wasWritten() bool
	wasSkipped() bool
	getError() error
}
// aggregateStats holds the collected results from a parallel fan-out/fan-in.
type aggregateStats struct {
	calls            []ClusteredCall
	speciesCount     map[string]int
	dataFilesWritten int
	dataFilesSkipped int
	filesProcessed   int
	filesDeleted     int
	firstErr         error
}
// aggregateResults collects results from a channel of parallelResult values,
// handling error tracking, species counting, optional file deletion, and
// progress reporting. Returns the aggregated stats.
func aggregateResults(
	results <-chan parallelResult,
	total int,
	processed *atomic.Int32,
	deleteFiles bool,
	progressHandler func(int, int, string),
) aggregateStats {
	var stats aggregateStats
	stats.speciesCount = make(map[string]int)
	for result := range results {
		if err := result.getError(); err != nil && stats.firstErr == nil {
			stats.firstErr = err
		}
		if result.wasWritten() {
			stats.dataFilesWritten++
		}
		if result.wasSkipped() {
			stats.dataFilesSkipped++
		}
		for _, call := range result.getCalls() {
			stats.calls = append(stats.calls, call)
			stats.speciesCount[call.EbirdCode]++
		}
		stats.filesProcessed++
		stats.maybeDeleteFile(deleteFiles, result)
		if progressHandler != nil {
			current := int(processed.Add(1))
			progressHandler(current, total, filepath.Base(result.filePath()))
		}
	}
	return stats
}
// maybeDeleteFile deletes the source file if requested and it was successfully processed.
func (s *aggregateStats) maybeDeleteFile(deleteFiles bool, result parallelResult) {
	if !deleteFiles || !result.wasWritten() {
		return
	}
	if err := os.Remove(result.filePath()); err != nil {
		if s.firstErr == nil {
			s.firstErr = fmt.Errorf("failed to delete %s: %w", result.filePath(), err)
		}
	} else {
		s.filesDeleted++
	}
}
// sortCallsByFileAndTime sorts calls by filename, then start time.
func sortCallsByFileAndTime(calls []ClusteredCall) {
	sort.Slice(calls, func(i, j int) bool {
		if calls[i].File != calls[j].File {
			return calls[i].File < calls[j].File
		}
		return calls[i].StartTime < calls[j].StartTime
	})
}

package tools
import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"sync/atomic"
)
// CallsFromSourceInput defines the common input for calls-from-source tools
type CallsFromSourceInput struct {
	Folder          string          `json:"folder"`
	File            string          `json:"file"`
	Delete          bool            `json:"delete"`
	ProgressHandler ProgressHandler `json:"-"` // Optional progress callback
}
// CallsFromSourceOutput defines the common output for calls-from-source tools
type CallsFromSourceOutput struct {
	Calls            []ClusteredCall `json:"calls"`
	TotalCalls       int             `json:"total_calls"`
	SpeciesCount     map[string]int  `json:"species_count"`
	DataFilesWritten int             `json:"data_files_written"`
	DataFilesSkipped int             `json:"data_files_skipped"`
	FilesProcessed   int             `json:"files_processed"`
	FilesDeleted     int             `json:"files_deleted"`
	Filter           string          `json:"filter"`
	Error            *string         `json:"error,omitempty"`
}
// CallSource abstracts a source of bird call data (Raven, BirdNET, etc.)
type CallSource interface {
	// Name returns the display name (e.g. "Raven", "BirdNET")
	Name() string
	// FindFiles discovers source files in the given folder
	FindFiles(folder string) ([]string, error)
	// ProcessFile processes a single source file and returns calls, write/skip status
	ProcessFile(path string, cache *DirCache) (calls []ClusteredCall, written, skipped bool, err error)
}
// callsFromSource is the shared entry point for all call source tools.
func callsFromSource(src CallSource, input CallsFromSourceInput) (CallsFromSourceOutput, error) {
	var output CallsFromSourceOutput
	output.Filter = src.Name()
	// Collect source files to process
	var files []string
	if input.File != "" {
		files = []string{input.File}
	} else if input.Folder != "" {
		var err error
		files, err = src.FindFiles(input.Folder)
		if err != nil {
			errMsg := fmt.Sprintf("Failed to find %s files: %v", src.Name(), err)
			output.Error = &errMsg
			return output, fmt.Errorf("%s", errMsg)
		}
	} else {
		errMsg := "Either --folder or --file must be specified"
		output.Error = &errMsg
		return output, fmt.Errorf("%s", errMsg)
	}
	if len(files) == 0 {
		errMsg := fmt.Sprintf("No %s files found", src.Name())
		output.Error = &errMsg
		return output, fmt.Errorf("%s", errMsg)
	}
	// Single file or small batch: process sequentially (avoid goroutine overhead)
	if len(files) < 10 {
		return callsFromSourceSequential(src, input, files)
	}
	// Large batch: parallel processing with DirCache
	return callsFromSourceParallel(src, input, files)
}
// callsFromSourceSequential processes source files one at a time (for small batches)
func callsFromSourceSequential(src CallSource, input CallsFromSourceInput, files []string) (CallsFromSourceOutput, error) {
	var output CallsFromSourceOutput
	output.Filter = src.Name()
	// Build DirCache once for the folder
	dirCaches := make(map[string]*DirCache)
	if input.Folder != "" {
		dirCaches[input.Folder] = NewDirCache(input.Folder)
	}
	speciesCount := make(map[string]int)
	var allCalls []ClusteredCall
	dataFilesWritten := 0
	dataFilesSkipped := 0
	filesProcessed := 0
	filesDeleted := 0
	for _, file := range files {
		dir := filepath.Dir(file)
		cache := dirCaches[dir]
		if cache == nil {
			cache = NewDirCache(dir)
			dirCaches[dir] = cache
		}
		calls, written, skipped, err := src.ProcessFile(file, cache)
		if err != nil {
			errMsg := fmt.Sprintf("Error processing %s: %v", file, err)
			output.Error = &errMsg
			return output, fmt.Errorf("%s", errMsg)
		}
		if written {
			dataFilesWritten++
		}
		if skipped {
			dataFilesSkipped++
		}
		for _, call := range calls {
			allCalls = append(allCalls, call)
			speciesCount[call.EbirdCode]++
		}
		filesProcessed++
		// Delete if requested and successfully processed
		if input.Delete && written {
			if err := os.Remove(file); err != nil {
				errMsg := fmt.Sprintf("Failed to delete %s: %v", file, err)
				output.Error = &errMsg
				return output, fmt.Errorf("%s", errMsg)
			}
			filesDeleted++
		}
		if input.ProgressHandler != nil {
			input.ProgressHandler(filesProcessed, len(files), filepath.Base(file))
		}
	}
	// Sort all calls by file, then start time
	sort.Slice(allCalls, func(i, j int) bool {
		if allCalls[i].File != allCalls[j].File {
			return allCalls[i].File < allCalls[j].File
		}
		return allCalls[i].StartTime < allCalls[j].StartTime
	})
	output.Calls = allCalls
	output.TotalCalls = len(allCalls)
	output.SpeciesCount = speciesCount
	output.DataFilesWritten = dataFilesWritten
	output.DataFilesSkipped = dataFilesSkipped
	output.FilesProcessed = filesProcessed
	output.FilesDeleted = filesDeleted
	return output, nil
}
// sourceJob represents a single file to process (generic over CallSource)
type sourceJob struct {
	filePath string
}
// sourceResult represents the result of processing a single source file
type sourceResult struct {
	path    string
	calls   []ClusteredCall
	written bool
	skipped bool
	err     error
}
func (r sourceResult) filePath() string          { return r.path }
func (r sourceResult) getCalls() []ClusteredCall { return r.calls }
func (r sourceResult) wasWritten() bool          { return r.written }
func (r sourceResult) wasSkipped() bool          { return r.skipped }
func (r sourceResult) getError() error           { return r.err }
// callsFromSourceParallel processes source files concurrently using a worker pool and DirCache
func callsFromSourceParallel(src CallSource, input CallsFromSourceInput, files []string) (CallsFromSourceOutput, error) {
	var output CallsFromSourceOutput
	output.Filter = src.Name()
	total := len(files)
	var processed atomic.Int32
	// Build DirCache for the folder
	dirCaches := &sync.Map{}
	if input.Folder != "" {
		cache := NewDirCache(input.Folder)
		dirCaches.Store(input.Folder, cache)
	}
	// Create job and result channels
	jobs := make(chan sourceJob, total)
	results := make(chan parallelResult, total)
	// Start workers
	var wg sync.WaitGroup
	for range DOT_DATA_WORKERS {
		wg.Add(1)
		go sourceWorker(src, dirCaches, jobs, results, &wg)
	}
	// Send jobs
	for _, file := range files {
		jobs <- sourceJob{filePath: file}
	}
	close(jobs)
	// Wait for workers to finish, then close results
	go func() {
		wg.Wait()
		close(results)
	}()
	// Collect results with progress reporting
	stats := aggregateResults(results, total, &processed, input.Delete, input.ProgressHandler)
	if stats.firstErr != nil {
		errMsg := stats.firstErr.Error()
		output.Error = &errMsg
		return output, stats.firstErr
	}
	sortCallsByFileAndTime(stats.calls)
	output.Calls = stats.calls
	output.TotalCalls = len(stats.calls)
	output.SpeciesCount = stats.speciesCount
	output.DataFilesWritten = stats.dataFilesWritten
	output.DataFilesSkipped = stats.dataFilesSkipped
	output.FilesProcessed = stats.filesProcessed
	output.FilesDeleted = stats.filesDeleted
	return output, nil
}
// sourceWorker processes source files from the jobs channel
func sourceWorker(src CallSource, dirCaches *sync.Map, jobs <-chan sourceJob, results chan<- parallelResult, wg *sync.WaitGroup) {
	defer wg.Done()
	for job := range jobs {
		dir := filepath.Dir(job.filePath)
		// Get or create DirCache for this directory
		var cache *DirCache
		if cached, ok := dirCaches.Load(dir); ok {
			cache = cached.(*DirCache)
		} else {
			cache = NewDirCache(dir)
			dirCaches.Store(dir, cache)
		}
		calls, written, skipped, err := src.ProcessFile(job.filePath, cache)
		results <- sourceResult{
			path:    job.filePath,
			calls:   calls,
			written: written,
			skipped: skipped,
			err:     err,
		}
	}
}

package tools
import (
	"fmt"
	"strings"
	"time"
	"github.com/sixdouglas/suncalc"
	"skraak/utils"
)
// IsNightInput defines the input parameters for the isnight tool
type IsNightInput struct {
	FilePath string  `json:"file_path"`
	Lat      float64 `json:"lat"`
	Lng      float64 `json:"lng"`
	Timezone string  `json:"timezone,omitempty"`
}
// IsNightOutput defines the output structure for the isnight tool
type IsNightOutput struct {
	FilePath      string  `json:"file_path"`
	TimestampUTC  string  `json:"timestamp_utc"`
	SolarNight    bool    `json:"solar_night"`
	CivilNight    bool    `json:"civil_night"`
	DiurnalActive bool    `json:"diurnal_active"`
	MoonPhase     float64 `json:"moon_phase"`
	DurationSec   float64 `json:"duration_seconds"`
	TimestampSrc  string  `json:"timestamp_source"`
	MidpointUTC   string  `json:"midpoint_utc"`
	SunriseUTC    string  `json:"sunrise_utc,omitempty"`
	SunsetUTC     string  `json:"sunset_utc,omitempty"`
	DawnUTC       string  `json:"dawn_utc,omitempty"`
	DuskUTC       string  `json:"dusk_utc,omitempty"`
}
// IsNight determines if a WAV file was recorded at night based on its
// metadata timestamp and the given GPS coordinates.
//
// Timestamp resolution order:
//  1. AudioMoth comment (timezone embedded)
//  2. Filename timestamp + timezone offset (requires --timezone)
//  3. File modification time (system local time)
func IsNight(input IsNightInput) (IsNightOutput, error) {
	var output IsNightOutput
	// Step 1: Parse WAV header
	metadata, err := utils.ParseWAVHeader(input.FilePath)
	if err != nil {
		return output, fmt.Errorf("WAV header parsing failed: %w", err)
	}
	output.DurationSec = metadata.Duration
	// Step 2: Resolve timestamp (use file mod time as fallback)
	tsResult, err := utils.ResolveTimestamp(metadata, input.FilePath, input.Timezone, true, nil)
	if err != nil {
		return output, fmt.Errorf("cannot determine recording timestamp: %w", err)
	}
	// Determine timestamp source label
	tsSource := "file_mod_time"
	if tsResult.IsAudioMoth {
		tsSource = "audiomoth_comment"
	} else if utils.HasTimestampFilename(input.FilePath) {
		tsSource = "filename"
	}
	// Step 3: Calculate astronomical data using recording midpoint
	astroData := utils.CalculateAstronomicalData(
		tsResult.Timestamp.UTC(),
		metadata.Duration,
		input.Lat,
		input.Lng,
	)
	// Step 4: Get sun event times for informational output
	midpoint := utils.CalculateMidpointTime(tsResult.Timestamp.UTC(), metadata.Duration)
	sunTimes := suncalc.GetTimes(midpoint, input.Lat, input.Lng)
	output.FilePath = input.FilePath
	output.TimestampUTC = tsResult.Timestamp.UTC().Format(time.RFC3339)
	output.SolarNight = astroData.SolarNight
	output.CivilNight = astroData.CivilNight
	output.MoonPhase = astroData.MoonPhase
	output.TimestampSrc = tsSource
	output.MidpointUTC = midpoint.Format(time.RFC3339)
	if dawn, ok := sunTimes[suncalc.Dawn]; ok && !dawn.Value.IsZero() {
		if sunset, ok := sunTimes[suncalc.Sunset]; ok && !sunset.Value.IsZero() {
			output.DiurnalActive = !midpoint.Before(dawn.Value) && !midpoint.After(sunset.Value)
		}
	}
	output.SunriseUTC = sunTimeUTC(sunTimes, suncalc.Sunrise)
	output.SunsetUTC = sunTimeUTC(sunTimes, suncalc.Sunset)
	output.DawnUTC = sunTimeUTC(sunTimes, suncalc.Dawn)
	output.DuskUTC = sunTimeUTC(sunTimes, suncalc.Dusk)
}
// String returns a human-readable summary of the isnight result
func (o IsNightOutput) String() string {
	var sb strings.Builder
	fmt.Fprintf(&sb, "File: %s\n", o.FilePath)
	fmt.Fprintf(&sb, "Timestamp (UTC): %s\n", o.TimestampUTC)
	fmt.Fprintf(&sb, "Midpoint (UTC):  %s\n", o.MidpointUTC)
	fmt.Fprintf(&sb, "Duration: %.1f seconds\n", o.DurationSec)
	fmt.Fprintf(&sb, "Source: %s\n", o.TimestampSrc)
	fmt.Fprintf(&sb, "Solar night: %v\n", o.SolarNight)
	fmt.Fprintf(&sb, "Civil night: %v\n", o.CivilNight)
	fmt.Fprintf(&sb, "Moon phase: %.2f\n", o.MoonPhase)
	if o.SunriseUTC != "" {
		fmt.Fprintf(&sb, "Sunrise (UTC): %s\n", o.SunriseUTC)
	}
	if o.SunsetUTC != "" {
		fmt.Fprintf(&sb, "Sunset (UTC):  %s\n", o.SunsetUTC)
	}
	if o.DawnUTC != "" {
		fmt.Fprintf(&sb, "Dawn (UTC):    %s\n", o.DawnUTC)
	}
	if o.DuskUTC != "" {
		fmt.Fprintf(&sb, "Dusk (UTC):    %s\n", o.DuskUTC)
	}
	return sb.String()
}
// populateSunTimes fills in sun event times and diurnal status from suncalc results.
func populateSunTimes(output *IsNightOutput, sunTimes map[suncalc.DayTimeName]suncalc.DayTime, midpoint time.Time) {
	// Diurnal: midpoint is between dawn and sunset
// sunTimeUTC returns the UTC RFC3339 string for a suncalc event, or "" if absent/zero.
func sunTimeUTC(sunTimes map[suncalc.DayTimeName]suncalc.DayTime, name suncalc.DayTimeName) string {
	if entry, ok := sunTimes[name]; ok && !entry.Value.IsZero() {
		return entry.Value.UTC().Format(time.RFC3339)
	}
	return ""
}
	populateSunTimes(&output, sunTimes, midpoint)
	return output, nil
}

package tools
import (
	"context"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"time"
	"skraak/db"
	"skraak/utils"
)
// ImportUnstructuredInput defines the input parameters for importing files into an unstructured dataset
type ImportUnstructuredInput struct {
	DatasetID  string `json:"dataset_id"`
	FolderPath string `json:"folder_path"`
	Recursive  *bool  `json:"recursive,omitempty"`
}
// ImportUnstructuredOutput defines the output structure
type ImportUnstructuredOutput struct {
	TotalFiles     int                     `json:"total_files"`
	ImportedFiles  int                     `json:"imported_files"`
	SkippedFiles   int                     `json:"skipped_files"` // Duplicates
	FailedFiles    int                     `json:"failed_files"`
	TotalDuration  float64                 `json:"total_duration_seconds"`
	ProcessingTime string                  `json:"processing_time"`
	Errors         []utils.FileImportError `json:"errors,omitempty"`
}
// ImportUnstructured imports WAV files into an unstructured dataset
// Files are stored with minimal metadata: hash, duration, sample_rate, file_mod_time as timestamp
// No location/cluster hierarchy, no astronomical data, no AudioMoth parsing
func ImportUnstructured(
	ctx context.Context,
	input ImportUnstructuredInput,
) (ImportUnstructuredOutput, error) {
	startTime := time.Now()
	var output ImportUnstructuredOutput
	// Default recursive to true
	recursive := true
	if input.Recursive != nil {
		recursive = *input.Recursive
	}
	// Validate input
	if err := validateUnstructuredInput(input); err != nil {
		return output, fmt.Errorf("validation failed: %w", err)
	}
	// Scan for WAV files (no DB needed)
	files, scanErrors := scanWavFiles(input.FolderPath, recursive)
	output.Errors = append(output.Errors, scanErrors...)
	output.TotalFiles = len(files)
	if len(files) == 0 {
		output.ProcessingTime = time.Since(startTime).String()
		return output, nil
	}
		// Process each file
		for _, filePath := range files {
			fileResult, procErr := processUnstructuredFile(tx, filePath, input.DatasetID)
			if procErr != nil {
				output.FailedFiles++
				output.Errors = append(output.Errors, utils.FileImportError{
					FileName: filepath.Base(filePath),
					Error:    procErr.Error(),
					Stage:    utils.StageProcess,
				})
				continue
			}
			if fileResult.Skipped {
				output.SkippedFiles++
			} else {
				output.ImportedFiles++
				output.TotalDuration += fileResult.Duration
			}
		}
		return nil
	})
	if err != nil {
		return output, err
	}
	output.ProcessingTime = time.Since(startTime).String()
	return output, nil
}
// unstructuredFileResult holds the result of processing a single file
type unstructuredFileResult struct {
	Skipped  bool    // True if duplicate
	Duration float64 // Duration in seconds
}
// processUnstructuredFile processes a single WAV file for unstructured import
func processUnstructuredFile(tx *db.LoggedTx, filePath, datasetID string) (*unstructuredFileResult, error) {
	result := &unstructuredFileResult{}
	// Step 1: Parse WAV header
	metadata, err := utils.ParseWAVHeader(filePath)
	if err != nil {
		return nil, fmt.Errorf("WAV header parsing failed: %w", err)
	}
	// Step 2: Calculate hash
	hash, err := utils.ComputeXXH64(filePath)
	if err != nil {
		return nil, fmt.Errorf("hash calculation failed: %w", err)
	}
	// Step 3: Check for duplicate - if exists, skip entirely (do not link to dataset)
	_, isDuplicate, err := utils.CheckDuplicateHash(tx, hash)
	if err != nil {
		return nil, fmt.Errorf("duplicate check failed: %w", err)
	}
	if isDuplicate {
		// File already exists in database - skip completely, do not link to dataset
		result.Skipped = true
		result.Duration = metadata.Duration
		return result, nil
	}
	// Step 4: Generate file ID
	fileID, err := utils.GenerateLongID()
	if err != nil {
		return nil, fmt.Errorf("ID generation failed: %w", err)
	}
	// Step 5: Use file modification time as timestamp (no timezone conversion)
	timestamp := metadata.FileModTime
	// Step 6: Insert into file table
	_, err = tx.Exec(`
		INSERT INTO file (
			id, file_name, xxh64_hash, location_id, cluster_id,
			timestamp_local, duration, sample_rate,
			maybe_solar_night, maybe_civil_night, moon_phase,
			active
		) VALUES (?, ?, ?, NULL, NULL, ?, ?, ?, NULL, NULL, NULL, TRUE)
	`,
		fileID,
		filepath.Base(filePath),
		hash,
		timestamp,
		metadata.Duration,
		metadata.SampleRate,
	)
	if err != nil {
		return nil, fmt.Errorf("file insert failed: %w", err)
	}
	// Step 7: Insert into file_dataset table
	_, err = tx.Exec(
		"INSERT INTO file_dataset (file_id, dataset_id) VALUES (?, ?)",
		fileID, datasetID,
	)
	if err != nil {
		return nil, fmt.Errorf("file_dataset insert failed: %w", err)
	}
	result.Duration = metadata.Duration
	return result, nil
}
// validateUnstructuredInput validates the input parameters
func validateUnstructuredInput(input ImportUnstructuredInput) error {
	// Validate dataset ID format
	if err := utils.ValidateShortID(input.DatasetID, "dataset_id"); err != nil {
		return err
	}
	// Verify folder exists
	info, err := os.Stat(input.FolderPath)
	if err != nil {
		return fmt.Errorf("folder not accessible: %w", err)
	}
	if !info.IsDir() {
		return fmt.Errorf("path is not a directory: %s", input.FolderPath)
	}
		// Verify dataset exists and is active
		if _, err := db.DatasetExistsAndActive(database, input.DatasetID); err != nil {
			return err
		}
		// Verify dataset is 'unstructured' type
		if err := db.ValidateDatasetTypeUnstructured(database, input.DatasetID); err != nil {
			return err
		}
		return nil
	})
}
// scanWavFiles scans a folder for WAV files
func scanWavFiles(folderPath string, recursive bool) ([]string, []utils.FileImportError) {
	var files []string
	var errors []utils.FileImportError
	walkFunc := func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			errors = append(errors, utils.FileImportError{
				FileName: path,
				Error:    err.Error(),
				Stage:    utils.StageScan,
			})
			return nil
		}
		// Skip directories if not recursive
		if d.IsDir() {
			if !recursive && path != folderPath {
				return fs.SkipDir
			}
			return nil
		}
		// Check for .wav extension (case-insensitive)
		if strings.HasSuffix(strings.ToLower(d.Name()), ".wav") {
			files = append(files, path)
		}
		return nil
	}
	if recursive {
		if err := filepath.WalkDir(folderPath, walkFunc); err != nil {
			errors = append(errors, utils.FileImportError{
				FileName: folderPath,
				Error:    err.Error(),
				Stage:    utils.StageScan,
			})
		}
	} else {
		// Non-recursive: only scan top-level
		entries, err := os.ReadDir(folderPath)
		if err != nil {
			errors = append(errors, utils.FileImportError{
				FileName: folderPath,
				Error:    err.Error(),
				Stage:    utils.StageScan,
			})
			return nil, errors
		}
		for _, entry := range entries {
			if !entry.IsDir() && strings.HasSuffix(strings.ToLower(entry.Name()), ".wav") {
				files = append(files, filepath.Join(folderPath, entry.Name()))
			}
		}
	}
	return files, errors
}
	return db.WithReadDB(resolveDBPath(input.DBPath), func(database *sql.DB) error {
	err := db.WithWriteTx(ctx, resolveDBPath(input.DBPath), "import_unstructured", func(database *sql.DB, tx *db.LoggedTx) error {
	DBPath     string `json:"db_path"`
	"database/sql"

package tools
import (
	"testing"
	"skraak/utils"
)
func TestValidateSegmentImportInput(t *testing.T) {
	t.Run("invalid dataset ID - too short", func(t *testing.T) {
		input := ImportSegmentsInput{
			DatasetID: "abc",
		}
		err := validateSegmentImportInput(input)
		if err == nil {
			t.Fatal("expected error for short dataset ID")
		}
	})
	t.Run("invalid dataset ID - too long", func(t *testing.T) {
		input := ImportSegmentsInput{
			DatasetID: "abc123def456ghi789",
		}
		err := validateSegmentImportInput(input)
		if err == nil {
			t.Fatal("expected error for long dataset ID")
		}
	})
	t.Run("invalid dataset ID - invalid characters", func(t *testing.T) {
		input := ImportSegmentsInput{
			DatasetID: "abc123!!!456",
		}
		err := validateSegmentImportInput(input)
		if err == nil {
			t.Fatal("expected error for invalid characters in dataset ID")
		}
	})
	t.Run("invalid location ID", func(t *testing.T) {
		input := ImportSegmentsInput{
			DatasetID:  "abc123def456",
			LocationID: "invalid",
		}
		err := validateSegmentImportInput(input)
		if err == nil {
			t.Fatal("expected error for invalid location ID")
		}
	})
	t.Run("invalid cluster ID", func(t *testing.T) {
		input := ImportSegmentsInput{
			DatasetID:  "abc123def456",
			LocationID: "xyz789uvw012",
			ClusterID:  "invalid",
		}
		err := validateSegmentImportInput(input)
		if err == nil {
			t.Fatal("expected error for invalid cluster ID")
		}
	})
}
func TestCountTotalSegments(t *testing.T) {
	t.Run("empty", func(t *testing.T) {
		count := countTotalSegments(map[string]scannedDataFile{})
		if count != 0 {
			t.Errorf("expected 0, got %d", count)
		}
	})
	t.Run("single file - no segments", func(t *testing.T) {
		files := map[string]scannedDataFile{
			"file1": {Segments: []*utils.Segment{}},
		}
		count := countTotalSegments(files)
		if count != 0 {
			t.Errorf("expected 0, got %d", count)
		}
	})
	t.Run("single file - multiple segments", func(t *testing.T) {
		files := map[string]scannedDataFile{
			"file1": {Segments: []*utils.Segment{{}, {}, {}}},
		}
		count := countTotalSegments(files)
		if count != 3 {
			t.Errorf("expected 3, got %d", count)
		}
	})
	t.Run("multiple files", func(t *testing.T) {
		files := map[string]scannedDataFile{
			"file1": {Segments: []*utils.Segment{{}, {}}},
			"file2": {Segments: []*utils.Segment{{}}},
			"file3": {Segments: []*utils.Segment{{}, {}, {}, {}}},
		}
		count := countTotalSegments(files)
		if count != 7 {
			t.Errorf("expected 7, got %d", count)
		}
	})
}

package tools
import (
	"context"
	"database/sql"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"
	"skraak/db"
	"skraak/utils"
)
// ImportSegmentsInput defines the input parameters for the import_segments tool
type ImportSegmentsInput struct {
	Folder          string `json:"folder"`
	Mapping         string `json:"mapping"`
	DatasetID       string `json:"dataset_id"`
	LocationID      string `json:"location_id"`
	ClusterID       string `json:"cluster_id"`
	ProgressHandler func(processed, total int, message string)
}
// ImportSegmentsOutput defines the output structure for the import_segments tool
type ImportSegmentsOutput struct {
	Summary  ImportSegmentsSummary `json:"summary"`
	Segments []SegmentImport       `json:"segments"`
	Errors   []ImportSegmentError  `json:"errors,omitempty"`
}
// ImportSegmentsSummary provides summary statistics for the import operation
type ImportSegmentsSummary struct {
	DataFilesFound     int   `json:"data_files_found"`
	DataFilesProcessed int   `json:"data_files_processed"`
	TotalSegments      int   `json:"total_segments"`
	ImportedSegments   int   `json:"imported_segments"`
	ImportedLabels     int   `json:"imported_labels"`
	ImportedSubtypes   int   `json:"imported_subtypes"`
	ProcessingTimeMs   int64 `json:"processing_time_ms"`
}
// SegmentImport represents an imported segment in the output
type SegmentImport struct {
	SegmentID string        `json:"segment_id"`
	FileName  string        `json:"file_name"`
	StartTime float64       `json:"start_time"`
	EndTime   float64       `json:"end_time"`
	FreqLow   float64       `json:"freq_low"`
	FreqHigh  float64       `json:"freq_high"`
	Labels    []LabelImport `json:"labels"`
}
// LabelImport represents an imported label in the output
type LabelImport struct {
	LabelID   string `json:"label_id"`
	Species   string `json:"species"`
	CallType  string `json:"calltype,omitempty"`
	Filter    string `json:"filter"`
	Certainty int    `json:"certainty"`
	Comment   string `json:"comment,omitempty"`
}
// ImportSegmentError records errors encountered during segment import
type ImportSegmentError struct {
	File    string            `json:"file,omitempty"`
	Stage   utils.ImportStage `json:"stage"`
	Message string            `json:"message"`
}
// scannedDataFile holds parsed data for a .data file
type scannedDataFile struct {
	DataPath string
	WavPath  string
	WavHash  string
	FileID   string
	Duration float64
	Segments []*utils.Segment
}
// ImportSegments imports segments from AviaNZ .data files into the database
func ImportSegments(ctx context.Context, input ImportSegmentsInput) (ImportSegmentsOutput, error) {
	startTime := time.Now()
	var output ImportSegmentsOutput
	output.Segments = make([]SegmentImport, 0)
	output.Errors = make([]ImportSegmentError, 0)
	// Phase A: Input Validation
	if err := validateSegmentImportInput(input); err != nil {
		return output, err
	}
	// Load mapping file
	mapping, err := utils.LoadMappingFile(input.Mapping)
	if err != nil {
		return output, fmt.Errorf("failed to load mapping file: %w", err)
	}
	// Find .data files
	dataFiles, err := utils.FindDataFiles(input.Folder)
	if err != nil {
		return output, fmt.Errorf("failed to find .data files: %w", err)
	}
	output.Summary.DataFilesFound = len(dataFiles)
	if len(dataFiles) == 0 {
		return output, fmt.Errorf("no .data files found in folder: %s", input.Folder)
	}
	// Phase B+C: Parse data files and validate against DB
	if err != nil {
		return output, fmt.Errorf("failed to open database: %w", err)
	}
	defer database.Close()
	val, valErrors, err := validateAndPrepareSegments(database, input, mapping, dataFiles)
	output.Errors = append(output.Errors, valErrors...)
	if err != nil {
		return output, err
	}
	if val == nil || len(val.fileIDMap) == 0 {
		output.Summary.ProcessingTimeMs = time.Since(startTime).Milliseconds()
		return output, nil
	}
	// Phase D: Transactional Import
	importedSegments, importedLabels, importedSubtypes, fileUpdates, importErrors := importSegmentsIntoDB(
		ctx, database, val.fileIDMap, val.scannedFiles, mapping, val.filterIDMap, val.speciesIDMap, val.calltypeIDMap, input.DatasetID, input.ProgressHandler,
	)
	output.Errors = append(output.Errors, importErrors...)
	output.Segments = append(output.Segments, importedSegments...)
	// Phase E: Write IDs back to .data files
	if len(fileUpdates) > 0 {
		writeErrors := writeIDsToDataFiles(fileUpdates)
		output.Errors = append(output.Errors, writeErrors...)
	}
	output.Summary.DataFilesProcessed = len(val.fileIDMap)
	output.Summary.TotalSegments = countTotalSegments(val.fileIDMap)
	output.Summary.ImportedSegments = len(importedSegments)
	output.Summary.ImportedLabels = importedLabels
	output.Summary.ImportedSubtypes = importedSubtypes
	output.Summary.ProcessingTimeMs = time.Since(startTime).Milliseconds()
	return output, nil
}
// validateSegmentImportInput validates input parameters
func validateSegmentImportInput(input ImportSegmentsInput) error {
	// Validate folder exists
	if info, err := os.Stat(input.Folder); err != nil {
		return fmt.Errorf("folder does not exist: %s", input.Folder)
	} else if !info.IsDir() {
		return fmt.Errorf("path is not a folder: %s", input.Folder)
	}
	// Validate mapping file exists
	if _, err := os.Stat(input.Mapping); err != nil {
		return fmt.Errorf("mapping file does not exist: %s", input.Mapping)
	}
	// Validate IDs
	if err := utils.ValidateShortID(input.DatasetID, "dataset_id"); err != nil {
		return err
	}
	if err := utils.ValidateShortID(input.LocationID, "location_id"); err != nil {
		return err
	}
	if err := utils.ValidateShortID(input.ClusterID, "cluster_id"); err != nil {
		return err
	}
	return nil
}
// validateSegmentHierarchy validates dataset/location/cluster relationships
func validateSegmentHierarchy(dbConn *sql.DB, datasetID, locationID, clusterID string) error {
	// Validate dataset exists and is structured
	if err := db.ValidateDatasetTypeForImport(dbConn, datasetID); err != nil {
		return err
	}
	// Validate location belongs to dataset
	if err := db.ValidateLocationBelongsToDataset(dbConn, locationID, datasetID); err != nil {
		return err
	}
	// Validate cluster belongs to location
	if err := db.ClusterBelongsToLocation(dbConn, clusterID, locationID); err != nil {
		return err
	}
	return nil
}
// scanAllDataFiles parses all .data files and collects unique values
func scanAllDataFiles(dataFiles []string, folder string) (
	[]scannedDataFile,
	[]ImportSegmentError,
	map[string]bool,
	map[string]bool,
	map[string]map[string]bool,
) {
	var scanned []scannedDataFile
	var errors []ImportSegmentError
	uniqueFilters := make(map[string]bool)
	uniqueSpecies := make(map[string]bool)
	uniqueCalltypes := make(map[string]map[string]bool) // species -> calltype -> true
	for _, dataPath := range dataFiles {
		// Find corresponding WAV file
		wavPath := strings.TrimSuffix(dataPath, ".data")
		if _, err := os.Stat(wavPath); err != nil {
			errors = append(errors, ImportSegmentError{
				File:    filepath.Base(dataPath),
				Stage:   utils.StageValidation,
				Message: fmt.Sprintf("corresponding WAV file not found: %s", filepath.Base(wavPath)),
			})
			continue
		}
		// Parse .data file
		df, err := utils.ParseDataFile(dataPath)
		if err != nil {
			errors = append(errors, ImportSegmentError{
				File:    filepath.Base(dataPath),
				Stage:   utils.StageValidation,
				Message: fmt.Sprintf("failed to parse .data file: %v", err),
			})
			continue
		}
		// Collect unique filters, species, calltypes
		for _, seg := range df.Segments {
			for _, label := range seg.Labels {
				uniqueFilters[label.Filter] = true
				uniqueSpecies[label.Species] = true
				if label.CallType != "" {
					if uniqueCalltypes[label.Species] == nil {
						uniqueCalltypes[label.Species] = make(map[string]bool)
					}
					uniqueCalltypes[label.Species][label.CallType] = true
				}
			}
		}
		scanned = append(scanned, scannedDataFile{
			DataPath: dataPath,
			WavPath:  wavPath,
			Duration: df.Meta.Duration,
			Segments: df.Segments,
		})
	}
	return scanned, errors, uniqueFilters, uniqueSpecies, uniqueCalltypes
}
// validateFiltersExist checks all filters exist in DB and returns ID map
func validateFiltersExist(dbConn *sql.DB, filterNames map[string]bool) (map[string]string, error) {
	filterIDMap := make(map[string]string)
	if len(filterNames) == 0 {
		return filterIDMap, nil
	}
	names := make([]string, 0, len(filterNames))
	for name := range filterNames {
		names = append(names, name)
	}
	query := `SELECT id, name FROM filter WHERE name IN (` + db.Placeholders(len(names)) + `) AND active = true`
	args := make([]any, len(names))
	for i, name := range names {
		args[i] = name
	}
	rows, err := dbConn.Query(query, args...)
	if err != nil {
		return nil, fmt.Errorf("failed to query filters: %w", err)
	}
	defer rows.Close()
	for rows.Next() {
		var id, name string
		if err := rows.Scan(&id, &name); err == nil {
			filterIDMap[name] = id
		}
	}
	// Check for missing filters
	var missing []string
	for name := range filterNames {
		if _, exists := filterIDMap[name]; !exists {
			missing = append(missing, name)
		}
	}
	if len(missing) > 0 {
		return nil, fmt.Errorf("filters not found in database: [%s]", strings.Join(missing, ", "))
	}
	return filterIDMap, nil
}
// loadSpeciesCalltypeIDs loads species and calltype ID maps
func loadSpeciesCalltypeIDs(
	dbConn *sql.DB,
	mapping utils.MappingFile,
	uniqueSpecies map[string]bool,
	uniqueCalltypes map[string]map[string]bool,
) (map[string]string, map[string]map[string]string, error) {
	speciesIDMap := make(map[string]string)
	calltypeIDMap := make(map[string]map[string]string) // (dbSpecies, dbCalltype) -> calltype_id
	// Collect all DB species labels from mapping
	dbSpeciesSet := make(map[string]bool)
	for dataSpecies := range uniqueSpecies {
		if dbSpecies, ok := mapping.GetDBSpecies(dataSpecies); ok {
			dbSpeciesSet[dbSpecies] = true
		}
	}
	// Load species IDs
	if len(dbSpeciesSet) > 0 {
		dbSpeciesList := make([]string, 0, len(dbSpeciesSet))
		for s := range dbSpeciesSet {
			dbSpeciesList = append(dbSpeciesList, s)
		}
		query := `SELECT id, label FROM species WHERE label IN (` + db.Placeholders(len(dbSpeciesList)) + `) AND active = true`
		args := make([]any, len(dbSpeciesList))
		for i, s := range dbSpeciesList {
			args[i] = s
		}
		rows, err := dbConn.Query(query, args...)
		if err != nil {
			return nil, nil, fmt.Errorf("failed to query species: %w", err)
		}
		defer rows.Close()
		for rows.Next() {
			var id, label string
			if err := rows.Scan(&id, &label); err == nil {
				speciesIDMap[label] = id
			}
		}
	}
	// Load calltype IDs
	for dataSpecies, ctSet := range uniqueCalltypes {
		dbSpecies, ok := mapping.GetDBSpecies(dataSpecies)
		if !ok {
			continue
		}
		if calltypeIDMap[dbSpecies] == nil {
			calltypeIDMap[dbSpecies] = make(map[string]string)
		}
		for dataCalltype := range ctSet {
			dbCalltype := mapping.GetDBCalltype(dataSpecies, dataCalltype)
			// Query calltype ID
			var calltypeID string
			err := dbConn.QueryRow(`
				SELECT ct.id
				FROM call_type ct
				JOIN species s ON ct.species_id = s.id
				WHERE s.label = ? AND ct.label = ? AND ct.active = true
			`, dbSpecies, dbCalltype).Scan(&calltypeID)
			if err == nil {
				calltypeIDMap[dbSpecies][dbCalltype] = calltypeID
			}
		}
	}
	return speciesIDMap, calltypeIDMap, nil
}
// validateAndMapFiles validates files exist by hash, are linked to dataset, and have no existing labels
func validateAndMapFiles(
	dbConn *sql.DB,
	scannedFiles []scannedDataFile,
	clusterID string,
	datasetID string,
) (map[string]scannedDataFile, []ImportSegmentError) {
	fileIDMap := make(map[string]scannedDataFile)
	var errors []ImportSegmentError
	for _, sf := range scannedFiles {
		// Compute hash
		hash, err := utils.ComputeXXH64(sf.WavPath)
		if err != nil {
			errors = append(errors, ImportSegmentError{
				File:    filepath.Base(sf.WavPath),
				Stage:   utils.StageHash,
				Message: fmt.Sprintf("failed to compute hash: %v", err),
			})
			continue
		}
		sf.WavHash = hash
		// Find file by hash in cluster
		var fileID string
		var duration float64
		err = dbConn.QueryRow(`
			SELECT id, duration FROM file WHERE xxh64_hash = ? AND cluster_id = ? AND active = true
		`, hash, clusterID).Scan(&fileID, &duration)
		if err == sql.ErrNoRows {
			errors = append(errors, ImportSegmentError{
				File:    filepath.Base(sf.WavPath),
				Stage:   utils.StageValidation,
				Message: fmt.Sprintf("file hash not found in database for cluster (hash: %s)", hash),
			})
			continue
		}
		if err != nil {
			errors = append(errors, ImportSegmentError{
				File:    filepath.Base(sf.WavPath),
				Stage:   utils.StageValidation,
				Message: fmt.Sprintf("failed to query file: %v", err),
			})
			continue
		}
		sf.FileID = fileID
		sf.Duration = duration
		// Verify file is linked to dataset via file_dataset junction table (composite FK)
		var fileLinkedToDataset bool
		err = dbConn.QueryRow(`
			SELECT EXISTS(SELECT 1 FROM file_dataset WHERE file_id = ? AND dataset_id = ?)
		`, fileID, datasetID).Scan(&fileLinkedToDataset)
		if err != nil {
			errors = append(errors, ImportSegmentError{
				File:    filepath.Base(sf.WavPath),
				Stage:   utils.StageValidation,
				Message: fmt.Sprintf("failed to verify file-dataset link: %v", err),
			})
			continue
		}
		if !fileLinkedToDataset {
			errors = append(errors, ImportSegmentError{
				File:    filepath.Base(sf.WavPath),
				Stage:   utils.StageValidation,
				Message: fmt.Sprintf("file exists in cluster but is not linked to dataset %s", datasetID),
			})
			continue
		}
		// Check no existing labels for this file
		var labelCount int
		err = dbConn.QueryRow(`
			SELECT COUNT(*) FROM label l
			JOIN segment s ON l.segment_id = s.id
			WHERE s.file_id = ? AND l.active = true
		`, fileID).Scan(&labelCount)
		if err != nil {
			errors = append(errors, ImportSegmentError{
				File:    filepath.Base(sf.WavPath),
				Stage:   utils.StageValidation,
				Message: fmt.Sprintf("failed to check existing labels: %v", err),
			})
			continue
		}
		if labelCount > 0 {
			errors = append(errors, ImportSegmentError{
				File:    filepath.Base(sf.WavPath),
				Stage:   utils.StageValidation,
				Message: fmt.Sprintf("file already has %d label(s) - fresh imports only", labelCount),
			})
			continue
		}
		fileIDMap[fileID] = sf
	}
	return fileIDMap, errors
}
// dataFileUpdate holds data to write back to .data file after import
type dataFileUpdate struct {
	DataPath string
	WavHash  string
	LabelIDs map[int]map[int]string // segmentIndex -> labelIndex -> labelID
}
// importSegmentsIntoDB performs the transactional import
func importSegmentsIntoDB(
	ctx context.Context,
	database *sql.DB,
	fileIDMap map[string]scannedDataFile,
	scannedFiles []scannedDataFile,
	mapping utils.MappingFile,
	filterIDMap map[string]string,
	speciesIDMap map[string]string,
	calltypeIDMap map[string]map[string]string,
	datasetID string,
	progressHandler func(processed, total int, message string),
) ([]SegmentImport, int, int, []dataFileUpdate, []ImportSegmentError) {
	var importedSegments []SegmentImport
	var errors []ImportSegmentError
	importedLabels := 0
	importedSubtypes := 0
	var fileUpdates []dataFileUpdate
	tx, err := db.BeginLoggedTx(ctx, database, "import_segments")
	if err != nil {
		errors = append(errors, ImportSegmentError{
			Stage:   utils.StageImport,
			Message: fmt.Sprintf("failed to begin transaction: %v", err),
		})
		return nil, 0, 0, nil, errors
	}
	defer tx.Rollback()
	totalFiles := len(fileIDMap)
	processedFiles := 0
	for _, sf := range fileIDMap {
		if sf.FileID == "" {
			continue
		}
		processedFiles++
		if progressHandler != nil {
			progressHandler(processedFiles, totalFiles, filepath.Base(sf.DataPath))
		}
		fileUpdate := dataFileUpdate{
			DataPath: sf.DataPath,
			WavHash:  sf.WavHash,
			LabelIDs: make(map[int]map[int]string),
		}
		for segIdx, seg := range sf.Segments {
			segImp, labelIDs, subtypes, segErrs := importSegment(ctx, tx, seg, segIdx, sf, datasetID, mapping, filterIDMap, speciesIDMap, calltypeIDMap)
			errors = append(errors, segErrs...)
			importedSubtypes += subtypes
			if len(segImp.Labels) == 0 {
				// Delete orphaned segment (no labels succeeded)
				if _, err := tx.ExecContext(ctx, `DELETE FROM segment WHERE id = ?`, segImp.SegmentID); err != nil {
					errors = append(errors, ImportSegmentError{
						File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
						Message: fmt.Sprintf("failed to delete orphaned segment: %v", err),
					})
				}
			} else {
				importedSegments = append(importedSegments, segImp)
				importedLabels += len(labelIDs)
				fileUpdate.LabelIDs[segIdx] = labelIDs
			}
		}
		fileUpdates = append(fileUpdates, fileUpdate)
	}
	if err := tx.Commit(); err != nil {
		errors = append(errors, ImportSegmentError{
			Stage:   utils.StageImport,
			Message: fmt.Sprintf("failed to commit transaction: %v", err),
		})
		return nil, 0, 0, nil, errors
	}
	return importedSegments, importedLabels, importedSubtypes, fileUpdates, errors
}
// countTotalSegments counts total segments from validated files
func countTotalSegments(fileIDMap map[string]scannedDataFile) int {
	count := 0
	for _, sf := range fileIDMap {
		count += len(sf.Segments)
	}
	return count
}
// writeIDsToDataFiles writes skraak_hash and skraak_label_ids back to .data files
func writeIDsToDataFiles(fileUpdates []dataFileUpdate) []ImportSegmentError {
	var errors []ImportSegmentError
	for _, fu := range fileUpdates {
		// Parse the .data file
		df, err := utils.ParseDataFile(fu.DataPath)
		if err != nil {
			errors = append(errors, ImportSegmentError{
				File:    filepath.Base(fu.DataPath),
				Stage:   utils.StageImport,
				Message: fmt.Sprintf("failed to re-parse .data file for writing: %v", err),
			})
			continue
		}
		// Write skraak_hash to metadata
		if df.Meta.Extra == nil {
			df.Meta.Extra = make(map[string]any)
		}
		df.Meta.Extra["skraak_hash"] = fu.WavHash
		// Write skraak_label_id to each label
		for segIdx, labelIDs := range fu.LabelIDs {
			if segIdx >= len(df.Segments) {
				continue
			}
			seg := df.Segments[segIdx]
			for labelIdx, labelID := range labelIDs {
				if labelIdx >= len(seg.Labels) {
					continue
				}
				label := seg.Labels[labelIdx]
				if label.Extra == nil {
					label.Extra = make(map[string]any)
				}
				label.Extra["skraak_label_id"] = labelID
			}
		}
		// Write the updated .data file
		if err := df.Write(fu.DataPath); err != nil {
			errors = append(errors, ImportSegmentError{
				File:    filepath.Base(fu.DataPath),
				Stage:   utils.StageImport,
				Message: fmt.Sprintf("failed to write updated .data file: %v", err),
			})
			continue
		}
	}
	return errors
}
	if seg.EndTime > sf.Duration {
		errors = append(errors, ImportSegmentError{
			File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
			Message: fmt.Sprintf("segment end time (%.2f) exceeds file duration (%.2f)", seg.EndTime, sf.Duration),
		})
		return SegmentImport{}, nil, 0, errors
	}
	segmentID, err := utils.GenerateLongID()
	if err != nil {
		errors = append(errors, ImportSegmentError{
			File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
			Message: fmt.Sprintf("failed to generate segment ID: %v", err),
		})
		return SegmentImport{}, nil, 0, errors
	}
	_, err = tx.ExecContext(ctx, `
		INSERT INTO segment (id, file_id, dataset_id, start_time, end_time, freq_low, freq_high, created_at, last_modified, active)
		VALUES (?, ?, ?, ?, ?, ?, ?, now(), now(), true)
	`, segmentID, sf.FileID, datasetID, seg.StartTime, seg.EndTime, seg.FreqLow, seg.FreqHigh)
	if err != nil {
		errors = append(errors, ImportSegmentError{
			File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
			Message: fmt.Sprintf("failed to insert segment: %v", err),
		})
		return SegmentImport{}, nil, 0, errors
	}
	segImport := SegmentImport{
		SegmentID: segmentID,
		FileName:  filepath.Base(sf.WavPath),
		StartTime: seg.StartTime,
		EndTime:   seg.EndTime,
		FreqLow:   seg.FreqLow,
		FreqHigh:  seg.FreqHigh,
		Labels:    make([]LabelImport, 0),
	}
	labelIDs := make(map[int]string)
	var subtypesImported int
	for labelIdx, label := range seg.Labels {
		result := importSingleLabel(ctx, tx, label, segmentID, segIdx, labelIdx, sf, mapping, filterIDMap, speciesIDMap, calltypeIDMap)
		if result.hasError {
			errors = append(errors, result.err)
			continue
		}
		labelIDs[labelIdx] = result.labelID
		segImport.Labels = append(segImport.Labels, result.labelImport)
		subtypesImported += result.subtypesImported
	}
	return segImport, labelIDs, subtypesImported, errors
}
// importSegment inserts a single segment and its labels into the DB.
func importSegment(
	ctx context.Context,
	tx *db.LoggedTx,
	seg *utils.Segment,
	segIdx int,
	sf scannedDataFile,
	datasetID string,
	mapping utils.MappingFile,
	filterIDMap map[string]string,
	speciesIDMap map[string]string,
	calltypeIDMap map[string]map[string]string,
) (SegmentImport, map[int]string, int, []ImportSegmentError) {
	var errors []ImportSegmentError
	if seg.StartTime >= seg.EndTime {
		errors = append(errors, ImportSegmentError{
			File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
			Message: fmt.Sprintf("invalid segment bounds: start=%.2f >= end=%.2f", seg.StartTime, seg.EndTime),
		})
		return SegmentImport{}, nil, 0, errors
	}
}
// importLabelResult holds the result of importing a single label.
type importLabelResult struct {
	labelImport      LabelImport
	labelID          string
	subtypesImported int
	err              ImportSegmentError
	hasError         bool
}
// importSingleLabel inserts a single label and its metadata/subtype into the DB.
func importSingleLabel(
	ctx context.Context,
	tx *db.LoggedTx,
	label *utils.Label,
	segmentID string,
	segIdx, labelIdx int,
	sf scannedDataFile,
	mapping utils.MappingFile,
	filterIDMap map[string]string,
	speciesIDMap map[string]string,
	calltypeIDMap map[string]map[string]string,
) importLabelResult {
	dbSpecies, ok := mapping.GetDBSpecies(label.Species)
	if !ok {
		return importLabelResult{err: ImportSegmentError{
			File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
			Message: fmt.Sprintf("species not found in mapping: %s", label.Species),
		}, hasError: true}
	}
	speciesID, ok := speciesIDMap[dbSpecies]
	if !ok {
		return importLabelResult{err: ImportSegmentError{
			File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
			Message: fmt.Sprintf("species ID not found: %s", dbSpecies),
		}, hasError: true}
	}
	filterID, ok := filterIDMap[label.Filter]
	if !ok {
		return importLabelResult{err: ImportSegmentError{
			File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
			Message: fmt.Sprintf("filter ID not found: %s", label.Filter),
		}, hasError: true}
	}
	labelID, err := utils.GenerateLongID()
	if err != nil {
		return importLabelResult{err: ImportSegmentError{
			File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
			Message: fmt.Sprintf("failed to generate label ID: %v", err),
		}, hasError: true}
	}
	_, err = tx.ExecContext(ctx, `
		INSERT INTO label (id, segment_id, species_id, filter_id, certainty, created_at, last_modified, active)
		VALUES (?, ?, ?, ?, ?, now(), now(), true)
	`, labelID, segmentID, speciesID, filterID, label.Certainty)
	if err != nil {
		return importLabelResult{err: ImportSegmentError{
			File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
			Message: fmt.Sprintf("failed to insert label: %v", err),
		}, hasError: true}
	}
	// Insert label_metadata if comment exists
	if label.Comment != "" {
		escapedComment := strings.ReplaceAll(label.Comment, `"`, `\"`)
		metadataJSON := fmt.Sprintf(`{"comment": "%s"}`, escapedComment)
		if _, err := tx.ExecContext(ctx, `
			INSERT INTO label_metadata (label_id, json, created_at, last_modified, active)
			VALUES (?, ?, now(), now(), true)
		`, labelID, metadataJSON); err != nil {
			return importLabelResult{err: ImportSegmentError{
				File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
				Message: fmt.Sprintf("failed to insert label_metadata: %v", err),
			}, hasError: true}
		}
	}
	labelImport := LabelImport{
		LabelID:   labelID,
		Species:   dbSpecies,
		Filter:    label.Filter,
		Certainty: label.Certainty,
	}
	if label.Comment != "" {
		labelImport.Comment = label.Comment
	}
	// Insert label_subtype if calltype exists
	if label.CallType != "" {
		if err := importCalltype(ctx, tx, labelID, label, dbSpecies, filterID, mapping, calltypeIDMap, sf); err != nil {
			return importLabelResult{err: *err, hasError: true}
		}
		labelImport.CallType = mapping.GetDBCalltype(label.Species, label.CallType)
		return importLabelResult{labelImport: labelImport, labelID: labelID, subtypesImported: 1}
	}
	return importLabelResult{labelImport: labelImport, labelID: labelID}
}
// importCalltype inserts a label_subtype row for a calltype label.
func importCalltype(
	ctx context.Context,
	tx *db.LoggedTx,
	labelID string,
	label *utils.Label,
	dbSpecies string,
	filterID string,
	mapping utils.MappingFile,
	calltypeIDMap map[string]map[string]string,
	sf scannedDataFile,
) *ImportSegmentError {
	dbCalltype := mapping.GetDBCalltype(label.Species, label.CallType)
	calltypeID := ""
	if calltypeIDMap[dbSpecies] != nil {
		calltypeID = calltypeIDMap[dbSpecies][dbCalltype]
	}
	if calltypeID == "" {
		return &ImportSegmentError{
			File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
			Message: fmt.Sprintf("calltype ID not found: %s/%s", dbSpecies, dbCalltype),
		}
	}
	subtypeID, err := utils.GenerateLongID()
	if err != nil {
		return &ImportSegmentError{
			File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
			Message: fmt.Sprintf("failed to generate label_subtype ID: %v", err),
		}
	}
	_, err = tx.ExecContext(ctx, `
		INSERT INTO label_subtype (id, label_id, calltype_id, filter_id, certainty, created_at, last_modified, active)
		VALUES (?, ?, ?, ?, ?, now(), now(), true)
	`, subtypeID, labelID, calltypeID, filterID, label.Certainty)
	if err != nil {
		return &ImportSegmentError{
			File: filepath.Base(sf.DataPath), Stage: utils.StageImport,
			Message: fmt.Sprintf("failed to insert label_subtype: %v", err),
		}
	}
	return nil
	database, err := db.OpenWriteableDB(resolveDBPath(input.DBPath))
// segmentValidation holds the results of pre-import validation (phases B+C).
type segmentValidation struct {
	scannedFiles  []scannedDataFile
	filterIDMap   map[string]string
	speciesIDMap  map[string]string
	calltypeIDMap map[string]map[string]string
	fileIDMap     map[string]scannedDataFile
}
// validateAndPrepareSegments performs phases B+C: parse data files, validate DB state, and prepare ID maps.
func validateAndPrepareSegments(
	database *sql.DB,
	input ImportSegmentsInput,
	mapping utils.MappingFile,
	dataFiles []string,
) (*segmentValidation, []ImportSegmentError, error) {
	// Phase B: Parse all .data files and collect unique values
	scannedFiles, parseErrors, uniqueFilters, uniqueSpecies, uniqueCalltypes := scanAllDataFiles(dataFiles, input.Folder)
	if len(scannedFiles) == 0 {
		return nil, parseErrors, nil
	}
	// Validate dataset/location/cluster hierarchy
	if err := validateSegmentHierarchy(database, input.DatasetID, input.LocationID, input.ClusterID); err != nil {
		return nil, parseErrors, err
	}
	// Validate all filters exist
	filterIDMap, err := validateFiltersExist(database, uniqueFilters)
	if err != nil {
		return nil, parseErrors, fmt.Errorf("filter validation failed: %w", err)
	}
	// Validate mapping covers all species/calltypes and they exist in DB
	validationResult, err := utils.ValidateMappingAgainstDB(database, mapping, uniqueSpecies, uniqueCalltypes)
	if err != nil {
		return nil, parseErrors, fmt.Errorf("mapping validation failed: %w", err)
	}
	if validationResult.HasErrors() {
		return nil, parseErrors, fmt.Errorf("mapping validation failed: %s", validationResult.Error())
	}
	// Load species and calltype ID maps
	speciesIDMap, calltypeIDMap, err := loadSpeciesCalltypeIDs(database, mapping, uniqueSpecies, uniqueCalltypes)
	if err != nil {
		return nil, parseErrors, fmt.Errorf("failed to load species/calltype IDs: %w", err)
	}
	// Validate files: hash exists, linked to dataset, no existing labels
	fileIDMap, hashErrors := validateAndMapFiles(database, scannedFiles, input.ClusterID, input.DatasetID)
	allErrors := append(parseErrors, hashErrors...)
	return &segmentValidation{
		scannedFiles:  scannedFiles,
		filterIDMap:   filterIDMap,
		speciesIDMap:  speciesIDMap,
		calltypeIDMap: calltypeIDMap,
		fileIDMap:     fileIDMap,
	}, allErrors, nil
}
	DBPath          string `json:"db_path"`

package tools
import (
	"context"
	"database/sql"
	"fmt"
	"os"
	"time"
	"skraak/db"
	"skraak/utils"
)
// ImportAudioFilesInput defines the input parameters for the import_audio_files tool
type ImportAudioFilesInput struct {
	FolderPath string `json:"folder_path"`
	DatasetID  string `json:"dataset_id"`
	LocationID string `json:"location_id"`
	ClusterID  string `json:"cluster_id"`
	Recursive  *bool  `json:"recursive,omitempty"` // *bool because default is true; plain bool would make "not provided" indistinguishable from "false"
}
// ImportAudioFilesOutput defines the output structure for the import_audio_files tool
type ImportAudioFilesOutput struct {
	Summary ImportSummary           `json:"summary"`
	FileIDs []string                `json:"file_ids"`
	Errors  []utils.FileImportError `json:"errors,omitempty"`
}
// ImportSummary provides summary statistics for the import operation
type ImportSummary struct {
	TotalFiles     int     `json:"total_files"`
	ImportedFiles  int     `json:"imported_files"`
	SkippedFiles   int     `json:"skipped_files"` // Duplicates
	FailedFiles    int     `json:"failed_files"`
	AudioMothFiles int     `json:"audiomoth_files"`
	TotalDuration  float64 `json:"total_duration_seconds"`
	ProcessingTime string  `json:"processing_time"`
}
// ImportAudioFiles batch imports WAV files from a folder with hash-based duplicate detection
func ImportAudioFiles(
	ctx context.Context,
	input ImportAudioFilesInput,
) (ImportAudioFilesOutput, error) {
	startTime := time.Now()
	var output ImportAudioFilesOutput
	// Default recursive to true
	recursive := true
	if input.Recursive != nil {
		recursive = *input.Recursive
	}
	// Validate database hierarchy (dataset → location → cluster)
		return output, fmt.Errorf("validation failed: %w", err)
	}
	// Open database
	if err != nil {
		return output, fmt.Errorf("failed to open database: %w", err)
	}
	defer database.Close()
	// Set cluster path if empty
	err = utils.EnsureClusterPath(database, input.ClusterID, input.FolderPath)
	if err != nil {
		return output, fmt.Errorf("failed to set cluster path: %w", err)
	}
	// Import the cluster (ALL THE LOGIC IS HERE)
		FolderPath: input.FolderPath,
		DatasetID:  input.DatasetID,
		LocationID: input.LocationID,
		ClusterID:  input.ClusterID,
		Recursive:  recursive,
	})
	if err != nil {
		return output, fmt.Errorf("cluster import failed: %w", err)
	}
	// Map to output format
	output = ImportAudioFilesOutput{
		Summary: ImportSummary{
			TotalFiles:     clusterOutput.TotalFiles,
			ImportedFiles:  clusterOutput.ImportedFiles,
			SkippedFiles:   clusterOutput.SkippedFiles,
			FailedFiles:    clusterOutput.FailedFiles,
			AudioMothFiles: clusterOutput.AudioMothFiles,
			TotalDuration:  clusterOutput.TotalDuration,
			ProcessingTime: time.Since(startTime).String(),
		},
		FileIDs: []string{}, // File IDs not tracked currently
		Errors:  clusterOutput.Errors,
	}
	return output, nil
}
// validateImportInput validates all input parameters and database relationships
func validateImportInput(input ImportAudioFilesInput, dbPath string) error {
	// Verify folder exists
	info, err := os.Stat(input.FolderPath)
	if err != nil {
		return fmt.Errorf("folder not accessible: %w", err)
	}
	if !info.IsDir() {
		return fmt.Errorf("path is not a directory: %s", input.FolderPath)
	}
	return validateHierarchyIDs(input.DatasetID, input.LocationID, input.ClusterID, dbPath)
}
// validateHierarchyIDs validates dataset/location/cluster ID formats and database relationships
func validateHierarchyIDs(datasetID, locationID, clusterID, dbPath string) error {
	// Validate ID formats first (fast fail before DB queries)
	if err := utils.ValidateShortID(datasetID, "dataset_id"); err != nil {
		return err
	}
	if err := utils.ValidateShortID(locationID, "location_id"); err != nil {
		return err
	}
	if err := utils.ValidateShortID(clusterID, "cluster_id"); err != nil {
		return err
	}
	return db.WithReadDB(dbPath, func(database *sql.DB) error {
		// Verify dataset exists, is active, and is 'structured' type
		if err := db.ValidateDatasetTypeForImport(database, datasetID); err != nil {
			return err
		}
		// Verify location exists and belongs to dataset
		if err := db.ValidateLocationBelongsToDataset(database, locationID, datasetID); err != nil {
			return err
		}
		// Verify cluster exists and belongs to location
		if err := db.ClusterBelongsToLocation(database, clusterID, locationID); err != nil {
			return err
		}
		return nil
	})
}
	}
	if err := tx.Commit(); err != nil {
		return output, fmt.Errorf("transaction commit failed: %w", err)
		tx.Rollback()
	tx, err := db.BeginLoggedTx(ctx, database, "import_audio_files")
	if err != nil {
		return output, fmt.Errorf("failed to begin transaction: %w", err)
	}
	clusterOutput, err := utils.ImportCluster(database, tx.UnderlyingTx(), utils.ClusterImportInput{
	database, err := db.OpenWriteableDB(resolveDBPath(input.DBPath))
	if err := validateImportInput(input, resolveDBPath(input.DBPath)); err != nil {
	DBPath     string `json:"db_path"`

package tools
import (
	"context"
	"database/sql"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"
	"skraak/db"
	"skraak/utils"
)
// ImportFileInput defines the input parameters for the import_file tool
type ImportFileInput struct {
	FilePath   string `json:"file_path"`
	DatasetID  string `json:"dataset_id"`
	LocationID string `json:"location_id"`
	ClusterID  string `json:"cluster_id"`
}
// ImportFileOutput defines the output structure for the import_file tool
type ImportFileOutput struct {
	FileID         string    `json:"file_id"`
	FileName       string    `json:"file_name"`
	Hash           string    `json:"hash"`
	Duration       float64   `json:"duration_seconds"`
	SampleRate     int       `json:"sample_rate"`
	TimestampLocal time.Time `json:"timestamp_local"`
	IsAudioMoth    bool      `json:"is_audiomoth"`
	IsDuplicate    bool      `json:"is_duplicate"`
	ProcessingTime string    `json:"processing_time"`
	Error          *string   `json:"error,omitempty"`
}
// ImportFile imports a single WAV file into the database with duplicate detection
func ImportFile(
	ctx context.Context,
	input ImportFileInput,
) (ImportFileOutput, error) {
	startTime := time.Now()
	var output ImportFileOutput
	// Phase 1: Validate file path
	_, err := validateFilePath(input.FilePath)
	if err != nil {
		return output, fmt.Errorf("file validation failed: %w", err)
	}
	output.FileName = filepath.Base(input.FilePath)
	// Phase 2: Validate database hierarchy
		return output, fmt.Errorf("hierarchy validation failed: %w", err)
	}
	// Phase 3: Open database connection (single connection for all DB operations)
	if err != nil {
		return output, fmt.Errorf("database connection failed: %w", err)
	}
	defer database.Close()
	// Phase 4: Get location data for astronomical calculations
	locData, err := utils.GetLocationData(database, input.LocationID)
	if err != nil {
		return output, fmt.Errorf("failed to get location data: %w", err)
	}
	// Phase 5: Process file metadata
	result, err := utils.ProcessSingleFile(input.FilePath, locData.Latitude, locData.Longitude, locData.TimezoneID, true)
	if err != nil {
		errMsg := err.Error()
		output.Error = &errMsg
		output.ProcessingTime = time.Since(startTime).String()
		return output, fmt.Errorf("file processing failed: %w", err)
	}
	// Populate output with extracted metadata
	output.FileName = result.FileName
	output.Hash = result.Hash
	output.Duration = result.Duration
	output.SampleRate = result.SampleRate
	output.TimestampLocal = result.TimestampLocal
	output.IsAudioMoth = result.IsAudioMoth
	// Phase 6: Ensure cluster path is set
	if err := utils.EnsureClusterPath(database, input.ClusterID, filepath.Dir(input.FilePath)); err != nil {
		return output, fmt.Errorf("failed to set cluster path: %w", err)
	}
	// Phase 7: Insert into database
	fileID, isDuplicate, err := insertFileIntoDB(ctx, database, result, input.DatasetID, input.ClusterID, input.LocationID)
	if err != nil {
		errMsg := err.Error()
		output.Error = &errMsg
		output.ProcessingTime = time.Since(startTime).String()
		return output, fmt.Errorf("database insertion failed: %w", err)
	}
	output.FileID = fileID
	output.IsDuplicate = isDuplicate
	output.ProcessingTime = time.Since(startTime).String()
	return output, nil
}
// validateFilePath validates the file exists, is a regular file, is a WAV file, and is not empty
func validateFilePath(filePath string) (os.FileInfo, error) {
	// Check file exists
	info, err := os.Stat(filePath)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("file does not exist: %s", filePath)
		}
		return nil, fmt.Errorf("cannot access file: %w", err)
	}
	// Check it's a regular file
	if !info.Mode().IsRegular() {
		return nil, fmt.Errorf("path is not a regular file: %s", filePath)
	}
	// Check extension is .wav (case-insensitive)
	ext := strings.ToLower(filepath.Ext(filePath))
	if ext != ".wav" {
		return nil, fmt.Errorf("file must be a WAV file (got extension: %s)", ext)
	}
	// Check file is not empty
	if info.Size() == 0 {
		return nil, fmt.Errorf("file is empty: %s", filePath)
	}
	return info, nil
}
// insertFileIntoDB inserts a single file into the database
// Returns (fileID, isDuplicate, error)
func insertFileIntoDB(
	ctx context.Context,
	database *sql.DB,
	result *utils.FileProcessingResult,
	datasetID, clusterID, locationID string,
) (string, bool, error) {
	// Begin logged transaction
	tx, err := db.BeginLoggedTx(ctx, database, "import_audio_file")
	if err != nil {
		return "", false, fmt.Errorf("failed to begin transaction: %w", err)
	}
	defer tx.Rollback() // Rollback if not committed
	// Check for duplicate hash
	existingID, isDup, err := utils.CheckDuplicateHash(tx, result.Hash)
	if err != nil {
		return "", false, err
	}
	if isDup {
		return existingID, true, nil
	}
	// Generate file ID
	fileID, err := utils.GenerateLongID()
	if err != nil {
		return "", false, fmt.Errorf("ID generation failed: %w", err)
	}
	// Insert file record
	_, err = tx.ExecContext(ctx, `
		INSERT INTO file (
			id, file_name, xxh64_hash, location_id, timestamp_local,
			cluster_id, duration, sample_rate, maybe_solar_night, maybe_civil_night,
			moon_phase, created_at, last_modified, active
		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, now(), now(), true)
	`,
		fileID, result.FileName, result.Hash, locationID,
		result.TimestampLocal, clusterID, result.Duration, result.SampleRate,
		result.AstroData.SolarNight, result.AstroData.CivilNight, result.AstroData.MoonPhase,
	)
	if err != nil {
		return "", false, fmt.Errorf("file insert failed: %w", err)
	}
	// Insert file_dataset junction
	_, err = tx.ExecContext(ctx, `
		INSERT INTO file_dataset (file_id, dataset_id, created_at, last_modified)
		VALUES (?, ?, now(), now())
	`, fileID, datasetID)
	if err != nil {
		return "", false, fmt.Errorf("file_dataset insert failed: %w", err)
	}
	// If AudioMoth, insert moth_metadata
	if result.IsAudioMoth && result.MothData != nil {
		_, err = tx.ExecContext(ctx, `
			INSERT INTO moth_metadata (
				file_id, timestamp, recorder_id, gain, battery_v, temp_c,
				created_at, last_modified, active
			) VALUES (?, ?, ?, ?, ?, ?, now(), now(), true)
		`,
			fileID,
			result.MothData.Timestamp,
			&result.MothData.RecorderID,
			&result.MothData.Gain,
			&result.MothData.BatteryV,
			&result.MothData.TempC,
		)
		if err != nil {
			return "", false, fmt.Errorf("moth_metadata insert failed: %w", err)
		}
	}
	// Commit transaction
	if err = tx.Commit(); err != nil {
		return "", false, fmt.Errorf("transaction commit failed: %w", err)
	}
	return fileID, false, nil
}
	database, err := db.OpenWriteableDB(resolveDBPath(input.DBPath))
	if err := validateHierarchyIDs(input.DatasetID, input.LocationID, input.ClusterID, resolveDBPath(input.DBPath)); err != nil {
	DBPath     string `json:"db_path"`

package tools
import (
	"sort"
	"strings"
	"skraak/utils"
)
// CallsSummariseInput defines the input for the calls-summarise tool
type CallsSummariseInput struct {
	Folder string `json:"folder"`
	Brief  bool   `json:"brief"`
	Filter string `json:"filter,omitempty"`
}
// CallsSummariseOutput defines the output for the calls-summarise tool
type CallsSummariseOutput struct {
	Segments         []SegmentSummary       `json:"segments"`
	Folder           string                 `json:"folder"`
	DataFilesRead    int                    `json:"data_files_read"`
	DataFilesSkipped []string               `json:"data_files_skipped"`
	TotalSegments    int                    `json:"total_segments"`
	Filters          map[string]FilterStats `json:"filters"`
	ReviewStatus     ReviewStatus           `json:"review_status"`
	Operators        []string               `json:"operators"`
	Reviewers        []string               `json:"reviewers"`
	Error            *string                `json:"error,omitempty"`
}
// SegmentSummary represents a single segment in the output
type SegmentSummary struct {
	File      string         `json:"file"`
	StartTime float64        `json:"start_time"`
	EndTime   float64        `json:"end_time"`
	Labels    []LabelSummary `json:"labels"`
}
// LabelSummary represents a label in the output (omits empty fields)
type LabelSummary struct {
	Filter    string `json:"filter"`
	Certainty int    `json:"certainty"`
	Species   string `json:"species"`
	CallType  string `json:"calltype,omitempty"`
	Comment   string `json:"comment,omitempty"`
	Bookmark  bool   `json:"bookmark,omitempty"`
}
// FilterStats contains per-filter statistics
type FilterStats struct {
	Segments  int                       `json:"segments"`
	Species   map[string]int            `json:"species"`
	Calltypes map[string]map[string]int `json:"calltypes,omitempty"` // species -> calltype -> count
}
// ReviewStatus contains review progress statistics
type ReviewStatus struct {
	Unreviewed   int `json:"unreviewed"` // certainty < 100
	Confirmed    int `json:"confirmed"`  // certainty = 100
	DontKnow     int `json:"dont_know"`  // certainty = 0
	WithCallType int `json:"with_calltype"`
	WithComments int `json:"with_comments"`
	Bookmarked   int `json:"bookmarked"`
}
// CallsSummarise reads all .data files in a folder and produces a summary
func CallsSummarise(input CallsSummariseInput) (CallsSummariseOutput, error) {
	var output CallsSummariseOutput
	// Find all .data files
	filePaths, err := utils.FindDataFiles(input.Folder)
	if err != nil {
		errMsg := err.Error()
		output.Error = &errMsg
		return output, err
	}
	// Initialize empty slices/maps (avoid null in JSON)
	output.Segments = make([]SegmentSummary, 0)
	output.Folder = input.Folder
	output.Filters = make(map[string]FilterStats)
	output.Operators = make([]string, 0)
	output.Reviewers = make([]string, 0)
	output.DataFilesSkipped = make([]string, 0)
	if len(filePaths) == 0 {
		return output, nil
	}
	// Track unique operators and reviewers
	operatorSet := make(map[string]bool)
	reviewerSet := make(map[string]bool)
	// Count segments for total
	if input.Brief {
		for _, fs := range output.Filters {
			output.TotalSegments += fs.Segments
		}
	} else {
		output.TotalSegments = len(output.Segments)
	}
	finaliseSummary(&output, operatorSet, reviewerSet, input.Brief)
	return output, nil
}
// summariseFiles processes all data files, populating output stats
func summariseFiles(filePaths []string, input CallsSummariseInput, output *CallsSummariseOutput, operatorSet, reviewerSet map[string]bool) {
	for _, path := range filePaths {
		df, err := utils.ParseDataFile(path)
		if err != nil {
			output.DataFilesSkipped = append(output.DataFilesSkipped, path)
			continue
		}
		output.DataFilesRead++
		trackMeta(df.Meta, operatorSet, reviewerSet)
		var relPath string
		if !input.Brief {
			relPath = extractRelativePath(input.Folder, path)
		}
		for _, seg := range df.Segments {
			filteredLabels := filterLabels(seg.Labels, input.Filter)
			if input.Filter != "" && len(filteredLabels) == 0 {
				continue
			}
			updateStatsFromLabels(filteredLabels, output)
			if !input.Brief {
				output.Segments = append(output.Segments, SegmentSummary{
					File:      relPath,
					StartTime: seg.StartTime,
					EndTime:   seg.EndTime,
					Labels:    buildLabelSummaries(filteredLabels),
				})
			}
// trackMeta records operator and reviewer from file metadata
func trackMeta(meta *utils.DataMeta, operatorSet, reviewerSet map[string]bool) {
	if meta == nil {
		return
	}
	if meta.Operator != "" {
		operatorSet[meta.Operator] = true
	}
	if meta.Reviewer != "" {
		reviewerSet[meta.Reviewer] = true
	}
}
// filterLabels returns labels matching the filter, or all labels if filter is empty
func filterLabels(labels []*utils.Label, filter string) []*utils.Label {
	if filter == "" {
		return labels
	}
	var filtered []*utils.Label
	for _, l := range labels {
		if l.Filter == filter {
			filtered = append(filtered, l)
		}
	}
	return filtered
}
// buildLabelSummaries converts labels to label summaries
func buildLabelSummaries(labels []*utils.Label) []LabelSummary {
	var summaries []LabelSummary
	for _, l := range labels {
		ls := LabelSummary{
			Filter:    l.Filter,
			Certainty: l.Certainty,
			Species:   l.Species,
		}
		if l.CallType != "" {
			ls.CallType = l.CallType
		}
		if l.Comment != "" {
			ls.Comment = l.Comment
		}
		if l.Bookmark {
			ls.Bookmark = true
		}
		summaries = append(summaries, ls)
	}
	return summaries
}
// updateStatsFromLabels updates filter stats and review status from a set of labels
func updateStatsFromLabels(labels []*utils.Label, output *CallsSummariseOutput) {
	for _, l := range labels {
		updateFilterStats(l, output)
		updateReviewStatus(l, output)
	}
}
// updateFilterStats increments filter-level statistics for a single label
func updateFilterStats(l *utils.Label, output *CallsSummariseOutput) {
	fs, exists := output.Filters[l.Filter]
	if !exists {
		fs = FilterStats{
			Segments:  0,
			Species:   make(map[string]int),
			Calltypes: make(map[string]map[string]int),
		}
	}
	if l.CallType != "" {
		if fs.Calltypes[l.Species] == nil {
			fs.Calltypes[l.Species] = make(map[string]int)
		}
		fs.Calltypes[l.Species][l.CallType]++
	}
	output.Filters[l.Filter] = fs
}
// updateReviewStatus increments review status counters for a single label
func updateReviewStatus(l *utils.Label, output *CallsSummariseOutput) {
	switch l.Certainty {
	case 100:
		output.ReviewStatus.Confirmed++
	case 0:
		output.ReviewStatus.DontKnow++
	default:
		output.ReviewStatus.Unreviewed++
	}
	if l.CallType != "" {
		output.ReviewStatus.WithCallType++
	}
	if l.Comment != "" {
		output.ReviewStatus.WithComments++
	}
	if l.Bookmark {
		output.ReviewStatus.Bookmarked++
	}
// finaliseSummary sorts output, cleans empty maps, and converts sets to sorted slices
func finaliseSummary(output *CallsSummariseOutput, operatorSet, reviewerSet map[string]bool, brief bool) {
	// Clean up empty calltypes maps
	for filter, fs := range output.Filters {
		if len(fs.Calltypes) == 0 {
			fs.Calltypes = nil
			output.Filters[filter] = fs
		}
	}
	// Convert sets to sorted slices
	for op := range operatorSet {
		output.Operators = append(output.Operators, op)
	}
	for r := range reviewerSet {
		output.Reviewers = append(output.Reviewers, r)
	}
	sort.Strings(output.Operators)
	sort.Strings(output.Reviewers)
	// Sort segments by file, then start time
	if !brief {
		sort.Slice(output.Segments, func(i, j int) bool {
			if output.Segments[i].File != output.Segments[j].File {
				return output.Segments[i].File < output.Segments[j].File
			}
			return output.Segments[i].StartTime < output.Segments[j].StartTime
		})
	}
}
// extractRelativePath extracts the audio filename from a .data file path
// e.g., "/folder/tx51_LISTENING_20260221_203004.WAV.data" -> "tx51_LISTENING_20260221_203004.WAV"
// Preserves the original case of the extension as-is.
func extractRelativePath(folder, dataPath string) string {
	// Get the filename
	filename := dataPath
	if idx := strings.LastIndex(dataPath, "/"); idx >= 0 {
		filename = dataPath[idx+1:]
	}
	// Remove .data extension, preserve everything else
	return strings.TrimSuffix(filename, ".data")
}
}
	fs.Segments++
	fs.Species[l.Species]++
		}
	}
}
	summariseFiles(filePaths, input, &output, operatorSet, reviewerSet)

package tools
import (
	"fmt"
	"os"
	"strings"
	"skraak/utils"
)
// CallsShowImagesInput defines the input for the show-images tool
type CallsShowImagesInput struct {
	DataFilePath string `json:"data_file_path"`
	Color        bool   `json:"color"`
	ImageSize    int    `json:"image_size"`
	Sixel        bool   `json:"sixel"`
	ITerm        bool   `json:"iterm"`
}
// CallsShowImagesOutput defines the output for the show-images tool
type CallsShowImagesOutput struct {
	SegmentsShown int    `json:"segments_shown"`
	WavFile       string `json:"wav_file"`
	Error         string `json:"error,omitempty"`
}
// CallsShowImages reads a .data file and displays spectrogram images for each segment
func CallsShowImages(input CallsShowImagesInput) (CallsShowImagesOutput, error) {
	var output CallsShowImagesOutput
	// Validate file exists
	if _, err := os.Stat(input.DataFilePath); os.IsNotExist(err) {
		output.Error = fmt.Sprintf("File not found: %s", input.DataFilePath)
		return output, fmt.Errorf("%s", output.Error)
	}
	// Derive WAV file path (strip .data suffix)
	wavPath := strings.TrimSuffix(input.DataFilePath, ".data")
	output.WavFile = wavPath
	// Check WAV file exists
	if _, err := os.Stat(wavPath); os.IsNotExist(err) {
		output.Error = fmt.Sprintf("WAV file not found: %s", wavPath)
		return output, fmt.Errorf("%s", output.Error)
	}
	// Parse .data file (includes labels for future filtering)
	dataFile, err := utils.ParseDataFile(input.DataFilePath)
	if err != nil {
		output.Error = err.Error()
		return output, fmt.Errorf("%s", output.Error)
	}
	if len(dataFile.Segments) == 0 {
		output.Error = "No segments found in .data file"
		return output, fmt.Errorf("%s", output.Error)
	}
	// Resolve image size
	imgSize := input.ImageSize
	if imgSize == 0 {
		imgSize = utils.SpectrogramDisplaySize
	}
	// Select graphics protocol
	protocol := utils.ProtocolKitty
	if input.ITerm {
		protocol = utils.ProtocolITerm
	} else if input.Sixel {
		protocol = utils.ProtocolSixel
	}
	// Generate spectrogram for each segment and output
	for i, seg := range dataFile.Segments {
		// Generate spectrogram image
		img, err := utils.GenerateSegmentSpectrogram(input.DataFilePath, seg.StartTime, seg.EndTime, input.Color, imgSize)
		if err != nil || img == nil {
			continue
		}
		// Print segment info
		labelInfo := formatSegmentLabels(seg.Labels)
		fmt.Fprintf(os.Stderr, "Segment %d: %.1fs - %.1fs (%.1fs)%s\n",
			i+1, seg.StartTime, seg.EndTime, seg.EndTime-seg.StartTime, labelInfo)
		// Write to stdout via terminal graphics protocol
		if err := utils.WriteImage(img, os.Stdout, protocol); err != nil {
			output.Error = fmt.Sprintf("Failed to write image: %v", err)
			return output, fmt.Errorf("%s", output.Error)
		}
		fmt.Println() // Newline after image
	}
	output.SegmentsShown = len(dataFile.Segments)
	return output, nil
}
// formatSegmentLabels formats labels for display in segment info
func formatSegmentLabels(labels []*utils.Label) string {
	if len(labels) == 0 {
		return ""
	}
	var parts []string
	for _, l := range labels {
		part := l.Species
		if l.CallType != "" {
			part += "/" + l.CallType
		}
		if l.Filter != "" {
			part += " [" + l.Filter + "]"
		}
		parts = append(parts, part)
	}
	return "  " + strings.Join(parts, ", ")
}

package tools
import (
	"encoding/json"
	"os"
	"path/filepath"
	"testing"
	"skraak/utils"
)
func TestPushCertaintyPromotesMatchingLabels(t *testing.T) {
	tempDir := t.TempDir()
	// File with two Kiwi segments: certainty=90 and certainty=70
	file1 := `[{"Operator": "test"}, [0, 10, 100, 1000, [{"species": "Kiwi", "certainty": 90}]], [10, 20, 100, 1000, [{"species": "Kiwi", "certainty": 70}]]]`
	file1Path := filepath.Join(tempDir, "file1.data")
	if err := os.WriteFile(file1Path, []byte(file1), 0644); err != nil {
		t.Fatal(err)
	}
	// File with one Tomtit at certainty=90 (must not be promoted when species=Kiwi)
	file2 := `[{"Operator": "test"}, [0, 10, 100, 1000, [{"species": "Tomtit", "certainty": 90}]]]`
	file2Path := filepath.Join(tempDir, "file2.data")
	if err := os.WriteFile(file2Path, []byte(file2), 0644); err != nil {
		t.Fatal(err)
	}
	result, err := PushCertainty(PushCertaintyConfig{
		Folder:   tempDir,
		Species:  "Kiwi",
		Reviewer: "TestReviewer",
	})
	if err != nil {
		t.Fatal(err)
	}
	if result.SegmentsUpdated != 1 {
		t.Errorf("expected 1 segment updated, got %d", result.SegmentsUpdated)
	}
	if result.FilesUpdated != 1 {
		t.Errorf("expected 1 file updated, got %d", result.FilesUpdated)
	}
	// Verify file1: certainty=90 Kiwi → 100, certainty=70 Kiwi → unchanged
	df, err := utils.ParseDataFile(file1Path)
	if err != nil {
		t.Fatal(err)
	}
	if df.Segments[0].Labels[0].Certainty != 100 {
		t.Errorf("expected certainty=100, got %d", df.Segments[0].Labels[0].Certainty)
	}
	if df.Segments[1].Labels[0].Certainty != 70 {
		t.Errorf("expected certainty=70 unchanged, got %d", df.Segments[1].Labels[0].Certainty)
	}
	if df.Meta.Reviewer != "TestReviewer" {
		t.Errorf("expected reviewer=TestReviewer, got %q", df.Meta.Reviewer)
	}
	// Verify Tomtit file was not modified
	df2, err := utils.ParseDataFile(file2Path)
	if err != nil {
		t.Fatal(err)
	}
	if df2.Segments[0].Labels[0].Certainty != 90 {
		t.Errorf("Tomtit certainty should be unchanged at 90, got %d", df2.Segments[0].Labels[0].Certainty)
	}
}
func TestPushCertaintyFilterScope(t *testing.T) {
	tempDir := t.TempDir()
	// Segment has two labels from different filters, both Kiwi certainty=90
	data := []any{
		map[string]any{"Operator": "test"},
		[]any{0.0, 10.0, 100.0, 1000.0, []any{
			map[string]any{"species": "Kiwi", "certainty": 90, "filter": "model-a"},
			map[string]any{"species": "Kiwi", "certainty": 90, "filter": "model-b"},
		}},
	}
	raw, _ := json.Marshal(data)
	filePath := filepath.Join(tempDir, "file1.data")
	if err := os.WriteFile(filePath, raw, 0644); err != nil {
		t.Fatal(err)
	}
	// Push only model-a
	result, err := PushCertainty(PushCertaintyConfig{
		Folder:   tempDir,
		Filter:   "model-a",
		Species:  "Kiwi",
		Reviewer: "TestReviewer",
	})
	if err != nil {
		t.Fatal(err)
	}
	if result.SegmentsUpdated != 1 {
		t.Errorf("expected 1 segment updated, got %d", result.SegmentsUpdated)
	}
	// Verify only model-a label was promoted; model-b stays at 90
	df, err := utils.ParseDataFile(filePath)
	if err != nil {
		t.Fatal(err)
	}
	for _, label := range df.Segments[0].Labels {
		if label.Filter == "model-a" && label.Certainty != 100 {
			t.Errorf("model-a label should be 100, got %d", label.Certainty)
		}
		if label.Filter == "model-b" && label.Certainty != 90 {
			t.Errorf("model-b label should be unchanged at 90, got %d", label.Certainty)
		}
	}
}

package tools
import (
	"fmt"
	"skraak/utils"
)
// PushCertaintyConfig holds the configuration for push-certainty
type PushCertaintyConfig struct {
	Folder   string
	File     string
	Filter   string
	Species  string
	CallType string
	Night    bool
	Day      bool
	Lat      float64
	Lng      float64
	Timezone string
	Reviewer string
}
// PushCertaintyResult holds the result of push-certainty
type PushCertaintyResult struct {
	SegmentsUpdated   int `json:"segments_updated"`
	FilesUpdated      int `json:"files_updated"`
	TimeFilteredCount int `json:"time_filtered_count"`
}
// PushCertainty promotes all certainty=90 segments matching the filter scope to certainty=100.
// Uses identical filtering logic to LoadDataFiles so the scope matches calls classify exactly.
func PushCertainty(config PushCertaintyConfig) (*PushCertaintyResult, error) {
	state, err := LoadDataFiles(ClassifyConfig{
		Folder:    config.Folder,
		File:      config.File,
		Filter:    config.Filter,
		Species:   config.Species,
		CallType:  config.CallType,
		Certainty: 90,
		Sample:    -1,
		Night:     config.Night,
		Day:       config.Day,
		Lat:       config.Lat,
		Lng:       config.Lng,
		Timezone:  config.Timezone,
	})
	if err != nil {
		return nil, err
	}
	var segsUpdated, filesUpdated int
	for i, df := range state.DataFiles {
		changed := false
		for _, seg := range state.FilteredSegs()[i] {
			for _, label := range seg.Labels {
				if labelMatchesPush(label, config.Filter, config.Species, config.CallType) {
					label.Certainty = 100
					changed = true
					segsUpdated++
				}
			}
		}
		if changed {
			df.Meta.Reviewer = config.Reviewer
			if err := df.Write(df.FilePath); err != nil {
				return nil, fmt.Errorf("write %s: %w", df.FilePath, err)
			}
			filesUpdated++
		}
	}
	return &PushCertaintyResult{
		SegmentsUpdated:   segsUpdated,
		FilesUpdated:      filesUpdated,
		TimeFilteredCount: state.TimeFilteredCount,
	}, nil
}
// labelMatchesPush returns true if the label matches the push scope and has certainty=90.
// Certainty is already guaranteed by LoadDataFiles, but we re-check to target only the
// specific label that matched (a segment may carry labels from multiple filters).
func labelMatchesPush(label *utils.Label, filter, species, callType string) bool {
	if filter != "" && label.Filter != filter {
		return false
	}
	if species != "" && label.Species != species {
		return false
	}
	if callType != "" && label.CallType != callType {
		return false
	}
	return label.Certainty == 90
}

package tools
import (
	"path/filepath"
	"testing"
	"skraak/utils"
)
// helpers
func seg(start, end float64, labels ...*utils.Label) *utils.Segment {
	return &utils.Segment{
		StartTime: start,
		EndTime:   end,
		FreqLow:   100,
		FreqHigh:  8000,
		Labels:    labels,
	}
}
func lbl(filter, species, calltype string, certainty int) *utils.Label {
	return &utils.Label{
		Filter:    filter,
		Species:   species,
		CallType:  calltype,
		Certainty: certainty,
	}
}
func writeFile(t *testing.T, segs ...*utils.Segment) string {
	t.Helper()
	dir := t.TempDir()
	path := filepath.Join(dir, "test.data")
	df := &utils.DataFile{
		Meta:     &utils.DataMeta{Operator: "ML", Reviewer: "David", Duration: 3600},
		Segments: segs,
	}
	if err := df.Write(path); err != nil {
		t.Fatalf("write fixture: %v", err)
	}
	return path
}
func readFile(t *testing.T, path string) *utils.DataFile {
	t.Helper()
	df, err := utils.ParseDataFile(path)
	if err != nil {
		t.Fatalf("parse %s: %v", path, err)
	}
	return df
}
// findLabel returns the label with matching filter and time on the parsed file, or nil.
func findLabel(df *utils.DataFile, filter string, start, end float64) *utils.Label {
	for _, s := range df.Segments {
		if s.StartTime != start || s.EndTime != end {
			continue
		}
		for _, l := range s.Labels {
			if l.Filter == filter {
				return l
			}
		}
	}
	return nil
}
const (
	fFrom = "opensoundscape-kiwi-1.2"
	fTo   = "opensoundscape-kiwi-1.5"
)
func TestPropagate_HappyPathSingle(t *testing.T) {
	path := writeFile(t,
		seg(100, 125, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(100, 125, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v (%s)", err, out.Error)
	}
	if out.Propagated != 1 || out.TargetsExamined != 1 || out.SkippedConflict != 0 || out.SkippedNoOverlap != 0 {
		t.Fatalf("counts wrong: %+v", out)
	}
	df := readFile(t, path)
	target := findLabel(df, fTo, 100, 125)
	if target == nil {
		t.Fatal("target label missing")
	}
	if target.Species != "Kiwi" || target.CallType != "Male" || target.Certainty != 90 {
		t.Errorf("target not updated correctly: species=%q calltype=%q cert=%d", target.Species, target.CallType, target.Certainty)
	}
	if df.Meta.Reviewer != "Skraak" {
		t.Errorf("reviewer = %q, want Skraak", df.Meta.Reviewer)
	}
}
func TestPropagate_NoOverlap(t *testing.T) {
	path := writeFile(t,
		seg(100, 125, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(500, 525, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.Propagated != 0 || out.TargetsExamined != 1 || out.SkippedNoOverlap != 1 {
		t.Fatalf("counts wrong: %+v", out)
	}
	df := readFile(t, path)
	target := findLabel(df, fTo, 500, 525)
	if target.Certainty != 70 {
		t.Errorf("target should not be modified, cert=%d", target.Certainty)
	}
	if df.Meta.Reviewer != "David" {
		t.Errorf("reviewer should stay David (no write), got %q", df.Meta.Reviewer)
	}
}
func TestPropagate_SourceWrongSpecies_Ignored(t *testing.T) {
	path := writeFile(t,
		seg(100, 125, lbl(fFrom, "Weka", "", 100)),
		seg(100, 125, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.Propagated != 0 || out.SkippedNoOverlap != 1 {
		t.Fatalf("counts wrong: %+v", out)
	}
}
func TestPropagate_SourceWrongCertainty_Ignored(t *testing.T) {
	// cert=70 and cert=0 source labels must NOT count as sources.
	path := writeFile(t,
		seg(100, 125, lbl(fFrom, "Kiwi", "Male", 70)),
		seg(200, 225, lbl(fFrom, "Don't Know", "", 0)),
		seg(100, 125, lbl(fTo, "Kiwi", "Duet", 70)),
		seg(200, 225, lbl(fTo, "Kiwi", "Male", 70)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.Propagated != 0 || out.SkippedNoOverlap != 2 {
		t.Fatalf("counts wrong: %+v", out)
	}
}
func TestPropagate_SourceWrongFilter_Ignored(t *testing.T) {
	path := writeFile(t,
		seg(100, 125, lbl("some-other-filter", "Kiwi", "Male", 100)),
		seg(100, 125, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if !out.FiltersMissing || out.Propagated != 0 || out.TargetsExamined != 0 {
		t.Fatalf("expected FiltersMissing=true with zero counts, got: %+v", out)
	}
}
func TestPropagate_TargetCert100_NotTouched(t *testing.T) {
	// Target with cert=100 is human-verified — must NOT be overwritten.
	path := writeFile(t,
		seg(100, 125, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(100, 125, lbl(fTo, "Kiwi", "Male", 100)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.TargetsExamined != 0 || out.Propagated != 0 {
		t.Fatalf("cert=100 target must not be examined: %+v", out)
	}
	df := readFile(t, path)
	if df.Meta.Reviewer != "David" {
		t.Errorf("reviewer should stay David (no write), got %q", df.Meta.Reviewer)
	}
}
func TestPropagate_TargetCert90_NotTouched(t *testing.T) {
	// Target with cert=90 (already propagated earlier) must NOT be re-propagated.
	path := writeFile(t,
		seg(100, 125, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(100, 125, lbl(fTo, "Kiwi", "Female", 90)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.TargetsExamined != 0 || out.Propagated != 0 {
		t.Fatalf("cert=90 target must not be examined: %+v", out)
	}
	df := readFile(t, path)
	target := findLabel(df, fTo, 100, 125)
	if target.Certainty != 90 || target.CallType != "Female" {
		t.Errorf("cert=90 target was modified: %+v", target)
	}
}
func TestPropagate_TargetCert0_Propagated(t *testing.T) {
	// Target at cert=0 ("Don't Know" / "Noise") SHOULD be propagated when an
	// overlapping cert=100 source exists — rescues labels from the noise bucket
	// so they surface for review even if occasionally wrong.
	path := writeFile(t,
		seg(100, 125, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(100, 125, lbl(fTo, "Don't Know", "", 0)),
		seg(200, 225, lbl(fFrom, "Kiwi", "Female", 100)),
		seg(200, 225, lbl(fTo, "Noise", "", 0)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.TargetsExamined != 2 || out.Propagated != 2 {
		t.Fatalf("cert=0 targets must be propagated: %+v", out)
	}
	df := readFile(t, path)
	for _, c := range []struct {
		start, end float64
		calltype   string
	}{{100, 125, "Male"}, {200, 225, "Female"}} {
		l := findLabel(df, fTo, c.start, c.end)
		if l == nil || l.Species != "Kiwi" || l.CallType != c.calltype || l.Certainty != 90 {
			t.Errorf("at %v-%v got %+v, want Kiwi+%s cert=90", c.start, c.end, l, c.calltype)
		}
	}
}
func TestPropagate_MultipleSourcesAgree(t *testing.T) {
	// Two overlapping sources with same calltype → propagate.
	path := writeFile(t,
		seg(100, 110, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(105, 120, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(100, 125, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.Propagated != 1 || out.SkippedConflict != 0 {
		t.Fatalf("counts wrong: %+v", out)
	}
	df := readFile(t, path)
	target := findLabel(df, fTo, 100, 125)
	if target.CallType != "Male" {
		t.Errorf("calltype should be Male, got %q", target.CallType)
	}
}
func TestPropagate_MultipleSourcesConflict(t *testing.T) {
	// Two overlapping sources with different calltypes → conflict, skip, report.
	path := writeFile(t,
		seg(100, 110, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(115, 120, lbl(fFrom, "Kiwi", "Female", 100)),
		seg(100, 125, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.Propagated != 0 || out.SkippedConflict != 1 {
		t.Fatalf("expected 1 conflict skip: %+v", out)
	}
	if len(out.Conflicts) != 1 {
		t.Fatalf("expected 1 conflict report, got %d", len(out.Conflicts))
	}
	if out.Conflicts[0].TargetStart != 100 || out.Conflicts[0].TargetEnd != 125 {
		t.Errorf("conflict target wrong: %+v", out.Conflicts[0])
	}
	if len(out.Conflicts[0].SourceChoices) != 2 {
		t.Errorf("expected 2 source choices, got %d", len(out.Conflicts[0].SourceChoices))
	}
	// Target must NOT be modified.
	df := readFile(t, path)
	target := findLabel(df, fTo, 100, 125)
	if target.CallType != "Duet" || target.Certainty != 70 {
		t.Errorf("conflicted target was modified: %+v", target)
	}
	if df.Meta.Reviewer != "David" {
		t.Errorf("reviewer should stay David (no write), got %q", df.Meta.Reviewer)
	}
}
func TestPropagate_EmptyCallTypePropagates(t *testing.T) {
	// Source with empty calltype → target gets empty calltype.
	path := writeFile(t,
		seg(100, 125, lbl(fFrom, "Kiwi", "", 100)),
		seg(100, 125, lbl(fTo, "Kiwi", "Male", 70)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.Propagated != 1 {
		t.Fatalf("expected propagated=1: %+v", out)
	}
	df := readFile(t, path)
	target := findLabel(df, fTo, 100, 125)
	if target.CallType != "" {
		t.Errorf("calltype should be cleared, got %q", target.CallType)
	}
	if target.Species != "Kiwi" || target.Certainty != 90 {
		t.Errorf("target fields wrong: %+v", target)
	}
}
func TestPropagate_SpeciesOverride(t *testing.T) {
	// Target species was different from --species; must be overwritten.
	path := writeFile(t,
		seg(100, 125, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(100, 125, lbl(fTo, "Don't Know", "", 70)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.Propagated != 1 {
		t.Fatalf("expected propagated=1: %+v", out)
	}
	df := readFile(t, path)
	target := findLabel(df, fTo, 100, 125)
	if target.Species != "Kiwi" || target.CallType != "Male" || target.Certainty != 90 {
		t.Errorf("target not overwritten correctly: %+v", target)
	}
}
func TestPropagate_OverlapBoundaryExclusive(t *testing.T) {
	// Segments touching at a point (src ends exactly where tgt starts) do NOT overlap.
	path := writeFile(t,
		seg(100, 125, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(125, 150, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.Propagated != 0 || out.SkippedNoOverlap != 1 {
		t.Fatalf("touching boundary must not count as overlap: %+v", out)
	}
}
func TestPropagate_OverlapPartial(t *testing.T) {
	// 1-second overlap is enough.
	path := writeFile(t,
		seg(100, 126, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(125, 150, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.Propagated != 1 {
		t.Fatalf("expected propagated=1: %+v", out)
	}
}
func TestPropagate_SupersetEitherDirection(t *testing.T) {
	// Source engulfs target.
	path1 := writeFile(t,
		seg(100, 200, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(110, 150, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	if out, _ := CallsPropagate(CallsPropagateInput{File: path1, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi"}); out.Propagated != 1 {
		t.Errorf("source-engulfs-target: %+v", out)
	}
	// Target engulfs source.
	path2 := writeFile(t,
		seg(110, 150, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(100, 200, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	if out, _ := CallsPropagate(CallsPropagateInput{File: path2, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi"}); out.Propagated != 1 {
		t.Errorf("target-engulfs-source: %+v", out)
	}
}
func TestPropagate_MissingFlags(t *testing.T) {
	cases := []struct {
		name string
		in   CallsPropagateInput
	}{
		{"no file", CallsPropagateInput{FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi"}},
		{"no from", CallsPropagateInput{File: "x", ToFilter: fTo, Species: "Kiwi"}},
		{"no to", CallsPropagateInput{File: "x", FromFilter: fFrom, Species: "Kiwi"}},
		{"no species", CallsPropagateInput{File: "x", FromFilter: fFrom, ToFilter: fTo}},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			_, err := CallsPropagate(c.in)
			if err == nil {
				t.Errorf("expected error")
			}
		})
	}
}
func TestPropagate_SameFromAndTo(t *testing.T) {
	_, err := CallsPropagate(CallsPropagateInput{
		File: "x", FromFilter: fFrom, ToFilter: fFrom, Species: "Kiwi",
	})
	if err == nil {
		t.Error("expected error when --from == --to")
	}
}
func TestPropagate_NonexistentFile(t *testing.T) {
	_, err := CallsPropagate(CallsPropagateInput{
		File: "/nonexistent/path.data", FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err == nil {
		t.Error("expected error for nonexistent file")
	}
}
func TestPropagate_RealisticMixed(t *testing.T) {
	// Mimics the 20260228_211500.WAV.data case: cert=0 "Don't Know" and cert=100 Kiwi sources
	// coexist; only cert=100 Kiwi gets propagated.
	path := writeFile(t,
		// Sources (kiwi-1.2)
		seg(45, 52.5, lbl(fFrom, "Don't Know", "", 0)),
		seg(142.5, 177.5, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(195, 217.5, lbl(fFrom, "Don't Know", "", 0)),
		seg(647.5, 682.5, lbl(fFrom, "Kiwi", "Female", 100)),
		seg(815, 855, lbl(fFrom, "Kiwi", "Duet", 100)),
		// Targets (kiwi-1.5)
		seg(147.5, 167.5, lbl(fTo, "Kiwi", "Male", 70)),
		seg(647.5, 672.5, lbl(fTo, "Kiwi", "Female", 70)),
		seg(815, 852.5, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.TargetsExamined != 3 || out.Propagated != 3 || out.SkippedConflict != 0 {
		t.Fatalf("counts wrong: %+v", out)
	}
	df := readFile(t, path)
	expect := []struct {
		start, end float64
		calltype   string
	}{
		{147.5, 167.5, "Male"},
		{647.5, 672.5, "Female"},
		{815, 852.5, "Duet"},
	}
	for _, e := range expect {
		l := findLabel(df, fTo, e.start, e.end)
		if l == nil || l.Certainty != 90 || l.CallType != e.calltype || l.Species != "Kiwi" {
			t.Errorf("at %v-%v got %+v, want Kiwi+%s cert=90", e.start, e.end, l, e.calltype)
		}
	}
}
func TestPropagate_NoWriteIfNothingChanged(t *testing.T) {
	// File with only non-target segments should not be rewritten (reviewer unchanged).
	path := writeFile(t,
		seg(100, 125, lbl(fFrom, "Kiwi", "Male", 100)),
	)
	out, err := CallsPropagate(CallsPropagateInput{
		File: path, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.Propagated != 0 || out.TargetsExamined != 0 {
		t.Fatalf("expected no activity: %+v", out)
	}
	df := readFile(t, path)
	if df.Meta.Reviewer != "David" {
		t.Errorf("reviewer should not be touched, got %q", df.Meta.Reviewer)
	}
}
// writeFileAt is like writeFile but puts the file inside an existing dir
// with a caller-provided basename (must end in .data).
func writeFileAt(t *testing.T, dir, base string, segs ...*utils.Segment) string {
	t.Helper()
	path := filepath.Join(dir, base)
	df := &utils.DataFile{
		Meta:     &utils.DataMeta{Operator: "ML", Reviewer: "David", Duration: 3600},
		Segments: segs,
	}
	if err := df.Write(path); err != nil {
		t.Fatalf("write fixture: %v", err)
	}
	return path
}
func TestPropagateFolder_AggregatesAndSkipsMissing(t *testing.T) {
	dir := t.TempDir()
	// File A: both filters present, one clean propagation.
	aPath := writeFileAt(t, dir, "a.wav.data",
		seg(100, 125, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(100, 125, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	// File B: only target filter — missing source, must be skipped silently.
	bPath := writeFileAt(t, dir, "b.wav.data",
		seg(200, 225, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	// File C: only source filter — missing target, must be skipped silently.
	writeFileAt(t, dir, "c.wav.data",
		seg(300, 325, lbl(fFrom, "Kiwi", "Male", 100)),
	)
	// File D: both filters, but no overlap → targets examined, none propagated.
	dPath := writeFileAt(t, dir, "d.wav.data",
		seg(400, 425, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(500, 525, lbl(fTo, "Kiwi", "Duet", 70)),
	)
	out, err := CallsPropagateFolder(CallsPropagateFolderInput{
		Folder: dir, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	assertPropagateStats(t, out, CallsPropagateFolderOutput{
		FilesTotal:           4,
		FilesWithBothFilters: 2,
		FilesSkippedNoFilter: 2,
		FilesChanged:         1,
		FilesErrored:         0,
		TargetsExamined:      2,
		Propagated:           1,
		SkippedNoOverlap:     1,
	})
	t.Run("file_a_propagated", func(t *testing.T) {
		aDf := readFile(t, aPath)
		if aDf.Meta.Reviewer != "Skraak" {
			t.Errorf("reviewer: got %q, want Skraak", aDf.Meta.Reviewer)
		}
		if l := findLabel(aDf, fTo, 100, 125); l == nil || l.Certainty != 90 || l.CallType != "Male" {
			t.Errorf("target label: got %+v, want cert=90 calltype=Male", l)
		}
	})
	t.Run("file_b_skipped", func(t *testing.T) {
		bDf := readFile(t, bPath)
		if bDf.Meta.Reviewer != "David" {
			t.Errorf("reviewer should not be touched, got %q", bDf.Meta.Reviewer)
		}
	})
	t.Run("file_d_no_overlap", func(t *testing.T) {
		dDf := readFile(t, dPath)
		if dDf.Meta.Reviewer != "David" {
			t.Errorf("reviewer should not be touched, got %q", dDf.Meta.Reviewer)
		}
		if l := findLabel(dDf, fTo, 500, 525); l == nil || l.Certainty != 70 {
			t.Errorf("target label should be unchanged cert=70, got %+v", l)
		}
	})
}
func TestPropagateFolder_EmptyFolder(t *testing.T) {
	dir := t.TempDir()
	out, err := CallsPropagateFolder(CallsPropagateFolderInput{
		Folder: dir, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.FilesTotal != 0 || out.Propagated != 0 {
		t.Errorf("expected empty result, got %+v", out)
	}
}
func TestPropagateFolder_MissingRequiredFlags(t *testing.T) {
	dir := t.TempDir()
	cases := []CallsPropagateFolderInput{
		{Folder: "", FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi"},
		{Folder: dir, FromFilter: "", ToFilter: fTo, Species: "Kiwi"},
		{Folder: dir, FromFilter: fFrom, ToFilter: "", Species: "Kiwi"},
		{Folder: dir, FromFilter: fFrom, ToFilter: fTo, Species: ""},
		{Folder: dir, FromFilter: fFrom, ToFilter: fFrom, Species: "Kiwi"},
	}
	for i, in := range cases {
		if _, err := CallsPropagateFolder(in); err == nil {
			t.Errorf("case %d: expected error for input %+v", i, in)
		}
	}
}
func TestPropagateFolder_NonexistentFolder(t *testing.T) {
	_, err := CallsPropagateFolder(CallsPropagateFolderInput{
		Folder: "/nonexistent/path/xyz", FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err == nil {
		t.Fatal("expected error for nonexistent folder")
	}
}
func TestPropagateFolder_ConflictsTaggedWithFile(t *testing.T) {
	dir := t.TempDir()
	// Two sources with different calltypes both overlapping one target.
	writeFileAt(t, dir, "conflict.wav.data",
		seg(100, 125, lbl(fFrom, "Kiwi", "Male", 100)),
		seg(110, 130, lbl(fFrom, "Kiwi", "Female", 100)),
		seg(100, 130, lbl(fTo, "Kiwi", "", 70)),
	)
	out, err := CallsPropagateFolder(CallsPropagateFolderInput{
		Folder: dir, FromFilter: fFrom, ToFilter: fTo, Species: "Kiwi",
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if out.SkippedConflict != 1 || len(out.Conflicts) != 1 {
		t.Fatalf("expected one conflict, got %+v", out)
	}
	if out.Conflicts[0].File == "" {
		t.Errorf("conflict should be tagged with file path, got %+v", out.Conflicts[0])
	}
}
}
// assertPropagateStats checks output stats against expected values.
func assertPropagateStats(t *testing.T, got, want CallsPropagateFolderOutput) {
	t.Helper()
	checks := []struct {
		name string
		got  int
		want int
	}{
		{"FilesTotal", got.FilesTotal, want.FilesTotal},
		{"FilesWithBothFilters", got.FilesWithBothFilters, want.FilesWithBothFilters},
		{"FilesSkippedNoFilter", got.FilesSkippedNoFilter, want.FilesSkippedNoFilter},
		{"FilesChanged", got.FilesChanged, want.FilesChanged},
		{"FilesErrored", got.FilesErrored, want.FilesErrored},
		{"TargetsExamined", got.TargetsExamined, want.TargetsExamined},
		{"Propagated", got.Propagated, want.Propagated},
		{"SkippedNoOverlap", got.SkippedNoOverlap, want.SkippedNoOverlap},
	}
	for _, c := range checks {
		if c.got != c.want {
			t.Errorf("%s: got %d, want %d", c.name, c.got, c.want)
		}
	}