}
	return nil
}
// ValidateDatasetTypeUnstructured checks that a dataset is 'unstructured' type
// Returns error if dataset doesn't exist or is not 'unstructured'
func ValidateDatasetTypeUnstructured(db *sql.DB, datasetID string) error {
	datasetType, exists, err := GetDatasetType(db, datasetID)
	if err != nil {
		return fmt.Errorf("failed to query dataset type: %w", err)
	}
	if !exists {
		return fmt.Errorf("dataset not found: %s", datasetID)
	}
	if datasetType != "unstructured" {
		return fmt.Errorf("dataset '%s' is type '%s' - this command only supports 'unstructured' datasets", datasetID, datasetType)

package tools
import (
	"context"
	"database/sql"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"time"
	"skraak/db"
	"skraak/utils"
)
// ImportUnstructuredInput defines the input parameters for importing files into an unstructured dataset
type ImportUnstructuredInput struct {
	DatasetID  string `json:"dataset_id" jsonschema:"required,Dataset ID (12 characters) - must be 'unstructured' type"`
	FolderPath string `json:"folder_path" jsonschema:"required,Absolute path to folder containing WAV files"`
	Recursive  *bool  `json:"recursive,omitempty" jsonschema:"Scan subfolders recursively (default: true)"`
}
// ImportUnstructuredOutput defines the output structure
type ImportUnstructuredOutput struct {
	TotalFiles     int                      `json:"total_files"`
	ImportedFiles  int                      `json:"imported_files"`
	SkippedFiles   int                      `json:"skipped_files"` // Duplicates
	FailedFiles    int                      `json:"failed_files"`
	TotalDuration  float64                  `json:"total_duration_seconds"`
	ProcessingTime string                   `json:"processing_time"`
	Errors         []utils.FileImportError  `json:"errors,omitempty"`
}
// ImportUnstructured imports WAV files into an unstructured dataset
// Files are stored with minimal metadata: hash, duration, sample_rate, file_mod_time as timestamp
// No location/cluster hierarchy, no astronomical data, no AudioMoth parsing
func ImportUnstructured(
	ctx context.Context,
	input ImportUnstructuredInput,
) (ImportUnstructuredOutput, error) {
	startTime := time.Now()
	var output ImportUnstructuredOutput
	// Default recursive to true
	recursive := true
	if input.Recursive != nil {
		recursive = *input.Recursive
	}
	// Validate input
	if err := validateUnstructuredInput(input); err != nil {
		return output, fmt.Errorf("validation failed: %w", err)
	}
	// Open database
	database, err := db.OpenWriteableDB(dbPath)
	if err != nil {
		return output, fmt.Errorf("failed to open database: %w", err)
	}
	defer database.Close()
	// Scan for WAV files
	files, scanErrors := scanWavFiles(input.FolderPath, recursive)
	output.Errors = append(output.Errors, scanErrors...)
	output.TotalFiles = len(files)
	if len(files) == 0 {
		output.ProcessingTime = time.Since(startTime).String()
		return output, nil
	}
	// Begin transaction
	tx, err := database.BeginTx(ctx, nil)
	if err != nil {
		return output, fmt.Errorf("failed to begin transaction: %w", err)
	}
	defer func() {
		if err != nil {
			tx.Rollback()
		}
	}()
	// Process each file
	for _, filePath := range files {
		fileResult, procErr := processUnstructuredFile(tx, filePath, input.DatasetID)
		
		if procErr != nil {
			output.FailedFiles++
			output.Errors = append(output.Errors, utils.FileImportError{
				FileName: filepath.Base(filePath),
				Error:    procErr.Error(),
				Stage:    "process",
			})
			continue
		}
		if fileResult.Skipped {
			output.SkippedFiles++
		} else {
			output.ImportedFiles++
			output.TotalDuration += fileResult.Duration
		}
	}
	// Commit transaction
	if err = tx.Commit(); err != nil {
		return output, fmt.Errorf("failed to commit transaction: %w", err)
	}
	output.ProcessingTime = time.Since(startTime).String()
	return output, nil
}
// unstructuredFileResult holds the result of processing a single file
type unstructuredFileResult struct {
	Skipped   bool    // True if duplicate
	Duration  float64 // Duration in seconds
}
// processUnstructuredFile processes a single WAV file for unstructured import
func processUnstructuredFile(tx *sql.Tx, filePath, datasetID string) (*unstructuredFileResult, error) {
	result := &unstructuredFileResult{}
	// Step 1: Parse WAV header
	metadata, err := utils.ParseWAVHeader(filePath)
	if err != nil {
		return nil, fmt.Errorf("WAV header parsing failed: %w", err)
	}
	// Step 2: Calculate hash
	hash, err := utils.ComputeXXH64(filePath)
	if err != nil {
		return nil, fmt.Errorf("hash calculation failed: %w", err)
	}
	// Step 3: Check for duplicate - if exists, skip entirely (do not link to dataset)
	_, isDuplicate, err := utils.CheckDuplicateHash(tx, hash)
	if err != nil {
		return nil, fmt.Errorf("duplicate check failed: %w", err)
	}
	if isDuplicate {
		// File already exists in database - skip completely, do not link to dataset
		result.Skipped = true
		result.Duration = metadata.Duration
		return result, nil
	}
	// Step 4: Generate file ID
	fileID, err := utils.GenerateLongID()
	if err != nil {
		return nil, fmt.Errorf("ID generation failed: %w", err)
	}
	// Step 5: Use file modification time as timestamp (no timezone conversion)
	timestamp := metadata.FileModTime
	// Step 6: Insert into file table
	_, err = tx.Exec(`
		INSERT INTO file (
			id, file_name, xxh64_hash, location_id, cluster_id,
			timestamp_local, duration, sample_rate,
			maybe_solar_night, maybe_civil_night, moon_phase,
			active
		) VALUES (?, ?, ?, NULL, NULL, ?, ?, ?, NULL, NULL, NULL, TRUE)
	`,
		fileID,
		filepath.Base(filePath),
		hash,
		timestamp,
		metadata.Duration,
		metadata.SampleRate,
	)
	if err != nil {
		return nil, fmt.Errorf("file insert failed: %w", err)
	}
	// Step 7: Insert into file_dataset table
	_, err = tx.Exec(
		"INSERT INTO file_dataset (file_id, dataset_id) VALUES (?, ?)",
		fileID, datasetID,
	)
	if err != nil {
		return nil, fmt.Errorf("file_dataset insert failed: %w", err)
	}
	result.Duration = metadata.Duration
	return result, nil
}
// validateUnstructuredInput validates the input parameters
func validateUnstructuredInput(input ImportUnstructuredInput) error {
	// Validate dataset ID format
	if err := utils.ValidateShortID(input.DatasetID, "dataset_id"); err != nil {
		return err
	}
	// Verify folder exists
	info, err := os.Stat(input.FolderPath)
	if err != nil {
		return fmt.Errorf("folder not accessible: %w", err)
	}
	if !info.IsDir() {
		return fmt.Errorf("path is not a directory: %s", input.FolderPath)
	}
	// Open database for validation
	database, err := db.OpenReadOnlyDB(dbPath)
	if err != nil {
		return fmt.Errorf("failed to open database: %w", err)
	}
	defer database.Close()
	// Verify dataset exists and is active
	var datasetExists bool
	err = database.QueryRow(
		"SELECT EXISTS(SELECT 1 FROM dataset WHERE id = ? AND active = true)",
		input.DatasetID,
	).Scan(&datasetExists)
	if err != nil {
		return fmt.Errorf("failed to query dataset: %w", err)
	}
	if !datasetExists {
		return fmt.Errorf("dataset not found or inactive: %s", input.DatasetID)
	}
	// Verify dataset is 'unstructured' type
	if err := utils.ValidateDatasetTypeUnstructured(database, input.DatasetID); err != nil {
		return err
	}
	return nil
}
// scanWavFiles scans a folder for WAV files
func scanWavFiles(folderPath string, recursive bool) ([]string, []utils.FileImportError) {
	var files []string
	var errors []utils.FileImportError
	walkFunc := func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			errors = append(errors, utils.FileImportError{
				FileName: path,
				Error:    err.Error(),
				Stage:    "scan",
			})
			return nil
		}
		// Skip directories if not recursive
		if d.IsDir() {
			if !recursive && path != folderPath {
				return fs.SkipDir
			}
			return nil
		}
		// Check for .wav extension (case-insensitive)
		if strings.HasSuffix(strings.ToLower(d.Name()), ".wav") {
			files = append(files, path)
		}
		return nil
	}
	if recursive {
		filepath.WalkDir(folderPath, walkFunc)
	} else {
		// Non-recursive: only scan top-level
		entries, err := os.ReadDir(folderPath)
		if err != nil {
			errors = append(errors, utils.FileImportError{
				FileName: folderPath,
				Error:    err.Error(),
				Stage:    "scan",
			})
			return nil, errors
		}
		for _, entry := range entries {
			if !entry.IsDir() && strings.HasSuffix(strings.ToLower(entry.Name()), ".wav") {
				files = append(files, filepath.Join(folderPath, entry.Name()))
			}
		}
	}
	return files, errors
}

	case "unstructured":
		runImportUnstructured(args[1:])

	fmt.Fprintf(os.Stderr, "  bulk         Bulk import WAV files from CSV\n")
	fmt.Fprintf(os.Stderr, "  file         Import a single WAV file\n")
	fmt.Fprintf(os.Stderr, "  folder       Import all WAV files from a folder\n")
	fmt.Fprintf(os.Stderr, "  selections   Import ML selections from folder structure\n")

	fmt.Fprintf(os.Stderr, "  bulk         Bulk import WAV files from CSV (structured datasets)\n")
	fmt.Fprintf(os.Stderr, "  file         Import a single WAV file (structured datasets)\n")
	fmt.Fprintf(os.Stderr, "  folder       Import all WAV files from a folder (structured datasets)\n")
	fmt.Fprintf(os.Stderr, "  selections   Import ML selections from folder structure (structured datasets)\n")
	fmt.Fprintf(os.Stderr, "  unstructured Import WAV files into unstructured dataset (no location/cluster)\n")

	fmt.Fprintf(os.Stderr, "  skraak import unstructured --db ./db/skraak.duckdb --dataset abc123 --path /path/to/folder\n")

func runImportUnstructured(args []string) {
	fs := flag.NewFlagSet("import unstructured", flag.ExitOnError)
	dbPath := fs.String("db", "", "Path to DuckDB database (required)")
	datasetID := fs.String("dataset", "", "Dataset ID (required - must be 'unstructured' type)")
	folderPath := fs.String("path", "", "Path to folder containing WAV files (required)")
	recursive := fs.Bool("recursive", true, "Scan subfolders recursively (default: true)")

	fs.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: skraak import unstructured [options]\n\n")
		fmt.Fprintf(os.Stderr, "Import WAV files into an unstructured dataset.\n")
		fmt.Fprintf(os.Stderr, "Files are stored with minimal metadata (hash, duration, sample_rate, file modification time).\n")
		fmt.Fprintf(os.Stderr, "No location/cluster hierarchy required.\n\n")
		fmt.Fprintf(os.Stderr, "Options:\n")
		fs.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\nExamples:\n")
		fmt.Fprintf(os.Stderr, "  skraak import unstructured --db ./db/skraak.duckdb --dataset abc123 --path /path/to/folder\n")
		fmt.Fprintf(os.Stderr, "  skraak import unstructured --db ./db/skraak.duckdb --dataset abc123 --path /path/to/folder --recursive=false\n")
	}
	if err := fs.Parse(args); err != nil {
		os.Exit(1)
	}
	// Validate required flags
	missing := []string{}
	if *dbPath == "" {
		missing = append(missing, "--db")
	}
	if *datasetID == "" {
		missing = append(missing, "--dataset")
	}
	if *folderPath == "" {
		missing = append(missing, "--path")
	}
	if len(missing) > 0 {
		fmt.Fprintf(os.Stderr, "Error: missing required flags: %v\n\n", missing)
		fs.Usage()
		os.Exit(1)
	}
	tools.SetDBPath(*dbPath)
	input := tools.ImportUnstructuredInput{
		DatasetID:  *datasetID,
		FolderPath: *folderPath,
		Recursive:  recursive,
	}
	fmt.Fprintf(os.Stderr, "Importing into unstructured dataset: %s\n", *datasetID)
	fmt.Fprintf(os.Stderr, "Scanning folder: %s\n", *folderPath)
	if *recursive {
		fmt.Fprintf(os.Stderr, "Scanning recursively...\n")
	}
	output, err := tools.ImportUnstructured(context.Background(), input)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
		os.Exit(1)
	}
	printJSON(output)
}

│   ├── import_unstructured.go    # import_unstructured (unstructured dataset import)

**Important**: All import tools only work with datasets of type `'structured'`. Attempting to import to `'unstructured'`, `'test'`, or `'train'` datasets will return a validation error.

**Dataset type requirements:**
- `import_audio_files`, `import_file`, `bulk_file_import`, `import_ml_selections`: Require `'structured'` datasets
- `import_unstructured`: Requires `'unstructured'` datasets

- `import_audio_files` - Batch import WAV files from folder

- `import_audio_files` - Batch import WAV files from folder (structured datasets)

- `import_audio_file` - Import a single WAV file into the database

- `import_audio_file` - Import a single WAV file into the database (structured datasets)

- `import_unstructured` - Import WAV files into unstructured dataset (CLI only)
  - **Input**: Folder path and dataset ID (must be 'unstructured' type)
  - **Processing**: Minimal metadata - hash, duration, sample_rate, file modification time as timestamp
  - **No hierarchy**: location_id and cluster_id are NULL
  - **No astronomical data**: maybe_solar_night, maybe_civil_night, moon_phase are NULL
  - **Duplicate detection**: By hash - skips files that already exist in database (no linking, no modification)
  - **Use case**: Import miscellaneous recordings without location/cluster structure
  - **CLI**: `skraak import unstructured --db ./db/skraak.duckdb --dataset abc123 --path /path/to/folder`

- **Dataset type validation for imports**: File imports (import_file, import_audio_files, bulk_file_import, import_ml_selections) only work with datasets of type 'structured'

- **Dataset type validation for imports**:
  - Structured imports (import_file, import_audio_files, bulk_file_import, import_ml_selections) require 'structured' datasets
  - Unstructured imports (import_unstructured) require 'unstructured' datasets

**Last Updated**: 2026-02-11 NZDT

**Last Updated**: 2026-02-14 NZDT

**Current Tools**: 10 (read: 2, write: 4, import: 4) — available via both MCP and CLI
**CLI Commands**: `skraak mcp`, `skraak sql`, `skraak import bulk`

**Current Tools**: 11 (read: 2, write: 4, import: 5) — 10 via MCP, 1 CLI-only (import_unstructured)
**CLI Commands**: `skraak mcp`, `skraak sql`, `skraak dataset`, `skraak location`, `skraak cluster`, `skraak import {bulk|file|folder|selections|unstructured}`