quietlight/skraak.app - Change C4PJ5BYCHKYQVPCXNESI5AZELV3TXCWXIVU7IFJ2F3MZVR23SQ3AC

first made changes to schema and migrations, made xxh hash unique, then increased chunk size for file import to 10

Created by AEj8dahVWy718uSSFPe9VSRJ5qX5G8pC2zvFzJJ8yzBd on June 26, 2025

C4PJ5BYCHKYQVPCXNESI5AZELV3TXCWXIVU7IFJ2F3MZVR23SQ3AC

Dependencies

In channels

main

Change contents

Replacement in src/worker/utils/chunkProcessor.ts at line 40 [3.1]
B:BD[3.831] → [3.831:847]
```
  chunkSize: 5,
```
[3.831]
[3.847]
```
  chunkSize: 10,
```
Replacement in src/worker/routes/fileImport.ts at line 55 [4.1]
B:BD[3.9657] → [3.9657:9694]
```
  files: FileImportRequest['files'];
```
[3.9657]
[4.1210]
```
  files: FileImportRequest["files"];
```

Replacement in src/worker/routes/fileImport.ts at line 195 [4.1]

B:BD[3.10061] → [3.10061:10136]

    const chunkSize = body.chunkSize || 5; // Default to 5 files per chunk

[3.10061]

[3.10136]

    const chunkSize = body.chunkSize || 10; // Default to 10 files per chunk

Replacement in src/worker/routes/fileImport.ts at line 198 [4.1]

B:BD[3.10247] → [3.10247:10335]

    
    const processChunk = async (chunk: typeof body.files, chunkIndex: number) => {

[3.10247]

[3.10335]


    const processChunk = async (
      chunk: typeof body.files,
      chunkIndex: number,
    ) => {

Replacement in src/worker/routes/fileImport.ts at line 205 [4.1]

B:BD[3.10404] → [3.10404:10502]

      
      console.log(`Processing chunk ${chunkIndex + 1} with ${chunk.length} files`);

[3.10404]

[3.10502]


      console.log(
        `Processing chunk ${chunkIndex + 1} with ${chunk.length} files`,
      );

Replacement in src/worker/routes/fileImport.ts at line 214 [4.1]
B:BD[3.10661] → [3.10661:10672]
```
          
```
[3.10661]
[3.10672]

Replacement in src/worker/routes/fileImport.ts at line 225 [4.1]

B:BD[3.11006] → [3.11006:11122]

              "Missing required file fields: fileName, xxh64Hash, locationId, timestampLocal, duration, sampleRate"

[3.11006]

[3.11122]

              "Missing required file fields: fileName, xxh64Hash, locationId, timestampLocal, duration, sampleRate",

Replacement in src/worker/routes/fileImport.ts at line 228 [4.1]
B:BD[3.11149] → [3.11149:11160]
```
          
```
[3.11149]
[3.11160]
Replacement in src/worker/routes/fileImport.ts at line 251 [4.1]
B:BD[3.12094] → [3.12094:12105]
```
          
```
[3.12094]
[3.12105]
Replacement in src/worker/routes/fileImport.ts at line 261 [4.1]
B:BD[3.12400] → [3.12400:12411]
```
          
```
[3.12400]
[3.12411]
Replacement in src/worker/routes/fileImport.ts at line 278 [4.1]
B:BD[3.13101] → [3.13101:13112]
```
          
```
[3.13101]
[3.13112]
Deletion in src/worker/routes/fileImport.ts at line 281 [4.1]
B:BD[3.13223] → [3.13223:13234]
```
          
```

Replacement in src/worker/routes/fileImport.ts at line 282 [4.1]

B:BD[3.13260] → [3.13260:13444]

          const errorMessage = error instanceof Error ? error.message : String(error);
          console.error(`Error processing file ${fileData.fileName}:`, errorMessage);

[3.13260]

[3.13444]

          const errorMessage =
            error instanceof Error ? error.message : String(error);
          console.error(
            `Error processing file ${fileData.fileName}:`,
            errorMessage,
          );

Replacement in src/worker/routes/fileImport.ts at line 293 [4.1]
B:BD[3.13561] → [3.13561:13572]
```
          
```
[3.13561]
[3.13572]
Replacement in src/worker/routes/fileImport.ts at line 298 [4.1]
B:BD[3.13742] → [3.13742:13753]
```
          
```
[3.13742]
[3.13753]
Replacement in src/worker/routes/fileImport.ts at line 303 [4.1]
B:BD[3.13860] → [3.13860:13867]
```
      
```
[3.13860]
[3.13867]
Replacement in src/worker/routes/fileImport.ts at line 306 [4.1]
B:BD[3.13912] → [3.13912:13917]
```
    
```
[3.13912]
[3.13917]

Replacement in src/worker/routes/fileImport.ts at line 308 [4.1]

B:BD[3.13965] → [3.13965:14052]

    const result = await processInChunks<typeof body.files[0], { fileIds: string[] }>(

[3.13965]

[3.14052]

    const result = await processInChunks<
      (typeof body.files)[0],
      { fileIds: string[] }
    >(

Replacement in src/worker/routes/fileImport.ts at line 325 [4.1]

B:BD[3.14463] → [3.14463:14578]

        console.log(`Import progress: ${progress.processedItems}/${progress.totalItems} files processed`);
      }

[3.14463]

[3.14578]

        console.log(
          `Import progress: ${progress.processedItems}/${progress.totalItems} files processed`,
        );
      },

Replacement in src/worker/routes/fileImport.ts at line 330 [4.1]
B:BD[3.14585] → [3.14585:14590]
```
    
```
[3.14585]
[3.14590]

Replacement in src/worker/routes/fileImport.ts at line 333 [4.1]

B:BD[3.14698] → [3.14698:14786]

      if (chunkResult && typeof chunkResult === 'object' && 'fileIds' in chunkResult) {

[3.14698]

[3.14786]

      if (
        chunkResult &&
        typeof chunkResult === "object" &&
        "fileIds" in chunkResult
      ) {

Replacement in src/worker/routes/fileImport.ts at line 341 [4.1]
B:BD[3.14853] → [3.14853:14858]
```
    
```
[3.14853]
[3.14858]

Replacement in src/worker/routes/fileImport.ts at line 343 [4.1]

B:BD[3.14911] → [3.14911:15043]

    result.errors.forEach(chunkError => {
      const fileName = (chunkError.item as { fileName?: string })?.fileName || 'unknown';

[3.14911]

[3.15043]

    result.errors.forEach((chunkError) => {
      const fileName =
        (chunkError.item as { fileName?: string })?.fileName || "unknown";

Replacement in src/worker/routes/fileImport.ts at line 351 [4.1]
B:BD[3.15145] → [3.15145:15150]
```
    
```
[3.15145]
[3.15150]

Replacement in src/worker/routes/fileImport.ts at line 353 [4.1]

B:BD[3.15193] → [3.15193:15265]

      message: result.success 
        ? "Files imported successfully"

[3.15193]

[3.15265]

      message: result.success
        ? "Files imported successfully"

Replacement in src/worker/routes/fileImport.ts at line 363 [4.1]
B:BD[3.15541] → [3.15541:15546]
```
    
```
[3.15541]
[3.15546]

Replacement in src/worker/routes/fileImport.ts at line 371 [4.1]

B:BD[3.15865] → [3.15865:15948]

      if (errorMessage.includes("401") || errorMessage.includes("Unauthorized")) {

[3.15865]

[3.15948]

      if (
        errorMessage.includes("401") ||
        errorMessage.includes("Unauthorized")
      ) {

Replacement in src/worker/routes/fileImport.ts at line 430 [4.1]

B:BD[3.17336] → [3.17336:17448]

    console.log(`Processing chunk ${body.chunkIndex + 1}/${body.totalChunks} with ${body.files.length} files`);

[3.17336]

[3.17448]

    console.log(
      `Processing chunk ${body.chunkIndex + 1}/${body.totalChunks} with ${body.files.length} files`,
    );

Replacement in src/worker/routes/fileImport.ts at line 453 [4.1]

B:BD[3.17622] → [3.17622:17736]

            "Missing required file fields: fileName, xxh64Hash, locationId, timestampLocal, duration, sampleRate"

[3.17622]

[4.4971]

            "Missing required file fields: fileName, xxh64Hash, locationId, timestampLocal, duration, sampleRate",

Deletion in src/worker/routes/fileImport.ts at line 509 [4.1]
B:BD[3.17811] → [3.17811:17812]

Replacement in src/worker/routes/fileImport.ts at line 510 [4.1]

B:BD[4.6883] → [3.17813:17982]

        const errorMessage = error instanceof Error ? error.message : String(error);
        console.error(`Error processing file ${fileData.fileName}:`, errorMessage);

[4.6883]

[3.17982]

        const errorMessage =
          error instanceof Error ? error.message : String(error);
        console.error(
          `Error processing file ${fileData.fileName}:`,
          errorMessage,
        );

Replacement in src/worker/routes/fileImport.ts at line 525 [4.1]

B:BD[3.18116] → [3.18116:18350]

      message: failedFiles.length === 0 
        ? `Chunk ${body.chunkIndex + 1}/${body.totalChunks} processed successfully` 
        : `Chunk ${body.chunkIndex + 1}/${body.totalChunks} completed with ${failedFiles.length} failures`,

[3.18116]

[4.7213]

      message:
        failedFiles.length === 0
          ? `Chunk ${body.chunkIndex + 1}/${body.totalChunks} processed successfully`
          : `Chunk ${body.chunkIndex + 1}/${body.totalChunks} completed with ${failedFiles.length} failures`,

Deletion in src/worker/routes/fileImport.ts at line 540 [4.1]
B:BD[3.18610] → [3.18610:18611]

Replacement in src/worker/routes/fileImport.ts at line 546 [4.1]

B:BD[3.18752] → [3.18752:18835]

      if (errorMessage.includes("401") || errorMessage.includes("Unauthorized")) {

[3.18752]

[3.18835]

      if (
        errorMessage.includes("401") ||
        errorMessage.includes("Unauthorized")
      ) {

Replacement in src/components/FileImportDialog.tsx at line 394 [6.1]

B:BD[5.4771] → [3.20503:20561]

        const chunkSize = 5; // Process 5 files at a time

[5.4771]

[3.20561]

        const chunkSize = 10; // Process 10 files at a time

Insertion in db/schema.ts at line 124 [8.50781]
[7.2581]
[7.2581]
```
  unique("unique_xxh64_hash").on(table.xxh64Hash),
```

Insertion in db/schema.sql at line 145 [8.70506]

[8.77778]

-- Unique constraint on xxh64_hash to prevent duplicate file hashes
ALTER TABLE file ADD CONSTRAINT unique_xxh64_hash UNIQUE (xxh64_hash);

Insertion in db/migrations.sql at line 876 [8.94401]

[9.6967]

[10.3898]

-- Delete duplicate file records with same xxh64_hash values
-- Removed 3 duplicate files ending in '_001' that had identical hashes to files ending in '_000'
DELETE FROM moth_metadata WHERE file_id IN ('V5gEQ6Fczhu9_GXPmR6I1', 'gskEfItZXJ93o-2Jpvucc', 'sfwj_jS3FQEh1SL1yYJRh');
DELETE FROM file_dataset WHERE file_id IN ('V5gEQ6Fczhu9_GXPmR6I1', 'gskEfItZXJ93o-2Jpvucc', 'sfwj_jS3FQEh1SL1yYJRh');
DELETE FROM file WHERE id IN ('V5gEQ6Fczhu9_GXPmR6I1', 'gskEfItZXJ93o-2Jpvucc', 'sfwj_jS3FQEh1SL1yYJRh');

Insertion in db/migrations.sql at line 882 [8.94401]

[10.3899]

-- Add unique constraint on xxh64_hash to prevent duplicate file hashes
ALTER TABLE file ADD CONSTRAINT unique_xxh64_hash UNIQUE (xxh64_hash);
SELECT NOW();
-- 2025-06-26 00:55:00.000+00

Replacement in TODO.md at line 17 [11.62710]

B:BD[3.32178] → [2.1227:1265]

[X] Zoom for waveform and spectrogram

[3.32178]

[11.62940]

[X] Check webhook still works
[ ] Zoom for waveform and spectrogram

Insertion in TODO.md at line 23 [11.62710]

[12.1249]

[13.437]

[ ] add check for duplicate hashes in file import
[ ] fine tune import files, refactor

first made changes to schema and migrations, made xxh hash unique, then increased chunk size for file import to 10

Dependencies

In channels

Change contents

Replacement in src/worker/utils/chunkProcessor.ts at line 40 [3.1]

Replacement in src/worker/routes/fileImport.ts at line 55 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 195 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 198 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 205 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 214 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 225 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 228 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 251 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 261 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 278 [4.1]

Deletion in src/worker/routes/fileImport.ts at line 281 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 282 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 293 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 298 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 303 [4.1]

Replacement in src/worker/routes/fileImport.ts at line 306 [4.1]