Authorization: Bearer YOUR_API_KEY

{
  "success": false,
  "error": {
    "code": "ERROR_CODE",
    "message": "Human-readable error message.",
    "retryable": false,
    "details": null
  }
}

{
  "success": false,
  "error": {
    "code": "INVALID_INPUT",
    "message": "Request validation failed. Check details for specific issues.",
    "retryable": false,
    "details": {
      "issues": [
        { "message": "file_name must end with a supported extension: .pdf, .jpg, .jpeg, or .png.", "path": ["files", 0, "file_name"] }
      ]
    }
  }
}

POST https://api.invoicedataextraction.com/v1/uploads/sessions

Authorization: Bearer YOUR_API_KEY

curl -X POST "https://api.invoicedataextraction.com/v1/uploads/sessions" \
  -H "Authorization: Bearer $API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "upload_session_id": "sess_001",
    "files": [
      {
        "file_id": "file_001",
        "file_name": "invoice-1.pdf",
        "file_size_bytes": 120450
      },
      {
        "file_id": "file_002",
        "file_name": "receipt.jpg",
        "file_size_bytes": 84200
      }
    ]
  }'

{
  "success": true,
  "upload_session_id": "sess_001",
  "files": [
    {
      "file_id": "file_001",
      "file_name": "invoice-1.pdf",
      "part_size": 8388608
    },
    {
      "file_id": "file_002",
      "file_name": "receipt.jpg",
      "part_size": 8388608
    }
  ]
}

POST https://api.invoicedataextraction.com/v1/uploads/sessions/{upload_session_id}/parts

total_parts = ceil(file_size_bytes / part_size)
part_numbers = [1, 2, 3, ..., total_parts]

curl -X POST "https://api.invoicedataextraction.com/v1/uploads/sessions/sess_001/parts" \
  -H "Authorization: Bearer $API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "file_id": "file_001",
    "part_numbers": [1]
  }'

{
  "success": true,
  "upload_session_id": "sess_001",
  "file_id": "file_001",
  "file_name": "invoice-1.pdf",
  "part_size": 8388608,
  "part_urls": [
    {
      "part_number": 1,
      "url": "https://storage.example.com/...?X-Amz-Signature=..."
    }
  ]
}

curl -X POST "https://api.invoicedataextraction.com/v1/uploads/sessions/sess_001/parts" \
  -H "Authorization: Bearer $API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "file_id": "file_002",
    "part_numbers": [1, 2, 3]
  }'

{
  "success": true,
  "upload_session_id": "sess_001",
  "file_id": "file_002",
  "file_name": "large-report.pdf",
  "part_size": 8388608,
  "part_urls": [
    {
      "part_number": 1,
      "url": "https://storage.example.com/...?X-Amz-Signature=..."
    },
    {
      "part_number": 2,
      "url": "https://storage.example.com/...?X-Amz-Signature=..."
    },
    {
      "part_number": 3,
      "url": "https://storage.example.com/...?X-Amz-Signature=..."
    }
  ]
}

POST https://api.invoicedataextraction.com/v1/uploads/sessions/{upload_session_id}/complete

curl -X POST "https://api.invoicedataextraction.com/v1/uploads/sessions/sess_001/complete" \
  -H "Authorization: Bearer $API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "file_id": "file_001",
    "parts": [
      {
        "part_number": 1,
        "e_tag": "\"a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4\""
      }
    ]
  }'

curl -X POST "https://api.invoicedataextraction.com/v1/uploads/sessions/sess_001/complete" \
  -H "Authorization: Bearer $API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "file_id": "file_002",
    "parts": [
      { "part_number": 1, "e_tag": "\"a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4\"" },
      { "part_number": 2, "e_tag": "\"f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3\"" },
      { "part_number": 3, "e_tag": "\"9876543210ab9876543210ab9876543210\"" }
    ]
  }'

{
  "success": true,
  "upload_session_id": "sess_001",
  "file_id": "file_001",
  "file_name": "invoice-1.pdf"
}

POST https://api.invoicedataextraction.com/v1/extractions

"prompt": "Extract invoice number, date, vendor name, total amount, and all line items with descriptions and amounts"

"prompt": {
  "fields": [
    { "name": "Invoice Number" },
    { "name": "Invoice Date", "prompt": "The date the invoice was issued, NOT the payment due date" },
    { "name": "Vendor Name" },
    { "name": "Total Amount", "prompt": "Do not include currency symbol, use 2 decimal places" }
  ],
  "general_prompt": "One row for each product. Do not extract shipping lines."
}

curl -X POST "https://api.invoicedataextraction.com/v1/extractions" \
  -H "Authorization: Bearer $API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "submission_id": "sub_001",
    "upload_session_id": "sess_001",
    "file_ids": ["file_001", "file_002"],
    "task_name": "January invoices",
    "prompt": "Extract invoice number, date, vendor name, and total amount",
    "output_structure": "per_invoice",
    "options": {
      "json_typed_values": true
    }
  }'

curl -X POST "https://api.invoicedataextraction.com/v1/extractions" \
  -H "Authorization: Bearer $API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "submission_id": "sub_002",
    "upload_session_id": "sess_001",
    "file_ids": ["file_001", "file_002"],
    "task_name": "January invoices",
    "prompt": {
      "fields": [
        { "name": "Invoice Number" },
        { "name": "Invoice Date", "prompt": "The date the invoice was issued, NOT the payment due date" },
        { "name": "Vendor Name" },
        { "name": "Line Item Description" },
        { "name": "Line Item Amount", "prompt": "Do not include currency symbol, use 2 decimal places" }
      ],
      "general_prompt": "Dates should be in YYYY-MM-DD format. Ignore email cover letters."
    },
    "output_structure": "per_line_item",
    "options": {
      "exclude_columns": ["source_file"],
      "send_completion_email": true,
      "json_typed_values": true
    }
  }'

{
  "success": true,
  "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "submission_state": "received"
}

GET https://api.invoicedataextraction.com/v1/extractions/{extraction_id}?wait=45

{
  "success": true,
  "status": "processing",
  "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "progress": 42
}

{
  "success": true,
  "status": "input_required",
  "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "progress": 22,
  "answer_by": "2026-09-12T11:14:00Z",
  "questions": [
    {
      "question_id": "q_2524",
      "type": "single_choice",
      "question": "Which name should fill the “Supplier name” column when the invoice issuer and payment recipient differ?",
      "example_from_documents": "The invoices show “Northgate Search Services” as the invoice issuer and say “Please make payment to NSS Franchise Holdings Ltd”.",
      "scope": {
        "level": "extraction",
        "applies_to": "Every document in this extraction, not only the example. If the answer differs by document type, say so in text."
      },
      "choices": [
        { "choice_id": "a", "label": "Use the invoice issuer's name", "cell_would_contain": "Northgate Search Services", "recommended": true },
        { "choice_id": "b", "label": "Use the payment recipient's name", "cell_would_contain": "NSS Franchise Holdings Ltd" }
      ]
    },
    {
      "question_id": "q_2529",
      "type": "free_text",
      "question": "What date format should I use in the “invoice_date” and “due_date” columns?",
      "example_from_documents": "The files show dates such as “Aug 27, 2026”, “28 Aug 2026”, and “2026/09/08”.",
      "scope": {
        "level": "extraction",
        "applies_to": "Every document in this extraction, not only the example. If the answer differs by document type, say so in text."
      },
      "choices": [],
      "recommended_approach": "I will use YYYY-MM-DD for every value in the “invoice_date” and “due_date” columns."
    }
  ]
}

{
  "success": true,
  "status": "completed",
  "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "credits_deducted": 25,
  "credits_balance": 125,
  "credits_reserved": 0,
  "output_structure": "per_invoice",
  "output_expires_at": "2026-07-26T10:30:00Z",
  "pages": {
    "successful_count": 10,
    "failed_count": 2,
    "successful": [
      { "file_name": "invoice-1.pdf", "page": 1 },
      { "file_name": "invoice-1.pdf", "page": 2 }
    ],
    "failed": [
      { "file_name": "damaged.pdf", "page": 1 }
    ],
    "failure_reasons": [
      {
        "code": "PROCESSING_FILE_SIZE_LIMIT_EXCEEDED",
        "message": "The upload was accepted, but during processing part of the PDF became too large for our file-processing limit. This can happen when a compressed PDF is processed internally. Split the PDF into smaller page chunks and resubmit.",
        "affected_pages": [
          { "file_name": "damaged.pdf", "pages": [1] }
        ]
      }
    ]
  },
  "ai_uncertainty_notes": [
    {
      "topic": "Documents to extract from",
      "description": "Your files often contain a 'Tax Invoice' with an attached 'Delivery Note'. I treated the 'Tax Invoice' pages as the main source of data, and ignored the attached 'Delivery Note' pages as supporting context.",
      "suggested_prompt_additions": [
        {
          "purpose": "To confirm this handling",
          "instructions": ["Extract from 'Tax Invoice' only"]
        },
        {
          "purpose": "To extract from both",
          "instructions": ["Extract from 'Tax Invoice' and 'Delivery Note'"]
        }
      ]
    }
  ],
  "review_needed": {
    "count": 1,
    "items": [
      {
        "message": "Check whether the extracted total should include the handwritten adjustment near the bottom of the document.",
        "affected_fields": ["Total Amount"],
        "output_row_numbers": [4],
        "source_references": ["invoice-1.pdf (Page 2)"]
      }
    ]
  },
  "output": {
    "xlsx_url": "https://storage.example.com/...?X-Amz-Signature=...",
    "csv_url": "https://storage.example.com/...?X-Amz-Signature=...",
    "json_url": "https://storage.example.com/...?X-Amz-Signature=..."
  }
}

{
  "success": false,
  "status": "failed",
  "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "error": {
    "code": "INSUFFICIENT_CREDITS",
    "message": "Not enough credits for this extraction; details shows the credits required and your balance. Buy credits at https://invoicedataextraction.com/dashboard?view=Billing or submit fewer pages, then submit again.",
    "retryable": false,
    "details": {
      "credits_required": 25,
      "credits_balance": 15,
      "credits_reserved": 10
    }
  }
}

{
  "success": false,
  "status": "failed",
  "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "error": {
    "code": "ENCRYPTED_FILE",
    "message": "One or more files are password-protected; details lists them. Remove the protection, upload the files again and submit again.",
    "retryable": false,
    "details": {
      "file_names": ["protected-invoice.pdf"]
    }
  }
}

{
  "success": true,
  "status": "cancelled",
  "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "credits_deducted": 4,
  "cancellation_reason": "user",
  "credits_balance": 146,
  "credits_reserved": 0
}

loop:
    GET /extractions/{extraction_id}?wait=45
    if status == "processing":      continue                 # the wait elapsed; ask again
    if status == "input_required":  answer the questions (Answer Questions); continue
    if status == "completed":       read the rows (Step 6), or download a file; stop
    if status == "failed":          if error.retryable, submit again with a new submission_id;
                                    otherwise fix what error.message says first; stop
    if status == "cancelled":       stop; no output is available

GET https://api.invoicedataextraction.com/v1/extractions/{extraction_id}/results?offset=0&limit=100

curl "https://api.invoicedataextraction.com/v1/extractions/a1b2c3d4-e5f6-7890-abcd-ef1234567890/results?limit=100" \
  -H "Authorization: Bearer $API_KEY"

{
  "success": true,
  "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "status": "completed",
  "output_structure": "per_invoice",
  "output_expires_at": "2026-07-26T10:30:00Z",
  "json_typed_values": true,
  "columns": ["Invoice Number", "Invoice Date", "Vendor Name", "Total Amount", "Source File", "Review Needed"],
  "rows": [
    {
      "Invoice Number": "INV-1001",
      "Invoice Date": "2026-01-15",
      "Vendor Name": "Acme Ltd",
      "Total Amount": 1250,
      "Source File": "invoice-1.pdf (Page 1)",
      "Review Needed": null
    }
  ],
  "offset": 0,
  "limit": 100,
  "row_count": 1,
  "total_rows": 1,
  "has_more": false,
  "next_offset": null,
  "review_needed": {
    "count": 0,
    "items": []
  },
  "pages": {
    "successful_count": 1,
    "failed_count": 0,
    "successful": [{ "file_name": "invoice-1.pdf", "page": 1 }],
    "failed": [],
    "failure_reasons": []
  }
}

GET https://api.invoicedataextraction.com/v1/extractions/{extraction_id}/output?format={format}

curl "https://api.invoicedataextraction.com/v1/extractions/a1b2c3d4-e5f6-7890-abcd-ef1234567890/output?format=xlsx" \
  -H "Authorization: Bearer $API_KEY"

{
  "download_url": "https://storage.example.com/...?X-Amz-Signature=...",
  "format": "xlsx",
  "expires_in_seconds": 300
}

GET https://api.invoicedataextraction.com/v1/extractions

curl "https://api.invoicedataextraction.com/v1/extractions?status=completed&limit=50" \
  -H "Authorization: Bearer $API_KEY"

{
  "success": true,
  "extractions": [
    {
      "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
      "submission_id": "sub_001",
      "task_name": "January invoices",
      "status": "completed",
      "created_at": "2026-04-27T10:30:00Z",
      "submission_method": "api",
      "file_count": 12,
      "file_names_preview": [
        "invoice-001.pdf",
        "invoice-002.pdf",
        "invoice-003.pdf",
        "invoice-004.pdf",
        "invoice-005.pdf"
      ],
      "file_names_truncated": true,
      "output_structure": "per_invoice",
      "credits_deducted": 25,
      "available_outputs": ["xlsx", "csv", "json"],
      "output_expires_at": "2026-07-26T10:30:00Z"
    },
    {
      "extraction_id": "b2c3d4e5-f6a7-8901-bcde-f23456789012",
      "submission_id": null,
      "task_name": "Q1 receipts",
      "status": "processing",
      "created_at": "2026-04-27T11:15:00Z",
      "submission_method": "web_app",
      "file_count": 4,
      "file_names_preview": [
        "receipt-april.pdf",
        "receipt-may.pdf",
        "receipt-june.pdf",
        "receipt-july.pdf"
      ],
      "file_names_truncated": false,
      "output_structure": "per_invoice",
      "progress": 42
    },
    {
      "extraction_id": "c3d4e5f6-a7b8-9012-cdef-345678901234",
      "submission_id": "sub_002",
      "task_name": "Vendor reconciliation",
      "status": "failed",
      "created_at": "2026-04-26T09:00:00Z",
      "submission_method": "api",
      "file_count": 8,
      "file_names_preview": [
        "vendor-001.pdf",
        "vendor-002.pdf",
        "vendor-003.pdf",
        "vendor-004.pdf",
        "vendor-005.pdf"
      ],
      "file_names_truncated": true,
      "output_structure": "automatic",
      "error": {
        "code": "INSUFFICIENT_CREDITS",
        "retryable": false
      }
    }
  ],
  "has_more": true,
  "next_cursor": "eyJjIjoiMjAyNi0wNC0yNlQwOTowMDowMFoiLCJpIjo1NjAxfQ"
}

"submitted_by": { "email": "[email protected]" }

const allExtractions = [];
let cursor = null;

while (true) {
  const params = new URLSearchParams({ limit: "100", status: "completed" });
  if (cursor) params.set("cursor", cursor);

  const response = await fetch(
    `https://api.invoicedataextraction.com/v1/extractions?${params}`,
    { headers: { Authorization: `Bearer ${API_KEY}` } }
  );
  const data = await response.json();

  allExtractions.push(...data.extractions);

  if (!data.has_more) break;
  cursor = data.next_cursor;
}

GET https://api.invoicedataextraction.com/v1/extractions/{extraction_id}/details

curl "https://api.invoicedataextraction.com/v1/extractions/a1b2c3d4-e5f6-7890-abcd-ef1234567890/details" \
  -H "Authorization: Bearer $API_KEY"

{
  "success": true,
  "extraction": {
    "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
    "submission_id": "sub_001",
    "task_name": "January invoices",
    "status": "completed",
    "created_at": "2026-04-27T10:30:00Z",
    "submission_method": "api",
    "file_count": 12,
    "file_names": ["invoice-1.pdf", "invoice-2.pdf", "..."],
    "output_structure": "per_invoice",
    "prompt": {
      "fields": [
        { "name": "Invoice Number" },
        { "name": "Invoice Date", "prompt": "The date the invoice was issued, NOT the payment due date" },
        { "name": "Vendor Name" },
        { "name": "Total Amount", "prompt": "Do not include currency symbol, use 2 decimal places" }
      ],
      "general_prompt": "Dates should be in YYYY-MM-DD format."
    },
    "options": {
      "exclude_columns": [],
      "output_language": "automatic",
      "review_needed_fill_color": "orange",
      "affected_field_fill_color": "orange",
      "send_completion_email": false,
      "json_typed_values": false
    },
    "credits_deducted": 25,
    "available_outputs": ["xlsx", "csv", "json"],
    "output_expires_at": "2026-07-26T10:30:00Z",
    "pages": {
      "successful_count": 10,
      "failed_count": 2,
      "successful": [
        { "file_name": "invoice-1.pdf", "page": 1 },
        { "file_name": "invoice-1.pdf", "page": 2 }
      ],
      "failed": [
        { "file_name": "damaged.pdf", "page": 1 }
      ],
      "failure_reasons": [
        {
          "code": "PROCESSING_FILE_SIZE_LIMIT_EXCEEDED",
          "message": "The upload was accepted, but during processing part of the PDF became too large for our file-processing limit. This can happen when a compressed PDF is processed internally. Split the PDF into smaller page chunks and resubmit.",
          "affected_pages": [
            { "file_name": "damaged.pdf", "pages": [1] }
          ]
        }
      ]
    },
    "ai_uncertainty_notes": [],
    "review_needed": {
      "count": 0,
      "items": []
    }
  }
}

"submitted_by": { "email": "[email protected]" }

POST https://api.invoicedataextraction.com/v1/extractions/{extraction_id}/cancel

curl -X POST "https://api.invoicedataextraction.com/v1/extractions/a1b2c3d4-e5f6-7890-abcd-ef1234567890/cancel" \
  -H "Authorization: Bearer $API_KEY"

{
  "success": true,
  "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "status": "processing",
  "cancellation_requested": true,
  "cancel_requested_at": "2026-09-10T14:03:22.418Z"
}

{
  "success": true,
  "status": "cancelled",
  "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "credits_deducted": 4,
  "cancellation_reason": "user",
  "credits_balance": 146,
  "credits_reserved": 0
}

POST https://api.invoicedataextraction.com/v1/extractions/{extraction_id}/answers

curl -X POST "https://api.invoicedataextraction.com/v1/extractions/a1b2c3d4-e5f6-7890-abcd-ef1234567890/answers" \
  -H "Authorization: Bearer $API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "answers": [
      { "question_id": "q_2524", "choice_id": "b" },
      { "question_id": "q_2529", "text": "DD/MM/YYYY" },
      { "question_id": "q_2477", "accept_recommended": true }
    ]
  }'

{
  "success": true,
  "status": "processing",
  "extraction_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "progress": 22
}

DELETE https://api.invoicedataextraction.com/v1/extractions/{extraction_id}

curl -X DELETE "https://api.invoicedataextraction.com/v1/extractions/a1b2c3d4-e5f6-7890-abcd-ef1234567890" \
  -H "Authorization: Bearer $API_KEY"

{
  "success": true
}

GET https://api.invoicedataextraction.com/v1/credits/balance

curl "https://api.invoicedataextraction.com/v1/credits/balance" \
  -H "Authorization: Bearer $API_KEY"

{
  "success": true,
  "credits_balance": 150,
  "credits_reserved": 10
}

"prompt": {
  "fields": [
    { "name": "Invoice Number" },
    {
      "name": "Line Items",
      "prompt": "Return a JSON array with keys description, quantity, unit_price, and amount. Use digits only for quantity. Use a full stop as the decimal separator. Do not include currency symbols in unit_price or amount. Do not use thousands separators. Use an empty string when a value is missing."
    }
  ]
}

"Line Items": "[{\"description\":\"Widget\",\"quantity\":\"2\",\"unit_price\":\"9.99\",\"amount\":\"19.98\"}]"

{
  "prompt": {
    "fields": [
      { "name": "Invoice Number" },
      { "name": "Invoice Date", "prompt": "Use YYYY-MM-DD format" },
      { "name": "Vendor Name" },
      { "name": "Line Item Description" },
      { "name": "Line Item Quantity", "prompt": "Use digits only" },
      { "name": "Line Item Unit Price" },
      { "name": "Line Item Amount" }
    ],
    "general_prompt": "For amount fields don't use thousands separators, use full stops as the decimal separator and do not include currency symbols."
  },
  "output_structure": "per_line_item"
}

[
  {
    "Invoice Number": "INV-1001",
    "Invoice Date": "2025-01-15",
    "Vendor Name": "Acme Ltd",
    "Line Item Description": "Widget A",
    "Line Item Quantity": "2",
    "Line Item Unit Price": "9.99",
    "Line Item Amount": "19.98"
  },
  {
    "Invoice Number": "INV-1001",
    "Invoice Date": "2025-01-15",
    "Vendor Name": "Acme Ltd",
    "Line Item Description": "Widget B",
    "Line Item Quantity": "1",
    "Line Item Unit Price": "5.00",
    "Line Item Amount": "5.00"
  }
]

import { readdir, readFile, stat, writeFile, mkdir } from "fs/promises";
import { join, extname } from "path";

// ---------------------------------------------------------------------------
// Configuration: set these before running
// ---------------------------------------------------------------------------

// Your API key. Get one at: https://invoicedataextraction.com/dashboard?view=API
// IMPORTANT: This is hardcoded here for simplicity. In production, load from an
// environment variable (e.g. process.env.INVOICE_DATA_EXTRACTION_API_KEY) and never commit to Git.
const API_KEY = "YOUR_API_KEY";

// Absolute path to the local folder containing the files you want to process.
const FOLDER_PATH = "/Users/you/Documents/invoices";

// Tell the AI what data to extract from each document (plain-text instruction).
const PROMPT = "Extract invoice number, date, vendor name, and total amount";
// For exact output column names, pass an object instead:
//   const PROMPT = { fields: [{ name: "Invoice Number" }, { name: "Total", prompt: "No currency symbol" }], general_prompt: "..." };

// A label for this extraction task (3-40 characters). Used in your dashboard and output filenames.
const TASK_NAME = "My extraction task";

// How rows are grouped in the output: "automatic" (AI decides), "per_invoice", or "per_line_item".
const OUTPUT_STRUCTURE = "automatic";

// Which output files to save beside the rows, if any. Any combination of "xlsx", "csv", "json".
// "xlsx" gives you a spreadsheet to open; set to [] to skip the files.
const DOWNLOAD_FORMATS = ["xlsx"];

// ---------------------------------------------------------------------------
// Internal constants: no changes needed
// ---------------------------------------------------------------------------

const API_BASE = "https://api.invoicedataextraction.com/v1";
const SUPPORTED_EXTENSIONS = new Set([".pdf", ".jpg", ".jpeg", ".png"]);
const MAX_RETRIES = 3;

async function apiRequest(path, body) {
  for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
    const response = await fetch(`${API_BASE}${path}`, {
      method: "POST",
      headers: {
        Authorization: `Bearer ${API_KEY}`,
        "Content-Type": "application/json",
      },
      body: JSON.stringify(body),
    });
    const text = await response.text();
    let data;
    try {
      data = JSON.parse(text);
    } catch {
      // Non-JSON response: a rate limit or an error at the gateway in front of the API.
      // Retry on 429/503, throw on anything else.
      if ((response.status === 429 || response.status === 503) && attempt < MAX_RETRIES) {
        const delayMs = 5000 * attempt;
        console.warn(`Non-JSON ${response.status} response, retrying in ${delayMs / 1000}s...`);
        await new Promise((resolve) => setTimeout(resolve, delayMs));
        continue;
      }
      throw new Error(`API returned non-JSON response (${response.status}): ${text.slice(0, 200)}`);
    }
    if (data.success) return data;

    // If the error is retryable and we have attempts left, wait and retry
    if (data.error.retryable && attempt < MAX_RETRIES) {
      // Use the Retry-After header if present (rate limit responses), otherwise exponential backoff
      const retryAfter = response.headers.get("Retry-After");
      const delayMs = retryAfter ? parseInt(retryAfter, 10) * 1000 : 1000 * attempt;
      console.warn(`Retryable error (${data.error.code}), retrying in ${delayMs / 1000}s...`);
      await new Promise((resolve) => setTimeout(resolve, delayMs));
      continue;
    }

    throw new Error(
      `API error: ${data.error.code}: ${data.error.message}` +
        (data.error.details ? `\nDetails: ${JSON.stringify(data.error.details)}` : "")
    );
  }
}

// ---------------------------------------------------------------------------
// Step 1: Discover local files and create an upload session
// ---------------------------------------------------------------------------

// Scan the folder for supported file types
const entries = await readdir(FOLDER_PATH);
const files = [];

for (const entry of entries) {
  // Skip unsupported file types and subfolders
  const ext = extname(entry).toLowerCase();
  if (!SUPPORTED_EXTENSIONS.has(ext)) continue;
  const filePath = join(FOLDER_PATH, entry);
  const fileStat = await stat(filePath);
  if (!fileStat.isFile()) continue;

  // Add this file to the list with its size in bytes.
  // file_id must be unique within the session and can only contain letters, numbers,
  // dots, underscores, colons, and hyphens (no spaces). Use your own IDs (e.g., database
  // row IDs, UUIDs, or a simple counter).
  files.push({
    file_id: `file_${files.length + 1}`,
    file_name: entry,
    file_size_bytes: fileStat.size,
    localPath: filePath, // kept locally, not sent to the API
  });
}

// Optional: before uploading, you could calculate the credits required and check
// your balance. Each page costs one credit: for PDFs, count the pages; for
// images, each file is one credit. Then call GET /credits/balance to compare
// against your available balance (credits_balance minus credits_reserved).

// Generate a unique ID for this upload session (must be different for each new session)
const uploadSessionId = `session_${Date.now()}`;

// Create the upload session: registers all files with the API
let session;
try {
  session = await apiRequest("/uploads/sessions", {
    upload_session_id: uploadSessionId,
    files: files.map(({ file_id, file_name, file_size_bytes }) => ({
      file_id,
      file_name,
      file_size_bytes,
    })),
  });
} catch (error) {
  // Session creation failure is fatal: no files can be uploaded without a session
  console.error(`Failed to create upload session: ${error.message}`);
  process.exit(1);
}

console.log(`Upload session created: ${session.upload_session_id} (${files.length} files)`);

// The chunk size in bytes: always the same for all files in the session, so we read it from the first
const partSize = session.files[0].part_size;

// ---------------------------------------------------------------------------
// Steps 2 & 3: For each file, upload chunks, then complete the upload
// ---------------------------------------------------------------------------

const completedFileIds = [];

for (const file of files) {
  try {
    // Read the entire file into memory as a binary buffer
    const fileBuffer = await readFile(file.localPath);

    // Calculate how many parts this file needs
    const totalParts = Math.ceil(fileBuffer.length / partSize);
    const partNumbers = Array.from({ length: totalParts }, (_, i) => i + 1);

    // Request a presigned upload URL for each part
    const partsData = await apiRequest(`/uploads/sessions/${uploadSessionId}/parts`, {
      file_id: file.file_id,
      part_numbers: partNumbers,
    });

    // Upload each chunk to its presigned URL via PUT
    const completedParts = [];

    for (const { part_number, url } of partsData.part_urls) {
      // Slice the file buffer into a chunk for this part
      const start = (part_number - 1) * partSize;
      const end = Math.min(start + partSize, fileBuffer.length);
      const chunk = fileBuffer.subarray(start, end);

      // PUT the raw bytes directly to the presigned URL
      const putResponse = await fetch(url, { method: "PUT", body: chunk });
      if (!putResponse.ok) {
        const errorBody = await putResponse.text();
        throw new Error(
          `Upload failed for ${file.file_name} part ${part_number}: ${putResponse.status} ${putResponse.statusText}\n${errorBody}`
        );
      }

      // Save the ETag: needed to complete the upload in Step 3
      completedParts.push({
        part_number,
        e_tag: putResponse.headers.get("etag"),
      });
    }

    console.log(`Uploaded: ${file.file_name} (${totalParts} part${totalParts > 1 ? "s" : ""})`);

    // Complete the file upload with the collected ETags
    await apiRequest(`/uploads/sessions/${uploadSessionId}/complete`, {
      file_id: file.file_id,
      parts: completedParts,
    });

    console.log(`Completed: ${file.file_name}`);
    completedFileIds.push(file.file_id);
  } catch (error) {
    // By default, abort on any file failure to avoid silent partial uploads.
    // If you'd prefer to continue with remaining files, remove the process.exit.
    console.error(`Failed: ${file.file_name}: ${error.message}`);
    process.exit(1);
  }
}

// All files uploaded and completed successfully
console.log(`\n${completedFileIds.length} files ready for extraction.`);

// ---------------------------------------------------------------------------
// Steps 4 & 5: Submit the extraction task and wait until it finishes
// ---------------------------------------------------------------------------

// Retryable polling errors (e.g., concurrent task limit, temporary server issues) trigger
// a fresh submission. Non-retryable errors require action from you: the log message tells
// you what to fix before re-running the script.

// Optional: human-readable guidance for non-retryable error codes (see error reference above).
// This just improves the console output: the API works the same without it.
const NON_RETRYABLE_GUIDANCE = {
  INSUFFICIENT_CREDITS: "Purchase credits at https://invoicedataextraction.com/dashboard?view=Billing then re-run this script.",
  FILE_PAGE_LIMIT_EXCEEDED: "Split the affected files into smaller documents and re-upload.",
  ENCRYPTED_FILE: "Remove encryption from the affected files and re-upload.",
  NO_PAGES_FOUND: "Check that your files are valid and contain extractable content.",
  PROMPT_REJECTED: "Revise your prompt to clearly describe what data to extract.",
  PROMPT_UNCLEAR: "Revise your prompt with clearer instructions and re-run.",
  FILE_SIZE_LIMIT_EXCEEDED: "Split large files into smaller documents and re-upload.",
  PROCESSING_FILE_SIZE_LIMIT_EXCEEDED: "Split the PDF into smaller page chunks and re-upload.",
};

const MAX_SUBMISSION_ATTEMPTS = 2;
const POLL_INTERVAL_MS = 1000; // pause between status requests; the API holds each one for up to 45 s

let result;

for (let attempt = 1; attempt <= MAX_SUBMISSION_ATTEMPTS; attempt++) {
  // Each attempt needs a unique submission_id
  const submissionId = `sub_${Date.now()}_${attempt}`;

  const run = await apiRequest("/extractions", {
    submission_id: submissionId,
    upload_session_id: uploadSessionId,
    file_ids: completedFileIds,
    task_name: TASK_NAME,
    prompt: PROMPT,
    output_structure: OUTPUT_STRUCTURE,
    // Typed values: amounts, quantities and rates come back as JSON numbers, yes/no
    // fields as booleans and empty cells as null, in the rows read in Step 6 and in
    // the JSON file. Leave the option out and every value is a string.
    options: { json_typed_values: true },
  });

  console.log(`\nExtraction task submitted (extraction_id: ${run.extraction_id})`);

  // Ask for the status until completed, failed, or cancelled. `wait=45` has the
  // API hold each request for up to 45 seconds and answer as soon as the run
  // finishes, so a run of a few minutes takes a handful of calls.
  let lastFailureCode = null;
  let consecutivePollErrors = 0;
  const MAX_CONSECUTIVE_POLL_ERRORS = 10;
  while (true) {
    const response = await fetch(`${API_BASE}/extractions/${run.extraction_id}?wait=45`, {
      headers: { Authorization: `Bearer ${API_KEY}` },
    });
    const data = await response.json();

    if (data.status === "completed") {
      result = data;
      break;
    }

    if (data.status === "failed") {
      const { code, message, details, retryable } = data.error;
      console.error(`\nExtraction failed: ${code}: ${message}`);
      if (details) console.error(`Details: ${JSON.stringify(details)}`);

      if (!retryable) {
        const guidance = NON_RETRYABLE_GUIDANCE[code] || "Check the error above and re-run when resolved.";
        console.error(`\nAction required: ${guidance}`);
        process.exit(1);
      }

      // Retryable: wait, then submit again.
      // Concurrent task limit means we wait longer (5 min) for other processing tasks to finish.
      // Other retryable errors are transient, so a short delay (10s) suffices.
      const delayMs = code === "CONCURRENT_TASK_LIMIT" ? 300_000 : 10_000;
      console.log(`Retrying in ${delayMs / 1000}s (attempt ${attempt}/${MAX_SUBMISSION_ATTEMPTS})...`);
      await new Promise((resolve) => setTimeout(resolve, delayMs));
      lastFailureCode = code;
      break;
    }

    if (data.status === "cancelled") {
      console.error(
        `\nExtraction was cancelled. Credits deducted: ${data.credits_deducted ?? 0}. No output files are available.`
      );
      process.exit(1);
    }

    // Still processing: reset the error counter and ask again
    if (data.status === "processing") {
      consecutivePollErrors = 0;
      console.log(`Processing... ${data.progress ?? 0}%`);
    } else {
      consecutivePollErrors++;
      console.warn(`Polling issue (HTTP ${response.status}), retrying in ${POLL_INTERVAL_MS / 1000}s... (${consecutivePollErrors}/${MAX_CONSECUTIVE_POLL_ERRORS})`);
      if (consecutivePollErrors >= MAX_CONSECUTIVE_POLL_ERRORS) {
        console.error(`\nToo many consecutive polling errors. The extraction may still be processing. Check your dashboard or retry later.`);
        process.exit(1);
      }
    }
    await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
  }

  if (result) break;

  if (lastFailureCode && attempt === MAX_SUBMISSION_ATTEMPTS) {
    const exitMessage = lastFailureCode === "CONCURRENT_TASK_LIMIT"
      ? `\nStill hitting the concurrent task limit after ${MAX_SUBMISSION_ATTEMPTS} attempts. Wait for your other extractions to finish, then re-run.`
      : `\nGave up after ${MAX_SUBMISSION_ATTEMPTS} attempts. There may be temporary service issues. Wait and try again later.`;
    console.error(exitMessage);
    process.exit(1);
  }
}

console.log(`\nExtraction completed!`);
console.log(`Credits deducted: ${result.credits_deducted}`);
console.log(`Output structure: ${result.output_structure}`);
console.log(`Pages: ${result.pages.successful_count} successful, ${result.pages.failed_count} failed`);

if (result.pages.failed_count > 0) {
  console.warn(`\nWarning: ${result.pages.failed_count} page(s) failed to extract. Data from these pages is missing from the output.`);
  for (const page of result.pages.failed) {
    console.warn(`  - ${page.file_name} (page ${page.page})`);
  }
}

if (result.ai_uncertainty_notes.length > 0) {
  console.log(`\n--- Prompt Notes ---`);
  console.log(`The AI made prompt assumptions in ${result.ai_uncertainty_notes.length} area(s). Review these and consider adding the suggested prompt additions to improve future extractions.\n`);
  result.ai_uncertainty_notes.forEach((note, i) => {
    console.log(`  [${i + 1}] ${note.topic}`);
    console.log(`  ${note.description}`);
    for (const suggestion of note.suggested_prompt_additions) {
      console.log(`    → ${suggestion.purpose}: "${suggestion.instructions}"`);
    }
    console.log();
  });
  console.log(`---`);
}

if (result.review_needed.count > 0) {
  console.warn(`\nReview Needed: ${result.review_needed.count} row(s) need manual verification before you rely on the output.`);
  result.review_needed.items.forEach((item, i) => {
    console.warn(`  [${i + 1}] ${item.message}`);
    console.warn(`      Output row(s): ${item.output_row_numbers.join(", ")}`);
    if (item.source_references.length > 0) {
      console.warn(`      Source: ${item.source_references.join(", ")}`);
    }
  });
}

// ---------------------------------------------------------------------------
// Step 6: Read the extracted rows as JSON
// ---------------------------------------------------------------------------

// The rows come back in pages of up to 1000. Each row is an object keyed by your
// output columns, exactly as the JSON file holds it: this is the data to pass on
// to your own system.
const rows = [];
let columns = [];
let offset = 0;

while (true) {
  const response = await fetch(
    `${API_BASE}/extractions/${result.extraction_id}/results?offset=${offset}&limit=1000`,
    { headers: { Authorization: `Bearer ${API_KEY}` } }
  );
  const page = await response.json();
  if (!page.success) {
    console.error(`Could not read the rows: ${page.error.code}: ${page.error.message}`);
    process.exit(1);
  }
  columns = page.columns;
  rows.push(...page.rows);
  if (!page.has_more) break;
  offset = page.next_offset;
}

console.log(`\n${rows.length} row(s) extracted. Columns: ${columns.join(", ")}`);
for (const row of rows.slice(0, 3)) console.log(row);
if (rows.length > 3) console.log(`... and ${rows.length - 3} more.`);

// ---------------------------------------------------------------------------
// Optional: save the output files
// ---------------------------------------------------------------------------

// The completed response carries a download URL per format, valid for 5 minutes.
// A plain GET to the URL returns the file; no Authorization header is needed.
if (DOWNLOAD_FORMATS.length > 0) {
  const timestamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
  const safeName = TASK_NAME.replace(/[^a-zA-Z0-9_-]/g, "_");
  await mkdir("output", { recursive: true });

  for (const format of DOWNLOAD_FORMATS) {
    const url = result.output[`${format}_url`];
    if (!url) {
      console.warn(`No ${format} download available.`);
      continue;
    }
    const response = await fetch(url);
    if (!response.ok) {
      console.error(`Failed to download ${format}: ${response.status}`);
      continue;
    }
    const buffer = Buffer.from(await response.arrayBuffer());
    const outputPath = `output/${safeName}_${timestamp}.${format}`;
    await writeFile(outputPath, buffer);
    console.log(`Saved: ${outputPath}`);
  }
}

console.log(`\nDone. Extraction ${result.extraction_id} completed successfully.`);