// Optimal batch configuration
const BATCH_SIZE = 100;        // Max per API call
const CONCURRENT_BATCHES = 3;  // Parallel uploads
const DELAY_BETWEEN_MS = 500;  // Avoid rate limits

async function indexWithThroughput(docs: GleanDocument[]) {
  const batches = chunk(docs, BATCH_SIZE);
  const queue = new PQueue({ concurrency: CONCURRENT_BATCHES, interval: DELAY_BETWEEN_MS });

  await Promise.all(batches.map((batch, i) =>
    queue.add(() => glean.indexDocuments('my_ds', batch))
  ));
}