Add items to batch queue operations on the fly

Question

I have some code which generates a batch with a single operation; to go off to an API and get some data. This data could be any number of items.

I would like to be able to populate the rest of the batch's operations during the first operation.

I have tried this, but it doesn't seem to work...

Batch initialisation (drush command callback):

function example_drush_sync_entries() {
  $batch = array(
    'operations' => array(
      array('_example_fetch_data', array())
    ),
    'finished' => '_example_batch_finished',
    'title' => t('Processing'),
    'progress_message' => '',
    'error_message' => t('The update has encountered an error.'),
  );

  batch_set($batch);
  $batch = &batch_get();
  $batch['progressive'] = FALSE;

  drush_backend_batch_process();
}

function _example_fetch_data(&$context) {
  $response = drupal_http_request('http://blah.com/data');
  $data = $response->data;
  $data = drupal_json_decode($data);

  $batch = &batch_get();
  $batch_set = &$batch['sets'][$batch['current_set']];
  foreach ($data as $entry) {
    $batch_set['operations'][] = array('_example_process_entry', array($entry));
  }
  print_r($batch);

  _batch_populate_queue($batch, $batch['current_set']);
}

function _example_process_entry($entry, &$context) {
  print_r(func_get_args());
}

I can see $batch has a load of new operations in it, however _example_process_entry never seems to fire.

Is it possible to dynamically add items to the batch queue while its processing?

Answer 1

I was really quite close! As always, asking the question on here caused the answer to reveal itself.

function example_drush_sync_entries() {
  $batch = array(
    'operations' => array(
      array('_example_fetch_data', array())
    ),
    'finished' => '_example_batch_finished',
    'title' => t('Processing'),
    'progress_message' => '',
    'error_message' => t('The update has encountered an error.'),
  );

  batch_set($batch);
  $batch = &batch_get();
  $batch['progressive'] = FALSE;

  drush_backend_batch_process();
}

function _example_fetch_data(&$context) {
  $response = drupal_http_request('http://blah.com/data');
  $data = $response->data;
  $data = drupal_json_decode($data);

  $batch = &batch_get();
  $batch_next_set = $batch['current_set'] + 1;
  $batch_set = &$batch['sets'][$batch_next_set];
  foreach ($data as $entry) {
    $batch_set['operations'][] = array('_example_process_entry', array($entry));
  }
  $batch_set['total'] = count($batch_set['operations']);
  $batch_set['count'] = $batch_set['total'];

  _batch_populate_queue($batch, $batch_next_set);
}

function _example_process_entry($entry, &$context) {
  var_dump($entry['id']);
}

Notable differences: * The operations need to be added to the next set, not the current one. * You need to define a total and count, otherwise the percentage progress is considered 100% and, therefore, complete before it even starts.

I hope this helps other people :)