Using migrate with a multi-page / paginated source

Question

I have a working module pulling a JSON feed into Drupal using the Migrate module. However, the feed is paginated, and I am only getting the first page of results (ordered DESC on creation, so giving the latest items).

I need to go back through all pages and get all/any changed items, but I can't see how to tell Migrate to iterate through the pages.

Answer 1

I would do a 2 step process -- I'm not sure there is a better way using solely migrate.

Use drupal cron and a Queue to fetch all pagination results to a folder containing all the files with json data. Then use your migration(s) to process the data. Drupal has built-in cron and queue APIs; but you could just use a Linux system cron, bash or CuRL if you like.

Answer 2

You can parse paginated sources with migrate but it takes a little work overriding MigrateListJSON - you may want to ignore the $jsonContainer part - that's just because the result set in our service is in a container (hence this class is abstract).

abstract class BSBaseMigrateListJSON extends MigrateListJSON {
  /**
   * String containing the JSON container name.
   */
  protected $jsonContainer;

  /**
   * Contains the default limit for pagination of the service.
   */
  protected $serviceLimit = 50;

  /**
   * Return an array of the IDs found in the list URL.
   */
  public function getIdList() {
    migrate_instrument_start("Retrieve $this->listUrl");
    $data = $this->getTree();
    migrate_instrument_stop("Retrieve $this->listUrl");
    if ($data) {
      return $this->getIDsFromJSON($data);
    }
    Migration::displayMessage(t('Loading of !listurl failed:', array('!listurl' => $this->listUrl)));
    return NULL;
  }

  /**
   * Returns the amount of items to be processed.
   */
  public function computeCount() {
    if ($data = $this->getTree()) {
      return count($data[$this->jsonContainer]);
    }
    // Else fail and return N/A results.
    return NULL;
  }

  /**
   * Returns an array of crids.
   */
  protected function getIDsFromJSON(array $data) {
    $ids = array();
    foreach ($data[$this->jsonContainer] as $item) {
      $ids[] = $item['id'];
    }
    return $ids;
  }

  /**
   * Returns the list URL decoded JSON response.
   */
  protected function getTree() {
    $items = array();
    $offset = 0;

    do {
      $url = $this->rebuildUrl($offset);
      $json = $this->getTreeData($url);
      $data = drupal_json_decode($json);
      $items = array_merge($items, $data[$this->jsonContainer]);
      $offset += $this->serviceLimit;
    }
    while ($data['moreResources'] == TRUE);

    // Return data if any is present.
    if (!empty($data)) {
      $data[$this->jsonContainer] = $items;
      // @todo Consider whether just returning $items is feasible.
      return $data;
    }

    Migration::displayMessage(t('Loading of !listurl failed:', array('!listurl' => $this->listUrl)));
    return NULL;
  }

  /**
   * Returns a constructed url with offset if required.
   */
  private function rebuildUrl($offset = 0) {
    $url_parts = parse_url($this->listUrl);

    // Build the query parameters.
    if (isset($url_parts['query'])) {
      parse_str($url_parts['query'], $url_parts['qparts']);
      foreach ($url_parts['qparts'] as $key => $value) {
        if ($value === '') {
          $url_parts['qparts'][$key] = NULL;
        }
        elseif ($key === 'q') {
          $url_parts['path'] = $value;
          unset($url_parts['qparts']['q']);
        }
      }
    }

    // If the offset is zero then we can simply return the original list URL.
    if ($offset == 0) {
      // Handles overriding the service limit if there's already a limit set in
      // the list URL.
      if (!empty($url_parts['qparts']['limit'])) {
        $this->serviceLimit = $url_parts['qparts']['limit'];
      }
      return $this->listUrl;
    }

    // Add the offset to the query parameters.
    $url_parts['qparts']['offset'] = $offset;

    // Build parameters to pass to the url function.
    $url_params = array(
      'path' => $url_parts['scheme'] . '://' . $url_parts['host'] . $url_parts['path'],
      'options' => array(
        'query' => isset($url_parts['qparts']) ? $url_parts['qparts'] : NULL,
        'fragment' => isset($url_parts['fragment']) ? $url_parts['fragment'] : NULL,
        'external' => TRUE,
        'https' => (bool) $url_parts['scheme'] == 'https',
      ),
    );

    return url($url_params['path'], $url_params['options']);
  }

  /**
   * Returns the data from the given list URL.
   */
  private function getTreeData($list_url) {
    if (empty($this->httpOptions)) {
      return file_get_contents($list_url);
    }
    $response = drupal_http_request($list_url, $this->httpOptions);
    if (!empty($response->data)) {
      return $response->data;
    }
  }
}

This code probably won't work as-is for you as it's specific for our service, you'll most definitely need to change this line: while ($data['moreResources'] == TRUE); and anything to do with offset depending on how your service works but it's a start for how to proceed.