How to migrate content using a web service with multiple calls?

Question

I need to migrate content from a web service to which I have to call several URLs. A call to receive a list of nodes and then a call for each node to receive information from the fields.

Something like this:

Calling domain.test/webservice/

<results>
  <count>10</count>
  <list>
    <id>100</id>
    <id>101</id>
    <id>102</id>
    ...
  </list>
</results>

Calling domain.test/webservice/101

<results>
  <id>101</id>
  <name>Name</name>
  <field1>Field example</field1>
    ...
</results>

So, in my migration yml file I have something like:

source:
  plugin: url
  data_fetcher_plugin: http
  headers:
    Accept: application/xml
  data_parser_plugin: simple_xml
  urls: https://domain.test/webservice/

How can I specified the migrate process to retrieve each content info from a URL like https://domain.test/webservice/[ID]?

Answer 1

There is an issue with the UrlPerRow plugin above.

The source data isn't populated until after the check for whether the source data has changed. Because of that fact, it's always looking at previous data, which means the hash will never change, so "rowChanged()" never returns true. It will import new data but not update if the data changes.

A fixed version is below:

<?php

namespace Drupal\fad_migration\Plugin\migrate\source;

use Drupal\migrate\Plugin\MigrationInterface;
use Drupal\migrate\Plugin\migrate\source\SourcePluginBase;
use Drupal\migrate_plus\Plugin\migrate\source\Url;
use Drupal\migrate\Row;

/**
 *
 * @MigrateSource(
 *   id = "url_per_row"
 * )
 */
class UrlPerRow extends Url {

  protected $dataNewParserPlugin;

  /**
   * {@inheritdoc}
   */
  public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration) {
    parent::__construct($configuration, $plugin_id, $plugin_definition, $migration);
  }

  /**
   * {@inheritdoc}
   */
  public function prepareRow(Row $row) {
    $prepare_row = parent::prepareRow($row);

    $item_info_settings = $row->getSourceProperty('item_info');
    if (!empty($item_info_settings)) {
      $this->getItemData($row, $item_info_settings);
    }

    if (!$row->getIdMap() || $row->needsUpdate() || $this->aboveHighwater($row) || $this->rowChanged($row)) {
      return $prepare_row;
    }
    else {
      return false;
    }

  }

  protected function getItemData(Row &$row, $settings) {

    $new_url = str_replace($settings['token'], $row->getSourceProperty($settings['token_replace']), $settings['url']);
    $this->getNewDataParserPlugin($new_url, $settings['item_selector'], $row->getSource());
    $this->dataNewParserPlugin->rewind();
    $xml_data = $this->dataNewParserPlugin->current();
    if (!empty($xml_data)) {
      $fields = $row->getSourceProperty('fields');
      foreach ($fields as $field) {
        if (!empty($field['item_info']) && !empty($xml_data[$field['name']])) {
          $row->setSourceProperty($field['name'], $xml_data[$field['name']]);
        }
      }
    }    
  }

  protected function getNewDataParserPlugin($url, $item_selector, $settings) {
    $settings['urls']          = [$url];
    $settings['item_selector'] = $item_selector;

    $this->dataNewParserPlugin = \Drupal::service('plugin.manager.migrate_plus.data_parser')
                                        ->createInstance($settings['data_parser_plugin'], $settings);

    return $this->dataNewParserPlugin;
  }

}

Answer 2

To solve this I had to create a source plugin that for each row calls a new data parse plugin with the URL for this row.

Here is the plugin:

/**
*
* @MigrateSource(
*   id = "url_per_row"
* )
*/
class UrlPerRow extends Url {

protected $dataNewParserPlugin;

  /**
   * {@inheritdoc}
   */
  public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration) {
    parent::__construct($configuration, $plugin_id, $plugin_definition, $migration);
  }

  /**
   * {@inheritdoc}
   */
  public function prepareRow(Row $row) {
    $prepare_row = parent::prepareRow($row);

    if (!$row->getIdMap() || $row->needsUpdate() || $this->aboveHighwater($row) || $this->rowChanged($row)) {
      $item_info_settings = $row->getSourceProperty('item_info');
      if (!empty($item_info_settings)) {
        $this->getItemData($row, $item_info_settings);
      }
    }

    return $prepare_row;
  }

  protected function getItemData(Row &$row, $settings) {
    $new_url = str_replace($settings['token'], $row->getSourceProperty($settings['token_replace']), $settings['url']);

    $this->getNewDataParserPlugin($new_url, $settings['item_selector'],     $row->getSource());
    $this->dataNewParserPlugin->rewind();
    $xml_data = $this->dataNewParserPlugin->current();
    if (!empty($xml_data)) {
      $fields = $row->getSourceProperty('fields');
      foreach ($fields as $field) {
        if (!empty($field['item_info']) && !empty($xml_data[$field['name']])) {
          $row->setSourceProperty($field['name'], $xml_data[$field['name']]);
        }
      }
    }
  }

  protected function getNewDataParserPlugin($url, $item_selector, $settings) {
    $settings['urls'] = [$url];
    $settings['item_selector'] = $item_selector;

    $this->dataNewParserPlugin = \Drupal::service('plugin.manager.migrate_plus.data_parser')
      ->createInstance($settings['data_parser_plugin'], $settings);

    return $this->dataNewParserPlugin;
  }

}

Then I put some info in the migration .yml definition:

source:
  plugin: url_per_row
  data_fetcher_plugin: http
  headers:
    Accept: application/xml
  data_parser_plugin: simple_xml
  urls: https://myws.com
  item_selector: /ResultWeb/data
  item_info:
    url: https://myws.com/[id]
    token: '[id]'
    token_replace: id
    item_selector: /ResultWeb/data

Notice that the id from token_replace is a field with name id. You can use other field.

Also, for every field where is needt to rereive information from the specified URL I put a item_info: true element.

-
  name: myfield
  label: 'My Field'
  selector: fields/field/my
  item_info: true