0

I'm trying to import all files from a D7 database via a custom migration. As thinks weren't working as expected I started off with a copy of the stanard d7_file migration template:

    # Every migration that references a file by Drupal 7 fid should specify this
# migration as an optional dependency.
id: lbdp7_files
label: Files
migration_group: lbdp7
migration_tags:
  - Drupal 7
source:
  plugin: lbdp7_file
  constants:
    # source_base_path must be set by the tool configuring this migration. It
    # represents the fully qualified path relative to which uris in the files
    # table are specified, and must end with a /. See source_full_path
    # configuration in this migration's process pipeline as an example.
    source_base_path: ''
process:
  # If you are using this file to build a custom migration consider removing
  # the fid field to allow incremental migrations.
  fid: fid
  filename: filename
  source_full_path:
    -
      plugin: concat
      delimiter: /
      source:
        - constants/source_base_path
        - filepath
    -
      plugin: urlencode
  uri:
    plugin: file_copy
    source:
      - '@source_full_path'
      - uri
  filemime: filemime
  # filesize is dynamically computed when file entities are saved, so there is
  # no point in migrating it.
  # filesize: filesize
  status: status
  # Drupal 7 didn't keep track of the file's creation or update time -- all it
  # had was the vague "timestamp" column. So we'll use it for both.
  created: timestamp
  changed: timestamp
  uid: uid
destination:
  plugin: entity:file

The source plguin lbdp7_file is also just the standard version:

<?php

namespace Drupal\lb_dp7_migration\Plugin\migrate\source;

use Drupal\Core\Database\Query\Condition;
use Drupal\migrate\Row;
use Drupal\migrate_drupal\Plugin\migrate\source\DrupalSqlBase;

/**
 * Drupal 7 file source from database.
 *
 * @MigrateSource(
 *   id = "lbdp7_file"
 * )
 */
class File extends DrupalSqlBase {

  /**
   * The public file directory path.
   *
   * @var string
   */
  protected $publicPath;

  /**
   * The private file directory path, if any.
   *
   * @var string
   */
  protected $privatePath;

  /**
   * The temporary file directory path.
   *
   * @var string
   */
  protected $temporaryPath;

  /**
   * {@inheritdoc}
   */
  public function query() {
    $query = $this->select('file_managed', 'f')
      ->fields('f')
      ;

    // Filter by scheme(s), if configured.
    if (isset($this->configuration['scheme'])) {
      $schemes = array();
      // Accept either a single scheme, or a list.
      foreach ((array) $this->configuration['scheme'] as $scheme) {
        $schemes[] = rtrim($scheme) . '://';
      }
      $schemes = array_map([$this->getDatabase(), 'escapeLike'], $schemes);

      // uri LIKE 'public://%' OR uri LIKE 'private://%'
      $conditions = new Condition('OR');
      foreach ($schemes as $scheme) {
        $conditions->condition('uri', $scheme . '%', 'LIKE');
      }
      $query->condition($conditions);
    }
    //echo $query->__toString();
    return $query;
  }

  /**
   * {@inheritdoc}
   */
  protected function initializeIterator() {
    $this->publicPath = $this->variableGet('file_public_path', 'sites/default/files');
    $this->privatePath = $this->variableGet('file_private_path', NULL);
    $this->temporaryPath = $this->variableGet('file_temporary_path', '/tmp');
    return parent::initializeIterator();
  }

  /**
   * {@inheritdoc}
   */
  public function prepareRow(Row $row) {
    // Compute the filepath property, which is a physical representation of
    // the URI relative to the Drupal root.
    $path = str_replace(['public:/', 'private:/', 'temporary:/'], [$this->publicPath, $this->privatePath, $this->temporaryPath], $row->getSourceProperty('uri'));
    // At this point, $path could be an absolute path or a relative path,
    // depending on how the scheme's variable was set. So we need to shear out
    // the source_base_path in order to make them all relative.
    $path = str_replace($this->configuration['constants']['source_base_path'], NULL, $path);
    $row->setSourceProperty('filepath', $path);
    return parent::prepareRow($row);
  }

  /**
   * {@inheritdoc}
   */
  public function fields() {
    return array(
      'fid' => $this->t('File ID'),
      'uid' => $this->t('The {users}.uid who added the file. If set to 0, this file was added by an anonymous user.'),
      'filename' => $this->t('File name'),
      'filepath' => $this->t('File path'),
      'filemime' => $this->t('File MIME Type'),
      'status' => $this->t('The published status of a file.'),
      'timestamp' => $this->t('The time that the file was added.'),
    );
  }

  /**
   * {@inheritdoc}
   */
  public function getIds() {
    $ids['fid']['type'] = 'integer';
    return $ids;
  }

}

If I evaluate the query in the source plugin query() function via echo $query->__toString(); I see that the query is SELECT f.* from file_managed f; just as expected.

The problem is that drush ms says that there are 0 items to import. Can't be a database config problem as I have other migrations which work perfectly using the same db connection from settings.php. The table file_managed is populated with over 7000 items...

Any ideas what the problem could be?

Many thanks in advance!

1
  • I dove into the SQL query logs and found out that drush mr indeed retrieves the correct data from the legacy db: SELECT f.*, map.sourceid1 AS migrate_map_sourceid1, map.source_row_status AS migrate_map_source_row_status FROM file_managed f LEFT OUTER JOIN lbdp8.migrate_map_lbdp7_files map ON fid = map.sourceid1 WHERE ( (map.sourceid1 IS NULL ) OR (map.source_row_status = '1') ) returns the correct amount of data sets.
    – Dennis M.
    Jan 10, 2017 at 14:45

2 Answers 2

0

Have you checked the migrate messages and message map for the migration? I'm guess you might see that all the migrations have failed. Does the D8 site have access to the files? And is source_base_path set (see the comment in the migration template)?

1
  • There are no messages at all for the migration. drush ms states that there are 0 files to migrate (and 0 files skipped). For the source_base_path: I set it to a directory in my workspace which is on the same level as the web folder. I refrenced it via relative path like so: '../legacy-files/'. Could it be that it doesn't find the path and therefore doesn't resolve the entities?
    – Dennis M.
    Jan 11, 2017 at 8:53
0

Ok, turns out it was a simple mistake I made. I forgot about setting the right database conenction in the migration.yml. So setting the attributes key and target for the source plugin resolved the issue.

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.

Not the answer you're looking for? Browse other questions tagged or ask your own question.