Bulk of HTML files to node Migration

Question

I am developing a module for migrating data from html file to drupal node. Each Html file is having 500+ logs. I want to create node for each log record. I have started development of module as it is explained in DRPAL DOCUMENTATION i.e.: https://www.drupal.org/node/2029713/

But I am not getting exact solution for resolving it. Getting this error:

An AJAX HTTP error occurred. HTTP Result Code: 500 Debugging information follows. Path: /batch?id=40&op=do StatusText: Service unavailable (with message) ResponseText: PDOException: SQLSTATE[22007]: Invalid datetime format: 1366 Incorrect integer value: '/Chat Log - 07:01:2014.html' for column 'sourceid1' at row 1: INSERT INTO {migrate_map_alertnodemigrate} (sourceid1, needs_update, rollback_action) VALUES (:db_insert_placeholder_0, :db_insert_placeholder_1, :db_insert_placeholder_2); Array ( [:db_insert_placeholder_0] => /Chat Log - 07:01:2014.html [:db_insert_placeholder_1] => 3 [:db_insert_placeholder_2] => 0 ) in MigrateSQLMap->saveIDMapping() (line 337 of /data/disk/o1/static/chartz_2/sites/all/modules/contrib/migrate/plugins/sources/sqlmap.inc).

Here is my code for this:

class AlertNodeMigration extends Migration    {
  public function __construct($arguments)
  {
    parent::__construct();

// SOURCE OBJECT
$directories = array('/data/disk/o1/static/chat_log/html');
$base_dir = '/data/disk/o1/static/chat_log/html';
$regex = '/(.*\.htm$|.*\.html$)/i';
$list_files = new MigrateListFiles($directories, $base_dir, $regex);
$item_file = new MigrateItemFile($base_dir);
$fields = array(
'title' => t('Node: Title'),
'uid' => t('Node: User Id'),
'created' => t('Node: Created Timestamp'),
'changed' => t('Node: Modified timestamp'),
'status' => t('Node: Published'),
'promote' => t('Node: Promoted to front page'),
'sticky' => t('Node: Sticky at top of lists'),
'revision' => t('Node: Create new revision'),
'log' => t('Node: Revision Log message'),
'language' => t('Node: Language (fr, en, ...)'),
'tnid' => t('Node: The translation set id for this node'),
'translate' => t('Node: A boolean indicating whether this translation page needs to be updated'),
'revision_uid' => t('Node: Modified (uid)'),
'is_new' => t('Option: Indicates a new node with the specified nid should be created'),
'field_instrument' => t('Field: Instrument (entityreference)'),
'field_alert' => t('Field: Alert (text_long)'),
'field_alert:language' => t('Subfield: Language for the field'),
'field_sentiment_analysis' => t('Field: Sentiment Analysis (list_text)'),
'field_tags' => t('Field: Tags (taxonomy_term_reference)'),
'field_tags:source_type' => t('Option: Set to \'tid\' when the value is a source ID'),
'field_tags:create_term' => t('Option: Set to TRUE to create referenced terms when necessary'),
'field_tags:ignore_case' => t('Option: Set to TRUE to ignore case differences between source data and existing term names'),
'field_source' => t('Field: Source (list_text)'),
'path' => t('Node: Path alias'),
'comment' => t('Whether comments may be posted to the node')
);
$this->team = array(new MigrateTeamMember('alertnode', '[email protected]', t('Implementor')));
    $this->description = t('Alert Node Migration');

$this->source = new MigrateSourceList($list_files, $item_file, $fields);


// DESTINATION Object
$this->destination = new MigrateDestinationNode('alert');


// MAP OBJECT
    $this->map = new MigrateSQLMap($this->machineName,
    array(
      'sourceid' => array('type' => 'int',
                      'unsigned' => TRUE,
                      'not null' => TRUE,
         )
    ),
    MigrateDestinationNode::getKeySchema()
  );

// FIELD MAPPING OBJECT 
$this->addFieldMapping('title', 'title');
$this->addFieldMapping('uid', 'uid');
$this->addFieldMapping('created', 'created');
$this->addFieldMapping('changed', 'changed');
$this->addFieldMapping('status', 'status');
$this->addFieldMapping('promote', 'promote');
$this->addFieldMapping('sticky', 'sticky');
$this->addFieldMapping('revision', 'revision');
$this->addFieldMapping('log', 'log');
$this->addFieldMapping('language', 'language');
$this->addFieldMapping('tnid', 'tnid');
$this->addFieldMapping('translate', 'translate');
$this->addFieldMapping('revision_uid', 'revision_uid');
$this->addFieldMapping('is_new', 'is_new');
$this->addFieldMapping('field_instrument', 'field_instrument');
$this->addFieldMapping('field_alert', 'field_alert');
$this->addFieldMapping('field_alert:language', 'field_alert:language');
$this->addFieldMapping('field_sentiment_analysis', 'field_sentiment_analysis');
$this->addFieldMapping('field_tags', 'field_tags');
$this->addFieldMapping('field_tags:source_type', 'field_tags:source_type');
$this->addFieldMapping('field_tags:create_term', 'field_tags:create_term');
$this->addFieldMapping('field_tags:ignore_case', 'field_tags:ignore_case');
$this->addFieldMapping('field_source', 'field_source');
$this->addFieldMapping('path', 'path');
$this->addFieldMapping('comment', 'comment');


}




public function prepareRow($row) 
 {
     return true;
 } 

 public function prepare($node, stdClass $row) {


// Set to admin for now.
$node->uid = 1;

// Create a new SourceParser to handle HTML content.
$source_parser = new SourceParser(substr($row->sourceid, 1), $row->filedata);
$alerthtml = $source_parser->parse_alert_html($row->html);


   $node->title = $alerthtml['title'];      // Node: Title
   $node->uid = '1';                //Node: User Id
   $node->created = $alerthtml['timestamp'];    //Node: Created Timestamp
   $node->changed = '0';            //Node: Modified timestamp
   $node->status = 'published';         //Node: Published
   $node->promote = '0';            //Node: Promoted to front page
   $node->sticky = '0';             //Node: Sticky at top of lists
   $node->revision = '0';           //Node: Create new revision
   $row->log = '0';             //Node: Revision Log message
   $node->language = 'en';          //Node: Language (fr, en, ...)
   $node->tnid = '0';               //Node: The translation set id for this node
   $node->translate = '0';          //Node: A boolean indicating whether this translation page
   $node->revision_uid = '1';           //Node: Modified (uid)
   $node->is_new = '1';             //Option: Indicates a new node with the specified nid
   $node->field_instrument = $alerthtml['inst'];//Field: Instrument (entityreference)
   $node->field_alert = $alerthtml['alert_text'];//Field: Alert (text_long)
   $node->field_alert['language'] = 'en';   //Subfield: Language for the field
   $node->field_sentiment_analysis = 'positive';//Field: Sentiment Analysis (list_text)
   $node->field_tags = '0';         //Field: Tags (taxonomy_term_reference)
   $node->field_tags['source_type'] = $alerthtml['src_typ'];//Option: Set to \'tid\' when the value is a source ID
   $node->field_tags['create_term'] = $alerthtml['term'];//Option: Set to TRUE to create referenced terms 
   $node->field_tags['ignore_case'] = $alerthtml['ignr'];//Option: Set to TRUE to ignore case differences
   $node->field_source = $alerthtml['src']; //Field: Source (list_text)
   $node->path = $alerthtml['title'];       //Node: Path alias
   $node->comment = $alerthtml['comments']; //Whether comments may be posted to the node
}

}

Is anyone can help me ? Thanks in advance.

Answer 1

The exception is telling you what the problem is:

Invalid datetime format: 1366 Incorrect integer value: '/Chat Log - 07:01:2014.html' for column 'sourceid1' at row 1

What this means is that the value that you're trying to assign to a datetime field is incorrect, and it gives you the illegal value that was encountered.

I suspect it's in this code:

<?php    
  $node->created = $alerthtml['timestamp'];    //Node: Created Timestamp**strong text** 
?>

The value of $alerthtml['timestamp'] appears to be "/Chat Log - 07:01:2014.html", not quite what you expected. You'll want to do some validation to make sure that when you parse out that date value, you get what you expect and clean up as necessary.