Programatically reorder referenced entities by field

Question

I have a content type Speaker that uses an Event content type as reference. The Event has a start date and time field. There can be any number of events that a speaker speaks at, when visiting the speaker page, the events are not ordered by date, they are ordered by id. How do I programmatically change this?

I am able to get the field value and sort them by date, but I am not able to set them back to the variable.

Related Drupal API Link

use \Drupal\Component\Utility\SortArray;


function mytheme_preprocess_node__speaker_page(&$variables) {
  $node = $variables['node'];
  $entity_ids = [];
  if ($node) {
    if (isset($node->field_events)) {
      $items = $node->field_events; //entity reference
      $unsorted=array();
      foreach ($items as $item) {
        $target_id = $item->target_id;
        $field_event = $item->entity;
        //creating a new unsorted array with the target id and event date, this is used for sorting
        array_push($unsorted, array('target_id' => $target_id, 'event_date' => strtotime($field_event->field_date_of_event->value)));
      }
      //sorting
      usort($unsorted, function ($a, $b) {
        return SortArray::sortByKeyInt($a, $b, 'event_date');
      });
      //sorted ids
      $new_entity_ids = array_column($unsorted, 'target_id');

      $items_new = \Drupal\node\Entity\Node::loadMultiple($new_entity_ids);
      $variables['node']->field_events = $items_new; //this doesn't work
      //$variables['node']->set($field_name)->referencedEntities(); 
    }
  }
}