Return to Answer

Drupal 8 uses hook_entity_access() to control access to any entity (including users) to various operations, hook_ENTITY_TYPE_access() to check the access to specific entities on various operations (for which hook_node_access() is the implementation for nodes), and hook_entity_create_access() to check which users can create an entity.
Drupal 8 code is much simpler than the Drupal 7 code, as as a single hook implementation does the task.

function mymodule_menu_alter(&$items) {
  if (isset($items['entityreference/autocomplete/single/%/%/%'])) {
    $items['user/%user/view']['page callback'] = 'mymodule_entity_autocomplete';
  }
}

function mymodule_entity_autocomplete($type, $field_name, $entity_type, $bundle_name, $entity_id = '', $string = '') {
  $field = field_info_field($field_name);
  $instance = field_info_instance($entity_type, $field_name, $bundle_name);
  $matches = array();

  if ($type = 'user') {
    // Populate $matches with the matching users the currently logged-in user can select.
    return $matches;
  }

  return sss$type, $field, $instance, $entity_type, $entity_id, $string);
}

function mymodule_menu_alter(&$items) {
  if (isset($items['entityreference/autocomplete/single/%/%/%'])) {
    $items['entityreference/autocomplete/single/%/%/%']['page callback'] = 'mymodule_entity_autocomplete';
  }
}

function mymodule_entity_autocomplete($type, $field_name, $entity_type, $bundle_name, $entity_id = '', $string = '') {
  $field = field_info_field($field_name);
  $instance = field_info_instance($entity_type, $field_name, $bundle_name);
  $matches = array();

  if ($type = 'user') {
    // Populate $matches with the matching users the currently logged-in user can select.
    return $matches;
  }

  return entityreference_autocomplete_callback_get_matches($type, $field, $instance, $entity_type, $entity_id, $string);
}

Probably you need to change the access callback of the autocomplete path too, with $items['entityreference/autocomplete/single/%/%/%']['access callback'] = 'mymodule_entity_autocomplete';. The access callback defined from the module is the following one.

function entityreference_autocomplete_access_callback($type, $field_name, $entity_type, $bundle_name) {
  $field = field_info_field($field_name);
  $instance = field_info_instance($entity_type, $field_name, $bundle_name);

  if (!$field || !$instance || $field['type'] != 'entityreference' || !field_access('edit', $field, $entity_type)) {
    return FALSE;
  }
  return TRUE;
}

There isn't an equivalent of hook_node_access() for users, or generically for any entity.

What you can do is checking who accesses user/%user/view using an access callback different from the default one.

function mymodule_menu_alter(&$items) {
  if (isset($items['user/%user/view'])) {
    $items['user/%user/view']['access callback'] = 'mymodule_user_view_access';
  }
}

In this way, you can avoid a user sees a user profile accessing it directly.

The default access callback is user_view_access(), which contains the following code.

  $uid = is_object($account) ? $account->uid : (int) $account;

  // Never allow access to view the anonymous user account.
  if ($uid) {
    // Admins can view all, users can view own profiles at all times.
    if ($GLOBALS['user']->uid == $uid || user_access('administer users')) {
      return TRUE;
    }
    elseif (user_access('access user profiles')) {
      // At this point, load the complete account object.
      if (!is_object($account)) {
        $account = user_load($uid);
      }
      return (is_object($account) && $account->status);
    }
  }
  return FALSE;

There isn't an equivalent of hook_node_access() for users, or generically for any entity.

To check which user accounts can be selected from the Entity reference autocomplete, you could change the menu callback for the autocomplete.

function mymodule_menu_alter(&$items) {
  if (isset($items['entityreference/autocomplete/single/%/%/%'])) {
    $items['user/%user/view']['page callback'] = 'mymodule_entity_autocomplete';
  }
}

function mymodule_entity_autocomplete($type, $field_name, $entity_type, $bundle_name, $entity_id = '', $string = '') {
  $field = field_info_field($field_name);
  $instance = field_info_instance($entity_type, $field_name, $bundle_name);
  $matches = array();

  if ($type = 'user') {
    // Populate $matches with the matching users the currently logged-in user can select.
    return $matches;
  }

  return sss$type, $field, $instance, $entity_type, $entity_id, $string);
}

If then you want to avoid users can view a user account directly using the link to the account, you can change the access callback for user/%user/view.

function mymodule_menu_alter(&$items) {
  if (isset($items['user/%user/view'])) {
    $items['user/%user/view']['access callback'] = 'mymodule_user_view_access';
  }
}

The default access callback is user_view_access(), which contains the following code.

  $uid = is_object($account) ? $account->uid : (int) $account;

  // Never allow access to view the anonymous user account.
  if ($uid) {
    // Admins can view all, users can view own profiles at all times.
    if ($GLOBALS['user']->uid == $uid || user_access('administer users')) {
      return TRUE;
    }
    elseif (user_access('access user profiles')) {
      // At this point, load the complete account object.
      if (!is_object($account)) {
        $account = user_load($uid);
      }
      return (is_object($account) && $account->status);
    }
  }
  return FALSE;