# a controller node through the OCI's ENC:
#
#---
#classes:
#   oci_controller:
#      is_first_master: true
#      first_master: zigo-controller-node-3.example.com
#      first_master_ip: 192.168.100.40
#      vip_hostname: zigo-api.example.com
#      vip_ipaddr: 192.168.101.2
#      network_ipaddr: 192.168.101.0
#      network_cidr: 24
#      other_masters:
#         - zigo-controller-node-4.example.com
#         - zigo-controller-node-5.example.com
#      other_masters_ip:
#         - 192.168.100.41
#         - 192.168.100.32
#      all_masters:
#         - zigo-controller-node-3.example.com
#         - zigo-controller-node-4.example.com
#         - zigo-controller-node-5.example.com
#      all_masters_ip:
#         - 192.168.100.40
#         - 192.168.100.41
#         - 192.168.100.32
#
# This is re-used in the oci_controller class below
#
class oci::controller(
  $region_name              = 'RegionOne',
  $openstack_release        = undef,		# rocky, stein, train
  $cluster_name             = undef,		# z
  $machine_hostname         = undef,		# z-controller-1.example.com
  $machine_ip               = undef,		# 192.168.101.2
  $bridge_mapping_list      = undef,		# [ 'external:br-ex', 'external1:br-lb']
  $external_network_list    = undef,		# [ 'external', 'external1' ]
  $machine_iface            = undef,		# br-ex
  $compute_nodes            = undef,		# [ 'z-compute-1.example.com', 'z-compute-2.example.com', 'z-compute-3.example.com' ]
  $compute_nodes_ip         = undef,            # [ '192.168.101.5', '192.168.101.6', '192.168.101.7' ]
  $volume_nodes             = undef,            # [ 'z-volume-1.example.com', 'z-volume-2.example.com' ]
  $volume_nodes_ip          = undef,            # [ '192.168.101.8', '192.168.101.9' ]
  $vmnet_ip                 = undef,		# 192.168.102.2

  $messaging_nodes_for_notifs = false,
  $all_rabbits                = [],
  $all_rabbits_ips            = [],

  $neutron_global_physnet_mtu        = 1500,
  $neutron_path_mtu                  = 1500,
  $neutron_executor_thread_pool_size = 512,
  $neutron_rpc_conn_pool_size        = 256,

  $is_first_master          = false,		# true
  $first_master             = undef,		# z-controller-1.example.com
  $first_master_ip          = undef,		# 192.168.101.2
  $non_first_master_controllers = undef,	# All non-primary masters: [ 'z-controller-2.example.com', 'z-controller-3.example.com' ]
  $non_first_master_controllers_ip = undef,	# [ '192.168.101.3', '192.168.101.4' ]
  $other_masters            = undef,		# Other masters in this cluster, excluding this host: [ 'z-controller-2.example.com', 'z-controller-3.example.com' ]
  $other_masters_ip         = undef,		# [ '192.168.101.3', '192.168.101.4' ]
  $vip_hostname             = undef,
  $vip_ipaddr               = undef,
  $vip_netmask              = undef,
  $self_signed_api_cert     = true,
  $swiftproxy_hostname      = 'none',
  $network_ipaddr           = undef,
  $network_cidr             = undef,
  $all_masters              = undef,
  $all_masters_ip           = undef,
  $all_masters_ids          = undef,

  $has_subrole_db           = true,

  $first_sql                = undef,
  $first_sql_ip             = undef,
  $is_first_sql             = undef,
  $sql_vip_ip               = undef,
  $sql_vip_netmask          = undef,
  $sql_vip_iface            = undef,

  $initial_cluster_setup    = false,

  $amp_secgroup_list        = undef,
  $amp_boot_network_list    = undef,

  $all_sql                  = undef,
  $all_sql_ip               = undef,
  $all_sql_ids              = undef,
  $non_master_sql           = undef,
  $non_master_sql_ip        = undef,

  $has_subrole_messaging    = true,
  $has_subrole_api_keystone = true,
  $has_subrole_heat         = true,
  $has_subrole_glance       = false,
  $has_subrole_nova         = false,
  $has_subrole_neutron      = false,
  $has_subrole_swift        = true,
  $has_subrole_horizon      = true,
  $has_subrole_barbican     = true,
  $has_subrole_cinder       = false,
  $has_subrole_gnocchi      = false,
  $has_subrole_gnocchi_services = true,
  $has_subrole_ceilometer   = false,
  $has_subrole_panko        = false,
  $has_subrole_cloudkitty   = false,
  $has_subrole_aodh         = false,
  $has_subrole_octavia      = false,
  $has_subrole_magnum       = false,
  $has_subrole_manila       = false,
  $has_subrole_designate    = false,
  $has_subrole_ironic       = false,

  $designate_pools_yaml     = undef,
  $dns_nodes                = [],
  $dns_nodes_ip             = [],

  $has_subrole_network_node = true,

  $glance_backend           = undef,

  $cinder_backup_backend    = 'none',

  $cinder_default_storage_availability_zone = 'nova',
  $cinder_default_volume_type       = 'LVM_1',

  $haproxy_custom_url       = undef,
  $haproxy_maintenance      = false,
  $use_ssl                  = true,
  $rabbit_env               = {},
  $all_swiftproxy           = undef,
  $all_swiftproxy_ip        = undef,
  $pass_haproxy_stats       = undef,
  $pass_mysql_rootuser      = undef,
  $pass_mysql_backup        = undef,
  $pass_rabbitmq_cookie     = undef,
  $pass_rabbitmq_monitoring = undef,
  $pass_keystone_db         = undef,
  $pass_keystone_messaging  = undef,
  $pass_keystone_adminuser  = undef,
  $pass_keystone_credkey1   = undef,
  $pass_keystone_credkey2   = undef,
  $pass_keystone_fernkey1   = undef,
  $pass_keystone_fernkey2   = undef,
  $pass_keystone_ssh_pub    = undef,
  $pass_keystone_ssh_priv   = undef,
  $pass_nova_db             = undef,
  $pass_nova_apidb          = undef,
  $pass_nova_messaging      = undef,
  $pass_nova_authtoken      = undef,
  $pass_metadata_proxy_shared_secret = undef,
  $pass_placement_db        = undef,
  $pass_placement_authtoken = undef,
  $pass_glance_db           = undef,
  $pass_glance_messaging    = undef,
  $pass_glance_authtoken    = undef,
  $pass_glance_ssh_pub      = undef,
  $pass_glance_ssh_priv     = undef,
  $pass_cinder_db           = undef,
  $pass_cinder_messaging    = undef,
  $pass_cinder_authtoken    = undef,
  $pass_neutron_db          = undef,
  $pass_neutron_messaging   = undef,
  $pass_neutron_authtoken   = undef,
  $pass_neutron_vrrpauth    = undef,
  $pass_heat_encryptkey     = undef,
  $pass_heat_db             = undef,
  $pass_heat_messaging      = undef,
  $pass_heat_authtoken      = undef,
  $pass_heat_keystone_domain = undef,
  $pass_swift_authtoken     = undef,
  $pass_swift_encryption    = undef,
  $pass_horizon_secretkey   = undef,
  $pass_barbican_db         = undef,
  $pass_barbican_messaging  = undef,
  $pass_barbican_authtoken  = undef,
  $pass_gnocchi_db          = undef,
  $pass_gnocchi_messaging   = undef,
  $pass_gnocchi_authtoken   = undef,
  $pass_gnocchi_rscuuid     = undef,
  $pass_panko_db            = undef,
  $pass_panko_messaging     = undef,
  $pass_panko_authtoken     = undef,
  $pass_ceilometer_db        = undef,
  $pass_ceilometer_messaging = undef,
  $pass_ceilometer_authtoken = undef,
  $pass_ceilometer_telemetry = undef,
  $pass_cloudkitty_db       = undef,
  $pass_cloudkitty_messaging = undef,
  $pass_cloudkitty_authtoken = undef,
  $pass_redis               = undef,
  $pass_aodh_db             = undef,
  $pass_aodh_messaging      = undef,
  $pass_aodh_authtoken      = undef,
  $pass_octavia_db          = undef,
  $pass_octavia_messaging   = undef,
  $pass_octavia_authtoken   = undef,
  $pass_octavia_heatbeatkey = undef,
  $pass_magnum_db           = undef,
  $pass_magnum_messaging    = undef,
  $pass_magnum_authtoken    = undef,
  $pass_magnum_domain       = undef,
  $pass_manila_db           = undef,
  $pass_manila_messaging    = undef,
  $pass_manila_authtoken    = undef,
  $pass_designate_db        = undef,
  $pass_designate_messaging = undef,
  $pass_designate_authtoken = undef,
  $pass_designate_rndckey   = undef, # hmac-sha256 rndc key
  $pass_ironic_db           = undef,
  $pass_ironic_messaging    = undef,
  $pass_ironic_authtoken    = undef,
  $pass_ironic_inspector_db = undef,
  $pass_ironic_inspector_messaging = undef,
  $pass_ironic_inspector_authtoken = undef,
  $pass_swift_messaging     = undef,
  $pass_zabbix_authtoken    = undef,

  $cluster_has_cinder_volumes = false,

  $cluster_has_osds         = false,
  $cluster_has_mons         = false,

  $ceph_fsid                = undef,
  $ceph_bootstrap_osd_key   = undef,
  $ceph_admin_key           = undef,
  $ceph_openstack_key       = undef,
  $ceph_mon_key             = undef,
  $ceph_mon_initial_members = undef,
  $ceph_mon_host            = undef,
  $ceph_mgr_key             = undef,
  $ceph_libvirtuuid         = undef,

  $disable_notifications    = false,
  $enable_monitoring_graphs = false,
  $monitoring_graphite_host = undef,
  $monitoring_graphite_port = '2003',

  $extswift_use_external        = false,
  $extswift_auth_url            = undef,
  $extswift_region              = undef,
  $extswift_proxy_url           = undef,
  $extswift_project_name        = undef,
  $extswift_project_domain_name = undef,
  $extswift_user_name           = undef,
  $extswift_user_domain_name    = undef,
  $extswift_password            = undef,
  $cinder_backup_use_external_swift = no,

  $use_gpu                  = false,
  $gpu_def                  = undef,

  $bgp_to_the_host          = false,
  $use_dynamic_routing      = false,
  $dhcp_domain              = '',

  $gnocchi_resources_yaml   = undef,
  $cloudkitty_metrics_yml   = undef,

  $no_api_rate_limit_networks = [],

  # Ceilometer dynamic pollsters definitions
  # located in /etc/openstack-cluster-installer/pollsters.d
  $dynamic_pollsters_files  = undef,
  $dynamic_pollsters_names  = [],
  $dynamic_pollsters_data   = undef,

  # Zookeeper ensemble
  $zookeeper_ensemble_string = join($all_masters_ip,':2181,'),

  $nameserver_v4_prim        = '9.9.9.9',
  $nameserver_v4_sec         = '149.112.112.112',
  $nameserver_v6_prim        = '2620:fe::fe:9',
  $nameserver_v6_sec         = '2620:fe::9',

  # These are from variables.json
  $nova_limit_tenants_to_placement_aggregate = false,
  $nova_scheduler_prefers_hosts_with_more_ram = true,
  $neutron_dns_domain       = 'example.com.',
  $admin_email_address      = 'admin@example.com',
  $glance_prefer_ceph_over_swift = true,
  $install_magnum           = false,
  $install_designate        = false,
  $install_panko            = false,
  $install_manila           = false,
  $install_octavia          = true,
  $install_telemetry        = true,
  $install_ironic           = false,
  $install_cloudkitty_dashboard = true,
  $neutron_use_dvr          = true,
  $api_disable_rate_limit   = false,
  $api_iptables_rate_limit  = 100,
  $api_haproxy_rate_limit   = 20,
  $api_rate_limit_whitelist = undef,
  $nova_metadata_cache_expiration = 60,
  $glance_metadef_admin_only = true,

  $aodh_notifier_use_proxy   = false,
  $aodh_notifier_proxy_url   = undef,

  $manila_network_cidr          = '10.240.0.0/12',
  $manila_network_division_mask = 28,

  $horizon_themes            = 'default@Default@themes/default:material@Material@themes/material',
  $horizon_limitreqbody      = 107374182400,

  $kernel_from_backports    = false,
  $brex_macaddr             = undef,

  $neutron_vxlan_vni_min     = '1000',
  $neutron_vxlan_vni_max     = '9999',

  $haproxy_timeout_http_request = '10s',
  $haproxy_timeout_queue        = '1m',
  $haproxy_timeout_connect      = '10s',
  $haproxy_timeout_client       = '1m',
  $haproxy_timeout_server       = '1m',
  $haproxy_timeout_check        = '10s',

  $designate_zone_admin_email = 'hostmaster@example.com',
  $designate_managed_resource_tenant_id = '123456',
  $controller_node_memcache_max_connections = 32768,
){

  # Generic performances tweak (will default to throughput-performance)
  # class { '::oci::tuned': }

  if $use_ssl {
    $proto = 'https'
    $messaging_default_port = '5671'
    $messaging_notify_port = '5671'
    $api_port = 443
  } else {
    $proto = 'http'
    $messaging_default_port = '5672'
    $messaging_notify_port = '5672'
    $api_port = 80
  }
  $messaging_default_proto = 'rabbit'
  $messaging_notify_proto = 'rabbit'

  if $has_subrole_db {
    $sql_host = $vip_ipaddr
  }else{
    $sql_host = $sql_vip_ip
  }

  if $messaging_nodes_for_notifs {
    $notif_rabbit_servers = $all_rabbits
  }else{
    $notif_rabbit_servers = $all_masters
  }

  $base_url = "${proto}://${vip_hostname}"
  if $self_signed_api_cert {
    $api_endpoint_ca_file = "/etc/ssl/certs/oci-pki-oci-ca-chain.pem"
  }else{
    $api_endpoint_ca_file = ''
  }
  $oci_pki_root_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'

  class { '::oci::puppet_oci_ca_cert':
    self_signed_api_cert => $self_signed_api_cert,
  }

  $keystone_auth_uri  = "${base_url}:${api_port}/identity"
  $keystone_admin_uri = "${base_url}:${api_port}/identity"
  #$memcached_servers  = ["127.0.0.1:11211"]
  $memcached_string = join([join($all_masters_ip,':11211,'), ':11211'],'')
  $memcached_servers  = ["${memcached_string}"]

    ensure_resource('file', '/root/oci-openrc', {
      'ensure'  => 'present',
      'content' => "
export OS_AUTH_TYPE=password
export OS_PROJECT_DOMAIN_NAME='default'
export OS_USER_DOMAIN_NAME='default'
export OS_PROJECT_NAME='admin'
export OS_USERNAME='admin'
export OS_PASSWORD='${pass_keystone_adminuser}'
export OS_AUTH_URL='https://${vip_hostname}/identity/v3'
export OS_REGION_NAME=${region_name}
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
export OS_CACERT=${$api_endpoint_ca_file}
",
      'mode'    => '0640',
    })

    ensure_resource('file', '/root/octavia-openrc', {
      'ensure'  => 'present',
      'content' => "
export OS_AUTH_TYPE=password
export OS_PROJECT_DOMAIN_NAME='default'
export OS_USER_DOMAIN_NAME='default'
export OS_PROJECT_NAME='services'
export OS_USERNAME='octavia'
export OS_PASSWORD='${pass_octavia_authtoken}'
export OS_AUTH_URL='https://${vip_hostname}/identity/v3'
export OS_REGION_NAME=${region_name}
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
export OS_CACERT=${$api_endpoint_ca_file}
",
      'mode'    => '0640',
    })

    ensure_resource('file', '/root/swift-openrc', {
      'ensure'  => 'present',
      'content' => "
export OS_AUTH_TYPE=password
export OS_PROJECT_DOMAIN_NAME='default'
export OS_USER_DOMAIN_NAME='default'
export OS_PROJECT_NAME='services'
export OS_USERNAME='swift'
export OS_PASSWORD='${pass_swift_authtoken}'
export OS_AUTH_URL='https://${vip_hostname}/identity/v3'
export OS_REGION_NAME=${region_name}
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
export OS_CACERT=${$api_endpoint_ca_file}
",
      'mode'    => '0640',
    })

    ensure_resource('file', '/root/manila-openrc', {
      'ensure'  => 'present',
      'content' => "
export OS_AUTH_TYPE=password
export OS_PROJECT_DOMAIN_NAME='default'
export OS_USER_DOMAIN_NAME='default'
export OS_PROJECT_NAME='services'
export OS_USERNAME='manila'
export OS_PASSWORD='${pass_manila_authtoken}'
export OS_AUTH_URL='https://${vip_hostname}/identity/v3'
export OS_REGION_NAME=${region_name}
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
export OS_CACERT=${$api_endpoint_ca_file}
",
      'mode'    => '0640',
    })

    ensure_resource('file', '/root/zabbix-openrc', {
      'ensure'  => 'present',
      'content' => "
export OS_AUTH_TYPE=password
export OS_PROJECT_DOMAIN_NAME='Default'
export OS_USER_DOMAIN_NAME='Default'
export OS_PROJECT_NAME='zabbix'
export OS_USERNAME='zabbix'
export OS_PASSWORD='${pass_zabbix_authtoken}'
export OS_AUTH_URL='https://${vip_hostname}/identity/v3'
export OS_REGION_NAME=${region_name}
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
export OS_CACERT=${$api_endpoint_ca_file}
",
      'mode'    => '0640',
    })

    ensure_resource('group', 'zabbix', {
      'ensure' => 'present',
      'gid'    => '966',
    })

    ensure_resource('file', '/etc/oci/monitoring_openstack_api_password', {
      'ensure'  => 'present',
      'content' => "${pass_zabbix_authtoken}",
      'group'   => 'zabbix',
      'mode'    => '0640',
    })

    ensure_resource('file', '/etc/oci/monitoring_openstack_api_url', {
      'ensure'  => 'present',
      'content' => "https://${vip_hostname}/identity/v3",
      'group'   => 'zabbix',
      'mode'    => '0640',
    })

  if $vip_netmask == 32 {
    $vip_iface = 'lo'
  } else {
    $vip_iface = $sql_vip_iface
  }

  $haproxy_api_backend_options = "check check-ssl ssl verify required ca-file ${oci_pki_root_ca_file}"
  $haproxy_api_backend_options_nossl = "check"
  $haproxy_api_backend_options_noverify = 'check check-ssl ssl verify none'

  if $is_first_master and $initial_cluster_setup{
    $do_db_sync = true
  }else{
    $do_db_sync = false
  }

  if $disable_notifications {
    $notification_driver = 'noop'
  }else{
    $notification_driver = 'messagingv2'
  }

  if $facts['os']['lsb'] != undef{
    $mycodename = $facts['os']['lsb']['distcodename']
  }else{
    $mycodename = $facts['os']['distro']['codename']
  }

  ##########################################################################
  ### Add a few package so that installing the Octavia service is easier ###
  ##########################################################################
  package { 'openstack-pkg-tools':
    ensure => present,
  }

  package { 'openstack-debian-images':
    ensure => present,
  }

  ################################
  ### Add a virtual IP address ###
  ################################
  # Setup corosync
  if ($openstack_release == 'rocky' or $openstack_release == 'stein') {
    class { 'corosync':
      authkey                  => '/var/lib/puppet/ssl/certs/ca.pem',
      bind_address             => $machine_ip,
      unicast_addresses        => $all_masters_ip,
      cluster_name             => 'mycluster',
      enable_secauth           => true,
      set_votequorum           => true,
      quorum_members           => $all_masters_ip,
      quorum_members_ids       => $all_masters_ids,
      log_stderr               => false,
      log_function_name        => true,
      syslog_priority          => 'debug',
      debug                    => true,
    }
    corosync::service { 'pacemaker':
      version => '0',
    }->
    cs_property { 'stonith-enabled':
      value   => 'false',
    }->
    cs_property { 'no-quorum-policy':
      value   => 'stop',
    }
  } else {
    class { 'corosync':
      authkey                  => '/var/lib/puppet/ssl/certs/ca.pem',
      bind_address             => $machine_ip,
      cluster_name             => 'mycluster',
      enable_secauth           => true,
      set_votequorum           => true,
      quorum_members           => $all_masters_ip,
      quorum_members_ids       => $all_masters_ids,
      log_stderr               => false,
      log_function_name        => true,
      syslog_priority          => 'debug',
      debug                    => true,
      enable_totem_interface   => false,
    }
    corosync::service { 'pacemaker':
      version => '0',
    }->
    cs_property { 'stonith-enabled':
      value   => 'false',
    }->
    cs_property { 'no-quorum-policy':
      value   => 'stop',
    }
  }
  if $initial_cluster_setup {
    cs_primitive { 'openstack-api-vip':
      primitive_class => 'ocf',
      primitive_type  => 'IPaddr2',
      provided_by     => 'heartbeat',
      parameters      => { 'ip' => $vip_ipaddr, 'cidr_netmask' => $vip_netmask, 'nic' => $vip_iface },
      operations      => { 'monitor' => { 'interval' => '10s' } },
      before          => Anchor['keystone::service::begin'],
      require         => Cs_property['no-quorum-policy'],
    }
  }
  # Eventually, a VIP for MariaDB
#  if $has_subrole_db {
#    oci::vip {'openstack-sql-vip':
#      vip_ip      => $sql_vip_ip,
#      vip_netmask => $sql_vip_netmask,
#      vip_iface   => $vip_sql_iface,
#    }
#  }

  #########################
  ### Setup firewalling ###
  #########################
  # Note: This only closes ports on the VIP, as everything else
  # is not accessible from the outside anyway.

  # Drop all services on public IP address, just let https APIs

  # First, general definirion (could be in global site.pp)
  resources { "firewall":
    purge   => false
  }
  class { 'firewall': }

  $whitelist_all_ports = concat(['127.0.0.0/8'],$all_masters_ip)

  $whitelist_all_ports.each |Integer $index, String $value| {
    $val1 = $index+80
    firewall { "${val1} Allow ${value} to access everything":
      proto       => tcp,
      action      => accept,
      source      => "${value}",
    }
  }

  if $volume_nodes_ip {
    $all_sql_allowed_ips = concat($compute_nodes_ip, $volume_nodes_ip)
  }else{
    $all_sql_allowed_ips = $compute_nodes_ip
  }
  $all_sql_allowed_ips2 = concat($all_masters_ip,$all_sql_allowed_ips)
  $all_sql_allowed_ips3 = concat(['127.0.0.0/8'],$all_sql_allowed_ips2)
  $all_sql_allowed_ips4 = concat(["${vip_ipaddr}/${vip_netmask}"],$all_sql_allowed_ips3)
  $all_sql_allowed_ips5 = concat($all_rabbits_ips,$all_sql_allowed_ips4)

  $all_sql_allowed_ips5.each |Integer $index, String $value| {
    $val1 = $index+100
    firewall { "${val1} Allow ${value} to access MySQL VIP":
      proto       => tcp,
      action      => accept,
      source      => "${value}",
      dport       => [3306, 4567],
    }
  }

  $api_rate_limit_whitelist2 = split($api_rate_limit_whitelist, ' ')
  $net_whitelist2 = concat($no_api_rate_limit_networks, $vip_ipaddr)
  $net_whitelist = concat($net_whitelist2, $api_rate_limit_whitelist2)
  if $api_disable_rate_limit {
    firewall { "791 rate limit to ${api_iptables_rate_limit} queries/s marked new":
      ensure          => absent,
      chain           => 'INPUT',
      dport           => 443,
      proto           => tcp,
      connlimit_above => $api_iptables_rate_limit,
      connlimit_mask  => 24,
      jump            => 'LOGDROP',
    }

    # Add a LOGDROP chain
    firewallchain { 'LOGDROP:filter:IPv4':
      ensure  => present,
    }->
    # LOG dropped packets
    firewall { '901 LOG rule for dropped packets':
      chain       => 'LOGDROP',
      proto       => tcp,
      jump        => 'LOG',
      log_level   => '6',
      log_prefix  => 'IPTables API rate limit:',
      limit       => '1/sec',
    }
  } else {

    $net_whitelist.each |Integer $index, String $value| {
      $val1 = $index+700
      firewall { "${val1} Allow ${value} to access OpenStack API without rate limit":
        proto       => tcp,
        action      => accept,
        source      => "${value}",
        dport       => [443],
      }
    }

    # Add a LOGDROP chain
    firewallchain { 'LOGDROP:filter:IPv4':
      ensure  => present,
    }->
    # LOG dropped packets
    firewall { '901 LOG rule for dropped packets':
      chain       => 'LOGDROP',
      proto       => tcp,
      jump        => 'LOG',
      log_level   => '6',
      log_prefix  => 'IPTables API rate limit:',
      limit       => '1/sec',
    }->
    # DROP packets
    firewall { "902 Deny access to OpenStack API":
      chain       => 'LOGDROP',
      proto       => tcp,
      action      => drop,
      dport       => '443',
    }->
    firewall { "903 Deny access to OpenStack HTTP":
      chain       => 'LOGDROP',
      proto       => tcp,
      action      => drop,
      dport       => '443',
    }->
    firewall { "904 Deny access to MySQL":
      chain       => 'LOGDROP',
      proto       => tcp,
      action      => drop,
      dport       => [3306, 4567],
    }->
    firewall { "905 Deny access to OpenStack FRR":
      chain       => 'LOGDROP',
      proto       => tcp,
      action      => drop,
      dport       => '179',
    }->
    firewall { "906 Deny access to OpenStack rabbit":
      chain       => 'LOGDROP',
      proto       => tcp,
      action      => drop,
      dport       => [4369, 5671, 5672, 15671, 25672],
    }->
    firewall { "907 Deny access to SMTP":
      chain       => 'LOGDROP',
      proto       => tcp,
      action      => drop,
      dport       => [25],
    }->
    firewall { "908 Deny access to RPCBIND":
      chain       => 'LOGDROP',
      proto       => tcp,
      action      => drop,
      dport       => [111],
    }->
    firewall { "909 Deny access to RPCBIND UDP":
      chain       => 'LOGDROP',
      proto       => udp,
      action      => reject,
      dport       => [111],
    }->
    firewall { "910 Deny access to xinetd":
      chain       => 'LOGDROP',
      proto       => tcp,
      action      => drop,
      dport       => [9200],
    }->
    firewall { "911 Deny access to Horizon without proxy":
      chain       => 'LOGDROP',
      proto       => tcp,
      action      => drop,
      dport       => [7080, 7443],
    }->
    firewall { "911 Deny access to half 1 of OpenStack services without proxy":
      chain       => 'LOGDROP',
      proto       => tcp,
      action      => drop,
      dport       => [5000, 6083, 8000, 8004, 8042, 8774, 8775, 8778,],
    }->
    firewall { "912 Deny access to half 2 of OpenStack services without proxy":
      chain       => 'LOGDROP',
      proto       => tcp,
      action      => drop,
      dport       => [8786, 9001, 9311, 9292, 9511, 9696, 9876, 10050],
    }

    firewall { "791 rate limit to ${api_iptables_rate_limit} queries/s marked new":
      chain           => 'INPUT',
      dport           => 443,
      proto           => tcp,
      connlimit_above => $api_iptables_rate_limit,
      connlimit_mask  => 24,
      jump            => 'LOGDROP',
    }
  }

  # Define firewall rules
  firewall { '801 deny public access to http':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => 80,
  }->
  firewall { '802 deny public access to mysql':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [3306, 4567],
  }->
  firewall { '803 deny public access to bgpd':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => 179,
  }->
  firewall { '804 deny public access to epmd and rabbitmq':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [4369, 5671, 5672, 15671, 25672],
  }->
  firewall { '805 Jump to LOGDROP for SMTP':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => '25',
  }->
  firewall { '806 deny public access to rpcbind':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [111],
  }->
  firewall { '807 deny public access to xinetd':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [9200],
  }->
  firewall { '808 deny public access to horizon without haproxy':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [7080, 7443],
  }->
  firewall { '809 deny public access to octavia API without haproxy':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/32",
    dport       => [9876],
  }->
  firewall { '810 deny public access to magnum API without haproxy':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [9511],
  }->
  firewall { '811 deny public access to designate API without haproxy':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [9001],
  }->
  firewall { '812 deny public access to keystone API without haproxy':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [5000],
  }->
  firewall { '813 deny public access to SMTP':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [25],
  }->
  firewall { '814 deny public access to Heat CFN API without haproxy':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [8000],
  }->
  firewall { '815 deny public access to Aodh API without haproxy':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [8042],
  }->
  firewall { '816 deny public access to manila API without haproxy':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [8786],
  }->
  firewall { '817 deny public access to Ironic API without haproxy':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [6385],
  }->
  firewall { '818 deny public access to Ironic Inspector API without haproxy':
    proto       => tcp,
    jump        => 'LOGDROP',
    destination => "${vip_ipaddr}/32",
    dport       => [5050],
  }

  ####################################
  ### Setup a Zookeeper in cluster ###
  ####################################
  # needed for Telemetry / metric / rating
  if $has_subrole_gnocchi or $has_subrole_designate or $has_subrole_cloudkitty{
    $all_masters_ip.each |Integer $index, String $value| {
      if($machine_ip == $value){
        class { 'zookeeper':
          id                      => String($index + 1),
          client_ip               => $machine_ip,
          servers                 => $all_masters_ip,
          purge_interval          => 6,
          max_allowed_connections => 600,
        }
      }
    }
  }

  # Add haproxy that will listen on the virtual IPs, and load balance
  # to the different API daemons using tcp mode (as the APIs will do
  # full SSL already).
  if $use_ssl {
    file { "/etc/haproxy/ssl":
      ensure                  => directory,
      owner                   => 'root',
      mode                    => '0755',
      selinux_ignore_defaults => true,
      require       => Package['haproxy'],
    }->
    file { "/etc/haproxy/ssl/private":
      ensure                  => directory,
      owner                   => 'root',
      mode                    => '0755',
      selinux_ignore_defaults => true,
    }->
    file { "/etc/haproxy/ssl/private/oci-pki-api.pem":
      ensure                  => present,
      owner                   => "haproxy",
      source                  => "/etc/ssl/private/oci-pki-api.pem",
      selinux_ignore_defaults => true,
      mode                    => '0600',
      notify                  => Service['haproxy'],
    }
  }
  $haproxy_schedule1 = ['heat::db::begin', 'barbican::db::begin']
  if $has_subrole_nova {
    if $openstack_release == 'rocky' {
      $haproxy_schedule2 = concat($haproxy_schedule1, ['nova::db::begin', 'neutron::db::begin'])
    }else{
      $haproxy_schedule2 = concat($haproxy_schedule1, ['nova::db::begin', 'placement::db::begin', 'neutron::db::begin'])
    }
  } else {
    $haproxy_schedule2 = $haproxy_schedule1
  }
  if $has_subrole_cinder {
    $haproxy_schedule3 = concat($haproxy_schedule2, ['cinder::db::begin'])
  } else {
    $haproxy_schedule3 = $haproxy_schedule2
  }
  if $has_subrole_glance {
    $haproxy_schedule4 = concat($haproxy_schedule3, ['glance::db::begin'])
  } else {
    $haproxy_schedule4 = $haproxy_schedule3
  }
  if $has_subrole_aodh and $install_telemetry{
    $haproxy_schedule5 = concat($haproxy_schedule4, ['aodh::db::begin'])
  } else {
    $haproxy_schedule5 = $haproxy_schedule4
  }
  if $has_subrole_octavia and $install_octavia{
    $haproxy_schedule6 = concat($haproxy_schedule5, ['octavia::db::begin'])
  } else {
    $haproxy_schedule6 = $haproxy_schedule5
  }
  if $has_subrole_panko and $install_panko and $install_telemetry {
    $haproxy_schedule7 = concat($haproxy_schedule6, ['panko::db::begin'])
  } else {
    $haproxy_schedule7 = $haproxy_schedule6
  }
  if $has_subrole_gnocchi and $install_telemetry{
    $haproxy_schedule8 = concat($haproxy_schedule7, ['gnocchi::db::begin'])
  } else {
    $haproxy_schedule8 = $haproxy_schedule7
  }
  if $has_subrole_cloudkitty and $install_telemetry{
    $haproxy_schedule9 = concat($haproxy_schedule8, ['cloudkitty::db::begin'])
  } else {
    $haproxy_schedule9 = $haproxy_schedule8
  }
  if $has_subrole_magnum and $install_magnum {
    $haproxy_schedule10 = concat($haproxy_schedule9, ['magnum::db::begin'])
  } else {
    $haproxy_schedule10 = $haproxy_schedule9
  }
  if $has_subrole_designate and $install_designate {
    $haproxy_schedule11 = concat($haproxy_schedule10, ['designate::db::begin'])
  } else {
    $haproxy_schedule11 = $haproxy_schedule10
  }
  if $has_subrole_manila and $install_manila {
    $haproxy_schedule12 = concat($haproxy_schedule11, ['manila::db::begin'])
  } else {
    $haproxy_schedule12 = $haproxy_schedule11
  }
  if $has_subrole_ironic and $install_ironic {
    $haproxy_schedule13 = concat($haproxy_schedule12, ['ironic::db::begin'])
  } else {
    $haproxy_schedule13 = $haproxy_schedule12
  }

  $haproxy_schedule = $haproxy_schedule13

  # Here, we assume that stretch & buster don't have
  # a version of haproxy >= 2.1.
  if $mycodename == 'stretch' or $mycodename == 'buster' {
    # This used to work with haproxy 1.8,
    # let's see if we can get rid of it
    $haproxy_replace_path = 'reqrep'
    $haproxy_replace_head = 'rspirep'
    $use_haproxy_21_format = false
  }else{
    # This works with haproxy >= 2.1
    $haproxy_replace_path  = 'http-request replace-path'
    $haproxy_replace_head  = 'http-response replace-header'
    $use_haproxy_21_format = true
  }

  class { 'haproxy':
    restart_command => 'systemctl reload haproxy',
    global_options   => {
      'log'     => '/dev/log local0',
      'chroot'  => '/var/lib/haproxy',
      'pidfile' => '/var/run/haproxy.pid',
      'maxconn' => '40960',
      'user'    => 'haproxy',
      'group'   => 'haproxy',
      'stats'   => [
        'socket /var/lib/haproxy/stats',
        'socket /var/lib/haproxy/admin.sock mode 600 level admin',
        'timeout 30s'
      ],
      'daemon'   => '',
      'nbthread' => '8',
      'ssl-default-bind-ciphers'   => 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256',
      'ssl-default-bind-options'   => 'no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets',
      'ssl-default-server-ciphers' => 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256',
      'ssl-default-server-options' => 'no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets',
    },
    defaults_options => {
      'log'     => 'global',
      'mode'      => 'http',
      'option'    => [
          'httplog',
        ],
      'retries' => '3',
      'timeout'   => [
          "http-request ${haproxy_timeout_http_request}",
          "queue ${haproxy_timeout_queue}",
          "connect ${haproxy_timeout_connect}",
          "client ${haproxy_timeout_client}",
          "server ${haproxy_timeout_server}",
          "check ${haproxy_timeout_check}",
        ],
      'maxconn' => '8000',
      'monitor-uri' => '/health'
    },
    merge_options => false,
    before        => Anchor[$haproxy_schedule],
    require       => Sysctl::Value['net.ipv4.ip_nonlocal_bind'],
  }->
  haproxy::listen { 'hapstats':
    section_name => 'statshttp',
    bind         => { "${machine_ip}:8088" => 'user root'},
    mode         => 'http',
    options      => {
        'stats' => [ 'uri /', "auth admin:${pass_haproxy_stats}", 'refresh 15', 'admin if TRUE', 'realm Haproxy Statistics', ],
      },
  }->
  haproxy::frontend { 'health_check':
    section_name => 'healthcheck',
    bind         => { "${machine_ip}:8081" => 'user root'},
    mode         => 'http',
    options      => {
      timeout      => 'client 5s',
      monitor-uri  => '/',
    }
  }
#  haproxy::listen { 'haphealthcheck':
#    section_name => 'healthcheck',
#    bind         => { "${machine_ip}:8081" => 'user root'},
#    mode         => 'health',
#    options => [
#       { 'option' => 'httpchk' },
#       { 'option' => 'http-keep-alive' },
#    ]
#  }

  # Fix haproxy log to keep 7 days of logs, instead of
  # the default which is 52.
  logrotate::rule { 'haproxy':
    path          => '/var/log/haproxy.log',
    rotate        => '7',
    rotate_every  => 'day',
    missingok     => true,
    compress      => true,
    delaycompress => true,
    postrotate    => '/usr/lib/rsyslog/rsyslog-rotate',
  }


  # If SQL is on the controllers, define front-end and back-end for SQL
  if $has_subrole_db {
    haproxy::frontend { 'galerafe':
      mode      => 'tcp',
      bind      => { "${vip_ipaddr}:3306" => [] },
      options   => [
        { 'timeout'         => 'client 3600s'},
        { 'default_backend' => 'galerabe'},
      ],
      before => Anchor[$haproxy_schedule],
    }
    haproxy::backend { 'galerabe':
      options => [
        { 'mode'    => 'tcp' },
        { 'balance' => 'roundrobin' },
        { 'timeout' => 'check 5000' },
        { 'timeout' => 'server 3600s' },
        { 'option'  => 'httpchk'},
      ],
      before => Anchor[$haproxy_schedule],
    }
    haproxy::balancermember { 'galerabm':
      listening_service => 'galerabe',
      ipaddresses       => $first_sql_ip,
      server_names      => $first_sql,
      options           => 'check inter 4000 port 9200 fall 3 rise 5',
      before => Anchor[$haproxy_schedule],
    }
    haproxy::balancermember { 'galerabmback':
      listening_service => 'galerabe',
      ipaddresses       => $non_master_sql_ip,
      server_names      => $non_master_sql,
      options           => 'check inter 4000 port 9200 fall 3 rise 5 backup',
      before => Anchor[$haproxy_schedule],
    }
  }

  $haproxy_options_keystone = [
    # Set HSTS (63072000 seconds = 2 years)
    { 'http-response' => 'set-header Strict-Transport-Security max-age=63072000'},
    # So that we log real customer's IPs in our logs
    { 'option'        => "forwardfor except $vip_ipaddr"},
    { 'acl'           => 'url_keystoneadmin path_beg -i /identity-admin'},
    { 'use_backend'   => 'keystonepubbe if url_keystoneadmin'},
    { 'acl'           => 'url_keystone path_beg -i /identity'},
    { 'use_backend'   => 'keystonepubbe if url_keystone'},
  ]

  $glance_haproxy_options = [
    { 'acl'         => 'url_glance path_beg -i /image'},
    { 'use_backend' => 'glancebe if url_glance'},
  ]

  $swift_haproxy_options = [
    { 'acl'         => 'url_swift path_beg -i /object'},
    { 'use_backend' => 'swiftbe if url_swift'},
  ]

  $heat_haproxy_options = [
    { 'acl'         => 'url_heatcfn path_beg -i /orchestration-cfn'},
    { 'use_backend' => 'heatcfnbe if url_heatcfn'},
    { 'acl'         => 'url_heat path_beg -i /orchestration-api'},
    { 'use_backend' => 'heatbe if url_heat'},
  ]

  $horizon_haproxy_options = [
    { 'acl'         => 'url_horizon path_beg -i /horizon'},
    { 'use_backend' => 'horizonbe if url_horizon'},
  ]

  $barbican_haproxy_options = [
    { 'acl'         => 'url_barbican path_beg -i /keymanager'},
    { 'use_backend' => 'barbicanbe if url_barbican'},
  ]

  $nova_haproxy_options = [
    { 'acl'         => 'url_nova path_beg -i /compute'},
    { 'use_backend' => 'novabe if url_nova'},
    { 'acl'         => 'url_placement path_beg -i /placement'},
    { 'use_backend' => 'placementbe if url_placement'},
    { 'acl'         => 'url_novnc path_beg -i /novnc'},
    { 'use_backend' => 'novncbe if url_novnc'},
    { 'acl'         => 'url_websockify path_beg -i /websockify'},
    { 'acl'         => 'url_websockify_token url_beg -i /?token'},
    { 'use_backend' => 'websockifybe if url_websockify or url_websockify_token'},
  ]

  $neutron_haproxy_options = [
    { 'acl'         => 'url_neutron path_beg -i /network'},
    { 'use_backend' => 'neutronbe if url_neutron'},
  ]

  $cinder_haproxy_options = [
    { 'acl'         => 'url_cinder path_beg -i /volume'},
    { 'use_backend' => 'cinderbe if url_cinder'},
  ]

  $gnocchi_haproxy_options = [
    { 'acl'         => 'url_gnocchi path_beg -i /metric'},
    { 'acl'         => 'url_gnocchi path_beg -i /v1/metric'},
    { 'use_backend' => 'gnocchibe if url_gnocchi'},
  ]

  $cloudkitty_haproxy_options = [
    { 'acl'         => 'url_cloudkitty path_beg -i /rating'},
    { 'use_backend' => 'cloudkittybe if url_cloudkitty'},
  ]

  $aodh_haproxy_options = [
    { 'acl'         => 'url_aodh path_beg -i /alarm'},
    { 'use_backend' => 'aodhbe if url_aodh'},
  ]

  $octavia_haproxy_options = [
    { 'acl'         => 'url_octavia path_beg -i /loadbalance'},
    { 'use_backend' => 'octaviabe if url_octavia'},
  ]

  $magnum_haproxy_options = [
    { 'acl'         => 'url_magnum path_beg -i /containers'},
    { 'use_backend' => 'magnumbe if url_magnum'},
    { 'acl'         => 'url_etcd_cluster path_beg -i /etcd-cluster'},
    { 'use_backend' => 'etcdclusterbe if url_etcd_cluster'},
    { 'acl'         => 'url_etcd_discovery path_beg -i /etcd-discovery'},
    { 'use_backend' => 'etcddiscoverybe if url_etcd_discovery'},
  ]

  $panko_haproxy_options = [
    { 'acl'         => 'url_panko path_beg -i /event'},
    { 'use_backend' => 'pankobe if url_panko'},
  ]

  $designate_haproxy_options = [
    { 'acl'         => 'url_designate path_beg -i /dns'},
    { 'use_backend' => 'designatebe if url_designate'},
  ]

  $manila_haproxy_options = [
    { 'acl'         => 'url_manila path_beg -i /share'},
    { 'use_backend' => 'manilabe if url_manila'},
  ]

  $ironic_haproxy_options = [
    { 'acl'         => 'url_ironic path_beg -i /baremetal'},
    { 'use_backend' => 'ironicbe if url_ironic'},
    { 'acl'         => 'url_ironicinspector path_beg -i /inspector'},
    { 'use_backend' => 'ironicinspectorbe if url_ironicinspector'},
  ]

  $custom_haproxy_options = [
    { 'acl'         => 'url_slash path /'},
    { 'redirect'    => "location $haproxy_custom_url if url_slash !url_websockify_token"},
  ]

  $maintenance_network_whitelist = $net_whitelist.join(' ')
  $maintenance_haproxy_options = [
    { 'acl'         => "network_allowed src ${maintenance_network_whitelist}" },
    { 'use_backend' => 'maintenancebe if !network_allowed' },
  ]

  if $haproxy_maintenance {
    $haproxy_options_with_maintenance = concat($maintenance_haproxy_options, $haproxy_options_keystone)
  } else {
    $haproxy_options_with_maintenance = $haproxy_options_keystone
  }

  if $has_subrole_glance {
    $haproxy_options_with_glance = concat($haproxy_options_with_maintenance, $glance_haproxy_options)
  } else {
    $haproxy_options_with_glance = $haproxy_options_with_maintenance
  }

  if $has_subrole_swift and $swiftproxy_hostname == 'none' {
    $haproxy_options_with_swift = concat($haproxy_options_with_glance, $swift_haproxy_options)
  } else {
    $haproxy_options_with_swift = $haproxy_options_with_glance
  }

  if $has_subrole_heat {
    $haproxy_options_with_heat = concat($haproxy_options_with_swift, $heat_haproxy_options)
  } else {
    $haproxy_options_with_heat = $haproxy_options_with_swift
  }

  if $has_subrole_horizon {
    $haproxy_options_with_horizon = concat($haproxy_options_with_heat, $horizon_haproxy_options)
  } else {
    $haproxy_options_with_horizon = $haproxy_options_with_heat
  }

  if $has_subrole_barbican {
    $haproxy_options_with_barbican = concat($haproxy_options_with_horizon, $barbican_haproxy_options)
  } else {
    $haproxy_options_with_barbican = $haproxy_options_with_horizon
  }

  if $has_subrole_nova {
    $haproxy_options_with_nova = concat($haproxy_options_with_barbican, $nova_haproxy_options)
  } else {
    $haproxy_options_with_nova = $haproxy_options_with_barbican
  }

  if $has_subrole_neutron {
    $haproxy_options_with_neutron = concat($haproxy_options_with_nova, $neutron_haproxy_options)
  } else {
    $haproxy_options_with_neutron = $haproxy_options_with_nova
  }

  if $has_subrole_cinder {
    $haproxy_options_with_cinder = concat($haproxy_options_with_neutron, $cinder_haproxy_options)
  } else {
    $haproxy_options_with_cinder = $haproxy_options_with_neutron
  }

  if $has_subrole_gnocchi and $install_telemetry{
    $haproxy_options_with_gnocchi = concat($haproxy_options_with_cinder, $gnocchi_haproxy_options)
  } else {
    $haproxy_options_with_gnocchi = $haproxy_options_with_cinder
  }

  if $has_subrole_cloudkitty and $install_telemetry{
    $haproxy_options_with_cloudkitty = concat($haproxy_options_with_gnocchi, $cloudkitty_haproxy_options)
  } else {
    $haproxy_options_with_cloudkitty = $haproxy_options_with_gnocchi
  }

  if $has_subrole_aodh and $install_telemetry {
    $haproxy_options_with_aodh = concat($haproxy_options_with_cloudkitty, $aodh_haproxy_options)
  } else {
    $haproxy_options_with_aodh = $haproxy_options_with_cloudkitty
  }

  if $has_subrole_octavia and $install_octavia {
    $haproxy_options_with_octavia = concat($haproxy_options_with_aodh, $octavia_haproxy_options)
  } else {
    $haproxy_options_with_octavia = $haproxy_options_with_aodh
  }

  if $has_subrole_magnum and $install_magnum {
    $haproxy_options_with_magnum = concat($haproxy_options_with_octavia, $magnum_haproxy_options)
  } else {
    $haproxy_options_with_magnum = $haproxy_options_with_octavia
  }

  if $has_subrole_panko and $install_panko and $install_telemetry {
    $haproxy_options_with_panko = concat($haproxy_options_with_magnum, $panko_haproxy_options)
  } else {
    $haproxy_options_with_panko = $haproxy_options_with_magnum
  }

  if $has_subrole_designate and $install_designate {
    $haproxy_options_with_designate = concat($haproxy_options_with_panko, $designate_haproxy_options)
  } else {
    $haproxy_options_with_designate = $haproxy_options_with_panko
  }

  if $has_subrole_manila and $install_manila {
    $haproxy_options_with_manila = concat($haproxy_options_with_designate, $manila_haproxy_options)
  } else {
    $haproxy_options_with_manila = $haproxy_options_with_designate
  }

  if $has_subrole_ironic and $install_ironic {
    $haproxy_options_with_ironic = concat($haproxy_options_with_manila, $ironic_haproxy_options)
  } else {
    $haproxy_options_with_ironic = $haproxy_options_with_manila
  }

  if $haproxy_custom_url {
    $haproxy_options_with_custom = concat($haproxy_options_with_ironic, $custom_haproxy_options)
  } else {
    $haproxy_options_with_custom = $haproxy_options_with_ironic
  }

  $api_haproxy_rate_limit_real = $api_haproxy_rate_limit * 10

  if $api_disable_rate_limit {
    # Global frontend for all services with dispatch depending on the URL
    haproxy::frontend { 'openstackfe':
      mode      => 'http',
      bind      => { "${vip_ipaddr}:${api_port}" => ['ssl', 'crt', '/etc/haproxy/ssl/private/oci-pki-api.pem', 'crt', '/etc/haproxy/ssl/private/'],
                     "${machine_ip}:${api_port}" => ['ssl', 'crt', '/etc/haproxy/ssl/private/oci-pki-api.pem', 'crt', '/etc/haproxy/ssl/private/'] },
      options   => $haproxy_options_with_custom,
    }
  } else {
    $haproxy_network_whitelist = $net_whitelist.join(' ')
    $haproxy_api_rate_limit = [
      {'acl'          => "network_allowed src ${haproxy_network_whitelist}"},
      {'stick-table'  => 'type ip size 10k expire 30s store http_req_rate(10s)'},
      {'http-request' => 'track-sc0 src'},
      {'http-request' => "deny deny_status 429 if { sc_http_req_rate(0) gt ${api_haproxy_rate_limit_real} } !network_allowed"},
    ]

    $haproxy_with_api_rate_limit = concat($haproxy_api_rate_limit, $haproxy_options_with_custom)

    # Global frontend for all services with dispatch depending on the URL
    haproxy::frontend { 'openstackfe':
      mode      => 'http',
      bind      => { "${vip_ipaddr}:${api_port}" => ['ssl', 'crt', '/etc/haproxy/ssl/private/oci-pki-api.pem', 'crt', '/etc/haproxy/ssl/private/'],
                     "${machine_ip}:${api_port}" => ['ssl', 'crt', '/etc/haproxy/ssl/private/oci-pki-api.pem', 'crt', '/etc/haproxy/ssl/private/'] },
      options   => $haproxy_with_api_rate_limit,
    }
  }

  # Keystone public
  if $use_haproxy_21_format {
    $tweak_path_identity_admin = '^/identity-admin/?(.*) /\1'
    $tweak_path_identity       = '^/identity/?(.*) /\1'
    $tweak_head_identity       = 'Location: (https://[^/]*)/(.*)$ \1/identity/\2'
  }else{
    $tweak_path_identity_admin = '^([^\ ]*\ /)identity-admin[/]?(.*) \1\2'
    $tweak_path_identity       = '^([^\ ]*\ /)identity[/]?(.*) \1\2'
    $tweak_head_identity       = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/identity/\2'
  }
  haproxy::backend { 'keystonepubbe':
    options => [
       { 'option'              => 'forwardfor' },
       { 'option'              => 'httpchk GET /healthcheck' },
       { 'http-request'        => "set-header X-Client-IP %[src]"},
       { 'mode'                => 'http' },
       { 'balance'             => 'roundrobin' },
       { $haproxy_replace_path => $tweak_path_identity_admin },
       { $haproxy_replace_path => $tweak_path_identity       },
       { $haproxy_replace_head => $tweak_head_identity       },
    ],
  }
  haproxy::balancermember { 'keystonepubbm':
    listening_service => 'keystonepubbe',
    ipaddresses       => $all_masters_ip,
    server_names      => $all_masters,
    ports             => 5000,
    options           => $haproxy_api_backend_options,
  }

  # glance API
  if $has_subrole_glance {
    if $use_haproxy_21_format {
      $teak_path_image = '^/image[/]?(.*) /\1'
    }else{
      $teak_path_image = '^([^\ ]*\ /)image[/]?(.*) \1\2'
    }
    if $openstack_release == 'rocky'{
      # It feels like /healthcheck is kind of broken with
      # glance and uwsgi under Rocky.
      haproxy::backend { 'glancebe':
        options => [
           { 'option'               => 'forwardfor' },
           { 'mode'                 => 'http' },
           { 'balance'              => 'source' },
           { $haproxy_replace_path  => $teak_path_image },
        ],
      }
    }else{
      haproxy::backend { 'glancebe':
        options => [
           { 'option'               => 'forwardfor' },
           { 'option'               => 'httpchk GET /healthcheck' },
           { 'mode'                 => 'http' },
           { 'balance'              => 'source' },
           { $haproxy_replace_path  => $teak_path_image},
        ],
      }
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' {
      $haproxy_api_backend_options_glance = $haproxy_api_backend_options
    }else{
      $haproxy_api_backend_options_glance = $haproxy_api_backend_options_nossl
    }
    if $glance_backend == 'file'{
      haproxy::balancermember { 'glancebm':
        listening_service => 'glancebe',
        ipaddresses       => [ "${first_master_ip}", ],
        server_names      => [ "${first_master}", ],
        ports             => 9292,
        options           => $haproxy_api_backend_options_glance,
      }
      haproxy::balancermember { 'glancebmbackup':
        listening_service => 'glancebe',
        ipaddresses       => $non_first_master_controllers_ip,
        server_names      => $non_first_master_controllers,
        ports             => 9292,
        options           => "${haproxy_api_backend_options_glance} backup",
      }
    } else {
      haproxy::balancermember { 'glancebm':
        listening_service => 'glancebe',
        ipaddresses       => $all_masters_ip,
        server_names      => $all_masters,
        ports             => 9292,
        options           => $haproxy_api_backend_options_glance,
      }
    }
  }

  # swift proxy
  if $has_subrole_swift and $swiftproxy_hostname == 'none' {
    if $use_haproxy_21_format {
      $tweak_path_object = '^/object/?(.*) /\1'
    }else{
      $tweak_path_object = '^([^\ ]*\ /)object[/]?(.*)     \1\2'
    }
    haproxy::backend { 'swiftbe':
      options => [
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'option'               => 'forwardfor' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_object },
      ],
    }
    haproxy::balancermember { 'swiftbm':
      listening_service => 'swiftbe',
      ipaddresses       => $all_swiftproxy_ip,
      server_names      => $all_swiftproxy,
      ports             => 8080,
      options           => 'check',
    }
  }

  # heat
  if $has_subrole_heat {
    if $use_haproxy_21_format {
      $tweak_path_orchestration_api_twice = '^/orchestration-api/orchestration-api/?(.*) /\1'
      $tweak_path_orchestration_api = '^/orchestration-api/?(.*) /\1'
      $tweak_head_orchestration_api = 'Location  ([^/:]*://[^/]*)?(.*) \1/orchestration-api\2'
      $tweak_path_orchestration_cfn = '^/orchestration-cfn/?(.*) /\1'
      $tweak_head_orchestration_cnf = 'Location  ([^/:]*://[^/]*)?(.*) \1/orchestration-cnf\2'
    }else{
      $tweak_path_orchestration_api_twice = '^([^\ ]*\ /)orchestration-api[/]orchestration-api[/]?(.*) \1\2'
      $tweak_path_orchestration_api = '^([^\ ]*\ /)orchestration-api[/]?(.*) \1\2'
      $tweak_head_orchestration_api = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/orchestration-api/\2'
      $tweak_path_orchestration_cfn = '^([^\ ]*\ /)orchestration-cfn[/]?(.*) \1\2'
      $tweak_head_orchestration_cfn = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/orchestration-cfn/\2'
    }
    haproxy::backend { 'heatbe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'option'              => 'httpchk GET /healthcheck' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_orchestration_api_twice },
         { $haproxy_replace_path => $tweak_path_orchestration_api },
         { $haproxy_replace_head => $tweak_head_orchestration_api },
      ],
    }
    haproxy::balancermember { 'heatbm':
      listening_service => 'heatbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8004,
      options           => $haproxy_api_backend_options,
    }
    haproxy::backend { 'heatcfnbe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'option'              => 'httpchk GET /healthcheck' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_orchestration_cfn },
         { $haproxy_replace_head => $tweak_head_orchestration_cfn },
      ],
    }
    haproxy::balancermember { 'heatcfnbm':
      listening_service => 'heatcfnbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8000,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_horizon {
#    if $use_haproxy_21_format {
#      $tweak_head_horizon = 'Location  ([^/:]*://[^/]*)?(.*) \1/horizon\2'
#    }else{
#      $tweak_head_horizon = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/horizon/\2'
#    }
    haproxy::backend { 'horizonbe':
      options => [
         { 'option'     => 'forwardfor' },
         { 'http-check' => 'expect rstring "Login - OpenStack Dashboard"' },
         { 'option'     => 'httpchk GET /horizon/auth/login/?next=/horizon/' },
         { 'mode'       => 'http' },
         { 'balance'    => 'source' },
#         { $haproxy_replace_head => $tweak_head_horizon },
      ],
    }
    haproxy::balancermember { 'horizonbm':
      listening_service => 'horizonbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 7443,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_barbican {
    if $use_haproxy_21_format {
      $tweak_path_keymanager = '^/keymanager/?(.*) /\1'
    }else{
      $tweak_path_keymanager = '^([^\ ]*\ /)keymanager[/]?(.*) \1\2'
    }
    haproxy::backend { 'barbicanbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_keymanager },
      ],
    }
    haproxy::balancermember { 'barbicanbm':
      listening_service => 'barbicanbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 9311,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_nova {
    if $use_haproxy_21_format {
      $tweak_path_compute = '^/compute/?(.*) /\1'
      $tweak_head_compute = 'Location  ([^/:]*://[^/]*)?(.*) \1/compute\2'
    }else{
      $tweak_path_compute = '^([^\ ]*\ /)compute/?(.*) \1\2'
      $tweak_head_compute = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/compute/\2'
    }
    haproxy::backend { 'novabe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'option'              => 'httpchk GET /healthcheck' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_compute },
         { $haproxy_replace_head => $tweak_head_compute },
      ],
    }
    haproxy::balancermember { 'novabm':
      listening_service => 'novabe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8774,
      options           => $haproxy_api_backend_options,
    }

    if $use_haproxy_21_format {
      $tweak_path_placement = '^/placement/?(.*) /\1'
    }else{
      $tweak_path_placement = '^([^\ ]*\ /)placement/?(.*) \1\2'
    }
    haproxy::backend { 'placementbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_placement },
      ],
    }
    haproxy::balancermember { 'placementbm':
      listening_service => 'placementbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8778,
      options           => $haproxy_api_backend_options,
    }

    if $use_haproxy_21_format {
      $tweak_path_novnc = '^/novnc/?(.*) /\1'
    }else{
      $tweak_path_novnc = '^([^\ ]*\ /)novnc/?(.*) \1\2'
    }
    haproxy::backend { 'novncbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'source' },
         { $haproxy_replace_path  => $tweak_path_novnc },
      ],
    }
    haproxy::balancermember { 'novncbm':
      listening_service => 'novncbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 6080,
      options           => 'check',
    }

    haproxy::backend { 'websockifybe':
      options => [
         { 'option'  => 'forwardfor' },
         { 'mode'    => 'http' },
         { 'balance' => 'source' },
         { 'acl'     => 'hdr_connection_upgrade hdr(Connection)                 -i upgrade'},
         { 'acl'     => 'hdr_upgrade_websocket  hdr(Upgrade)                    -i websocket'},
         { 'acl'     => 'hdr_websocket_key      hdr_cnt(Sec-WebSocket-Key)      eq 1'},
         { 'acl'     => 'hdr_websocket_version  hdr_cnt(Sec-WebSocket-Version)  eq 1'},
      ],
    }
    haproxy::balancermember { 'websockifybm':
      listening_service => 'websockifybe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 6080,
      options           => 'check',
    }
  }

  if $has_subrole_neutron {
    if $use_haproxy_21_format {
      $tweak_path_network = '^/network/?(.*) /\1'
    }else{
      $tweak_path_network = '^([^\ ]*\ /)network/?(.*) \1\2'
    }
    haproxy::backend { 'neutronbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_network },
      ],
    }
    haproxy::balancermember { 'neutronbm':
      listening_service => 'neutronbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 9696,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_cinder {
    if $use_haproxy_21_format {
      $tweak_path_volume = '^/volume/?(.*) /\1'
    }else{
      $tweak_path_volume = '^([^\ ]*\ /)volume/?(.*) \1\2'
    }
    haproxy::backend { 'cinderbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_volume },
      ],
    }
    haproxy::balancermember { 'cinderbm':
      listening_service => 'cinderbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8776,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_gnocchi and $install_telemetry{
    if $use_haproxy_21_format {
      $tweak_path_metric = '^/metric/?(.*) /\1'
    }else{
      $tweak_path_metric = '^([^\ ]*\ /)metric/?(.*) \1\2'
    }
    haproxy::backend { 'gnocchibe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_metric },
      ],
    }
    if $has_subrole_gnocchi_services {
      $gnocchi_backend_ipaddresses = $all_masters_ip
      $gnocchi_backend_server_names = $all_masters
    }else{
      $gnocchi_backend_ipaddresses = $all_rabbits_ips
      $gnocchi_backend_server_names = $all_rabbits
    }
    haproxy::balancermember { 'gnocchibm':
      listening_service => 'gnocchibe',
      ipaddresses       => $gnocchi_backend_ipaddresses,
      server_names      => $gnocchi_backend_server_names,
      ports             => 8041,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_cloudkitty and $install_telemetry{
    if $use_haproxy_21_format {
      $tweak_path_rating = '^/rating/?(.*) /\1'
      $tweak_head_rating = 'Location  ([^/:]*://[^/]*)?(.*) \1/rating\2'
    }else{
      $tweak_path_rating = '^([^\ ]*\ /)rating/?(.*) \1\2'
      $tweak_head_rating = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/rating/\2'
    }
    haproxy::backend { 'cloudkittybe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'option'              => 'httpchk GET /healthcheck' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_rating },
         { $haproxy_replace_head => $tweak_head_rating },
      ],
    }
    if $has_subrole_gnocchi_services {
      $cloudkitty_backend_ipaddresses = $all_masters_ip
      $cloudkitty_backend_server_names = $all_masters
    }else{
      $cloudkitty_backend_ipaddresses = $all_rabbits_ips
      $cloudkitty_backend_server_names = $all_rabbits
    }
    haproxy::balancermember { 'cloudkittybm':
      listening_service => 'cloudkittybe',
      ipaddresses       => $cloudkitty_backend_ipaddresses,
      server_names      => $cloudkitty_backend_server_names,
      ports             => 8889,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_aodh and $install_telemetry {
    if $use_haproxy_21_format {
      $tweak_path_alarm = '^/alarm/?(.*) /\1'
    }else{
      $tweak_path_alarm = '^([^\ ]*\ /)alarm/?(.*) \1\2'
    }
    haproxy::backend { 'aodhbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_alarm },
      ],
    }
    haproxy::balancermember { 'aodhbm':
      listening_service => 'aodhbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8042,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_octavia and $install_octavia{
    if $use_haproxy_21_format {
      $tweak_path_loadbalance = '^/loadbalance/?(.*) /\1'
    }else{
      $tweak_path_loadbalance = '^([^\ ]*\ /)loadbalance/?(.*) \1\2'
    }
    haproxy::backend { 'octaviabe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_loadbalance },
      ],
    }
    haproxy::balancermember { 'octaviabm':
      listening_service => 'octaviabe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 9876,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_magnum and $install_magnum {
    if $use_haproxy_21_format {
      $tweak_path_container = '^/containers/?(.*) /\1'
      $tweak_head_container = 'Location  ([^/:]*://[^/]*)?(.*) \1/containers\2'
    }else{
      $tweak_path_container = '^([^\ ]*\ /)containers/?(.*) \1\2'
      $tweak_head_container = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/containers/\2'
    }
    haproxy::backend { 'magnumbe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'option'              => 'httpchk GET /healthcheck' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_container },
         { $haproxy_replace_head => $tweak_head_container },
      ],
    }
    haproxy::balancermember { 'magnumbm':
      listening_service => 'magnumbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 9511,
      options           => $haproxy_api_backend_options,
    }

    # If we're setting-up Magnum, then we also want the ETCD ...
    if $use_haproxy_21_format {
      $tweak_path_etcdc = '^/etcd-cluster/?(.*) /\1'
      $tweak_head_etcdc = 'Location  ([^/:]*://[^/]*)?(.*) \1/etcd-cluster\2'
    }else{
      $tweak_path_etcdc = '^([^\ ]*\ /)etcd-cluster/?(.*) \1\2'
      $tweak_head_etcdc = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/etcd-clusters/\2'
    }
    haproxy::backend { 'etcdclusterbe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_etcdc },
         { $haproxy_replace_head => $tweak_head_etcdc },
      ],
    }
    haproxy::balancermember { 'etcdclusterbm':
      listening_service => 'etcdclusterbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 2379,
      options           => 'check',
    }

    # ...and etcd-discovery services.
    if $use_haproxy_21_format {
      $tweak_path_etcdd = '^/etcd-discovery/?(.*) /\1'
      $tweak_head_etcdd = 'Location  ([^/:]*://[^/]*)?(.*) \1/etcd-discovery\2'
    }else{
      $tweak_path_etcdd = '^([^\ ]*\ /)etcd-discovery/?(.*) \1\2'
      $tweak_head_etcdd = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/etcd-discoverys/\2'
    }
    haproxy::backend { 'etcddiscoverybe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_etcdd },
         { $haproxy_replace_head => $tweak_head_etcdd },
      ],
    }
    haproxy::balancermember { 'etcddiscoverybm':
      listening_service => 'etcddiscoverybe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8087,
      options           => 'check',
    }
  }

  if $has_subrole_panko and $install_panko and $install_telemetry {
    if $use_haproxy_21_format {
      $tweak_path_event = '^/event/?(.*) /\1'
    }else{
      $tweak_path_event = '^([^\ ]*\ /)event/?(.*) \1\2'
    }
    haproxy::backend { 'pankobe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_event },
      ],
    }
    haproxy::balancermember { 'pankobm':
      listening_service => 'pankobe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8977,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_designate and $install_designate {
    if $use_haproxy_21_format {
      $tweak_path_dns = '^/dns/?(.*) /\1'
    }else{
      $tweak_path_dns = '^([^\ ]*\ /)dns/?(.*) \1\2'
    }
    haproxy::backend { 'designatebe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_dns },
      ],
    }
    haproxy::balancermember { 'designatebm':
      listening_service => 'designatebe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 9001,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_manila and $install_manila {
    if $use_haproxy_21_format {
      $tweak_path_share = '^/share/?(.*) /\1'
    }else{
      $tweak_path_share = '^([^\ ]*\ /)share/?(.*) \1\2'
    }
    haproxy::backend { 'manilabe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_share },
      ],
    }
    haproxy::balancermember { 'manilabm':
      listening_service => 'manilabe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8786,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_ironic and $install_ironic {
    if $use_haproxy_21_format {
      $tweak_path_share = '^/baremetal/?(.*) /\1'
    }else{
      $tweak_path_share = '^([^\ ]*\ /)baremetal/?(.*) \1\2'
    }
    haproxy::backend { 'ironicbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_share },
      ],
    }
    haproxy::balancermember { 'ironicbm':
      listening_service => 'ironicbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 6385,
      options           => $haproxy_api_backend_options,
    }

    if $use_haproxy_21_format {
      $tweak_path_share2 = '^/inspector/?(.*) /\1'
    }else{
      $tweak_path_share2 = '^([^\ ]*\ /)inspector/?(.*) \1\2'
    }
    haproxy::backend { 'ironicinspectorbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_share2 },
      ],
    }
    haproxy::balancermember { 'ironicinspectorbm':
      listening_service => 'ironicinspectorbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 5050,
      options           => $haproxy_api_backend_options,
    }
  }

  haproxy::backend { 'maintenancebe':
    options => [
      { 'mode'      => 'http' },
      { 'errorfile' => '503 /etc/haproxy/errors/503.http' },
    ],
  }

  #######################
  ### Setup databases ###
  #######################
  if $has_subrole_db and $initial_cluster_setup {
    if $mycodename != 'stretch'{
      package { 'mariadb-backup':
        ensure => present,
        before => Class['galera'],
      }
      $wsrep_sst_method = 'mariabackup'
    }else{
      $wsrep_sst_method = 'mariabackup'
    }

    if $is_first_sql {
      # If this is the first node, it must publish itself as working
      # before waiting for the other nodes to join.
      Class['galera::status'] -> Exec['galera-size-is-correct']
      Service['xinetd'] -> Exec['galera-size-is-correct']
    }else{
      # If this machine isn't the first Galera machine, then it must
      # wait until the first master replies it's setup.
      exec {'galera-first-master-is-up':
        command => "/usr/bin/oci-wait-for-first-galera-node ${first_master_ip} 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        timeout => 5000,
      }
      Exec['galera-first-master-is-up'] -> Class['galera']
    }

    if $mycodename == 'stretch' or $mycodename == 'buster' {
      $galera_package_name = 'galera-3'
    }else{
      $galera_package_name = 'galera-4'
    }
# This cannot work as-is, as on the first controller,
# restarting MySQL doesn't just work, it needs to run
# galera_new_cluster instead.
#    systemd::dropin_file { 'override.conf':
#      unit    => 'mariadb.service',
#      notify  => Service['mysql'],
#      content => "[Service]
#LimitNOFILE=100000
#TimeoutStartSec=43200",
#    }
    $ram_total_in_mb = Integer($facts['memorysize_mb'])
    $ram_total_in_gb = $ram_total_in_mb / 1024
    $ram_total_in_gb_div_4 = $ram_total_in_gb / 4

    if $ram_total_in_gb_div_4 > 64 {
      $innodb_buffer_pool_size = 64
    }elsif $ram_total_in_gb_div_4 > 32{
      $innodb_buffer_pool_size = 32
    }elsif $ram_total_in_gb_div_4 > 16{
      $innodb_buffer_pool_size = 16
    }elsif $ram_total_in_gb_div_4 > 8{
      $innodb_buffer_pool_size = 8
    }elsif $ram_total_in_gb_div_4 > 4{
      $innodb_buffer_pool_size = 4
    }else{
      $innodb_buffer_pool_size = 2
    }

    class { 'galera':
      galera_servers      => $all_masters_ip,
      galera_master       => $first_master,
      local_ip            => $machine_ip,
      mysql_package_name  => 'mariadb-server',
      client_package_name => 'default-mysql-client',
      vendor_type         => 'mariadb',
      root_password       => $pass_mysql_rootuser,
      status_password     => $pass_mysql_rootuser,
      deb_sysmaint_password => $pass_mysql_rootuser,
      configure_repo      => false,
      configure_firewall  => false,
      galera_package_name => $galera_package_name,
      override_options => {
        'mysqld' => {
          'bind_address'                    => $machine_ip,
          'wait_timeout'                    => '28800',
          'interactive_timeout'             => '30',
          'connect_timeout'                 => '30',
          'character_set_server'            => 'utf8',
          'collation_server'                => 'utf8_general_ci',
          'innodb_buffer_pool_size'         => "${innodb_buffer_pool_size}G",
          'innodb_flush_log_at_trx_commit'  => '2',
          'max_connections'                 => '5000',
          'max_user_connections'            => '1000',
          'binlog_cache_size'               => '1M',
          'log-bin'                         => 'mysql-bin',
          'binlog_format'                   => 'ROW',
          'performance_schema'              => '1',
          'log_warnings'                    => '2',
          'wsrep_sst_auth'                  => "backup:${pass_mysql_backup}",
          'wsrep_sst_method'                => $wsrep_sst_method,
          'wsrep_cluster_name'              => $cluster_name,
          'wsrep_node_name'                 => $machine_hostname,
          'wsrep_provider_options'          => 'cert.log_conflicts=YES;gcache.recover=yes;gcache.size=5G',
        }
      },
      require             => Cs_primitive['openstack-api-vip'],
    }->
    mysql_user { 'backup@%':
      ensure        => present,
      password_hash => mysql_password($pass_mysql_backup),
    }->
    mysql_grant{'backup@%/*.*':
      ensure     => 'present',
      options    => ['GRANT'],
      privileges => ['BINLOG MONITOR', 'CREATE TEMPORARY TABLES', 'LOCK TABLES', 'PROCESS', 'RELOAD', 'SELECT', 'SHOW VIEW', 'TRIGGER'],
      table      => '*.*',
      user       => 'backup@%',
    }->

    # Wait until SHOW STATUS LIKE 'wsrep_cluster_status' shows Primary
    exec {'galera-is-up':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_cluster_status'\" Primary mysql 4800",
      unless  => "/usr/bin/mysql --defaults-extra-file=/root/.my.cnf --database=mysql -e \"SHOW STATUS LIKE 'wsrep_cluster_status'\" | grep -q Primary",
      timeout => 5000,
    }

    # Wait until SHOW STATUS LIKE 'wsrep_connected' shows ON
    exec {'galera-wsrep-connected-on':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_connected'\" ON mysql 4800",
      unless  => "/usr/bin/mysql --defaults-extra-file=/root/.my.cnf --database=mysql -e \"SHOW STATUS LIKE 'wsrep_connected'\" | grep -q ON",
      require => Exec['galera-is-up'],
      timeout => 5000,
    }

    # Wait until SHOW STATUS LIKE 'wsrep_local_state_comment' shows Synced
    exec {'galera-is-synced':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_local_state_comment'\" Synced mysql 4800",
      unless  => "/usr/bin/mysql --defaults-extra-file=/root/.my.cnf --database=mysql -e \"SHOW STATUS LIKE 'wsrep_local_state_comment'\" | grep -q Synced",
      require => Exec['galera-wsrep-connected-on'],
      timeout => 5000,
    }

    # Wait until all nodes are connected to the cluster
    $galera_cluster_num_of_nodes = sprintf('%i', $all_masters.size)
    exec {'galera-size-is-correct':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_cluster_size'\" ${galera_cluster_num_of_nodes} mysql 4800",
      unless  => "/usr/bin/mysql --defaults-extra-file=/root/.my.cnf --database=mysql -e \"SHOW STATUS LIKE 'wsrep_cluster_size'\" | grep -q ${galera_cluster_num_of_nodes}",
      require => Exec['galera-is-synced'],
      timeout => 5000,
    }


    if $is_first_master {
      class { '::keystone::db::mysql':
        dbname   => 'keystonedb',
        password => $pass_keystone_db,
        allowed_hosts => '%',
        require  => Exec['galera-size-is-correct'],
#        require  => Class['oci::sql::galera'],
        before   => Anchor['keystone::service::begin'],
      }
    } else {
      exec {'wait-for-keystone-dbuser':
        command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='keystone'\" keystone mysql 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        require => Exec['galera-size-is-correct'],
        before  => Anchor['keystone::service::begin'],
        timeout => 5000,
      }
    }
    if $has_subrole_glance {
      if $is_first_master {
        class { '::glance::db::mysql':
          dbname   => 'glancedb',
          password => $pass_glance_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['glance::service::begin'],
        }
      } else {
        exec { 'glance-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='glance'\" glance mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Anchor['glance::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_heat {
      if $is_first_master {
        class { '::heat::db::mysql':
          dbname   => 'heatdb',
          password => $pass_heat_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['heat::service::begin'],
        }
      } else {
        exec { 'heat-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='heat'\" heat mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['heat::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_barbican {
      if $is_first_master {
        class { '::barbican::db::mysql':
          dbname   => 'barbicandb',
          password => $pass_barbican_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['barbican::service::begin'],
        }
      } else {
        exec { 'barbican-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='barbican'\" barbican mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['barbican::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_nova {
      if $is_first_master {
        class { '::nova::db::mysql':
          user     => 'nova',
          dbname   => 'novadb',
          password => $pass_nova_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
          before   => Class['::nova::cell_v2::simple_setup'],
          notify   => Service['nova-conductor', 'nova-scheduler'],
        }
        class { '::nova::db::mysql_api':
          user     => 'novaapi',
          dbname   => 'novaapidb',
          password => $pass_nova_apidb,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
          before   => Class['::nova::cell_v2::simple_setup'],
        }
        if $openstack_release == 'rocky'{
          class { '::nova::db::mysql_placement':
            user     => 'placement',
            dbname   => 'placementdb',
            password => $pass_placement_db,
            allowed_hosts => '%',
            require  => Exec['galera-size-is-correct'],
            before   => Anchor['nova::service::begin'],
          }
        }else{
          class { '::placement::db::mysql':
            user => 'placement',
            dbname => 'placementdb',
            password => $pass_placement_db,
            allowed_hosts => '%',
            require  => Exec['galera-size-is-correct'],
            before   => Anchor['nova::service::begin'],
          }
        }
      } else {
        exec { 'nova-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='nova'\" nova mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Class['nova'],
          timeout => 5000,
        }
        exec { 'novaapi-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='novaapi'\" novaapi mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Class['nova'],
          timeout => 5000,
        }
        exec { 'placement-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='placement'\" placement mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Anchor['nova::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_neutron {
      if $is_first_master {
        class { '::neutron::db::mysql':
          dbname   => 'neutrondb',
          password => $pass_neutron_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
          before   => Anchor['neutron::service::begin'],
        }
      } else {
        exec { 'neutron-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='neutron'\" neutron mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Anchor['neutron::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_cinder {
      if $is_first_master {
        class { '::cinder::db::mysql':
          dbname        => 'cinderdb',
          password      => $pass_cinder_db,
          allowed_hosts => '%',
          require       => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before        => Anchor['cinder::service::begin'],
        }
      } else {
        exec { 'cinder-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='cinder'\" cinder mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['cinder::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_gnocchi and $install_telemetry {
      if $is_first_master {
        class { '::gnocchi::db::mysql':
          dbname   => 'gnocchidb',
          password => $pass_gnocchi_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['gnocchi::service::begin'],
        }
      } else {
        exec { 'gnocchi-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='gnocchi'\" gnocchi mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['gnocchi::service::begin'],
          timeout => 5000,
        }
      }
    }

    if $has_subrole_panko and $install_panko and $install_telemetry {
      if $is_first_master {
        class { '::panko::db::mysql':
          dbname   => 'pankodb',
          password => $pass_panko_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
          before   => Anchor['panko::service::begin'],
        }
      } else {
        exec { 'panko-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='panko'\" panko mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Anchor['panko::service::begin'],
          timeout => 5000,
        }
      }
    }

    if $has_subrole_cloudkitty and $install_telemetry{
      if $is_first_master {
        class { '::cloudkitty::db::mysql':
          dbname   => 'cloudkittydb',
          password => $pass_cloudkitty_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['cloudkitty::service::begin'],
        }
      } else {
        exec { 'cloudkitty-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='cloudkitty'\" cloudkitty mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['cloudkitty::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_aodh and $install_telemetry {
      if $is_first_master {
        class { '::aodh::db::mysql':
          dbname   => 'aodhdb',
          password => $pass_aodh_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['aodh::service::begin'],
        }
      } else {
        exec { 'aodh-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='aodh'\" aodh mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['aodh::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_octavia and $install_octavia{
      if $is_first_master {
        class { '::octavia::db::mysql':
          dbname   => 'octaviadb',
          password => $pass_octavia_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['octavia::service::begin'],
        }
      } else {
        exec { 'octavia-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='octavia'\" octavia mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['octavia::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_magnum and $install_magnum {
      if $is_first_master {
        class { '::magnum::db::mysql':
          dbname   => 'magnumdb',
          password => $pass_magnum_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['magnum::service::begin'],
        }
      } else {
        exec { 'magnum-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='magnum'\" magnum mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['magnum::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_designate and $install_designate {
      if $is_first_master {
        class { '::designate::db::mysql':
          dbname   => 'designatedb',
          password => $pass_designate_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
          before   => Anchor['designate::service::begin'],
        }
      } else {
        exec { 'designate-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='designate'\" designate mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Anchor['designate::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_manila and $install_manila {
      if $is_first_master {
        class { '::manila::db::mysql':
          dbname   => 'maniladb',
          password => $pass_manila_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
          before   => Anchor['manila::service::begin'],
        }
      } else {
        exec { 'manila-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='manila'\" manila mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Anchor['manila::service::begin'],
          timeout => 5000,
        }
      }
    }


    if $has_subrole_ironic and $install_ironic {
      if $is_first_master {
        class { '::ironic::db::mysql':
          dbname        => 'ironicdb',
          password      => $pass_ironic_db,
          allowed_hosts => '%',
          require       => Exec['galera-size-is-correct'],
          before        => Anchor['ironic::service::begin'],
        }
        class { '::ironic::inspector::db::mysql':
          dbname        => 'ironicinspectordb',
          user          => 'ironicinspector',
          password      => $pass_ironic_inspector_db,
          allowed_hosts => '%',
          require       => Exec['galera-size-is-correct'],
          before        => Anchor['ironic::service::begin'],
        }
      } else {
        exec { 'ironic-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='ironic'\" ironic mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Anchor['ironic::service::begin'],
          timeout => 5000,
        }
        exec { 'ironic-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='ironicinspector'\" ironicinspector mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Anchor['ironic::service::begin'],
          timeout => 5000,
        }
      }
    }

  }

  ######################
  ### Setup RabbitMQ ###
  ######################
  if $has_subrole_messaging {
    if $has_subrole_db and $initial_cluster_setup {
      Exec['galera-size-is-correct'] -> Class['::rabbitmq']
    }
    if $use_ssl {
      file { "/etc/rabbitmq/ssl/private":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        require                 => File['/etc/rabbitmq/ssl'],
        selinux_ignore_defaults => true,
      }->
      file { "/etc/rabbitmq/ssl/public":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { "/etc/rabbitmq/ssl/private/${::fqdn}.key":
        ensure                  => present,
        owner                   => "rabbitmq",
        source                  => "/etc/ssl/private/ssl-cert-snakeoil.key",
        selinux_ignore_defaults => true,
        mode                    => '0600',
      }->
      file { "/etc/rabbitmq/ssl/public/${::fqdn}.crt":
        ensure                  => present,
        owner                   => "rabbitmq",
        source                  => '/etc/ssl/certs/ssl-cert-snakeoil.pem',
        selinux_ignore_defaults => true,
        mode                    => '0644',
        notify        => Service['rabbitmq-server'],
      }
      $rabbit_ssl_cert = "/etc/rabbitmq/ssl/public/${::fqdn}.crt"
      $rabbit_ssl_key  = "/etc/rabbitmq/ssl/private/${::fqdn}.key"
      $rabbit_ssl_ca   = $oci_pki_root_ca_file
    } else {
      $rabbit_ssl_cert = UNSET
      $rabbit_ssl_key  = UNSET
      $rabbit_ssl_ca   = UNSET
    }

    class { '::rabbitmq':
      delete_guest_user           => true,
      node_ip_address             => $machine_ip,
      ssl_interface               => $machine_ip,
      ssl                         => $use_ssl,
      ssl_only                    => $use_ssl,
      ssl_cacert                  => $rabbit_ssl_ca,
      ssl_cert                    => $rabbit_ssl_cert,
      ssl_key                     => $rabbit_ssl_key,
      environment_variables       => $rabbit_env,
      repos_ensure                => false,
      # Clustering options...
      config_cluster              => true,
      cluster_nodes               => $all_masters,
      cluster_node_type           => 'ram',
      erlang_cookie               => $pass_rabbitmq_cookie,
      wipe_db_on_cookie_change    => true,
      collect_statistics_interval => 60000,
      cluster_partition_handling  => 'autoheal',
      config_variables            => {
          'vm_memory_high_watermark' => '0.4',
        },
    }->
    rabbitmq_vhost { '/':
      provider => 'rabbitmqctl',
      require  => Class['::rabbitmq'],
    }->
    rabbitmq_user { 'keystone':
      admin    => true,
      password => $pass_keystone_messaging,
    }->
    rabbitmq_user_permissions { "keystone@/":
      configure_permission => '.*',
      write_permission     => '.*',
      read_permission      => '.*',
      provider             => 'rabbitmqctl',
      require              => Class['::rabbitmq'],
    }

    if $initial_cluster_setup {
      exec { 'auto-join-rabbit-cluster':
        command => "/usr/bin/oci-auto-join-rabbitmq-cluster ${first_master}",
        unless  => "/bin/false",
        require => Class['::rabbitmq'],
      }

      rabbitmq_user { 'monitoring':
        admin    => true,
        password => $pass_rabbitmq_monitoring,
        require  => Class['::rabbitmq'],
        tags     => ['monitoring']
      }->
      rabbitmq_user_permissions { "monitoring@/":
        configure_permission => '.*',
        write_permission     => '.*',
        read_permission      => '.*',
        provider             => 'rabbitmqctl',
      }

      if $has_subrole_glance {
        rabbitmq_user { 'glance':
          admin    => true,
          password => $pass_glance_messaging,
        }->
        rabbitmq_user_permissions { "glance@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
          before               => Anchor['glance::service::begin'],
        }
      }

      if $is_first_master {
        if $has_subrole_heat {
          rabbitmq_user { 'heat':
            admin    => true,
            password => $pass_heat_messaging,
          }->
          rabbitmq_user_permissions { "heat@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['heat::service::begin'],
          }
        }

        if $has_subrole_barbican {
          rabbitmq_user { 'barbican':
            admin    => true,
            password => $pass_barbican_messaging,
          }->
          rabbitmq_user_permissions { "barbican@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['barbican::service::begin'],
          }
        }

        if $has_subrole_nova {
          rabbitmq_user { 'nova':
            admin    => true,
            password => $pass_nova_messaging,
          }->
          rabbitmq_user_permissions { "nova@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['nova::service::begin'],
          }
        }
        if $has_subrole_neutron {
          rabbitmq_user { 'neutron':
            admin    => true,
            password => $pass_neutron_messaging,
          }->
          rabbitmq_user_permissions { "neutron@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['neutron::service::begin'],
          }
        }
        if $has_subrole_cinder {
          rabbitmq_user { 'cinder':
            admin    => true,
            password => $pass_cinder_messaging,
          }->
          rabbitmq_user_permissions { "cinder@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['cinder::service::begin'],
          }
        }
        if $has_subrole_gnocchi and $install_telemetry {
          rabbitmq_user { 'gnocchi':
            admin    => true,
            password => $pass_gnocchi_messaging,
          }->
          rabbitmq_user_permissions { "gnocchi@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['gnocchi::service::begin'],
          }
        }
        if $has_subrole_panko and $install_panko and $install_telemetry {
          rabbitmq_user { 'panko':
            admin    => true,
            password => $pass_panko_messaging,
          }->
          rabbitmq_user_permissions { "panko@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['panko::service::begin'],
          }
        }
        if $has_subrole_ceilometer and $install_telemetry {
          rabbitmq_user { 'ceilometer':
            admin    => true,
            password => $pass_ceilometer_messaging,
          }->
          rabbitmq_user_permissions { "ceilometer@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['ceilometer::service::begin'],
          }

          rabbitmq_user { 'swift':
            admin    => true,
            password => $pass_swift_messaging,
          }->
          rabbitmq_user_permissions { "swift@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
          }
        }
        if $has_subrole_cloudkitty and $install_telemetry{
          rabbitmq_user { 'cloudkitty':
            admin    => true,
            password => $pass_cloudkitty_messaging,
          }->
          rabbitmq_user_permissions { "cloudkitty@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['cloudkitty::service::begin'],
          }
        }
        if $has_subrole_aodh and $install_telemetry {
          rabbitmq_user { 'aodh':
            admin    => true,
            password => $pass_aodh_messaging,
          }->
          rabbitmq_user_permissions { "aodh@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['aodh::service::begin'],
          }
        }
        if $has_subrole_octavia and $install_octavia{
          rabbitmq_user { 'octavia':
            admin    => true,
            password => $pass_octavia_messaging,
          }->
          rabbitmq_user_permissions { "octavia@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['octavia::service::begin'],
          }
        }
        if $has_subrole_magnum and $install_magnum {
          rabbitmq_user { 'magnum':
            admin    => true,
            password => $pass_magnum_messaging,
          }->
          rabbitmq_user_permissions { "magnum@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['magnum::service::begin'],
          }
        }
        if $has_subrole_designate and $install_designate {
          rabbitmq_user { 'designate':
            admin    => true,
            password => $pass_designate_messaging,
          }->
          rabbitmq_user_permissions { "designate@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['designate::service::begin'],
          }
        }
        if $has_subrole_manila and $install_manila {
          rabbitmq_user { 'manila':
            admin    => true,
            password => $pass_manila_messaging,
          }->
          rabbitmq_user_permissions { "manila@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['manila::service::begin'],
          }
        }
        if $has_subrole_ironic and $install_ironic {
          rabbitmq_user { 'ironic':
            admin    => true,
            password => $pass_ironic_messaging,
          }->
          rabbitmq_user_permissions { "ironic@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['ironic::service::begin'],
          }

          rabbitmq_user { 'ironicinspector':
            admin    => true,
            password => $pass_ironic_inspector_messaging,
          }->
          rabbitmq_user_permissions { "ironicinspector@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['ironic::service::begin'],
          }

        }
      }
    }
  }

  #######################
  ### Setup memcached ###
  #######################
  class { '::memcached':
    listen_ip       => $machine_ip,
    udp_port        => 0,
    max_memory      => '20%',
    max_connections => $controller_node_memcache_max_connections,
    pidfile         => '/var/run/memcached/memcached.pid',
    user            => 'memcache',
  }

  # Configure Apache to use mod-wsgi-py3, not mod-wsgi
  if $has_subrole_api_keystone {
    # Set python3 for mod-wsgi
    include ::apache::params
    class { '::apache':
      mod_packages => merge($::apache::params::mod_packages, {
        'wsgi' => 'libapache2-mod-wsgi-py3',
      })
    }
  }

  if $has_subrole_cinder or $has_subrole_nova or $has_subrole_horizon {
    include ::apache
  }

  if $has_subrole_api_keystone {
    ##############################
    ### Setup keystone cluster ###
    ##############################
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['keystone::install::begin']
    }

    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope'){
      class { '::keystone::client': }
    }
    # This cleans-up previous setup with:
    # class { '::keystone::cron::token_flush': }
    file { "/var/spool/cron/crontabs/keystone":
      ensure                  => absent,
    }

    if $use_ssl {
      if $openstack_release == 'rocky'{
        file { "/etc/keystone/ssl":
          ensure                  => directory,
          owner                   => 'root',
          mode                    => '0755',
          selinux_ignore_defaults => true,
          require       => Package['keystone'],
        }->
        file { "/etc/keystone/ssl/private":
          ensure                  => directory,
          owner                   => 'root',
          mode                    => '0755',
          selinux_ignore_defaults => true,
        }->
        file { "/etc/keystone/ssl/public":
          ensure                  => directory,
          owner                   => 'root',
          mode                    => '0755',
          selinux_ignore_defaults => true,
        }->
        file { "/etc/keystone/ssl/private/${::fqdn}.key":
          ensure                  => present,
          owner                   => "keystone",
          source                  => "/etc/ssl/private/ssl-cert-snakeoil.key",
          selinux_ignore_defaults => true,
          mode                    => '0600',
        }->
        file { "/etc/keystone/ssl/public/${::fqdn}.crt":
          ensure                  => present,
          owner                   => "keystone",
          source                  => '/etc/ssl/certs/ssl-cert-snakeoil.pem',
          selinux_ignore_defaults => true,
          mode                    => '0644',
          notify        => Service['httpd'],
        }
      }else{
        ::oci::sslkeypair {'keystone':
          notify_service_name => 'keystone',
        }
      }
      $keystone_key_file = "/etc/keystone/ssl/private/${::fqdn}.pem"
      $keystone_crt_file = "/etc/keystone/ssl/public/${::fqdn}.crt"
    } else {
      $keystone_key_file = undef
      $keystone_crt_file = undef
    }

    if $openstack_release == 'rocky'{
      ensure_resource('file', '/etc/apache2/sites-available/wsgi-keystone.conf', {
        'ensure'  => 'present',
        'content' => '',
        require   => Package['keystone'],
      })
    }

    if $disable_notifications {
      $keystone_notif_transport_url = ''
    } else {
      $keystone_notif_transport_url = os_transport_url({
                                        'transport' => 'rabbit',
                                        'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                        'port'      => '5671',
                                        'username'  => 'keystone',
                                        'password'  => $pass_keystone_messaging,
                                      })
    }
    if $openstack_release == 'rocky'{
      class { '::keystone':
        debug                      => true,
        database_connection        => "mysql+pymysql://keystone:${pass_keystone_db}@${sql_host}/keystonedb",
        database_idle_timeout      => 1800,
        catalog_type               => 'sql',
        admin_token                => $pass_keystone_adminuser,
        admin_password             => $pass_keystone_adminuser,
        enabled                    => true,
        service_name               => 'httpd',
        enable_ssl                 => $use_ssl,
        public_bind_host           => "${::fqdn}",
        admin_bind_host            => "${::fqdn}",
        manage_policyrcd           => true,
        enable_credential_setup    => true,
        credential_key_repository  => '/etc/keystone/credential-keys',
        credential_keys            => { '/etc/keystone/credential-keys/0' => { 'content' => $pass_keystone_credkey1 },
                                        '/etc/keystone/credential-keys/1' => { 'content' => $pass_keystone_credkey2 },
                                      },
        enable_fernet_setup        => true,
        fernet_replace_keys	 => false,
        fernet_key_repository      => '/etc/keystone/fernet-keys',
        fernet_max_active_keys     => 4,
        fernet_keys                => { '/etc/keystone/fernet-keys/0' => { 'content' => $pass_keystone_fernkey1 },
# With fernet_replace_keys => false, if  we don't have the "1" key
# then re-applying puppet is fine, and initial setup is ok too.
#                                      '/etc/keystone/fernet-keys/1' => { 'content' => $pass_keystone_fernkey2 },
                                      },
        token_expiration           => 604800,
        admin_endpoint             => "${keystone_admin_uri}",
        public_endpoint            => "https://${vip_hostname}:${api_port}/identity",
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'keystone',
          'password'  => $pass_keystone_messaging,
        }),
        notification_transport_url => $keystone_notif_transport_url,
        notification_driver        => $notification_driver,
        rabbit_use_ssl             => $use_ssl,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        rabbit_ha_queues           => true,
        cache_backend              => 'dogpile.cache.memcached',
        memcache_servers           => $memcached_servers,
        sync_db                    => $do_db_sync,
      }
      class { '::keystone::wsgi::apache':
        bind_host         => $machine_ip,
        admin_bind_host   => $machine_ip,
        ssl               => $use_ssl,
        ssl_key           => "/etc/keystone/ssl/private/${::fqdn}.key",
        ssl_cert          => "/etc/keystone/ssl/public/${::fqdn}.crt",
        workers           => 2,
        access_log_format => '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"',
      }
    } elsif ($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
      class { '::keystone':
        database_connection        => "mysql+pymysql://keystone:${pass_keystone_db}@${sql_host}/keystonedb",
        database_idle_timeout      => 1800,
        catalog_type               => 'sql',
        admin_token                => $pass_keystone_adminuser,
        admin_password             => $pass_keystone_adminuser,
        enabled                    => true,
        service_name               => 'keystone',
        enable_ssl                 => $use_ssl,
        public_bind_host           => "${::fqdn}",
        admin_bind_host            => "${::fqdn}",
        manage_policyrcd           => false,
        enable_credential_setup    => true,
        credential_key_repository  => '/etc/keystone/credential-keys',
        credential_keys            => { '/etc/keystone/credential-keys/0' => { 'content' => $pass_keystone_credkey1 },
                                        '/etc/keystone/credential-keys/1' => { 'content' => $pass_keystone_credkey2 },
                                      },
        enable_fernet_setup        => true,
        fernet_replace_keys	 => false,
        fernet_key_repository      => '/etc/keystone/fernet-keys',
        fernet_max_active_keys     => 4,
        fernet_keys                => { '/etc/keystone/fernet-keys/0' => { 'content' => $pass_keystone_fernkey1 },
# With fernet_replace_keys => false, if  we don't have the "1" key
# then re-applying puppet is fine, and initial setup is ok too.
#                                      '/etc/keystone/fernet-keys/1' => { 'content' => $pass_keystone_fernkey2 },
                                      },
        token_expiration           => 604800,
        admin_endpoint             => "${keystone_admin_uri}",
        public_endpoint            => "https://${vip_hostname}:${api_port}/identity",
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'keystone',
          'password'  => $pass_keystone_messaging,
        }),
        notification_transport_url => $keystone_notif_transport_url,
        notification_driver        => $notification_driver,
        rabbit_use_ssl             => $use_ssl,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        rabbit_ha_queues           => true,
        cache_backend              => 'dogpile.cache.memcached',
        cache_memcache_servers     => $memcached_servers,
        require                    => Service['haproxy'],
        sync_db                    => $do_db_sync,
      }
    }else{
      if ($openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena') {
        class { '::keystone':
          database_connection        => "mysql+pymysql://keystone:${pass_keystone_db}@${sql_host}/keystonedb",
          database_idle_timeout      => 1800,
          catalog_type               => 'sql',
          enabled                    => true,
          service_name               => 'keystone',
          enable_ssl                 => $use_ssl,
          manage_policyrcd           => false,
          enable_credential_setup    => true,
          credential_key_repository  => '/etc/keystone/credential-keys',
          credential_keys            => { '/etc/keystone/credential-keys/0' => { 'content' => $pass_keystone_credkey1 },
                                          '/etc/keystone/credential-keys/1' => { 'content' => $pass_keystone_credkey2 },
                                        },
          enable_fernet_setup        => true,
          fernet_replace_keys	 => false,
          fernet_key_repository      => '/etc/keystone/fernet-keys',
          fernet_max_active_keys     => 4,
          fernet_keys                => { '/etc/keystone/fernet-keys/0' => { 'content' => $pass_keystone_fernkey1 },
  # With fernet_replace_keys => false, if  we don't have the "1" key
  # then re-applying puppet is fine, and initial setup is ok too.
  #                                      '/etc/keystone/fernet-keys/1' => { 'content' => $pass_keystone_fernkey2 },
                                        },
          token_expiration           => 604800,
          admin_endpoint             => "${keystone_admin_uri}",
          public_endpoint            => "${base_url}/identity",
          default_transport_url      => os_transport_url({
            'transport' => 'rabbit',
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => '5671',
            'username'  => 'keystone',
            'password'  => $pass_keystone_messaging,
          }),
          notification_transport_url => $keystone_notif_transport_url,
          notification_driver        => $notification_driver,
          rabbit_use_ssl             => $use_ssl,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          rabbit_ha_queues           => true,
          require                    => Service['haproxy'],
          sync_db                    => $do_db_sync,
        }
      # Yoga and up
      }else{
        class { '::keystone::db':
          database_connection              => "mysql+pymysql://keystone:${pass_keystone_db}@${sql_host}/keystonedb",
          database_connection_recycle_time => 1800,
        }
        if ($openstack_release == 'yoga') {
          class { '::keystone':
            catalog_driver             => 'sql',
            enabled                    => true,
            service_name               => 'keystone',
            enable_ssl                 => $use_ssl,
            manage_policyrcd           => false,
            enable_credential_setup    => true,
            credential_key_repository  => '/etc/keystone/credential-keys',
            credential_keys            => { '/etc/keystone/credential-keys/0' => { 'content' => $pass_keystone_credkey1 },
                                            '/etc/keystone/credential-keys/1' => { 'content' => $pass_keystone_credkey2 },
                                          },
            enable_fernet_setup        => true,
            fernet_replace_keys        => false,
            fernet_key_repository      => '/etc/keystone/fernet-keys',
            fernet_max_active_keys     => 4,
            fernet_keys                => { '/etc/keystone/fernet-keys/0' => { 'content' => $pass_keystone_fernkey1 },
                                          },
            token_expiration           => 604800,
            public_endpoint            => "${base_url}/identity",
            default_transport_url      => os_transport_url({
              'transport' => 'rabbit',
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => '5671',
              'username'  => 'keystone',
              'password'  => $pass_keystone_messaging,
            }),
            notification_transport_url => $keystone_notif_transport_url,
            notification_driver        => $notification_driver,
            rabbit_use_ssl             => $use_ssl,
            kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
            rabbit_ha_queues           => true,
            require                    => Service['haproxy'],
            sync_db                    => $do_db_sync,
          }
        # No parameter enable_ssl on zed and up.
        }else{
          class { '::keystone':
            catalog_driver             => 'sql',
            enabled                    => true,
            service_name               => 'keystone',
            manage_policyrcd           => false,
            enable_credential_setup    => true,
            credential_key_repository  => '/etc/keystone/credential-keys',
            credential_keys            => { '/etc/keystone/credential-keys/0' => { 'content' => $pass_keystone_credkey1 },
                                            '/etc/keystone/credential-keys/1' => { 'content' => $pass_keystone_credkey2 },
                                          },
            enable_fernet_setup        => true,
            fernet_replace_keys        => false,
            fernet_key_repository      => '/etc/keystone/fernet-keys',
            fernet_max_active_keys     => 4,
            fernet_keys                => { '/etc/keystone/fernet-keys/0' => { 'content' => $pass_keystone_fernkey1 },
                                          },
            token_expiration           => 604800,
            public_endpoint            => "${base_url}/identity",
            default_transport_url      => os_transport_url({
              'transport' => 'rabbit',
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => '5671',
              'username'  => 'keystone',
              'password'  => $pass_keystone_messaging,
            }),
            notification_transport_url => $keystone_notif_transport_url,
            notification_driver        => $notification_driver,
            rabbit_use_ssl             => $use_ssl,
            kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
            rabbit_ha_queues           => true,
            require                    => Service['haproxy'],
            sync_db                    => $do_db_sync,
          }
        }
      }
      # This class has only been backported up to Victoria.
      # It setups the number of worker to $::os_workers. See
      # how that is calculated in puppet-module-openstacklib
      # in lib/facter/os_workers.rb.
      class { '::keystone::wsgi::uwsgi': }
      class { '::keystone::cache':
        backend          => 'oslo_cache.memcache_pool',
        enabled          => true,
        memcache_servers => $memcached_servers,
      }
      if $initial_cluster_setup {
        if $is_first_master {
          class { '::keystone::bootstrap':
            password   => $pass_keystone_adminuser,
            email      => $admin_email_address,
            admin_url  => "${base_url}/identity",
            public_url => "${base_url}/identity",
            region     => $region_name,
            require    => Anchor['keystone::install::end'],
            before     => Keystone_role['creator'],
          }
        }else{
          if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
            ensure_resource('file', '/etc/keystone/puppet.conf', {
              'ensure'  => 'present',
              'content' => "
[keystone_authtoken]
region_name=${region_name}
auth_url=${base_url}/identity
username=admin
password=${pass_keystone_adminuser}
project_name=admin
interface=public
",
              'mode'    => '0600',
              require   => Anchor['keystone::install::end'],
              before    => Anchor['keystone::config::begin'],
            })
          }else{
            ensure_resource('file', '/etc/keystone/puppet.conf', {
              'ensure'  => 'absent',
              require   => Anchor['keystone::install::end'],
              before    => Anchor['keystone::config::begin'],
            })

            ensure_resource('file', '/etc/openstack', {
              'ensure' => 'directory',
              'mode'   => '0755',
              'owner'  => 'root',
              'group'  => 'root',
            })

            ensure_resource('file', '/etc/openstack/puppet', {
              'ensure' => 'directory',
              'mode'   => '0755',
              'owner'  => 'root',
              'group'  => 'root',
            })

            openstacklib::clouds { '/etc/openstack/puppet/admin-clouds.yaml':
              username     => 'admin',
              password     => $pass_keystone_adminuser,
              auth_url     => "${base_url}/identity",
              project_name => 'admin',
              system_scope => 'all',
              region_name  => $region_name,
              interface    => 'public',
            }
            Anchor['keystone::config::begin']
              -> Openstacklib::Clouds['/etc/openstack/puppet/admin-clouds.yaml']
              -> Anchor['keystone::config::end']
          }
        }
      }
    }
    if($is_first_master and ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri')){
      class { '::keystone::roles::admin':
        email    => $admin_email_address,
        password => $pass_keystone_adminuser,
      }->
      class { '::keystone::endpoint':
        public_url => "${base_url}/identity",
        admin_url  => "${keystone_admin_uri}",
      }
    }
    if $openstack_release == 'rocky'{
      class { '::keystone::disable_admin_token_auth': }
    }
#    class { '::openstack_extras::auth_file':
#      password       => $pass_keystone_adminuser,
#      project_domain => 'default',
#      user_domain    => 'default',
#      auth_url       => "${base_url}/identity/v3/",
#    }
    if($is_first_master){
      keystone_role { 'creator':
        ensure => present,
      }->
      keystone_role { 'member':
        ensure => present,
      }->
      keystone_role { 'SwiftOperator':
        ensure => present,
      }->
      keystone_role { 'ResellerAdmin':
        ensure => present,
      }->
      keystone_role { 'load-balancer_member':
        ensure => present,
      }->
      keystone_role { 'load-balancer_admin':
        ensure => present,
      }->
      keystone_role { 'load-balancer_global_observer':
        ensure => present,
      }->
      keystone_tenant { 'zabbix':
        ensure  => 'present',
        enabled => true,
      }->
      keystone_user { 'zabbix':
        ensure   => 'present',
        enabled  => true,
        password => $pass_zabbix_authtoken,
        email    => $admin_email_address,
        domain   => 'Default',
      }->
      keystone_user_role { 'zabbix@zabbix':
        ensure => 'present',
        roles  => [ 'member', 'creator', 'SwiftOperator', 'ResellerAdmin', 'load-balancer_member', 'load-balancer_global_observer', 'load-balancer_admin', ],
      }
    }
  }

  #######################
  ### Setup Ceph keys ###
  #######################
  if $cluster_has_osds {
    $mon_keyring_path = "/tmp/ceph-${machine_hostname}.keyring"
    class { 'ceph':
      fsid                => $ceph_fsid,
      ensure              => 'present',
      authentication_type => 'cephx',
      mon_initial_members => $ceph_mon_initial_members,
      mon_host            => $ceph_mon_host,
    }->
    ceph::key { 'client.admin':
      secret  => $ceph_admin_key,
      cap_mon => 'allow *',
      cap_osd => 'allow *',
      cap_mgr => 'allow *',
      cap_mds => 'allow',
    }->
    ceph::key { 'client.openstack':
      secret  => $ceph_openstack_key,
      mode    => '0644',
      cap_mon => 'profile rbd',
      cap_osd => 'profile rbd pool=cinder, profile rbd pool=nova, profile rbd pool=glance, profile rbd pool=gnocchi, profile rbd pool=cinderback, profile rbd pool=manila',
    }->
    ceph::key { 'client.bootstrap-osd':
      secret       => $ceph_bootstrap_osd_key,
      keyring_path => '/var/lib/ceph/bootstrap-osd/ceph.keyring',
      cap_mon      => 'allow profile bootstrap-osd',
    }

    if $cluster_has_mons {
      warning('Cluster has OSD nodes: will not setup MON and MGR on the controller nodes.')
    }else{
      exec { 'create-tmp-keyring':
        command => "/bin/true # comment to satisfy puppet syntax requirements
set -ex

cat > /etc/ceph/ceph.conf<< EOF
[global]
fsid = ${ceph_fsid}
mon_initial_members = ${ceph_mon_initial_members}
mon_host = ${ceph_mon_host}
auth cluster required = cephx
auth service required = cephx
auth client required = cephx
osd journal size = 1024
osd pool default size = 3
osd pool default min size = 2
osd pool default pg num = 333
osd pool default pgp num = 333
osd crush chooseleaf type = 1
EOF

cat > ${mon_keyring_path}<< EOF
[mon.]
    key = ${ceph_mon_key}
    caps mon = \"allow *\"
EOF

mon_data=\$(ceph-mon --cluster ceph --id ${machine_hostname} --show-config-value mon_data)

ceph-authtool ${mon_keyring_path} --import-keyring /etc/ceph/ceph.client.admin.keyring
ceph-authtool ${mon_keyring_path} --import-keyring /etc/ceph/ceph.client.openstack.keyring
ceph-authtool ${mon_keyring_path} --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
chmod 0444 ${mon_keyring_path}
chown ceph:ceph ${mon_keyring_path}

rm -f /tmp/monmap
monmaptool --create --add ${machine_hostname} ${machine_ip} --fsid ${ceph_fsid} /tmp/monmap
MACHINE_IPS=\$(echo ${ceph_mon_host} | tr ',' ' ')
CNT=1
for i in \$(echo ${ceph_mon_initial_members} | tr ',' ' ') ; do
        if [ \$i = ${machine_hostname} ] ; then
                echo -n ''
        else
                IP=\$(echo \$MACHINE_IPS | awk '{print \$'\$CNT'}')
                monmaptool --add \$i \$IP /tmp/monmap
        fi
        CNT=\$((\${CNT} + 1))
done

chown ceph:ceph /tmp/monmap

sudo -u ceph mkdir -p \$mon_data
sudo -u ceph ceph-mon --cluster ceph --mkfs -i ${machine_hostname} --monmap /tmp/monmap --keyring ${mon_keyring_path}
touch \$mon_data/done
",
        unless  => "/bin/true # comment to satisfy puppet syntax requirements
set -ex
mon_data=\$(ceph-mon --cluster ceph --id ${machine_hostname} --show-config-value mon_data) || exit 1
# if ceph-mon fails then the mon is probably not configured yet
test -e \$mon_data/done
",
        require => Ceph::Key['client.bootstrap-osd'],
      }->
      ceph::mon { $machine_hostname:
        keyring     => $mon_keyring_path,
        public_addr => $machine_ip,
      }->
      ceph::mgr { $machine_hostname:
        key        => $ceph_mgr_key,
        inject_key => true,
      }
    }
    if $has_subrole_nova {
      $ceph_pools = ['glance', 'nova', 'cinder', 'gnocchi', 'cinderback', 'manila']
      ceph::pool { $ceph_pools: }
    }
  }

  ####################
  ### Setup Glance ###
  ####################
  if $has_subrole_glance {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['glance::install::begin']
    }
    if $use_ssl {
      ::oci::sslkeypair {'glance':
        notify_service_name => 'glance-api',
      }
      $glance_key_file = "/etc/glance/ssl/private/${::fqdn}.pem"
      $glance_crt_file = "/etc/glance/ssl/public/${::fqdn}.crt"
    } else {
      $glance_key_file = undef
      $glance_crt_file = undef
    }

    include ::glance
    include ::glance::client
    if $is_first_master {
      class { '::glance::keystone::auth':
        public_url   => "${base_url}/image",
        internal_url => "${base_url}/image",
        admin_url    => "${base_url}/image",
        password     => $pass_glance_authtoken,
        region       => $region_name,
      }
    }

    class { '::glance::api::authtoken':
      password             => $pass_glance_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }

    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
      class { '::glance::backend::file':
        multi_store => true,
      }
    }else{
      ::glance::backend::multistore::file {'oci-glance-file-backend': }
    }
    if $extswift_use_external {
      notice("Will use an external swift as a backend for Glance")
      if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
        $backend_stores = ['swift', 'file']
      }else{
        $backend_stores = ['swift:swift', 'file:file']
      }
      $glance_default_store = 'swift'
    }else{
      case $glance_backend {
        # This is there just in case, but doesn't really work, since
        # there's no mech to sync files between controller nodes.
        'file': {
          if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
            $backend_stores = ['file']
          }else{
            $backend_stores = ['file:file']
          }
          $glance_default_store = 'file'
        }
        # This will be in use in priority, if OCI sees some OSD nodes.
        'ceph': {
          if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
            class { '::glance::backend::rbd':
              rbd_store_user => 'openstack',
              rbd_store_pool => 'glance',
              multi_store    => true,
            }
          }
          if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
            $backend_stores = ['rbd']
          }else{
            $backend_stores = ['rbd:rbd']
            ::glance::backend::multistore::rbd { 'rbd':
              rbd_store_user        => 'openstack',
              rbd_store_pool        => 'glance',
              rbd_store_chunk_size  => 4,
              rbd_thin_provisioning => true,
              rados_connect_timeout => 0,
              store_description     => 'RBD backend',
            }
          }
          $glance_default_store = 'rbd'
          # make sure ceph pool exists before running Glance API
          Exec['create-glance'] -> Service['glance-api']
        }
        # Used if no OSD nodes are setup, but some swift store are present.
        'swift': {
          if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
            $backend_stores = ['swift', 'file']
          }else{
            $backend_stores = ['swift:swift', 'file:file']
          }
          $glance_default_store = 'swift'
          if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
            if $extswift_use_external {
              $region_real = $extswift_region
            } else {
              $region_real = $region_name
            }
            class { '::glance::backend::swift':
              swift_store_user                    => 'services:glance',
              swift_store_key                     => $pass_glance_authtoken,
              swift_store_create_container_on_put => 'True',
              swift_store_auth_address            => $keystone_auth_uri,
              swift_store_auth_version            => '3',
              multi_store                         => true,
              swift_store_region                  => $region_real,
            }
          }else{
            ::glance::backend::multistore::swift { 'swift':
              swift_store_user                    => 'services:glance',
              swift_store_auth_address            => "${keystone_auth_uri}/v3",
              swift_store_auth_project_domain_id  => 'default',
              swift_store_auth_user_domain_id     => 'default',
              swift_store_key                     => $pass_glance_authtoken,
              swift_store_create_container_on_put => 'True',
              swift_store_auth_version            => '3',
              store_description                   => 'SWIFT store',
              swift_store_config_file             => '/etc/glance/glance-swift.conf',
              swift_store_region                  => $region_real,
            }
            glance_api_config {
              "swift/swift_store_cacert": value => $api_endpoint_ca_file;
            }
          }
	  if $self_signed_api_cert {
	    glance_api_config {
              'glance_store/swift_store_auth_insecure': value => true;
            }
	  }else{
	    glance_api_config {
              'glance_store/swift_store_auth_insecure': value => false;
            }
	  }
        }
        'cinder': {
          if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
            $backend_stores = ['cinder', 'file']
          }else{
            $backend_stores = ['cinder:cinder', 'file:file']
          }
          if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
            class { '::glance::backend::cinder':
              cinder_ca_certificates_file => $api_endpoint_ca_file,
              multi_store                 => true,
            }
          }else{
            class { '::glance::backend::multistore::cinder':
              cinder_ca_certificates_file => $api_endpoint_ca_file,
              cinder_use_multipath        => true,
            }
          }
        }
      }
    }

    if $openstack_release == 'rocky'{
      class { '::glance::api':
        debug                        => true,
        database_connection          => "mysql+pymysql://glance:${pass_glance_db}@${sql_host}/glancedb?charset=utf8",
        database_idle_timeout        => 1800,
        workers                      => $::os_workers,
        use_stderr                   => true,
        stores                       => $backend_stores,
        default_store                => $glance_default_store,
        bind_host                    => $machine_ip,
        cert_file                    => $glance_crt_file,
        key_file                     => $glance_key_file,
        enable_v1_api                => false,
        enable_v2_api                => true,
        sync_db                      => $do_db_sync,
        enable_proxy_headers_parsing => true,
      }
    } else {
      if ($openstack_release == 'stein' or $openstack_release == 'train') {
        class { '::glance::api':
          database_connection          => "mysql+pymysql://glance:${pass_glance_db}@${sql_host}/glancedb?charset=utf8",
          database_idle_timeout        => 1800,
          workers                      => 2,
          stores                       => $backend_stores,
          default_store                => $glance_default_store,
          bind_host                    => $machine_ip,
          cert_file                    => $glance_crt_file,
          key_file                     => $glance_key_file,
          enable_v1_api                => false,
          enable_v2_api                => true,
          multi_store                  => true,
          sync_db                      => $do_db_sync,
          enable_proxy_headers_parsing => true,
        }
      }else{
        if $openstack_release == 'ussuri' {
          class { '::glance::api':
            database_connection          => "mysql+pymysql://glance:${pass_glance_db}@${sql_host}/glancedb?charset=utf8",
            database_idle_timeout        => 1800,
            workers                      => 2,
            bind_host                    => $machine_ip,
            cert_file                    => $glance_crt_file,
            key_file                     => $glance_key_file,
            enable_v1_api                => false,
            enable_v2_api                => true,
            sync_db                      => $do_db_sync,
            enable_proxy_headers_parsing => true,
            enabled_backends             => $backend_stores,
            default_backend              => $glance_default_store,
            multi_store                  => true,
          }
        # Victoria and up
        }else{
          # To be called before ::glance::api that includes it.
          class { '::glance::policy':
            policy_path => '/etc/glance/policy.d/00_default_policy.yaml',
            policy_dirs => '/etc/glance/policy.d',
          }

          if ($openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena') {
            class { '::glance::api':
              database_connection          => "mysql+pymysql://glance:${pass_glance_db}@${sql_host}/glancedb?charset=utf8",
              database_idle_timeout        => 1800,
              workers                      => 2,
              bind_host                    => $machine_ip,
              sync_db                      => $do_db_sync,
              enable_proxy_headers_parsing => true,
              enabled_backends             => $backend_stores,
              default_backend              => $glance_default_store,
              multi_store                  => true,
            }
          # Yoga and up
          }else{
            class { '::glance::api::db':
              database_connection              => "mysql+pymysql://glance:${pass_glance_db}@${sql_host}/glancedb?charset=utf8",
              database_connection_recycle_time => 1800,
            }
            if ($openstack_release == 'yoga') {
              class { '::glance::api':
                workers                      => 2,
                bind_host                    => $machine_ip,
                sync_db                      => $do_db_sync,
                enable_proxy_headers_parsing => true,
                enabled_backends             => $backend_stores,
                default_backend              => $glance_default_store,
                multi_store                  => true,
              }
            # Zed and up adds public_endpoint directly in ::glance::api
            }else{
              class { '::glance::api':
                workers                      => 2,
                bind_host                    => $machine_ip,
                sync_db                      => $do_db_sync,
                enable_proxy_headers_parsing => true,
                enabled_backends             => $backend_stores,
                default_backend              => $glance_default_store,
                multi_store                  => true,
                public_endpoint              => "${base_url}/image",
              }
            }
          }

          if $glance_metadef_admin_only {
            file { '/etc/glance/policy.d/metadef-admin-only.yaml':
              ensure  => present,
              source  => 'puppet:///modules/oci/glance/metadef-api-disabled.yaml',
              group   => 'root',
              owner   => 'glance',
              mode    => '0640',
              require => Anchor['glance::install::end'],
              notify  => Service['glance-api'],
            }
          }else{
            file { '/etc/glance/policy.d/metadef-admin-only.yaml':
              ensure  => absent,
              require => Anchor['glance::install::end'],
              notify  => Service['glance-api'],
            }
          }
        }
      }
      class { '::glance::api::logging':
        debug => true,
      }
    }
    # Fix the number of uwsgi processes
    class { '::glance::wsgi::uwsgi': }
    # Disable uwsgi logs
    glance_api_uwsgi_config {
      'uwsgi/disable-logging': value => true;
    }

    if $extswift_use_external {
      class { '::glance::backend::swift':
        swift_store_user                    => "${extswift_project_name}:${extswift_user_name}",
        swift_store_key                     => $extswift_password,
        swift_store_create_container_on_put => 'True',
        swift_store_auth_address            => "${extswift_auth_url}/v3",
        swift_store_auth_version            => '3',
        swift_store_region                  => $extswift_region,
        multi_store                         => true,
      }
    }

    if $disable_notifications {
      $glance_notif_transport_url = ''
    } else {
      $glance_notif_transport_url = os_transport_url({
                                      'transport' => $messaging_notify_proto,
                                      'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                      'port'      => $messaging_notify_port,
                                      'username'  => 'glance',
                                      'password'  => $pass_glance_messaging,
                                    })
    }
    class { '::glance::notify::rabbitmq':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'glance',
        'password'  => $pass_glance_messaging,
      }),
      notification_transport_url => $glance_notif_transport_url,
      notification_driver        => $notification_driver,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
    }

    # Starting with Zed, DEFAULT/public_endpoint is set when calling glange::api.
    if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena') {
      class { '::glance::config':
        api_config => {
          'DEFAULT/public_endpoint' => { value => "${base_url}/image"},
        }
      }
    }
  }

  ###################
  ### Setup Swift ###
  ###################
  if $has_subrole_swift  {
    if $swiftproxy_hostname == 'none' {
      $swiftproxy_baseurl = $base_url
    } else {
      $swiftproxy_baseurl = "${proto}://${swiftproxy_hostname}"
    }
    if $is_first_master {
      class { '::swift::keystone::auth':
        public_url            => "${swiftproxy_baseurl}/object/v1/AUTH_%(tenant_id)s",
        admin_url             => "${swiftproxy_baseurl}/object",
        internal_url          => "${swiftproxy_baseurl}/object/v1/AUTH_%(tenant_id)s",
        password              => $pass_swift_authtoken,
        operator_roles        => ['admin', 'SwiftOperator', 'ResellerAdmin'],
        configure_s3_endpoint => false,
        region                => $region_name,
      }
    }
  }

  ##################
  ### Setup Heat ###
  ##################
  if $has_subrole_heat {
    if $has_subrole_db  and $initial_cluster_setup {
      Class['galera'] -> Anchor['heat::install::begin']
      Exec['galera-size-is-correct'] -> Package['heat dashboard']
#      Class['oci::sql::galera'] -> Package['heat dashboard']
    }
    if $use_ssl {
      oci::sslkeypair { 'heat':
        notify_service_name => 'heat-api',
      }
      $heat_key_file = "/etc/heat/ssl/private/${::fqdn}.pem"
      $heat_crt_file = "/etc/heat/ssl/public/${::fqdn}.crt"
    } else {
      $heat_key_file = undef
      $heat_crt_file = undef
    }
    class { '::heat::keystone::authtoken':
      password             => $pass_heat_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    if $is_first_master {
      class { '::heat::keystone::auth':
        public_url   => "${base_url}/orchestration-api/v1/%(tenant_id)s",
        internal_url => "${base_url}/orchestration-api/v1/%(tenant_id)s",
        admin_url    => "${base_url}/orchestration-api/v1/%(tenant_id)s",
        password     => $pass_heat_authtoken,
        region       => $region_name,
      }
      class { '::heat::keystone::auth_cfn':
        public_url   => "${base_url}/orchestration-cfn/v1",
        internal_url => "${base_url}/orchestration-cfn/v1",
        admin_url    => "${base_url}/orchestration-cfn/v1",
        password     => $pass_heat_authtoken,
        region       => $region_name,
      }
    }

    if $disable_notifications {
      $heat_notif_transport_url = ''
    } else {
      $heat_notif_transport_url = os_transport_url({
                                    'transport' => $messaging_notify_proto,
                                    'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                    'port'      => $messaging_notify_port,
                                    'username'  => 'heat',
                                    'password'  => $pass_heat_messaging,
                                  })
    }
    if $openstack_release == 'rocky'{
      class { '::heat':
        debug                        => true,
        default_transport_url        => os_transport_url({
                                          'transport' => $messaging_default_proto,
                                          'hosts'     => fqdn_rotate($all_masters),
                                          'port'      => $messaging_default_port,
                                          'username'  => 'heat',
                                          'password'  => $pass_heat_messaging,
                                        }),
        notification_transport_url   => $heat_notif_transport_url,
        host                         => $machine_hostname,
        rabbit_use_ssl               => $use_ssl,
        rabbit_ha_queues             => true,
        kombu_ssl_ca_certs           => $oci_pki_root_ca_file,
        database_connection          => "mysql+pymysql://heat:${pass_heat_db}@${sql_host}/heatdb?charset=utf8",
        database_idle_timeout        => 1800,
        notification_driver          => $notification_driver,
        sync_db                      => $do_db_sync,
        max_stacks_per_tenant        => 5000,
        enable_proxy_headers_parsing => true,
      }
    } else {
      if $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' {
        class { '::heat':
          default_transport_url      => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'heat',
            'password'  => $pass_heat_messaging,
          }),
          notification_transport_url => $heat_notif_transport_url,
          host                       => $machine_hostname,
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          database_connection        => "mysql+pymysql://heat:${pass_heat_db}@${sql_host}/heatdb?charset=utf8",
          database_idle_timeout      => 1800,
          notification_driver        => $notification_driver,
          sync_db                    => $do_db_sync,
          max_stacks_per_tenant      => 5000,
          enable_stack_abandon       => true,
        }
      }elsif $openstack_release == 'xena'{
        # max_stacks_per_tenant is now a parameter of heat::engine
        class { '::heat':
          default_transport_url      => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'heat',
            'password'  => $pass_heat_messaging,
          }),
          notification_transport_url => $heat_notif_transport_url,
          host                       => $machine_hostname,
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          database_connection        => "mysql+pymysql://heat:${pass_heat_db}@${sql_host}/heatdb?charset=utf8",
          database_idle_timeout      => 1800,
          notification_driver        => $notification_driver,
          sync_db                    => $do_db_sync,
          enable_stack_abandon       => true,
        }
      # Yoga and up
      }else{
        class { '::heat::db':
          database_connection              => "mysql+pymysql://heat:${pass_heat_db}@${sql_host}/heatdb?charset=utf8",
          database_connection_recycle_time => 1800,
          sync_db                          => $do_db_sync,
        }
        if $openstack_release == 'yoga'{
          class { '::heat':
            default_transport_url      => os_transport_url({
              'transport' => $messaging_default_proto,
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => $messaging_default_port,
              'username'  => 'heat',
              'password'  => $pass_heat_messaging,
            }),
            notification_transport_url => $heat_notif_transport_url,
            host                       => $machine_hostname,
            rabbit_use_ssl             => $use_ssl,
            rabbit_ha_queues           => true,
            kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
            notification_driver        => $notification_driver,
            enable_stack_abandon       => true,
          }
        # Zed and up
        }else{
          class { '::heat::clients':
            ca_file => $api_endpoint_ca_file,
          }
          class { '::heat':
            default_transport_url      => os_transport_url({
              'transport' => $messaging_default_proto,
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => $messaging_default_port,
              'username'  => 'heat',
              'password'  => $pass_heat_messaging,
            }),
            notification_transport_url => $heat_notif_transport_url,
            host                       => $machine_hostname,
            rabbit_use_ssl             => $use_ssl,
            rabbit_ha_queues           => true,
            kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
            notification_driver        => $notification_driver,
            enable_stack_abandon       => true,
          }
        }
      }
      class { '::heat::logging':
        debug => true,
      }
      heat_config {
        'heat_api/heat_api_root':    value => "${base_url}/orchestration-api/v1";
      }
    }
    if $initial_cluster_setup {
      class { '::heat::keystone::domain':
        domain_password => $pass_heat_keystone_domain,
      }
    }
    class { '::heat::client': }
    class { '::heat::api':
      service_name => 'heat-api',
    }
    # Fix the number of uwsgi processes
    class { '::heat::wsgi::uwsgi_api': }
    # Disable uwsgi logs
    heat_api_uwsgi_config {
      'uwsgi/disable-logging': value => true;
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' {
      class { '::heat::engine':
        num_engine_workers            => $::os_workers,
        auth_encryption_key           => $pass_heat_encryptkey[0,32],
#        heat_metadata_server_url      => "${base_url}:8000/orchestration-cfn",
#        heat_waitcondition_server_url => "${base_url}:8000/orchestration-cfn/v1/waitcondition",
      }
    }else{
      class { '::heat::engine':
        num_engine_workers            => $::os_workers,
        auth_encryption_key           => $pass_heat_encryptkey[0,32],
#        heat_metadata_server_url      => "${base_url}:8000/orchestration-cfn",
#        heat_waitcondition_server_url => "${base_url}:8000/orchestration-cfn/v1/waitcondition",
        max_stacks_per_tenant         => 5000,
      }
    }
    class { '::heat::api_cfn':
      service_name => 'heat-api-cfn',
    }
    # Fix the number of uwsgi processes
    class { '::heat::wsgi::uwsgi_api_cfn': }
    # Disable uwsgi logs
    heat_api_cfn_uwsgi_config {
      'uwsgi/disable-logging': value => true;
    }
    class { '::heat::cron::purge_deleted': }
    package { 'heat dashboard':
      name => 'python3-heat-dashboard',
      ensure => installed,
    }
    heat_config {
      'clients_aodh/ca_file':      value => $api_endpoint_ca_file;
      'clients_barbican/ca_file':  value => $api_endpoint_ca_file;
      'clients_cinder/ca_file':    value => $api_endpoint_ca_file;
      'clients_designate/ca_file': value => $api_endpoint_ca_file;
      'clients_glance/ca_file':    value => $api_endpoint_ca_file;
      'clients_keystone/ca_file':  value => $api_endpoint_ca_file;
      'clients_magnum/ca_file':    value => $api_endpoint_ca_file;
      'clients_manila/ca_file':    value => $api_endpoint_ca_file;
      'clients_mistral/ca_file':   value => $api_endpoint_ca_file;
      'clients_monasca/ca_file':   value => $api_endpoint_ca_file;
      'clients_neutron/ca_file':   value => $api_endpoint_ca_file;
      'clients_nova/ca_file':      value => $api_endpoint_ca_file;
      'clients_octavia/ca_file':   value => $api_endpoint_ca_file;
      'clients_sahara/ca_file':    value => $api_endpoint_ca_file;
      'clients_senlin/ca_file':    value => $api_endpoint_ca_file;
      'clients_swift/ca_file':     value => $api_endpoint_ca_file;
      'clients_trove/ca_file':     value => $api_endpoint_ca_file;
      'clients_zaqar/ca_file':     value => $api_endpoint_ca_file;
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'or $openstack_release == 'yoga'{
      heat_config {
        'clients/ca_file':      value => $api_endpoint_ca_file;
        'clients_heat/ca_file': value => $api_endpoint_ca_file;
      }
    }
  }

  #####################
  ### Setup Horizon ###
  #####################
  if $has_subrole_horizon {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['horizon::install::begin']
    }
    if $use_ssl {
      file { "/etc/openstack-dashboard/ssl":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
        require                 => Class['::horizon'],
      }->
      file { "/etc/openstack-dashboard/ssl/private":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { "/etc/openstack-dashboard/ssl/public":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { "/etc/openstack-dashboard/ssl/private/${::fqdn}.pem":
        ensure                  => present,
        owner                   => "www-data",
        source                  => "/etc/ssl/private/ssl-cert-snakeoil.key",
        selinux_ignore_defaults => true,
        mode                    => '0600',
      }->
      file { "/etc/openstack-dashboard/ssl/public/${::fqdn}.crt":
        ensure                  => present,
        owner                   => "www-data",
        source                  => '/etc/ssl/certs/ssl-cert-snakeoil.pem',
        selinux_ignore_defaults => true,
        mode                    => '0644',
        notify                  => Service['httpd'],
      }

      $horizon_key_file = "/etc/openstack-dashboard/ssl/private/${::fqdn}.pem"
      $horizon_crt_file = "/etc/openstack-dashboard/ssl/public/${::fqdn}.crt"
    } else {
      $horizon_key_file = undef
      $horizon_crt_file = undef
    }

    case $cinder_backup_backend {
      'ceph': {
        $enable_backup             = true
        $create_volume             = true
        $disable_instance_snapshot = false
        $disable_volume_snapshot   = false
      }
      'swift': {
        $enable_backup             = true
        $create_volume             = true
        $disable_instance_snapshot = false
        $disable_volume_snapshot   = false
      }
      'none': {
        $enable_backup             = false
        $create_volume             = false
        $disable_instance_snapshot = true
        $disable_volume_snapshot   = true
      }
    }

    if $openstack_release == 'rocky' or $openstack_release == 'stein'{
      $horizon_log_hanlder = 'file'
    }else{
      $horizon_log_hanlder = 'console'
    }
    $horizon_themes_array = split($horizon_themes, /:/)
    $available_themes = $horizon_themes_array.map |$index,$value| {
      $vals_array = $value.split(/@/)
      $retval = { 'name' => $vals_array[0], 'label' => $vals_array[1], 'path' => $vals_array[2]}
    }
    $password_retrieve = true
    if $openstack_release == 'rocky' or $openstack_release == 'stein'{
      class { '::horizon':
        secret_key       => $pass_horizon_secretkey,
        # TODO: Fix this with a more secure stuff, probably the FQDN of the API.
        allowed_hosts    => '*',
        listen_ssl       => true,
        ssl_redirect     => false,
        http_port        => '7080',
        https_port       => '7443',
        horizon_cert     => $horizon_crt_file,
        horizon_key      => $horizon_key_file,
  # Is this $api_endpoint_ca_file or $oci_pki_root_ca_file ?
        horizon_ca       => '/etc/ssl/certs/oci-pki-oci-ca-chain.pem',
        keystone_url     => "${keystone_auth_uri}/v3",
        compress_offline => true,
        cinder_options   => { 'enable_backup' => $enable_backup },
        neutron_options  => { 'enable_ha_router' => true },
        instance_options => {
          'create_volume'             => $create_volume,
          'disable_instance_snapshot' => $disable_instance_snapshot,
          'disable_volume_snapshot'   => $disable_volume_snapshot },
        # Horizon is broken in Train otherwise:
        log_handler      => $horizon_log_hanlder,
        available_themes => $available_themes,
        vhost_extra_params => { 'limitreqbody' => $horizon_limitreqbody },
      }
      file { '/var/lib/openstack-dashboard/upload_tmp':
        ensure => 'directory',
        mode   => '0755',
        owner  => 'www-data',
        group  => 'www-data',
        require => Anchor['horizon::service::end'],
      }
    }else{
      if $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria'{
        class { '::horizon':
          secret_key              => $pass_horizon_secretkey,
          # TODO: Fix this with a more secure stuff, probably the FQDN of the API.
          allowed_hosts           => '*',
          listen_ssl              => true,
          ssl_redirect            => false,
          http_port               => '7080',
          https_port              => '7443',
          horizon_cert            => $horizon_crt_file,
          horizon_key             => $horizon_key_file,
          # Is this $api_endpoint_ca_file or $oci_pki_root_ca_file ?
          horizon_ca              => '/etc/ssl/certs/oci-pki-oci-ca-chain.pem',
          keystone_url            => "${keystone_auth_uri}/v3",
          compress_offline        => true,
          cinder_options          => { 'enable_backup' => $enable_backup },
          neutron_options         => { 'enable_ha_router' => true },
          instance_options        => {
            'create_volume'             => $create_volume,
            'disable_instance_snapshot' => $disable_instance_snapshot,
            'disable_volume_snapshot'   => $disable_volume_snapshot },
          # Horizon is broken in Train otherwise:
          log_handler             => $horizon_log_hanlder,
          manage_memcache_package => false,
          django_session_engine   => 'django.contrib.sessions.backends.cache',
          cache_backend           => 'django.core.cache.backends.memcached.MemcachedCache',
          cache_server_url        => "${machine_hostname}:11211",
          available_themes        => $available_themes,
          vhost_extra_params      => { 'limitreqbody' => $horizon_limitreqbody },
          file_upload_temp_dir    => '/var/lib/openstack-dashboard/upload_tmp',
          password_retrieve       => true,
          default_nameserver_1    => $nameserver_v4_prim,
          default_nameserver_2    => $nameserver_v4_sec,
          #policy_files            => { 'dns' => 'designate_policy.json' },
        }
        #-> file { '/etc/openstack-dashboard/policy/designate_policy.json':
         # ensure => file,
          #owner                   => 'root',
          #group                   => 'root',
          #mode                    => '0644',
          #selinux_ignore_defaults => true,
          #content                 => '{
#"dnsmember": "role:dnsmember",
#"admin": "role:admin or is_admin:True",
#"admin_or_dnsmember": "role:admin or role:dnsmember"
#}',
     #   }
      }elsif $openstack_release == 'wallaby' or $openstack_release == 'xena'{
        class { '::horizon':
          secret_key              => $pass_horizon_secretkey,
          # TODO: Fix this with a more secure stuff, probably the FQDN of the API.
          allowed_hosts           => '*',
          listen_ssl              => true,
          ssl_redirect            => false,
          http_port               => '7080',
          https_port              => '7443',
          horizon_cert            => $horizon_crt_file,
          horizon_key             => $horizon_key_file,
          ssl_verify_client       => 'off',
          # Is this $api_endpoint_ca_file or $oci_pki_root_ca_file ?
          horizon_ca              => '/etc/ssl/certs/oci-pki-oci-ca-chain.pem',
          keystone_url            => "${keystone_auth_uri}/v3",
          compress_offline        => true,
          cinder_options          => { 'enable_backup' => $enable_backup },
          neutron_options         => { 'enable_ha_router' => true },
          instance_options        => {
            'create_volume'             => $create_volume,
            'disable_instance_snapshot' => $disable_instance_snapshot,
            'disable_volume_snapshot'   => $disable_volume_snapshot },
          # Horizon is broken in Train otherwise:
          manage_memcache_package => false,
          django_session_engine   => 'django.contrib.sessions.backends.cache',
          cache_backend           => 'django.core.cache.backends.memcached.MemcachedCache',
          cache_server_url        => "${machine_hostname}:11211",
          available_themes        => $available_themes,
          vhost_extra_params      => { 'limitreqbody' => $horizon_limitreqbody },
          file_upload_temp_dir    => '/var/lib/openstack-dashboard/upload_tmp',
          password_retrieve       => true,
          default_nameserver_1    => $nameserver_v4_prim,
          default_nameserver_2    => $nameserver_v4_sec,
          #policy_files            => { 'dns' => 'designate_policy.json' },
        }
        #-> file { '/etc/openstack-dashboard/policy/designate_policy.json':
         # ensure => file,
          #owner                   => 'root',
          #group                   => 'root',
          #mode                    => '0644',
          #selinux_ignore_defaults => true,
          #content                 => '{
#"dnsmember": "role:dnsmember",
#"admin": "role:admin or is_admin:True",
#"admin_or_dnsmember": "role:admin or role:dnsmember"
#}',
        #}
      # Yoga and up
      }else{
        class { '::horizon':
          secret_key              => $pass_horizon_secretkey,
          # TODO: Fix this with a more secure stuff, probably the FQDN of the API.
          allowed_hosts           => '*',
          listen_ssl              => true,
          ssl_redirect            => false,
          http_port               => '7080',
          https_port              => '7443',
          ssl_cert                => $horizon_crt_file,
          ssl_key                 => $horizon_key_file,
          ssl_verify_client       => 'off',
          # Is this $api_endpoint_ca_file or $oci_pki_root_ca_file ?
          ssl_ca                  => '/etc/ssl/certs/oci-pki-oci-ca-chain.pem',
          keystone_url            => "${keystone_auth_uri}/v3",
          compress_offline        => true,
          cinder_options          => { 'enable_backup' => $enable_backup },
          neutron_options         => { 'enable_ha_router' => true },
          instance_options        => {
            'create_volume'             => $create_volume,
            'disable_instance_snapshot' => $disable_instance_snapshot,
            'disable_volume_snapshot'   => $disable_volume_snapshot },
          # Horizon is broken in Train otherwise:
          manage_memcache_package => false,
          django_session_engine   => 'django.contrib.sessions.backends.cache',
          cache_backend           => 'django.core.cache.backends.memcached.MemcachedCache',
          cache_server_url        => "${machine_hostname}:11211",
          available_themes        => $available_themes,
          vhost_extra_params      => { 'limitreqbody' => $horizon_limitreqbody },
          file_upload_temp_dir    => '/var/lib/openstack-dashboard/upload_tmp',
          password_retrieve       => true,
          default_nameserver_1    => $nameserver_v4_prim,
          default_nameserver_2    => $nameserver_v4_sec,
          #policy_files            => { 'dns' => 'designate_policy.json' },
        }
        #-> file { '/etc/openstack-dashboard/policy/designate_policy.json':
         # ensure => file,
          #owner                   => 'root',
          #group                   => 'root',
          #mode                    => '0644',
          #selinux_ignore_defaults => true,
          #content                 => '{
#"dnsmember": "role:dnsmember",
#"admin": "role:admin or is_admin:True",
#"admin_or_dnsmember": "role:admin or role:dnsmember"
#}',
        #}
      }
      file { '/var/lib/openstack-dashboard/secret-key/.secret_key_store':
        ensure => 'file',
        mode   => '0600',
        owner  => 'www-data',
        group  => 'root',
        require => Anchor['horizon::service::end'],
      }
    }
  }
  ######################
  ### Setup Barbican ###
  ######################
  if $has_subrole_barbican {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['barbican::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'barbican':
        notify_service_name => 'barbican-api',
      }
      $barbican_key_file = "/etc/barbican/ssl/private/${::fqdn}.pem"
      $barbican_crt_file = "/etc/barbican/ssl/public/${::fqdn}.crt"
    } else {
      $barbican_key_file = undef
      $barbican_crt_file = undef
    }
    include ::barbican
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::barbican::db':
        database_connection   => "mysql+pymysql://barbican:${pass_barbican_db}@${sql_host}/barbicandb?charset=utf8",
        database_idle_timeout => 1800,
      }
    }else{
      class { '::barbican::db':
        database_connection              => "mysql+pymysql://barbican:${pass_barbican_db}@${sql_host}/barbicandb?charset=utf8",
        database_connection_recycle_time => 1800,
      }
    }
    if $is_first_master {
      class { '::barbican::keystone::auth':
        public_url   => "${base_url}/keymanager",
        internal_url => "${base_url}/keymanager",
        admin_url    => "${base_url}/keymanager",
        password     => $pass_barbican_authtoken,
        region       => $region_name,
      }
    }
    include ::barbican::quota
    include ::barbican::keystone::notification
    class { '::barbican::api::logging':
      debug => true,
    }
    class { '::barbican::keystone::authtoken':
      password             => $pass_barbican_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }

    if $disable_notifications {
      $barbican_notif_transport_url = ''
    } else {
      $barbican_notif_transport_url = os_transport_url({
                                        'transport' => $messaging_default_proto,
                                        'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                        'port'      => $messaging_default_port,
                                        'username'  => 'barbican',
                                        'password'  => $pass_barbican_messaging,
                                      })
    }
    class { '::barbican::api':
      default_transport_url       => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'barbican',
        'password'  => $pass_barbican_messaging,
      }),
      notification_transport_url  => $barbican_notif_transport_url,
      host_href                   => "${base_url}/keymanager",
      auth_strategy               => 'keystone',
      enabled_certificate_plugins => ['simple_certificate'],
      db_auto_create              => false,
      rabbit_use_ssl              => $use_ssl,
      rabbit_ha_queues            => true,
      kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
      sync_db                     => $do_db_sync,
      max_allowed_secret_in_bytes => 25600,
    }
    # Fix the number of uwsgi processes
    class { '::barbican::wsgi::uwsgi': }
    # Disable uwsgi logs
    barbican_api_uwsgi_config {
      'uwsgi/disable-logging': value => true;
    }

    class { 'barbican::worker': }
  }

  #######################
  ### Setup Placement ###
  #######################
  if $has_subrole_nova and $openstack_release != 'rocky'{
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['placement::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'placement':
        notify_service_name => 'placement-api',
      }
      $placement_key_file = "/etc/placement/ssl/private/${::fqdn}.pem"
      $placement_crt_file = "/etc/placement/ssl/public/${::fqdn}.crt"
    } else {
      $placement_key_file = undef
      $placement_crt_file = undef
    }

    class { '::placement':
      sync_db => $do_db_sync,
    }

    if $is_first_master {
      class { '::placement::keystone::auth':
        public_url   => "${base_url}/placement",
        internal_url => "${base_url}/placement",
        admin_url    => "${base_url}/placement",
        password     => $pass_placement_authtoken,
        region       => $region_name,
      }
    }
    class { '::placement::keystone::authtoken':
      password             => $pass_placement_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    class { '::placement::logging':
      debug => true,
    }
    class { '::placement::db':
      database_connection   => "mysql+pymysql://placement:${pass_placement_db}@${sql_host}/placementdb?charset=utf8",
    }
    package { 'python3-osc-placement':
      ensure => 'present',
    }
    if $is_first_master and $initial_cluster_setup{
      $do_placement_db_sync = true
    }else{
      $do_placement_db_sync = false
    }
    package { 'nova-placement-api':
      ensure => 'absent',
    }->
    class { '::placement::api':
      sync_db => $do_placement_db_sync,
    }
    # Fix the number of uwsgi processes
    class { '::placement::wsgi::uwsgi': }
    # Disable uwsgi logs
    placement_api_uwsgi_config {
      'uwsgi/disable-logging': value => true;
    }
  }

  ##################
  ### Setup Nova ###
  ##################
  if $has_subrole_nova {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['nova::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'nova':
        notify_service_name => 'httpd',
      }
      $nova_key_file = "/etc/nova/ssl/private/${::fqdn}.pem"
      $nova_crt_file = "/etc/nova/ssl/public/${::fqdn}.crt"
    } else {
      $nova_key_file = undef
      $nova_crt_file = undef
    }
    if $is_first_master and $initial_cluster_setup {
      class { '::nova::cell_v2::simple_setup':
        require => Exec['nova-db-sync-api'],
      }
      Class['nova'] -> Class['::nova::cell_v2::simple_setup']
    }

    if $is_first_master {
      class { '::nova::keystone::auth':
        public_url   => "${base_url}/compute/v2.1",
        internal_url => "${base_url}/compute/v2.1",
        admin_url    => "${base_url}/compute/v2.1",
        password     => $pass_nova_authtoken,
        region       => $region_name,
      }
    }

    if $is_first_master {
      if $openstack_release == 'rocky'{
        class { '::nova::keystone::auth_placement':
          public_url   => "${base_url}/placement",
          internal_url => "${base_url}/placement",
          admin_url    => "${base_url}/placement",
          password     => $pass_placement_authtoken,
          region       => $region_name,
        }
      }
    }

    class { '::nova::keystone::authtoken':
      password             => $pass_nova_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }

    # This is mandatory starting with bobcat, and this is supported in puppet-nova only in train and up.
    # This is related to CVE-2023-2088.
    if ($openstack_release != 'rocky' and $openstack_release != 'stein'){
      class { '::nova::keystone::service_user':
        username                => 'nova',
        password                => $pass_nova_authtoken,
        auth_url                => $keystone_admin_uri,
        project_name            => 'services',
        region_name             => $region_name,
        send_service_user_token => true,
        cafile                  => $api_endpoint_ca_file,
      }
    }

    class { '::nova::logging':
      debug => true,
    }

    if $disable_notifications {
      $nova_notif_transport_url = ''
    } else {
      $nova_notif_transport_url = os_transport_url({
                                    'transport' => $messaging_notify_proto,
                                    'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                    'port'      => $messaging_notify_port,
                                    'username'  => 'nova',
                                    'password'  => $pass_nova_messaging,
                                  })
    }
    if $openstack_release == 'rocky'{
      class { '::nova':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'nova',
          'password'  => $pass_nova_messaging,
        }),
        notification_transport_url    => $nova_notif_transport_url,
        database_connection           => "mysql+pymysql://nova:${pass_nova_db}@${sql_host}/novadb?charset=utf8",
        api_database_connection       => "mysql+pymysql://novaapi:${pass_nova_apidb}@${sql_host}/novaapidb?charset=utf8",
        placement_database_connection => "mysql+pymysql://placement:${pass_placement_db}@${sql_host}/placementdb?charset=utf8",
        database_idle_timeout         => 1800,
        rabbit_use_ssl                => $use_ssl,
        rabbit_ha_queues              => true,
        kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms          => 'PLAIN',
        use_ipv6                      => false,
        glance_api_servers            => "${base_url}/image",
        notification_driver           => $notification_driver,
        notify_on_state_change        => 'vm_and_task_state',
      }
    }else{
      if $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'{
        class { '::nova':
          default_transport_url      => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'nova',
            'password'  => $pass_nova_messaging,
          }),
          notification_transport_url    => $nova_notif_transport_url,
          database_connection           => "mysql+pymysql://nova:${pass_nova_db}@${sql_host}/novadb?charset=utf8",
          api_database_connection       => "mysql+pymysql://novaapi:${pass_nova_apidb}@${sql_host}/novaapidb?charset=utf8",
          database_idle_timeout         => 1800,
          rabbit_use_ssl                => $use_ssl,
          rabbit_ha_queues              => true,
          kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms          => 'PLAIN',
          glance_api_servers            => "${base_url}/image",
          notification_driver           => $notification_driver,
          notify_on_state_change        => 'vm_and_task_state',
        }
      }else{
        if $openstack_release == 'victoria' {
          class { '::nova':
            default_transport_url      => os_transport_url({
              'transport' => $messaging_default_proto,
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => $messaging_default_port,
              'username'  => 'nova',
              'password'  => $pass_nova_messaging,
            }),
            notification_transport_url    => $nova_notif_transport_url,
            database_connection           => "mysql+pymysql://nova:${pass_nova_db}@${sql_host}/novadb?charset=utf8",
            api_database_connection       => "mysql+pymysql://novaapi:${pass_nova_apidb}@${sql_host}/novaapidb?charset=utf8",
            database_idle_timeout         => 1800,
            rabbit_use_ssl                => $use_ssl,
            rabbit_ha_queues              => true,
            kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
            amqp_sasl_mechanisms          => 'PLAIN',
            notification_driver           => $notification_driver,
            notify_on_state_change        => 'vm_and_task_state',
            glance_endpoint_override      => "${base_url}/image",
          }
        }elsif $openstack_release == 'wallaby' or $openstack_release == 'xena'{
          # ::nova::glance must be called before ::nova, because
          # the ::nova class contains an include ::nova::glance
          class { '::nova::glance':
            endpoint_override => "${base_url}/image",
          }
          class { '::nova':
            default_transport_url      => os_transport_url({
              'transport' => $messaging_default_proto,
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => $messaging_default_port,
              'username'  => 'nova',
              'password'  => $pass_nova_messaging,
            }),
            notification_transport_url    => $nova_notif_transport_url,
            database_connection           => "mysql+pymysql://nova:${pass_nova_db}@${sql_host}/novadb?charset=utf8",
            api_database_connection       => "mysql+pymysql://novaapi:${pass_nova_apidb}@${sql_host}/novaapidb?charset=utf8",
            database_idle_timeout         => 1800,
            rabbit_use_ssl                => $use_ssl,
            rabbit_ha_queues              => true,
            kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
            amqp_sasl_mechanisms          => 'PLAIN',
            notification_driver           => $notification_driver,
            notify_on_state_change        => 'vm_and_task_state',
          }
          nova_config {
            'glance/api_servers': value => "${base_url}/image";
          }
        # Yoga and up
        }else{
          # ::nova::glance must be called before ::nova, because
          # the ::nova class contains an include ::nova::glance
          class { '::nova::glance':
            endpoint_override => "${base_url}/image",
          }
          class { '::nova::db':
            database_connection              => "mysql+pymysql://nova:${pass_nova_db}@${sql_host}/novadb?charset=utf8",
            api_database_connection          => "mysql+pymysql://novaapi:${pass_nova_apidb}@${sql_host}/novaapidb?charset=utf8",
            database_connection_recycle_time => 1800,
          }
          class { '::nova':
            default_transport_url      => os_transport_url({
              'transport' => $messaging_default_proto,
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => $messaging_default_port,
              'username'  => 'nova',
              'password'  => $pass_nova_messaging,
            }),
            notification_transport_url    => $nova_notif_transport_url,
            rabbit_use_ssl                => $use_ssl,
            rabbit_ha_queues              => true,
            kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
            amqp_sasl_mechanisms          => 'PLAIN',
            notification_driver           => $notification_driver,
            notify_on_state_change        => 'vm_and_task_state',
          }
          nova_config {
            'glance/api_servers': value => "${base_url}/image";
          }
        }
      }
    }

    if $use_gpu {
      $gpu_alias_table = $gpu_def.map |Integer $index, String $value| {
        {"vendor_id" => values_at(split($value, '[.]'),1)[0], "product_id" => values_at(split($value, '[.]'),2)[0], "name" => values_at(split($value, '[.]'),0)[0], "device_type" => values_at(split($value, '[.]'),3)[0]}
      }
#      notify { "GPU alias table finished: ${gpu_alias_table}": withpath => true }
      class { '::nova::pci':
        aliases => $gpu_alias_table,
      }
    }

    # The nova-api and nova-api-metadata services.
    if $openstack_release == 'rocky'{
      class { '::nova::api':
        api_bind_address                     => $machine_ip,
        neutron_metadata_proxy_shared_secret => $pass_metadata_proxy_shared_secret,
        metadata_workers                     => 2,
        sync_db_api                          => $do_db_sync,
        service_name                         => 'httpd',
        allow_resize_to_same_host            => true,
        enable_proxy_headers_parsing         => true,
        max_limit                            => 10000,
      }

      class { '::nova::wsgi::apache_api':
        bind_host => $machine_ip,
        ssl_key   => $nova_key_file,
        ssl_cert  => $nova_crt_file,
        ssl       => $use_ssl,
        workers   => '2',
      }

      class { '::nova::placement':
        auth_url    => $keystone_admin_uri,
        password    => $pass_placement_authtoken,
        region_name => $region_name,
      }
    }else{
      # In >= Stein, use uwsgi, not httpd.
      if $openstack_release == 'stein' {
        class { '::nova::api':
          api_bind_address                     => $machine_ip,
          sync_db                              => $do_db_sync,
          sync_db_api                          => $do_db_sync,
          allow_resize_to_same_host            => true,
          neutron_metadata_proxy_shared_secret => $pass_metadata_proxy_shared_secret,
          enable_proxy_headers_parsing         => true,
          max_limit                            => 10000,
        }
      }else{
        class { '::nova::api':
          api_bind_address                     => $machine_ip,
          sync_db                              => $do_db_sync,
          sync_db_api                          => $do_db_sync,
          allow_resize_to_same_host            => true,
          enable_proxy_headers_parsing         => true,
          max_limit                            => 10000,
        }
        if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
          class { '::nova::metadata':
            neutron_metadata_proxy_shared_secret => $pass_metadata_proxy_shared_secret,
            dhcp_domain                          => $dhcp_domain,
            enable_proxy_headers_parsing         => true,
            metadata_cache_expiration            => $nova_metadata_cache_expiration,
          }
        }else{
          class { '::nova::metadata':
            neutron_metadata_proxy_shared_secret => $pass_metadata_proxy_shared_secret,
            dhcp_domain                          => $dhcp_domain,
            metadata_cache_expiration            => $nova_metadata_cache_expiration,
          }
        }
        # Fix the number of uwsgi processes
        class { '::nova::wsgi::uwsgi_api': }
        class { '::nova::wsgi::uwsgi_api_metadata': }
        # Disable uwsgi logs
        nova_api_uwsgi_config {
          'uwsgi/disable-logging': value => true;
        }
        nova_api_metadata_uwsgi_config {
          'uwsgi/disable-logging': value => true;
        }
        nova_config {
          'glance/region_name': value => $region_name;
        }
      }
      nova_config {
        'placement/auth_type':           value => 'password';
        'placement/auth_url':            value => $keystone_admin_uri;
        'placement/password':            value => $pass_placement_authtoken, secret => true;
        'placement/project_domain_name': value => 'Default';
        'placement/project_name':        value => 'services';
        'placement/user_domain_name':    value => 'Default';
        'placement/username':            value => 'placement';
        'placement/region_name':         value => $region_name;
      }

      if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga'{
        class { '::nova::cinder':
          password    => $pass_cinder_authtoken,
          region_name => $region_name,
          auth_url    => $keystone_admin_uri,
        }
      }else{
        class { '::nova::cinder':
          password       => $pass_cinder_authtoken,
          os_region_name => $region_name,
          auth_url       => $keystone_admin_uri,
        }
      }
    }

    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope'){
      class { '::nova::client': }
    }
    class { '::nova::conductor':
      workers             => $::os_workers,
      enable_new_services => false, # Disable new compute by default
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::nova::consoleauth': }
    }
    class { '::nova::cron::archive_deleted_rows': }

    if $openstack_release == 'rocky'{
      class { '::nova::scheduler': }
    }else{
      if $openstack_release == 'stein'{
        class { '::nova::scheduler':
          workers => $::os_workers,
        }
      } else {
        class { '::nova::scheduler':
          workers                              => $::os_workers,
          limit_tenants_to_placement_aggregate => $nova_limit_tenants_to_placement_aggregate,
        }
      }
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein'{
      nova_config {
        'scheduler/limit_tenants_to_placement_aggregate':   value => $nova_limit_tenants_to_placement_aggregate;
      }
    }

    if $nova_scheduler_prefers_hosts_with_more_ram {
      $ram_weight_multiplier = 1
    }else{
      $ram_weight_multiplier = -1
    }

    if $use_gpu {
      if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby'{
        class { '::nova::scheduler::filter':
          scheduler_default_filters => [ 'AvailabilityZoneFilter', 'ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'PciPassthroughFilter', ],
          ram_weight_multiplier     => $ram_weight_multiplier,
        }
      # Xena and up
      }else{
        class { '::nova::scheduler::filter':
          scheduler_enabled_filters => [ 'AvailabilityZoneFilter', 'ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'PciPassthroughFilter', ],
          ram_weight_multiplier     => $ram_weight_multiplier,
        }
      }
    }else{
      class { '::nova::scheduler::filter':
        ram_weight_multiplier     => $ram_weight_multiplier,
      }
    }
    class { '::nova::vncproxy':
      host           => $machine_ip,
      vncproxy_path  => "/novnc/vnc_auto.html",
      allow_noauth   => true,
      allow_vencrypt => true,
      vencrypt_ca    => '/etc/pki/nova-novncproxy/ca-cert.pem',
      vencrypt_key   => '/etc/pki/nova-novncproxy/client-key.pem',
      vencrypt_cert  => '/etc/pki/nova-novncproxy/client-cert.pem',
    }
    class { '::nova::vncproxy::common':
	vncproxy_protocol => 'https',
	vncproxy_host     => "${vip_hostname}",
	vncproxy_path     => "/novnc/vnc_auto.html",
	vncproxy_port     => "443",
    }

    nova_config {
      'neutron/cafile':                   value => $api_endpoint_ca_file;
      'glance/cafile':                    value => $api_endpoint_ca_file;
      'keystone/cafile':                  value => $api_endpoint_ca_file;
      'placement/cafile':                 value => $api_endpoint_ca_file;
      'cinder/cafile':                    value => $api_endpoint_ca_file;
      'DEFAULT/default_ephemeral_format': value => 'ext4';
    }
    if $openstack_release == 'rocky'{
      nova_config {
        'scheduler/workers':         value => $::os_workers;
      }
    }

    if $openstack_release == 'rocky'{
      class { '::nova::network::neutron':
        neutron_auth_url      => "${keystone_auth_uri}/v3",
        neutron_url           => "${base_url}/network",
        neutron_password      => $pass_neutron_authtoken,
        default_floating_pool => 'public',
        dhcp_domain           => $dhcp_domain,
        neutron_region_name   => $region_name,
      }
      nova_config {
        'neutron/endpoint_override': value => "${base_url}/network";
      }
    }else{
      if $openstack_release == 'stein' or $openstack_release == 'train'{
        class { '::nova::network::neutron':
          neutron_auth_url          => "${keystone_auth_uri}/v3",
          neutron_url               => "${base_url}/network",
          neutron_password          => $pass_neutron_authtoken,
          default_floating_pool     => 'public',
          neutron_endpoint_override => "${base_url}/network",
          dhcp_domain               => $dhcp_domain,
          neutron_region_name       => $region_name,
        }
      }else{
        class { '::nova::network::neutron':
          auth_url              => "${keystone_auth_uri}/v3",
          password              => $pass_neutron_authtoken,
          default_floating_pool => 'public',
          endpoint_override     => "${base_url}/network",
          region_name           => $region_name,
        }
      }
    }
  }
  ####################
  ### Setup Ironic ###
  ####################
  if $has_subrole_ironic and $install_ironic {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['ironic::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'ironic':
        notify_service_name => 'ironic-api',
      }
      $ironic_key_file = "/etc/ironic/ssl/private/${::fqdn}.pem"
      $ironic_crt_file = "/etc/ironic/ssl/public/${::fqdn}.crt"
    }else{
      $ironic_key_file = undef
      $ironic_crt_file = undef
    }

    if $is_first_master {
      class { '::ironic::keystone::auth':
        public_url   => "${base_url}/baremetal",
        internal_url => "${base_url}/baremetal",
        admin_url    => "${base_url}/baremetal",
        password     => $pass_ironic_authtoken,
        region       => $region_name,
      }
      class { '::ironic::keystone::auth_inspector':
        public_url   => "${base_url}/inspector",
        internal_url => "${base_url}/inspector",
        admin_url    => "${base_url}/inspector",
        password     => $pass_ironic_inspector_authtoken,
        region       => $region_name,
      }
    }
    if $disable_notifications {
      $ironic_notif_transport_url = ''
      $ironic_notification_driver = 'noop'
    } else {
      $ironic_notif_transport_url = os_transport_url({
                                      'transport' => $messaging_notify_proto,
                                      'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                      'port'      => $messaging_notify_port,
                                      'username'  => 'ironic',
                                      'password'  => $pass_ironic_messaging,
                                    })
      $ironic_notification_driver = 'messagingv2'
    }
    class { '::ironic::logging':
      debug => true,
    }
    class { '::ironic::db':
      database_connection => "mysql+pymysql://ironic:${pass_ironic_db}@${sql_host}/ironicdb?charset=utf8",
    }
    class { 'ironic::inspector::db':
      database_connection => "mysql+pymysql://ironicinspector:${pass_ironic_inspector_db}@${sql_host}/ironicinspectordb?charset=utf8",
    }
    class { '::ironic':
      default_transport_url => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'neutron',
          'password'  => $pass_ironic_messaging,
        }),
      notification_transport_url => $ironic_notif_transport_url,
      notification_driver        => $ironic_notification_driver,
      enabled                    => true,
      my_ip                      => $machine_ip,
      auth_strategy              => 'keystone',
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
      amqp_sasl_mechanisms       => 'PLAIN',
    }
    class { '::ironic::api::authtoken':
      password             => $pass_ironic_authtoken,
      user_domain_name     => 'Default',
      project_domain_name  => 'Default',
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    class { 'ironic::inspector::authtoken':
      password             => $pass_ironic_inspector_authtoken,
      user_domain_name     => 'Default',
      project_domain_name  => 'Default',
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    class { 'ironic::client': }
    class { 'ironic::api':
      service_name                 => 'ironic-api',
      host_ip                      => $machine_ip,
      max_limit                    => 10000,
      enable_proxy_headers_parsing => true,
    }
    # TODO: write a ironic::wsgi::uwsgi class in puppet-ironic
    # to configure the number of API process automatically.
    class { 'ironic::conductor':
      enabled_hardware_types => [
#                                  'idrac',
                                  'ipmi',
#                                  'ilo5',
#                                  'redfish',
#                                  'xclarity',
                                ],
      cleaning_disk_erase    => 'full',
    }
    class { 'ironic::drivers::ipmi': }
    class { 'ironic::inspector':
      listen_address        => $machine_ip,
      default_transport_url => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'ironic',
          'password'  => $pass_ironic_inspector_messaging,
      }),
      dnsmasq_interface     => 'eth0',
    }

  }else{
    package {'python3-ironic':
      ensure => 'purged',
    }
  }
  #####################
  ### Setup Neutron ###
  #####################
  if $has_subrole_neutron {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['neutron::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'neutron':
        notify_service_name => 'neutron-api',
      }
      $neutron_key_file = "/etc/neutron/ssl/private/${::fqdn}.pem"
      $neutron_crt_file = "/etc/neutron/ssl/public/${::fqdn}.crt"
    } else {
      $neutron_key_file = undef
      $neutron_crt_file = undef
    }

    if $install_designate {
      $notif_topics_neutron = ['notifications','notifications_designate']
    } else {
      $notif_topics_neutron = ['notifications']
    }

    if $is_first_master {
      class { '::neutron::keystone::auth':
        public_url   => "${base_url}/network",
        internal_url => "${base_url}/network",
        admin_url    => "${base_url}/network",
        password     => $pass_neutron_authtoken,
        region       => $region_name,
      }
    }

    if $disable_notifications {
      $neutron_notif_transport_url = ''
    } else {
      $neutron_notif_transport_url = os_transport_url({
                                       'transport' => $messaging_notify_proto,
                                       'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                       'port'      => $messaging_notify_port,
                                       'username'  => 'neutron',
                                       'password'  => $pass_neutron_messaging,
                                     })
    }

    $bgp_dr_plugin = $use_dynamic_routing ? {
      true    => 'bgp',
      default => undef,
    }
    $plugins_list = delete_undef_values(['router', 'metering', 'qos', 'trunk', 'segments', $bgp_dr_plugin])

    if $openstack_release == 'rocky'{
      class { '::neutron':
        debug                      => true,
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'neutron',
          'password'  => $pass_neutron_messaging,
        }),
        dhcp_agents_per_network    => 2,
        notification_transport_url => $neutron_notif_transport_url,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        allow_overlapping_ips      => true,
        core_plugin                => 'ml2',
        service_plugins            => $plugins_list,
        bind_host                  => $machine_ip,
        use_ssl                    => $use_ssl,
        cert_file                  => $neutron_crt_file,
        key_file                   => $neutron_key_file,
        notification_driver        => $notification_driver,
        global_physnet_mtu         => $neutron_global_physnet_mtu,
        dns_domain                 => $neutron_dns_domain,
      }
    }else{
      if $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga'{
        class { '::neutron':
          default_transport_url      => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'neutron',
            'password'  => $pass_neutron_messaging,
          }),
          dhcp_agents_per_network    => 2,
          notification_transport_url => $neutron_notif_transport_url,
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms       => 'PLAIN',
          allow_overlapping_ips      => true,
          core_plugin                => 'ml2',
          service_plugins            => $plugins_list,
          bind_host                  => $machine_ip,
          use_ssl                    => $use_ssl,
          cert_file                  => $neutron_crt_file,
          key_file                   => $neutron_key_file,
          notification_driver        => $notification_driver,
          global_physnet_mtu         => $neutron_global_physnet_mtu,
          dns_domain                 => $neutron_dns_domain,
          executor_thread_pool_size  => $neutron_executor_thread_pool_size,
          notification_topics        => $notif_topics_neutron,
        }
      # Zed and up desn't have the allow_overlapping_ips param
      }else{
        class { '::neutron':
          default_transport_url      => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'neutron',
            'password'  => $pass_neutron_messaging,
          }),
          dhcp_agents_per_network    => 2,
          notification_transport_url => $neutron_notif_transport_url,
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms       => 'PLAIN',
          core_plugin                => 'ml2',
          service_plugins            => $plugins_list,
          bind_host                  => $machine_ip,
          use_ssl                    => $use_ssl,
          cert_file                  => $neutron_crt_file,
          key_file                   => $neutron_key_file,
          notification_driver        => $notification_driver,
          global_physnet_mtu         => $neutron_global_physnet_mtu,
          dns_domain                 => $neutron_dns_domain,
          executor_thread_pool_size  => $neutron_executor_thread_pool_size,
          notification_topics        => $notif_topics_neutron,
        }
      }
      neutron_config {
        'DEFAULT/rpc_conn_pool_size': value => $neutron_rpc_conn_pool_size;
      }
      class { '::neutron::logging':
        debug => true,
      }
    }
    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope'){
      class { '::neutron::client': }
    }
    class { '::neutron::keystone::authtoken':
      password             => $pass_neutron_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    Service<| title == 'neutron-server'|> -> Openstacklib::Service_validation<| title == 'neutron-server' |> -> Neutron_network<||>
    if $use_dynamic_routing {
      $ensure_dr_package = true
    }else{
      $ensure_dr_package = false
    }

    if $neutron_use_dvr {
      $router_distributed         = true
      $enable_distributed_routing = true
      $l3_agent_mode              = 'dvr_snat'
    }else{
      $router_distributed         = false
      $enable_distributed_routing = false
      $l3_agent_mode              = 'legacy'
    }

    if $openstack_release == 'rocky'{
      class { '::neutron::server':
        database_connection              => "mysql+pymysql://neutron:${pass_neutron_db}@${sql_host}/neutrondb?charset=utf8",
        database_idle_timeout            => 1800,
        sync_db                          => $do_db_sync,
        api_workers                      => 2,
        rpc_workers                      => 2,
        validate                         => true,
        router_distributed               => $router_distributed,
        allow_automatic_l3agent_failover => true,
        enable_dvr                       => true,
        ensure_fwaas_package             => true,
        service_providers                => [ 'LOADBALANCERV2:Octavia:neutron_lbaas.drivers.octavia.driver.OctaviaDriver:default', ],
        enable_proxy_headers_parsing     => true,
      }
    }elsif($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
      class { '::neutron::server':
        database_connection              => "mysql+pymysql://neutron:${pass_neutron_db}@${sql_host}/neutrondb?charset=utf8",
        database_idle_timeout            => 1800,
        sync_db                          => $do_db_sync,
        api_workers                      => 2,
        rpc_workers                      => $::os_workers,
        validate                         => true,
        router_distributed               => $router_distributed,
        allow_automatic_l3agent_failover => true,
        enable_dvr                       => true,
        ensure_fwaas_package             => true,
        ensure_dr_package                => $ensure_dr_package,
        service_providers                => [ 'LOADBALANCERV2:Octavia:neutron_lbaas.drivers.octavia.driver.OctaviaDriver:default', ],
        enable_proxy_headers_parsing     => true,
      }
      # Fix the number of uwsgi processes
      class { '::neutron::wsgi::uwsgi': }
      # Disable uwsgi logs
      neutron_api_uwsgi_config {
        'uwsgi/disable-logging': value => true;
      }
    }else{
      class { 'neutron::db':
        database_connection              => "mysql+pymysql://neutron:${pass_neutron_db}@${sql_host}/neutrondb?charset=utf8",
        database_connection_recycle_time => 1800,
      }
      if($openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed'){
        class { '::neutron::server':
          sync_db                          => $do_db_sync,
          api_workers                      => 2,
          rpc_workers                      => $::os_workers,
          validate                         => true,
          router_distributed               => $router_distributed,
          allow_automatic_l3agent_failover => true,
          enable_dvr                       => true,
          ensure_dr_package                => $ensure_dr_package,
          service_providers                => [ 'LOADBALANCERV2:Octavia:neutron_lbaas.drivers.octavia.driver.OctaviaDriver:default', ],
          enable_proxy_headers_parsing     => true,
          max_l3_agents_per_router         => 2,
          l3_ha                            => true,
        }
      # Antelope and up don't have the 'validate' param
      }else{
        class { '::neutron::server':
          sync_db                          => $do_db_sync,
          api_workers                      => 2,
          rpc_workers                      => $::os_workers,
          router_distributed               => $router_distributed,
          allow_automatic_l3agent_failover => true,
          enable_dvr                       => true,
          ensure_dr_package                => $ensure_dr_package,
          service_providers                => [ 'LOADBALANCERV2:Octavia:neutron_lbaas.drivers.octavia.driver.OctaviaDriver:default', ],
          enable_proxy_headers_parsing     => true,
          max_l3_agents_per_router         => 2,
          l3_ha                            => true,
        }
      }
      # Fix the number of uwsgi processes
      class { '::neutron::wsgi::uwsgi': }
      # Disable uwsgi logs
      neutron_api_uwsgi_config {
        'uwsgi/disable-logging': value => true;
      }
    }
    # If we have dynamic routing agent installed anywhere
    # in the cluster, then we must also install the plugin
    # python files in the controller.
    if $use_dynamic_routing {
      ensure_packages('neutron-dynamic-routing', {
        ensure => installed,
        name   => $::neutron::params::dynamic_routing_package,
        tag    => ['openstack', 'neutron-package'],
      })
      class { '::neutron::server::placement':
        password            => $pass_nova_authtoken,
        auth_type           => 'password',
        username            => 'nova',
        project_domain_name => 'Default',
        project_name        => 'services',
        user_domain_name    => 'Default',
        auth_url            => "${base_url}:${api_port}/identity",
        region_name         => $region_name,
      }
    }else{
      package { 'Python 3 neutron dr agent package':
        name => 'python3-neutron-dynamic-routing',
        ensure => purged,
      }
    }

    neutron_config {
      'nova/cafile':                       value => $api_endpoint_ca_file;
    }

    if $has_subrole_network_node {
      class { '::neutron::agents::ml2::ovs':
        local_ip                   => $vmnet_ip,
        tunnel_types               => ['vxlan'],
        bridge_uplinks             => ['eth0'],
        bridge_mappings            => $bridge_mapping_list,
        extensions                 => '',
        l2_population              => true,
        arp_responder              => true,
        firewall_driver            => 'iptables_hybrid',
        drop_flows_on_start        => false,
        enable_distributed_routing => $enable_distributed_routing,
        manage_vswitch             => false,
      }

      class { '::neutron::agents::l3':
        interface_driver      => 'openvswitch',
        debug                 => true,
        agent_mode            => $l3_agent_mode,
        ha_enabled            => false,
        extensions            => '',
        ha_vrrp_auth_type     => 'PASS',
        ha_vrrp_auth_password => $pass_neutron_vrrpauth,
      }
      neutron_l3_agent_config {
        'DEFAULT/external_network_bridge': value => '';
      }
    }

    if $has_subrole_designate {
      $extension_drivers = 'port_security,qos,dns_domain_ports'
    }else{
      $extension_drivers = 'port_security,qos'
    }

    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
      class { '::neutron::plugins::ml2':
        type_drivers         => ['flat', 'vxlan', 'vlan', ],
        tenant_network_types => ['flat', 'vxlan', 'vlan', ],
        extension_drivers    => $extension_drivers,
        mechanism_drivers    => 'openvswitch,l2population',
        firewall_driver      => 'iptables_v2',
        flat_networks        => $external_network_list,
        network_vlan_ranges  => $external_network_list,
        vni_ranges           => "${neutron_vxlan_vni_min}:${neutron_vxlan_vni_max}",
        path_mtu             => $neutron_path_mtu,
      }
    }else{
      class { '::neutron::plugins::ml2':
        type_drivers         => ['flat', 'vxlan', 'vlan', ],
        tenant_network_types => ['flat', 'vxlan', 'vlan', ],
        extension_drivers    => $extension_drivers,
        mechanism_drivers    => 'openvswitch,l2population',
        flat_networks        => $external_network_list,
        network_vlan_ranges  => $external_network_list,
        vni_ranges           => "${neutron_vxlan_vni_min}:${neutron_vxlan_vni_max}",
        path_mtu             => $neutron_path_mtu,
      }
    }

    if $openstack_release == 'rocky'{
      class { '::neutron::services::lbaas::octavia':
        base_url          => "${base_url}/loadbalance",
        allocates_vip     => true,
        auth_url          => "${base_url}:${api_port}/identity/v3/",
        admin_user        => 'octavia',
        admin_tenant_name => 'services',
        admin_password    => $pass_octavia_authtoken,
      }
    }
    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'){
      class { '::neutron::server::notifications':
        auth_url    => $keystone_admin_uri,
        password    => $pass_nova_authtoken,
        region_name => $region_name,
      }
    }else{
      class { '::neutron::server::notifications::nova':
        auth_url    => $keystone_admin_uri,
        password    => $pass_nova_authtoken,
        region_name => $region_name,
      }
    }
    if $has_subrole_designate {
      class { '::neutron::designate':
        url                       => "${base_url}/dns/v2",
        username                  => 'admin',
        project_name              => 'admin',
        password                  => $pass_keystone_adminuser,
        auth_url                  => "${base_url}/identity/v3/",
        ipv4_ptr_zone_prefix_size => 24,
        ipv6_ptr_zone_prefix_size => 64,
        ptr_zone_email            => $designate_zone_admin_email,
        allow_reverse_dns_lookup  => true,
      }
    }
  }
  ####################
  ### Setup Cinder ###
  ####################
  if $has_subrole_cinder {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['cinder::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'cinder':
        notify_service_name => 'httpd',
      }
      $cinder_key_file = "/etc/cinder/ssl/private/${::fqdn}.pem"
      $cinder_crt_file = "/etc/cinder/ssl/public/${::fqdn}.crt"
    } else {
      $cinder_key_file = undef
      $cinder_crt_file = undef
    }

    # Cinder endpoints
    if $is_first_master {
      if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' {
        class { '::cinder::keystone::auth':
          public_url_v2   => "${base_url}/volume/v2/%(tenant_id)s",
          internal_url_v2 => "${base_url}/volume/v2/%(tenant_id)s",
          admin_url_v2    => "${base_url}/volume/v2/%(tenant_id)s",
          public_url_v3   => "${base_url}/volume/v3/%(tenant_id)s",
          internal_url_v3 => "${base_url}/volume/v3/%(tenant_id)s",
          admin_url_v3    => "${base_url}/volume/v3/%(tenant_id)s",
          password        => $pass_cinder_authtoken,
          region          => $region_name,
        }
      }else{
        class { '::cinder::keystone::auth':
          public_url_v3   => "${base_url}/volume/v3/%(tenant_id)s",
          internal_url_v3 => "${base_url}/volume/v3/%(tenant_id)s",
          admin_url_v3    => "${base_url}/volume/v3/%(tenant_id)s",
          password        => $pass_cinder_authtoken,
          region          => $region_name,
        }
      }
    }
    $cinder_notif_transport_url = os_transport_url({
                                  'transport' => $messaging_notify_proto,
                                  'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                  'port'      => $messaging_notify_port,
                                  'username'  => 'cinder',
                                  'password'  => $pass_cinder_messaging,
                                })

    if $openstack_release == 'rocky'{
      class { '::cinder':
        debug                 => true,
        default_transport_url => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'cinder',
          'password'  => $pass_cinder_messaging,
        }),
        database_connection              => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_host}/cinderdb?charset=utf8",
        database_connection_recycle_time => 1800,
        rabbit_use_ssl                   => $use_ssl,
        rabbit_ha_queues                 => true,
        kombu_ssl_ca_certs               => $oci_pki_root_ca_file,
        storage_availability_zone        => $cinder_default_storage_availability_zone,
        allow_availability_zone_fallback => true,
      }
    }else{
      if($openstack_release == 'stein' or $openstack_release == 'train'){
        class { '::cinder':
          default_transport_url => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'cinder',
            'password'  => $pass_cinder_messaging,
          }),
          database_connection       => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_host}/cinderdb?charset=utf8",
          database_idle_timeout     => 1800,
          rabbit_use_ssl            => $use_ssl,
          rabbit_ha_queues          => true,
          kombu_ssl_ca_certs        => $oci_pki_root_ca_file,
          storage_availability_zone => $cinder_default_storage_availability_zone,
          allow_availability_zone_fallback => true,
        }
        # Fix the number of uwsgi processes
        class { '::cinder::wsgi::uwsgi': }
        # Disable uwsgi logs
        cinder_api_uwsgi_config {
          'uwsgi/disable-logging': value => true;
        }
      }else{
        if $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' {
          class { '::cinder':
            default_transport_url => os_transport_url({
              'transport' => $messaging_default_proto,
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => $messaging_default_port,
              'username'  => 'cinder',
              'password'  => $pass_cinder_messaging,
            }),
            database_connection        => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_host}/cinderdb?charset=utf8",
            database_idle_timeout      => 1800,
            rabbit_use_ssl             => $use_ssl,
            rabbit_ha_queues           => true,
            kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
            storage_availability_zone  => $cinder_default_storage_availability_zone,
            notification_transport_url => $cinder_notif_transport_url,
            notification_driver        => $notification_driver,
            allow_availability_zone_fallback => true,
          }
        }else{
          if $openstack_release == 'xena'{
            class { '::cinder':
              default_transport_url => os_transport_url({
                'transport' => $messaging_default_proto,
                'hosts'     => fqdn_rotate($all_masters),
                'port'      => $messaging_default_port,
                'username'  => 'cinder',
                'password'  => $pass_cinder_messaging,
              }),
              database_connection       => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_host}/cinderdb?charset=utf8",
              database_idle_timeout     => 1800,
              rabbit_use_ssl            => $use_ssl,
              rabbit_ha_queues          => true,
              kombu_ssl_ca_certs        => $oci_pki_root_ca_file,
              storage_availability_zone => $cinder_default_storage_availability_zone,
              notification_transport_url => $cinder_notif_transport_url,
              notification_driver        => $notification_driver,
              keymgr_encryption_api_url    => "${base_url}/keymanager",
              keymgr_encryption_auth_url   => "${keystone_auth_uri}/v3",
              allow_availability_zone_fallback => true,
            }
          }else{
            class { '::cinder::db':
              database_connection              => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_host}/cinderdb?charset=utf8",
              database_connection_recycle_time => 1800,
            }
            if $openstack_release == 'yoga'{
              class { '::cinder':
                default_transport_url => os_transport_url({
                  'transport' => $messaging_default_proto,
                  'hosts'     => fqdn_rotate($all_masters),
                  'port'      => $messaging_default_port,
                  'username'  => 'cinder',
                  'password'  => $pass_cinder_messaging,
                }),
                rabbit_use_ssl            => $use_ssl,
                rabbit_ha_queues          => true,
                kombu_ssl_ca_certs        => $oci_pki_root_ca_file,
                storage_availability_zone => $cinder_default_storage_availability_zone,
                notification_transport_url => $cinder_notif_transport_url,
                notification_driver        => $notification_driver,
                keymgr_encryption_api_url    => "${base_url}/keymanager",
                keymgr_encryption_auth_url   => "${keystone_auth_uri}/v3",
                allow_availability_zone_fallback => true,
              }
            # Zed and up: keymgr_encryption_api_url and keymgr_encryption_auth_url don't exist
            }else{
              class { '::cinder':
                default_transport_url => os_transport_url({
                  'transport' => $messaging_default_proto,
                  'hosts'     => fqdn_rotate($all_masters),
                  'port'      => $messaging_default_port,
                  'username'  => 'cinder',
                  'password'  => $pass_cinder_messaging,
                }),
                rabbit_use_ssl            => $use_ssl,
                rabbit_ha_queues          => true,
                kombu_ssl_ca_certs        => $oci_pki_root_ca_file,
                storage_availability_zone => $cinder_default_storage_availability_zone,
                notification_transport_url => $cinder_notif_transport_url,
                notification_driver        => $notification_driver,
                allow_availability_zone_fallback => true,
              }
            }
          }
          class { '::cinder::key_manager':
            backend => 'barbican',
          }
        }
        # Fix the number of uwsgi processes
        class { '::cinder::wsgi::uwsgi': }
        # Disable uwsgi logs
        cinder_api_uwsgi_config {
          'uwsgi/disable-logging': value => true;
        }
      }
      class { '::cinder::logging':
        debug => true,
      }
    }
    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
      # Starting with puppet-cinder 16.2.0, this has to go in the main init.pp call.
      class { 'cinder::ceilometer':
        notification_transport_url => $cinder_notif_transport_url,
        notification_driver        => $notification_driver,
      }
    }

    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
      class { '::cinder::keystone::authtoken':
        password             => $pass_cinder_authtoken,
        auth_url             => $keystone_admin_uri,
        www_authenticate_uri => $keystone_auth_uri,
        memcached_servers    => $memcached_servers,
        cafile               => $api_endpoint_ca_file,
        region_name          => $region_name,
      }
      cinder_config {
        'keystone_authtoken/service_type': value => 'volumev3';
      }
    }else{
      class { '::cinder::keystone::authtoken':
        password             => $pass_cinder_authtoken,
        auth_url             => $keystone_admin_uri,
        www_authenticate_uri => $keystone_auth_uri,
        memcached_servers    => $memcached_servers,
        cafile               => $api_endpoint_ca_file,
        region_name          => $region_name,
        service_type         => 'volumev3',
      }
    }

    # This is mandatory starting with bobcat, and this is supported in puppet-cinder only in train and up.
    # This is related to CVE-2023-2088.
    if ($openstack_release != 'rocky' and $openstack_release != 'stein'){
      class { '::cinder::keystone::service_user':
        username                => 'cinder',
        password                => $pass_cinder_authtoken,
        auth_url                => $keystone_admin_uri,
        project_name            => 'services',
        region_name             => $region_name,
        send_service_user_token => true,
        cafile                  => $api_endpoint_ca_file,
      }
    }

    if $cluster_has_cinder_volumes{
      $default_volume_type = $cinder_default_volume_type
    }else{
      $default_volume_type = 'CEPH_1_perf1'
    }
    if $openstack_release == 'rocky'{
      class { '::cinder::api':
        # Must match what we have in the volumes as enabled backend.
        default_volume_type          => $default_volume_type,
        public_endpoint              => "${base_url}/volume",
        service_name                 => 'httpd',
        keymgr_backend               => 'barbican',
        keymgr_encryption_api_url    => "${base_url}/keymanager",
        keymgr_encryption_auth_url   => "${keystone_auth_uri}/v3",
        sync_db                      => $do_db_sync,
        enable_proxy_headers_parsing => true,
      }
      class { '::cinder::wsgi::apache':
        bind_host => $machine_ip,
        ssl       => $use_ssl,
        ssl_key   => $cinder_key_file,
        ssl_cert  => $cinder_crt_file,
        workers   => 2,
      }
    }else{
      if $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby'{
        class { '::cinder::api':
          # Must match what we have in the volumes as enabled backend.
          default_volume_type          => $default_volume_type,
          public_endpoint              => "${base_url}/volume",
          service_name                 => 'cinder-api',
          keymgr_backend               => 'barbican',
          keymgr_encryption_api_url    => "${base_url}/keymanager",
          keymgr_encryption_auth_url   => "${keystone_auth_uri}/v3",
          sync_db                      => $do_db_sync,
          enable_proxy_headers_parsing => true,
        }
      }else{
        class { '::cinder::api':
          # Must match what we have in the volumes as enabled backend.
          default_volume_type          => $default_volume_type,
          public_endpoint              => "${base_url}/volume",
          service_name                 => 'cinder-api',
          sync_db                      => $do_db_sync,
          enable_proxy_headers_parsing => true,
        }
      }
    }
    class { '::cinder::quota': }
    class { '::cinder::scheduler': }
    class { '::cinder::scheduler::filter': }
    class { '::cinder::cron::db_purge': }
    class { '::cinder::glance':
      glance_api_servers => "${base_url}/image",
    }
    if $extswift_use_external and $cinder_backup_use_external_swift{
      class { '::cinder::backup::swift':
        backup_swift_url             => $extswift_proxy_url,
        backup_swift_auth_url        => "${extswift_auth_url}",
        backup_swift_container       => 'volumebackups',
#        backup_swift_object_size     => 52428800,
#        backup_swift_retry_attempts  => $::os_service_default,
#        backup_swift_retry_backoff   => $::os_service_default,
        backup_swift_user_domain     => $extswift_user_domain_name,
        backup_swift_project_domain  => $extswift_project_domain_name,
        backup_swift_project         => $extswift_project_name,
        backup_compression_algorithm => 'None',
      }
      cinder_config {
        'DEFAULT/backup_swift_user': value => $extswift_user_name;
        'DEFAULT/backup_swift_key':  value => $extswift_password;
        'DEFAULT/backup_swift_auth':         value => 'single_user';
        'DEFAULT/backup_swift_auth_version': value => '3';
      }
    }else{
      cinder_config {
        'DEFAULT/backup_swift_auth_url':     value => $keystone_auth_uri;
        'DEFAULT/backup_swift_auth':         value => 'per_user';
      }
    }
    cinder_config {
      'DEFAULT/scheduler_default_weighers': value => 'GoodnessWeigher';
    }
  }
  #####################
  ### Setup Gnocchi ###
  #####################
  if $has_subrole_gnocchi and $install_telemetry {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['gnocchi::install::begin']
    }
    if $use_ssl {
      if $has_subrole_gnocchi_services {
        oci::sslkeypair {'gnocchi':
          notify_service_name => 'gnocchi-api',
        }
      }else{
        oci::sslkeypair {'gnocchi': }
      }
      $gnocchi_key_file = "/etc/gnocchi/ssl/private/${::fqdn}.pem"
      $gnocchi_crt_file = "/etc/gnocchi/ssl/public/${::fqdn}.crt"
    } else {
      $gnocchi_key_file = undef
      $gnocchi_crt_file = undef
    }
    if $openstack_release == 'rocky'{
      class { '::gnocchi':
        debug                 => true,
        database_connection   => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8",
      }
    }else{
      if ($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
        class { '::gnocchi':
          database_connection   => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8",
        }
      } else {
        class { '::gnocchi':
          coordination_url => "zookeeper://${zookeeper_ensemble_string}:2181/",
        }
        class { 'gnocchi::db':
          database_connection => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8",
        }
      }
      class { '::gnocchi::logging':
        debug => true,
      }
    }
    if $is_first_master {
      class { '::gnocchi::keystone::auth':
        public_url   => "${base_url}/metric",
        internal_url => "${base_url}/metric",
        admin_url    => "${base_url}/metric",
        password     => $pass_gnocchi_authtoken,
        region       => $region_name,
      }
    }
    class { '::gnocchi::keystone::authtoken':
      password             => $pass_gnocchi_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    class { '::gnocchi::client': }
    if $has_subrole_gnocchi_services {
      class { '::gnocchi::api':
        enabled                      => true,
        service_name                 => 'gnocchi-api',
        sync_db                      => true,
        enable_proxy_headers_parsing => true,
      }
      # Fix the number of uwsgi processes
      class { '::gnocchi::wsgi::uwsgi': }
      # Disable uwsgi logs
      gnocchi_api_uwsgi_config {
        'uwsgi/disable-logging': value => true;
      }
      class { '::gnocchi::metricd':
        workers                 => $::os_workers,
        cleanup_delay           => 20,
        metric_processing_delay => 10,
      }
    }else{
      package { 'gnocchi api removed':
        name   => 'gnocchi-api',
        ensure => purged,
      }
      package { 'gnocchi metricd removed':
        name   => 'gnocchi-metricd',
        ensure => purged,
      }
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
      class { '::gnocchi::storage::ceph':
        ceph_username => 'openstack',
        ceph_keyring  => '/etc/ceph/ceph.client.openstack.keyring',
        manage_cradox => false,
        manage_rados  => true,
      }
    }else{
      class { '::gnocchi::storage::ceph':
        ceph_username => 'openstack',
        ceph_keyring  => '/etc/ceph/ceph.client.openstack.keyring',
        manage_rados  => true,
      }
    }
    gnocchi_config {
      'database/connection': value => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8", secret => true;
    }

    # make sure ceph pool exists before running gnocchi (dbsync & services)
    if $has_subrole_gnocchi_services {
      Exec['create-gnocchi'] -> Exec['gnocchi-db-sync']

      class { '::gnocchi::statsd':
        archive_policy_name => 'high',
        flush_delay         => '100',
        # random datas:
        resource_id         => $pass_gnocchi_rscuuid,
      }
    }else{
      package { 'gnocchi statsd removed':
        name   => 'gnocchi-statsd',
        ensure => purged,
      }
    }
  }
  ###################
  ### Setup Panko ###
  ###################
  if $has_subrole_panko and $install_panko and $install_telemetry{
    if $use_ssl {
      oci::sslkeypair {'panko':
        notify_service_name => 'panko-api',
      }
      $panko_key_file = "/etc/panko/ssl/private/${::fqdn}.pem"
      $panko_crt_file = "/etc/panko/ssl/public/${::fqdn}.crt"
    } else {
      $panko_key_file = undef
      $panko_crt_file = undef
    }
    include ::panko
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::panko::db':
        database_connection   => "mysql+pymysql://panko:${pass_panko_db}@${sql_host}/pankodb?charset=utf8",
        database_idle_timeout => 1800,
      }
    }else{
      class { '::panko::db':
        database_connection              => "mysql+pymysql://panko:${pass_panko_db}@${sql_host}/pankodb?charset=utf8",
        database_connection_recycle_time => 1800,
      }
    }
    if $is_first_master {
      class { '::panko::keystone::auth':
        public_url   => "${base_url}/event",
        internal_url => "${base_url}/event",
        admin_url    => "${base_url}/event",
        password     => $pass_panko_authtoken,
        region       => $region_name,
      }
    }
    class { '::panko::keystone::authtoken':
      password             => $pass_panko_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    class { '::panko::api':
      sync_db                      => $do_db_sync,
      enabled                      => true,
      enable_proxy_headers_parsing => true,
    }
    # Fix the number of uwsgi processes
    class { '::panko::wsgi::uwsgi': }
    # Disable uwsgi logs
    panko_api_uwsgi_config {
      'uwsgi/disable-logging': value => true;
    }
    if $openstack_release != 'rocky'{
      class { '::panko::logging':
        debug => true,
      }
    }
    class { '::panko::client': }
  }

  ########################
  ### Setup Ceilometer ###
  ########################
  if $has_subrole_ceilometer and $install_telemetry {
    if $openstack_release == 'rocky'{
      class { '::ceilometer':
        debug                      => true,
        telemetry_secret           => $pass_ceilometer_telemetry,
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        notification_transport_url => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($notif_rabbit_servers),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        memcache_servers           => $memcached_servers,
      }
    }else{
      if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
        class { '::ceilometer':
          telemetry_secret           => $pass_ceilometer_telemetry,
          default_transport_url      => os_transport_url({
            'transport' => 'rabbit',
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => '5671',
            'username'  => 'ceilometer',
            'password'  => $pass_ceilometer_messaging,
          }),
          notification_transport_url => os_transport_url({
            'transport' => 'rabbit',
            'hosts'     => fqdn_rotate($notif_rabbit_servers),
            'port'      => '5671',
            'username'  => 'ceilometer',
            'password'  => $pass_ceilometer_messaging,
          }),
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms       => 'PLAIN',
          memcache_servers           => $memcached_servers,
        }
      # Xena and up should use ::ceilometer::cache for the memcache_servers param.
      }else{
        class { '::ceilometer':
          telemetry_secret           => $pass_ceilometer_telemetry,
          default_transport_url      => os_transport_url({
            'transport' => 'rabbit',
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => '5671',
            'username'  => 'ceilometer',
            'password'  => $pass_ceilometer_messaging,
          }),
          notification_transport_url => os_transport_url({
            'transport' => 'rabbit',
            'hosts'     => fqdn_rotate($notif_rabbit_servers),
            'port'      => '5671',
            'username'  => 'ceilometer',
            'password'  => $pass_ceilometer_messaging,
          }),
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms       => 'PLAIN',
        }
        class { '::ceilometer::cache':
          memcache_servers           => $memcached_servers,
        }
      }
      class { '::ceilometer::logging':
        debug => true,
      }

    }
    # This don't work for floating ips polling
    #ceilometer_config {
    #  'service_types/neutron': value => 'load-balancer';
    #}

    # We don't need the endpoint, but we need it as this
    # configures the user and pass for keystone_authtoken.
    if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'or $openstack_release == 'ussuri'or $openstack_release == 'victoria') {
      class { '::ceilometer::keystone::auth':
        password           => $pass_ceilometer_authtoken,
        configure_endpoint => false,
        region             => $region_name,
      }
    }else{
      class { '::ceilometer::keystone::auth':
        password           => $pass_ceilometer_authtoken,
        region             => $region_name,
      }
    }

    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
      class { '::ceilometer::keystone::authtoken':
        password             => $pass_ceilometer_authtoken,
        auth_url             => $keystone_admin_uri,
        www_authenticate_uri => $keystone_auth_uri,
        memcached_servers    => $memcached_servers,
        cafile               => $api_endpoint_ca_file,
        region_name          => $region_name,
      }
    }
    include ::ceilometer::db::sync
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby'{
      Class['ceilometer::agent::auth'] -> Exec['ceilometer-upgrade']
    }else{
      Class['ceilometer::agent::service_credentials'] -> Exec['ceilometer-upgrade']
    }
    if $is_first_master {
      Class['gnocchi::keystone::auth']    -> Exec['ceilometer-upgrade']
      Class['ceilometer::keystone::auth'] -> Exec['ceilometer-upgrade']
      if $has_subrole_gnocchi_services {
        Service['gnocchi-api']              -> Exec['ceilometer-upgrade']
      }
    }

    if $gnocchi_resources_yaml {
      file { "/etc/ceilometer/gnocchi_resources.yaml":
        ensure                  => present,
        owner                   => root,
        group                   => root,
        content                 => base64('decode', $gnocchi_resources_yaml),
        selinux_ignore_defaults => true,
        mode                    => '0644',
        require                 => Anchor['ceilometer::install::end'],
        notify                  => Exec['ceilometer-upgrade'],
      }
    }

    $sample_pipeline_publishers = ['gnocchi://']
    $event_pipeline_publishers = ['gnocchi://', 'notifier://?topic=alarm.all']

    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria'{
      class { '::ceilometer::agent::notification':
        notification_workers      => $::os_workers,
        manage_pipeline           => true,
        pipeline_publishers       => $sample_pipeline_publishers,
        manage_event_pipeline     => true,
        event_pipeline_publishers => $event_pipeline_publishers,
        messaging_urls            => os_transport_url({
            'transport' => 'rabbit',
            'hosts'     => fqdn_rotate($notif_rabbit_servers),
            'port'      => '5671',
            'username'  => 'ceilometer',
            'password'  => $pass_ceilometer_messaging,
          }),
      }
    }else{
      class { '::ceilometer::agent::notification':
        workers                   => $::os_workers,
        manage_pipeline           => true,
        pipeline_publishers       => $sample_pipeline_publishers,
        manage_event_pipeline     => true,
        event_pipeline_publishers => $event_pipeline_publishers,
        messaging_urls            => os_transport_url({
            'transport' => 'rabbit',
            'hosts'     => fqdn_rotate($notif_rabbit_servers),
            'port'      => '5671',
            'username'  => 'ceilometer',
            'password'  => $pass_ceilometer_messaging,
          }),
      }
    }
    if $openstack_release == 'rocky'{
      class { '::ceilometer::agent::central':
        coordination_url => "zookeeper://${zookeeper_ensemble_string}:2181/",
      }
    }else{
      $pollsters_default_list = [
			# Libvirt / compute
                              'cpu',
                              'cpu_l3_cache',
                              'vcpus',
                              'memory.usage',
                              'network.incoming.bytes',
                              'network.incoming.packets',
                              'network.outgoing.bytes',
                              'network.outgoing.packets',
                              'disk.read.bytes',
                              'disk.read.requests',
                              'disk.write.bytes',
                              'disk.write.requests',
                        # Networking
                              'ip.floating',
                              'switch.port',
                              'port',
                              'switch.ports',
			# Cinder
                              'volume.size',
                              'volume.snapshot.size',
                              'volume.backup.size',
			# Glance
                              'image.size',
                              'image.download',
                              'image.serve',
			# Swift
                              'storage.objects',
                              'storage.objects.size',
                              'storage.objects.outgoing.bytes',
                              'storage.objects.incoming.bytes',
			# LBs
                        # Buggy with Octavia
                              #'network.services.lb.pool',
                              #'network.services.lb.listener',
                              #'network.services.lb.member',
                              #'network.services.lb.health_monitor',
                              #'network.services.lb.loadbalancer',
                              #'network.services.lb.total.connections',
                              #'network.services.lb.incoming.bytes',
                              #'network.services.lb.outgoing.bytes',
                             ]
      $pollsters_list = concat($pollsters_default_list, $dynamic_pollsters_names)

      file { '/etc/ceilometer/pollsters.d':
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
        require                 => Package[$::ceilometer::params::agent_polling_package_name],
      }

      if $dynamic_pollsters_files {
        $dynamic_pollsters_files.each |Integer $index, String $value| {
          $pollster_content = base64('decode', $dynamic_pollsters_data[$index])
          file { "/etc/ceilometer/pollsters.d/${value}":
            ensure  => present,
            owner   => 'root',
            group   => 'ceilometer',
            mode    => '0640',
            content => $pollster_content,
            require => File['/etc/ceilometer/pollsters.d'],
            notify  => Service['ceilometer-polling'],
          }
        }
      }
      if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby'{
        class { '::ceilometer::agent::polling':
          central_namespace => true,
          compute_namespace => false,
          ipmi_namespace    => false,
          manage_polling    => true,
          polling_meters    => $pollsters_list,
          # That's each 5 minutes:
          polling_interval  => 300,
          coordination_url  => "zookeeper://${zookeeper_ensemble_string}:2181/",
        }
      }else{
        class { '::ceilometer::agent::polling':
          central_namespace => true,
          compute_namespace => false,
          ipmi_namespace    => false,
          manage_polling    => true,
          polling_meters    => $pollsters_list,
          # That's each 5 minutes:
          polling_interval  => 300,
        }
        class { 'ceilometer::coordination':
          backend_url => "zookeeper://${zookeeper_ensemble_string}:2181/",
        }
      }
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria'{
      class { '::ceilometer::agent::auth':
        auth_password => $pass_ceilometer_authtoken,
        auth_url      => $keystone_auth_uri,
        auth_cacert   => $api_endpoint_ca_file,
        auth_region   => $region_name,
      }
    }else{
      class { '::ceilometer::agent::service_credentials':
        password    => $pass_ceilometer_authtoken,
        auth_url    => $keystone_auth_uri,
        cafile      => $api_endpoint_ca_file,
        region_name => $region_name,
      }
    }
    # Looks like the above sets ca_file instead of cafile.
    if $openstack_release == 'rocky'{
      ceilometer_config {
        'service_credentials/cafile': value => $api_endpoint_ca_file;
      }
    }
    ceilometer_config {
      'service_credentials/www_authenticate_uri': value => "${keystone_auth_uri}";
    }
  }
  ########################
  ### Setup CloudKitty ###
  ########################
  if $has_subrole_cloudkitty and $install_telemetry{
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['cloudkitty::install::begin']
      Exec['galera-size-is-correct'] -> Package['cloudkitty dashboard']
#      Class['oci::sql::galera'] -> Package['cloudkitty dashboard']
    }
    if ! $messaging_nodes_for_notifs {
      if $use_ssl {
        oci::sslkeypair {'cloudkitty':
          notify_service_name => 'cloudkitty-api',
        }
        $cloudkitty_key_file = "/etc/cloudkitty/ssl/private/${::fqdn}.pem"
        $cloudkitty_crt_file = "/etc/cloudkitty/ssl/public/${::fqdn}.crt"
      } else {
        $cloudkitty_key_file = undef
        $cloudkitty_crt_file = undef
      }
    }
    # Note: ::cloudkitty::db *MUST* be declared before ::cloudkitty,
    # because of the include in the ::cloudkitty class.
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::cloudkitty::db':
        database_connection   => "mysql+pymysql://cloudkitty:${pass_cloudkitty_db}@${sql_host}/cloudkittydb?charset=utf8",
        database_idle_timeout => 1800,
      }
    }else{
      class { '::cloudkitty::db':
        database_connection              => "mysql+pymysql://cloudkitty:${pass_cloudkitty_db}@${sql_host}/cloudkittydb?charset=utf8",
        database_connection_recycle_time => 1800,
      }
    }
    if $disable_notifications {
      $cloudkitty_notif_transport_url = ''
    } else {
      $cloudkitty_notif_transport_url = os_transport_url({
                                          'transport' => 'rabbit',
                                          'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                          'port'      => '5671',
                                          'username'  => 'cloudkitty',
                                          'password'  => $pass_cloudkitty_messaging,
                                        })
    }
    if $openstack_release == 'rocky'{
      class { '::cloudkitty':
        default_transport_url      => os_transport_url({
                                                        'transport' => 'rabbit',
                                                        'hosts'     => fqdn_rotate($all_masters),
                                                        'port'      => '5671',
                                                        'username'  => 'cloudkitty',
                                                        'password'  => $pass_cloudkitty_messaging,
                                                      }),
        notification_transport_url => $cloudkitty_notif_transport_url,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        storage_backend            => 'sqlalchemy',
        tenant_fetcher_backend     => 'keystone',
        auth_section               => 'keystone_authtoken',
        host                       => $machine_hostname,
      }
    }else{
      if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
        class { '::cloudkitty':
          default_transport_url      => os_transport_url({
                                                          'transport' => 'rabbit',
                                                          'hosts'     => fqdn_rotate($all_masters),
                                                          'port'      => '5671',
                                                          'username'  => 'cloudkitty',
                                                          'password'  => $pass_cloudkitty_messaging,
                                                        }),
          notification_transport_url => $cloudkitty_notif_transport_url,
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms       => 'PLAIN',
          storage_backend            => 'sqlalchemy',
          storage_version            => '1',
          tenant_fetcher_backend     => 'keystone',
          auth_section               => 'keystone_authtoken',
          host                       => $machine_hostname,
        }
      }else{
        if ($openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena') {
          class { '::cloudkitty':
            default_transport_url      => os_transport_url({
                                                            'transport' => 'rabbit',
                                                            'hosts'     => fqdn_rotate($all_masters),
                                                            'port'      => '5671',
                                                            'username'  => 'cloudkitty',
                                                            'password'  => $pass_cloudkitty_messaging,
                                                          }),
            notification_transport_url => $cloudkitty_notif_transport_url,
            rabbit_use_ssl             => $use_ssl,
            rabbit_ha_queues           => true,
            kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
            amqp_sasl_mechanisms       => 'PLAIN',
            storage_backend            => 'sqlalchemy',
            storage_version            => '1',
            fetcher_backend            => 'keystone',
            auth_section               => 'ociadmin_authtoken',
            host                       => $machine_hostname,
          }
        }else{
          class { '::cloudkitty::fetcher::keystone':
            auth_section            => 'ociadmin_authtoken',
            ignore_rating_role      => true,
            ignore_disabled_tenants => true,
          }
          class { '::cloudkitty':
            default_transport_url      => os_transport_url({
                                                            'transport' => 'rabbit',
                                                            'hosts'     => fqdn_rotate($all_masters),
                                                            'port'      => '5671',
                                                            'username'  => 'cloudkitty',
                                                            'password'  => $pass_cloudkitty_messaging,
                                                          }),
            notification_transport_url => $cloudkitty_notif_transport_url,
            rabbit_use_ssl             => $use_ssl,
            rabbit_ha_queues           => true,
            kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
            amqp_sasl_mechanisms       => 'PLAIN',
            storage_backend            => 'sqlalchemy',
            storage_version            => '1',
            fetcher_backend            => 'keystone',
            host                       => $machine_hostname,
          }
        }
        class { '::cloudkitty::logging':
          debug => true,
        }
      }
      # Fix the number of uwsgi processes
      class { '::cloudkitty::wsgi::uwsgi': }
      # Disable uwsgi logs
      cloudkitty_api_uwsgi_config {
        'uwsgi/disable-logging': value => true;
      }
    }
    if $is_first_master {
      class { '::cloudkitty::keystone::auth':
        public_url   => "${base_url}/rating",
        internal_url => "${base_url}/rating",
        admin_url    => "${base_url}/rating",
        password     => $pass_cloudkitty_authtoken,
        region       => $region_name,
      }
    }
    class { '::cloudkitty::keystone::authtoken':
      password             => $pass_cloudkitty_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    class { '::cloudkitty::client': }
    if $messaging_nodes_for_notifs {
      package { 'cloudkitty api removed':
        name => 'cloudkitty-api',
        ensure => purged,
      }
    }else{
      class { '::cloudkitty::api':
        sync_db      => $do_db_sync,
      }
    }
    cloudkitty_config {
      'keystone_fetcher/cafile':       value => $api_endpoint_ca_file;
      'gnocchi_collector/cafile':      value => $api_endpoint_ca_file;
      'coordination/backend_url':      value => "zookeeper://${zookeeper_ensemble_string}:2181/";
      'orchestrator/coordination_url': value => "zookeeper://${zookeeper_ensemble_string}:2181/";
      'fetcher_gnocchi/cafile':        value => $api_endpoint_ca_file;
      'fetcher_gnocchi/auth_section':  value => 'keystone_authtoken';
    }

    # This is needed for the Keystone fetcher to list users
    cloudkitty_config {
      'ociadmin_authtoken/auth_type':            value => 'password';
      'ociadmin_authtoken/auth_url':             value => $keystone_admin_uri;
      'ociadmin_authtoken/project_name':         value => 'admin';
      'ociadmin_authtoken/project_domain_name':  value => 'Default';
      'ociadmin_authtoken/username':             value => 'admin';
      'ociadmin_authtoken/user_domain_name':     value => 'Default';
      'ociadmin_authtoken/password':             value => $pass_keystone_adminuser;
      'ociadmin_authtoken/www_authenticate_uri': value => $keystone_auth_uri;
      'ociadmin_authtoken/cafile':               value => $api_endpoint_ca_file;
      'ociadmin_authtoken/memcached_servers':    value => $memcached_string;
      'ociadmin_authtoken/region_name':          value => $region_name;
    }
    if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
      cloudkitty_config {
        'fetcher/backend': value => 'keystone';
      }
    }

    if $messaging_nodes_for_notifs {
      package { 'cloudkitty processor removed':
        name => 'cloudkitty-processor',
        ensure => purged,
      }
    }else{
      if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby') {
        class { '::cloudkitty::processor':
          auth_section => 'ociadmin_authtoken',
        }
        cloudkitty_config {
          'collector_gnocchi/region_name': value => $region_name;
        }
      }else{
        class { '::cloudkitty::processor':
          auth_section => 'ociadmin_authtoken',
          region_name  => $region_name,
        }
      }


      if $cloudkitty_metrics_yml {
        file { "/etc/cloudkitty/metrics.yml":
          ensure                  => present,
          owner                   => root,
          group                   => root,
          content                 => base64('decode', $cloudkitty_metrics_yml),
          selinux_ignore_defaults => true,
          mode                    => '0644',
          require                 => Anchor['cloudkitty::install::end'],
          notify                  => Service['cloudkitty-processor'],
        }
      }
    }
    if $install_cloudkitty_dashboard {
      package { 'cloudkitty dashboard':
        name => 'python3-cloudkitty-dashboard',
        ensure => installed,
      }
    }else{
      package { 'cloudkitty dashboard':
        name => 'python3-cloudkitty-dashboard',
        ensure => purged,
      }
    }
  }

  ##################
  ### Setup Aodh ###
  ##################
  if $has_subrole_aodh and $install_telemetry{
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['aodh::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'aodh':
        notify_service_name => 'aodh-api',
      }
      $aodh_key_file = "/etc/aodh/ssl/private/${::fqdn}.pem"
      $aodh_crt_file = "/etc/aodh/ssl/public/${::fqdn}.crt"
    } else {
      $aodh_key_file = undef
      $aodh_crt_file = undef
    }
    if $disable_notifications {
      $aodh_notif_transport_url = ''
    }else{
      $aodh_notif_transport_url = os_transport_url({
                                  'transport' => $messaging_notify_proto,
                                  'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                  'port'      => $messaging_notify_port,
                                  'username'  => 'aodh',
                                  'password'  => $pass_aodh_messaging,
                                })
    }
    if $openstack_release == 'rocky'{
      class { '::aodh':
        debug                      => true,
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'aodh',
          'password'  => $pass_aodh_messaging,
        }),
        notification_transport_url => $aodh_notif_transport_url,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        database_connection        => "mysql+pymysql://aodh:${pass_aodh_db}@${sql_host}/aodhdb?charset=utf8",
        database_idle_timeout      => 1800,
        notification_driver        => $notification_driver,
      }
    }else{
      if $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
        class { '::aodh':
          default_transport_url      => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'aodh',
            'password'  => $pass_aodh_messaging,
          }),
          notification_transport_url => $aodh_notif_transport_url,
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms       => 'PLAIN',
          database_connection        => "mysql+pymysql://aodh:${pass_aodh_db}@${sql_host}/aodhdb?charset=utf8",
          database_idle_timeout      => 1800,
          notification_driver        => $notification_driver,
        }
      }else{
        class { '::aodh::db':
          database_connection              => "mysql+pymysql://aodh:${pass_aodh_db}@${sql_host}/aodhdb?charset=utf8",
          database_connection_recycle_time => 1800,
        }
        class { '::aodh':
          default_transport_url      => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'aodh',
            'password'  => $pass_aodh_messaging,
          }),
          notification_transport_url => $aodh_notif_transport_url,
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms       => 'PLAIN',
          notification_driver        => $notification_driver,
        }
      }
      class { '::aodh::logging':
        debug => true,
      }
    }
    if $is_first_master {
      class { '::aodh::keystone::auth':
        public_url   => "${base_url}/alarm",
        internal_url => "${base_url}/alarm",
        admin_url    => "${base_url}/alarm",
        password     => $pass_aodh_authtoken,
        region       => $region_name,
      }
    }
    class { '::aodh::keystone::authtoken':
      password             => $pass_aodh_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    class { '::aodh::api':
      enabled                      => true,
      service_name                 => 'aodh-api',
      sync_db                      => $do_db_sync,
      enable_proxy_headers_parsing => true,
    }
    # Fix the number of uwsgi processes
    class { '::aodh::wsgi::uwsgi': }
    # Disable uwsgi logs
    aodh_api_uwsgi_config {
      'uwsgi/disable-logging': value => true;
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
      class { '::aodh::auth':
        auth_url      => $keystone_auth_uri,
        auth_cacert   => $api_endpoint_ca_file,
        auth_password => $pass_aodh_authtoken,
        auth_region   => $region_name,
      }
    }else{
      class { '::aodh::service_credentials':
        auth_url    => $keystone_auth_uri,
        cacert      => $api_endpoint_ca_file,
        password    => $pass_aodh_authtoken,
        region_name => $region_name,
      }
    }
    class { '::aodh::client': }
    if $aodh_notifier_use_proxy {
      file { '/etc/aodh/aodh-notifier.enviroment':
        ensure                  => present,
        owner                   => 'root',
        content                 => "http_proxy=${aodh_notifier_proxy_url}
https_proxy=${aodh_notifier_proxy_url}
HTTP_PROXY=${aodh_notifier_proxy_url}
HTTPS_PROXY=${aodh_notifier_proxy_url}
no_proxy=localhost,127.0.0.1,localaddress,${$vip_hostname},${vip_ipaddr}",
        selinux_ignore_defaults => true,
        mode                    => '0640',
        require                 => Package['aodh-notifier'],
      }->
      systemd::dropin_file { 'aodh-http-proxy-enviroment-file.conf':
        unit    => 'aodh-notifier.service',
        notify  => Service['aodh-notifier'],
        content => "[Service]
EnvironmentFile=/etc/aodh/aodh-notifier.enviroment
",
      }
    }else{
      file { '/etc/aodh/aodh-notifier.enviroment':
        ensure => absent,
      }->
      file { '/etc/systemd/system/aodh-notifier.service.d/aodh-http-proxy-enviroment-file.conf':
        ensure => absent,
        notify => Service['aodh-notifier'],
      }
    }
    class { '::aodh::notifier': }
    class { '::aodh::listener': }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
      class { '::aodh::evaluator':
        coordination_url    => "zookeeper://${zookeeper_ensemble_string}:2181/",
        evaluation_interval => 10,
      }
    }else{
      class { '::aodh::evaluator':
        evaluation_interval => 10,
      }
      class { '::aodh::coordination':
        backend_url => "zookeeper://${zookeeper_ensemble_string}:2181/",
      }
    }
  }
  #####################
  ### Setup Octavia ###
  #####################
  if $has_subrole_octavia and $install_octavia{
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['octavia::install::begin']
      Exec['galera-size-is-correct'] -> Package['octavia dashboard']
#      Class['oci::sql::galera'] -> Package['octavia dashboard']
    }
    if $use_ssl {
      oci::sslkeypair {'octavia':
        notify_service_name => 'octavia-api',
      }
      $octavia_key_file = "/etc/octavia/ssl/private/${::fqdn}.pem"
      $octavia_crt_file = "/etc/octavia/ssl/public/${::fqdn}.crt"
    } else {
      $octavia_key_file = undef
      $octavia_crt_file = undef
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::octavia::db':
        database_connection   => "mysql+pymysql://octavia:${pass_octavia_db}@${sql_host}/octaviadb?charset=utf8",
        database_idle_timeout => 1800,
      }
    }else{
      class { '::octavia::db':
        database_connection              => "mysql+pymysql://octavia:${pass_octavia_db}@${sql_host}/octaviadb?charset=utf8",
        database_connection_recycle_time => 1800,
      }
    }
    if $disable_notifications {
      $octavia_notif_transport_url = ''
    }else{
      $octavia_notif_transport_url = os_transport_url({
                                       'transport' => $messaging_notify_proto,
                                       'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                       'port'      => $messaging_notify_port,
                                       'username'  => 'octavia',
                                       'password'  => $pass_octavia_messaging,
                                     })
    }
    class { '::octavia':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'octavia',
        'password'  => $pass_octavia_messaging,
      }),
      notification_transport_url => $octavia_notif_transport_url,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
      amqp_sasl_mechanisms       => 'PLAIN',
      notification_driver        => $notification_driver,
    }
    if $is_first_master {
      class { '::octavia::keystone::auth':
        public_url   => "${base_url}/loadbalance",
        internal_url => "${base_url}/loadbalance",
        admin_url    => "${base_url}/loadbalance",
        password     => $pass_octavia_authtoken,
        region       => $region_name,
      }
    }
    class { '::octavia::keystone::authtoken':
      password             => $pass_octavia_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    class { '::octavia::service_auth':
      auth_url            => $keystone_admin_uri,
      username            => 'octavia',
      project_name        => 'services',
      password            => $pass_octavia_authtoken,
      user_domain_name    => 'default',
      project_domain_name => 'default',
      auth_type           => 'password',
      region_name         => $region_name,
    }

    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' {
      class { '::octavia::api':
        enabled      => true,
        sync_db      => $do_db_sync,
      }
    # default_listener_ciphers only supported on Victoria and up
    } else {
      class { '::octavia::api':
        enabled                  => true,
        sync_db                  => $do_db_sync,
        default_listener_ciphers => [ 'TLS_AES_256_GCM_SHA384',
                                      'TLS_CHACHA20_POLY1305_SHA256',
                                      'TLS_AES_128_GCM_SHA256',
                                      'DHE-RSA-AES256-GCM-SHA384',
                                      'DHE-RSA-AES128-GCM-SHA256',
                                      'ECDHE-RSA-AES256-GCM-SHA384',
                                      'ECDHE-RSA-AES128-GCM-SHA256',
                                      'DHE-RSA-AES256-SHA256',
                                      'DHE-RSA-AES128-SHA256',
                                    ]
      }
    }
    # Fix the number of uwsgi processes
    class { '::octavia::wsgi::uwsgi': }
    # Disable uwsgi logs
    octavia_api_uwsgi_config {
      'uwsgi/disable-logging': value => true;
    }

    octavia_config {
      'certificates/ca_certificates_file':      value => $api_endpoint_ca_file;
      'certificates/ca_certificate':            value => '/etc/octavia/certs/server_ca.cert.pem';
      'certificates/ca_private_key':            value => '/etc/octavia/certs/server_ca.key.pem';
      'certificates/ca_private_key_passphrase': value => 'octavia';
      'certificates/region_name':               value => $region_name;
      'controller_worker/client_ca':            value => '/etc/octavia/certs/client_ca.cert.pem';
      'haproxy_amphora/client_cert':            value => '/etc/octavia/certs/client.cert-and-key.pem';
      'haproxy_amphora/server_ca':              value => '/etc/octavia/certs/server_ca.cert.pem';
      'glance/ca_certificates_file':            value => $api_endpoint_ca_file;
      'neutron/ca_certificates_file':           value => $api_endpoint_ca_file;
      'nova/ca_certificates_file':              value => $api_endpoint_ca_file;
      'service_auth/cafile':                    value => $api_endpoint_ca_file;
    }

    $octavia_controller_ip_port_list = join([join($all_masters_ip,':5555,'), ':5555'],'')
    #$all_masters_ip.map |Integer $index, String $value| {
    #  [ "${value}:555" ]
    #}
    # Flavor creation is currently broken in Victoria
    if ($initial_cluster_setup and ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri')){
      $manage_nova_flavor = true
    }else{
      $manage_nova_flavor = false
    }
    if $openstack_release == 'rocky' {
      class { '::octavia::worker':
        amp_image_tag         => 'amphora',
        loadbalancer_topology => 'ACTIVE_STANDBY',
        amp_secgroup_list     => $amp_secgroup_list,
        amp_boot_network_list => $amp_boot_network_list,
        manage_nova_flavor    => $manage_nova_flavor,
        # The default is not enough, haproxy logs may fill the HDD quickly.
        nova_flavor_config    => { 'disk' => '4', 'ram' => '4096' },
      }
      # puppet-openstack in Rocky doesn't have a way to configure controller_ip_port_list
      octavia_config {
        'health_manager/controller_ip_port_list': value => $octavia_controller_ip_port_list;
      }
    }else{
      # Take care: ::octavia::controller *MUST* be declared before
      # the ::octavia::worker, otherwise it's called twice and
      # compilation of the catalogue will failt, because the
      # ::octavia::worker will include it.
      # Starting with Zed, ::octavia::health_manager also includes it.
      if $openstack_release == 'rocky' or $openstack_release == 'stein' {
        class { '::octavia::controller':
          amp_image_tag           => 'amphora',
          loadbalancer_topology   => 'ACTIVE_STANDBY',
          amp_secgroup_list       => $amp_secgroup_list,
          amp_boot_network_list   => $amp_boot_network_list,
          controller_ip_port_list => $octavia_controller_ip_port_list,
        }
        octavia_config {
          'amphora_agent/disable_local_log_storage': value => 'true';
          'haproxy_amphora/connection_logging':      value => 'false';
        }
      }elsif $openstack_release == 'train'{
        class { '::octavia::controller':
          amp_image_tag             => 'amphora',
          loadbalancer_topology     => 'ACTIVE_STANDBY',
          amp_secgroup_list         => $amp_secgroup_list,
          amp_boot_network_list     => $amp_boot_network_list,
          controller_ip_port_list   => $octavia_controller_ip_port_list,
          connection_logging        => false,
        }
        octavia_config {
          'amphora_agent/disable_local_log_storage': value => 'true';
        }
      }elsif($openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga'){
        class { '::octavia::controller':
          amp_image_tag             => 'amphora',
          loadbalancer_topology     => 'ACTIVE_STANDBY',
          amp_secgroup_list         => $amp_secgroup_list,
          amp_boot_network_list     => $amp_boot_network_list,
          controller_ip_port_list   => $octavia_controller_ip_port_list,
          disable_local_log_storage => true,
          connection_logging        => false,
        }
      # In Zed and up, heartbeat_key is in ::octavia::controller, and not anymore in ::octavia::health_manager.
      }else{
        class { '::octavia::controller':
          heartbeat_key             => $pass_octavia_heatbeatkey,
          amp_image_tag             => 'amphora',
          loadbalancer_topology     => 'ACTIVE_STANDBY',
          amp_secgroup_list         => $amp_secgroup_list,
          amp_boot_network_list     => $amp_boot_network_list,
          controller_ip_port_list   => $octavia_controller_ip_port_list,
          disable_local_log_storage => true,
          connection_logging        => false,
        }
      }
      class { '::octavia::worker':
        manage_nova_flavor    => $manage_nova_flavor,
        # The default is not enough, haproxy logs may fill the HDD quickly.
        nova_flavor_config    => { 'disk' => '4', 'ram' => '2048' },
      }
    }
    if ($openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga'){
      class { '::octavia::health_manager':
        heartbeat_key => $pass_octavia_heatbeatkey,
        ip            => $machine_ip,
      }
    }else{
      class { '::octavia::health_manager':
        ip            => $machine_ip,
      }
    }
    if $openstack_release == 'rocky' {
      octavia_config {
        'nova/enable_anti_affinity': value => 'true';
      }
    }else{
      class { '::octavia::nova':
        enable_anti_affinity => 'true',
      }
    }
    # cleans old database records, rotates the certificates issued to the amphora when they come close to expiration, manage the spares pool, etc.
    # Without it your database tables will grow forever and your amphora will eventually drop off the network as they will no longer be trusted.
    class { '::octavia::housekeeping': }
    package { 'octavia dashboard':
      name => 'python3-octavia-dashboard',
      ensure => installed,
    }
  }else{
    package { 'octavia dashboard':
      name => 'python3-octavia-dashboard',
      ensure => purged,
    }
    package { 'octavia purged':
      name => 'python3-octavia',
      ensure => purged,
    }
  }
  ####################
  ### Setup Magnum ###
  ####################
  if $has_subrole_magnum and $install_magnum{
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['magnum::install::begin']
      Exec['galera-size-is-correct'] -> Package['magnum ui']
    }
    if $use_ssl {
      oci::sslkeypair {'magnum':
        notify_service_name => 'magnum-api',
      }
      $magnum_key_file = "/etc/magnum/ssl/private/${::fqdn}.pem"
      $magnum_crt_file = "/etc/magnum/ssl/public/${::fqdn}.crt"
    } else {
      $magnum_key_file = undef
      $magnum_crt_file = undef
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::magnum::db':
        database_connection   => "mysql+pymysql://magnum:${pass_magnum_db}@${sql_host}/magnumdb?charset=utf8",
        database_idle_timeout => 1800,
      }
    }else{
      class { '::magnum::db':
        database_connection              => "mysql+pymysql://magnum:${pass_magnum_db}@${sql_host}/magnumdb?charset=utf8",
        database_connection_recycle_time => 1800,
      }
    }
    if $initial_cluster_setup {
      if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
        class { '::magnum::keystone::domain':
          cluster_user_trust => 'true',
          domain_admin       => 'magnum_admin',
          domain_admin_email => 'root@localhost',
          domain_password    => $pass_magnum_domain,
          roles              => 'admin',
        }
      }else{
        class { '::magnum::keystone::domain':
          cluster_user_trust   => 'true',
          domain_admin         => 'magnum_admin',
          domain_admin_email   => 'root@localhost',
          domain_password      => $pass_magnum_domain,
          roles                => 'admin',
#          keystone_region_name => $region_name,
        }
      }
    }
    class { '::magnum::keystone::authtoken':
      password             => $pass_magnum_authtoken,
      auth_url             => "${keystone_admin_uri}/v3",
      www_authenticate_uri => "${keystone_auth_uri}/v3",
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    class { '::magnum::api':
      service_name  => 'magnum-api',
      host          => $machine_ip,
      enabled_ssl   => $use_ssl,
      ssl_cert_file => $magnum_crt_file,
      ssl_key_file  => $magnum_key_file,
      sync_db       => $do_db_sync,
    }
    # Fix the number of uwsgi processes
    class { '::magnum::wsgi::uwsgi': }

    if $is_first_master {
      class { '::magnum::keystone::auth':
        public_url   => "${base_url}/containers/v1",
        internal_url => "${base_url}/containers/v1",
        admin_url    => "${base_url}/containers/v1",
        password     => $pass_magnum_authtoken,
        region       => $region_name,
      }
    }

    if $disable_notifications {
      $magnum_notif_transport_url = ''
    }else{
      $magnum_notif_transport_url = os_transport_url({
                                      'transport' => $messaging_notify_proto,
                                      'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                      'port'      => $messaging_notify_port,
                                      'username'  => 'magnum',
                                      'password'  => $pass_magnum_messaging,
                                    })
    }
    if $openstack_release == 'rocky' {
      class { '::magnum':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'magnum',
          'password'  => $pass_magnum_messaging,
        }),
        notification_transport_url => $magnum_notif_transport_url,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        notification_driver        => $notification_driver,
      }
    }else{

      # This defines $initial_cluster as an array of hosts
      # to be used in the initial_cluster parameter.
      $initial_cluster = $all_masters.map |Integer $indexetcd, String $valueetcd| {
        "${valueetcd}=http://${valueetcd}:2380"
      }

      $etcd_key_file = "/etc/etcd/ssl/private/${::fqdn}.pem"
      $etcd_crt_file = "/etc/etcd/ssl/public/${::fqdn}.crt"

      user { 'etcd':
        ensure   => present,
        system   => true,
      }->
      file { "/etc/etcd":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { '/var/lib/etcd':
        ensure                  => directory,
        owner                   => 'etcd',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { '/var/lib/etcd/default':
        ensure                  => directory,
        owner                   => 'etcd',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      oci::sslkeypair { 'etcd':
        notify_service_name => 'etcd',
        require_anchor      => Package['etcd'],
      }

      class { 'etcd':
        ensure                      => 'present',
        etcd_name                   => $machine_hostname,
        # clients
        listen_client_urls          => "http://127.0.0.1:2379,http://${machine_ip}:2379",
        advertise_client_urls       => "http://${machine_hostname}:2379",
        # clients ssl
#        cert_file                   => $etcd_crt_file,
#        key_file                    => $etcd_key_file,
        # authorize clients
#        client_cert_auth            => true,
        # verify clients certificates
        trusted_ca_file             => '/etc/ssl/certs/oci-pki-oci-ca-chain.pem',
        # cluster
        listen_peer_urls            => "http://${machine_ip}:2380",
        initial_advertise_peer_urls => "http://${machine_hostname}:2380",
        initial_cluster             => $initial_cluster,
        # peers ssl
#        peer_cert_file              => $etcd_crt_file,
#        peer_key_file               => $etcd_key_file,
        # authorize peers
#        peer_client_cert_auth       => true,
        # verify peers certificates
#        peer_trusted_ca_file        => '/etc/ssl/certs/oci-pki-oci-ca-chain.pem',
        debug                       => true,
        require                    => Service['haproxy'],
      }

/*      class { '::oci::etcddiscovery':
        prefix_url             => "${base_url}/etcd-discovery",
        web_service_address    => "${machine_ip}:8087",
        etcd_endpoint_location => "http://${machine_hostname}:2379",
        require                => Service['haproxy'],
      }
*/
      class { '::magnum':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'magnum',
          'password'  => $pass_magnum_messaging,
        }),
        notification_transport_url => $magnum_notif_transport_url,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        notification_driver        => $notification_driver,
      }
    }

    class { '::magnum::conductor':
      workers => $::os_workers,
    }

    class { '::magnum::client': }

    class { '::magnum::clients::barbican':
      region_name => $region_name,
    }
    class { '::magnum::clients::cinder':
      region_name => $region_name,
    }
    class { '::magnum::clients::glance':
      region_name => $region_name,
      ca_file     => $api_endpoint_ca_file,
    }
    class { '::magnum::clients::heat':
      region_name => $region_name,
      ca_file     => $api_endpoint_ca_file,
    }
    class { '::magnum::clients::magnum':
      region_name => $region_name,
    }
    class { '::magnum::clients::neutron':
      region_name => $region_name,
      ca_file     => $api_endpoint_ca_file,
    }
    class { '::magnum::clients::nova':
      region_name => $region_name,
      ca_file     => $api_endpoint_ca_file,
    }

    class { '::magnum::certificates':
      cert_manager_type => 'barbican'
#      cert_manager_type => 'local'
    }

    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      magnum_config {
        'trust/trustee_keystone_region_name': value => $region_name;
      }
    }
    magnum_config {
      'keystone_auth/auth_section': value => 'keystone_authtoken';
      'octavia_client/region_name': value => $region_name;
      'octavia_client/ca_file':     value => $api_endpoint_ca_file;
      'cluster/etcd_discovery_service_endpoint_format': value => "${base_url}/etcd-discovery/new?size=%(size)d";
    }

    package { 'magnum ui':
      name => 'python3-magnum-ui',
      ensure => installed,
    }

    class { '::etcddiscovery':
      prefix_url          => "${base_url}/etcd-discovery",
      web_service_address => "${machine_ip}:8087",
      endpoint_location   => "http://${machine_ip}:2379",
    }
  }

  #######################
  ### Setup Designate ###
  #######################
  if $has_subrole_designate and $install_designate {
    # We only need rndc from bind9-utils, but we don't want bind9 itself.
    package {'bind9':
      ensure  => 'purged',
      before  => Group['bind'],
    }
    ensure_resource('group', 'bind', {
      'ensure' => 'present',
      'system' => 'true',
    })
    package {'bind9-utils':
      ensure => 'present',
      require => Group['bind'],
    }->
    file {'/etc/bind':
      ensure                  => directory,
      owner                   => 'root',
      mode                    => '0755',
      selinux_ignore_defaults => true,
    }->
    # Configure the designate-rndc.key of bind9-util's rndc
    file {'/etc/bind/designate-rndc.key':
      ensure                  => 'present',
      owner                   => 'root',
      group                   => 'bind',
      mode                    => '0640',
      selinux_ignore_defaults => true,
      content                 => "key \"designate-rndc-key\" {
        algorithm hmac-sha256;
        secret \"${pass_designate_rndckey}\";
};
",
    }->
    file { '/etc/bind/rndc.conf':
      ensure  => present,
      owner   => 'root',
      group   => 'bind',
      content => template("${module_name}/rndc.conf.erb"),
    }

    ### SETUP DESIGNATE ###
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['designate::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'designate':
        notify_service_name => 'designate-api',
      }
      $designate_key_file = "/etc/designate/ssl/private/${::fqdn}.pem"
      $designate_crt_file = "/etc/designate/ssl/public/${::fqdn}.crt"
    } else {
      $designate_key_file = undef
      $designate_crt_file = undef
    }
    class { '::designate::db':
      database_connection   => "mysql+pymysql://designate:${pass_designate_db}@${sql_host}/designatedb?charset=utf8",
    }
    class { 'designate::logging':
      debug => true,
    }
    if $disable_notifications {
      $designate_notif_transport_url = ''
    }else{
      $designate_notif_transport_url = os_transport_url({
                                       'transport' => $messaging_notify_proto,
                                       'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                       'port'      => $messaging_notify_port,
                                       'username'  => 'designate',
                                       'password'  => $pass_designate_messaging,
                                     })
    }

    class { '::designate':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'designate',
        'password'  => $pass_designate_messaging,
      }),
      notification_transport_url => $designate_notif_transport_url,
      notification_driver        => $notification_driver,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
      notification_topics       => ['notifications'],
#      amqp_sasl_mechanisms       => 'PLAIN',
      
    }

    include 'designate::client'

    class { '::designate::keystone::authtoken':
      password             => $pass_designate_authtoken,
      user_domain_name     => 'Default',
      project_domain_name  => 'Default',
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }

    if $is_first_master {
      class { '::designate::keystone::auth':
        public_url   => "${base_url}/dns",
        internal_url => "${base_url}/dns",
        admin_url    => "${base_url}/dns",
        password     => $pass_designate_authtoken,
        region       => $region_name,
      }
    }

    class { 'designate::api':
      listen             => "${machine_ip}:9001",
      api_base_uri       => "${base_url}/dns",
      auth_strategy      => 'keystone',
      enable_api_v2      => true,
      enable_api_admin   => true,
      enable_host_header => false
    }
    # Fix the number of uwsgi processes
    class { '::designate::wsgi::uwsgi': }

    class { 'designate::mdns':
      listen => "${machine_ip}:5354"
    }

    class { 'designate::central':
      managed_resource_tenant_id => $designate_managed_resource_tenant_id,
    }

    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
      class { 'designate::producer':
        backend_url => "zookeeper://${zookeeper_ensemble_string}:2181/",
      }
    }else{
      class { 'designate::producer': }
      class { '::designate::coordination':
        backend_url => "zookeeper://${zookeeper_ensemble_string}:2181/",
      }
    }

    class { 'designate::worker': }

    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'{
      class { 'designate::backend::bind9':
        rndc_host        => $machine_ip,
        rndc_port        => '953',
        rndc_config_file => '/etc/bind/rndc.conf',
        rndc_key_file    => '/etc/bind/designate-rndc.key',
      }
    }else{
      if $openstack_release == 'victoria' {
        class { 'designate::backend::bind9':
          rndc_host        => $machine_ip,
          rndc_port        => '953',
          rndc_config_file => '/etc/bind/rndc.conf',
          rndc_key_file    => '/etc/bind/designate-rndc.key',
          configure_bind   => false,
        }
      }elsif $openstack_release == 'wallaby' or $openstack_release == 'xena'{
        class { 'designate::backend::bind9':
          rndc_port        => '953',
          rndc_config_file => '/etc/bind/rndc.conf',
          rndc_key_file    => '/etc/bind/designate-rndc.key',
          configure_bind   => false,
        }
      }else{
        # We use manage_pool => false in Zed and up
        # otherwise the File['...pools.yaml'] is defined twice
        # (see its definition below).
        class { 'designate::backend::bind9':
          rndc_port        => '953',
          rndc_config_file => '/etc/bind/rndc.conf',
          rndc_key_file    => '/etc/bind/designate-rndc.key',
          configure_bind   => false,
          manage_pool      => false,
        }
      }
      if $is_first_master {
        package { 'designate-tlds':
          ensure => present,
        }
        if $aodh_notifier_use_proxy {
          designate_tlds_config {
            'DEFAULT/http_proxy':     value => "true";
            'DEFAULT/http_proxy_url': value => $aodh_notifier_proxy_url;
            'DEFAULT/dry_run':        value => "false";
          }
        } else {
          designate_tlds_config {
            'DEFAULT/http_proxy':     value => "false";
            'DEFAULT/dry_run':        value => "false";
          }
        }
        class { '::designate::keystone::authtoken_tlds':
          password             => $pass_designate_authtoken,
          user_domain_name     => 'Default',
          project_domain_name  => 'Default',
          auth_url             => $keystone_admin_uri,
          www_authenticate_uri => $keystone_auth_uri,
          memcached_servers    => $memcached_servers,
          cafile               => $api_endpoint_ca_file,
          region_name          => $region_name,
        }
      }else{
        package { 'designate-tlds':
          ensure => 'purged',
        }
      }
    }

    file {'/etc/designate/rndc.key':
      ensure => 'present',
      owner                   => 'root',
      group                   => 'root',
      mode                    => '0640',
      selinux_ignore_defaults => true,
      require                 => Package['designate-mdns'],
      content                 => "key \"designate-rndc-key\" {
	algorithm hmac-sha256;
	secret \"${pass_designate_rndckey}\";
};
",
    }

    $command = "openstack --os-auth-url ${base_url}/identity/v3 \
--os-region-name ${region_name} \
--os-identity-api-version 3 \
--os-project-name services --os-username designate --os-password ${pass_designate_authtoken} \
--os-project-domain-name Default --os-user-domain-name Default zone list"
    openstacklib::service_validation { 'designate-central':
      command     => $command,
      timeout     => '15',
      refreshonly => true,
      subscribe   => Anchor['designate::service::end'],
    }
    file { '/etc/designate/pools.yaml':
      ensure  => present,
      content => base64('decode', $designate_pools_yaml),
      require => Service['designate-central'],
    }
    exec { 'Update designate pools':
      command     => 'designate-manage pool update --file /etc/designate/pools.yaml',
      path        => '/usr/bin',
      refreshonly => true,
      logoutput   => 'on_failure',
      subscribe   => File['/etc/designate/pools.yaml'],
      require     => Openstacklib::Service_validation['designate-central'],
    }
    designate_config {
      'network_api:neutron/ca_certificates_file':           value => $api_endpoint_ca_file;
      'keystone/cafile':                                    value => $api_endpoint_ca_file;
      'handler:neutron_floatingip_ng/notification_topics':  value => 'notifications_designate'
    }
    package { 'python3-designate-dashboard':
      ensure => 'installed',
    }
    class { '::designate::sink':
      enabled_notification_handlers => [ 'neutron_floatingip_ng' ],
    }
  }else{
    package { 'python3-designate':
      ensure => 'purged',
    }
    package { 'designate-tlds':
      ensure => 'purged',
    }
    package { 'python3-designate-dashboard':
      ensure => 'purged',
    }
  }

  ####################
  ### Setup Manila ###
  ####################
  if $has_subrole_manila and $install_manila {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['manila::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'manila':
        notify_service_name => 'manila-api',
      }
      $manila_key_file = "/etc/manila/ssl/private/${::fqdn}.pem"
      $manila_crt_file = "/etc/manila/ssl/public/${::fqdn}.crt"
    } else {
      $manila_key_file = undef
      $manila_crt_file = undef
    }

    class { '::manila::db':
      database_connection   => "mysql+pymysql://manila:${pass_manila_db}@${sql_host}/maniladb?charset=utf8",
    }
    class { '::manila::logging':
      debug => true,
    }
    if $disable_notifications {
      $manila_notif_transport_url = ''
    }else{
      $manila_notif_transport_url = os_transport_url({
                                       'transport' => $messaging_notify_proto,
                                       'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                       'port'      => $messaging_notify_port,
                                       'username'  => 'manila',
                                       'password'  => $pass_manila_messaging,
                                     })
    }
    class { '::manila':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'manila',
        'password'  => $pass_manila_messaging,
      }),
      notification_transport_url => $manila_notif_transport_url,
      notification_driver        => $notification_driver,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
      host                       => $machine_hostname,
    }
    # This *must* be called before ::manila::api that contains an include
    class { '::manila::keystone::authtoken':
      password             => $pass_manila_authtoken,
      user_domain_name     => 'Default',
      project_domain_name  => 'Default',
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => $region_name,
    }
    class { '::manila::api':
      default_share_type           => 'default_share_type',
      sync_db                      => true,
      service_name                 => 'manila-api',
      enabled_share_protocols      => 'NFS',
      enable_proxy_headers_parsing => true,
    }
    if $is_first_master {
      class { '::manila::keystone::auth':
        public_url      => "${base_url}/share/v1/%(tenant_id)s",
        internal_url    => "${base_url}/share/v1/%(tenant_id)s",
        admin_url       => "${base_url}/share/v1/%(tenant_id)s",
        public_url_v2   => "${base_url}/share/v2/%(tenant_id)s",
        internal_url_v2 => "${base_url}/share/v2/%(tenant_id)s",
        admin_url_v2    => "${base_url}/share/v2/%(tenant_id)s",
        email           => $admin_email_address,
        password        => $pass_manila_authtoken,
        email_v2        => $admin_email_address,
        auth_name_v2    => 'manila',
        password_v2     => $pass_manila_authtoken,
        region          => $region_name,
      }
      class { '::manila::cron::db_purge': }
    }
    class { '::manila::compute::nova':
      cafile      => $api_endpoint_ca_file,
      username    => 'nova',
      password    => $pass_nova_authtoken,
      region_name => $region_name,
      auth_url    => $keystone_admin_uri,
      auth_type   => 'password',
    }
    class { '::manila::image::glance':
      cafile      => $api_endpoint_ca_file,
      region_name => $region_name,
      username    => 'glance',
      password    => $pass_glance_authtoken,
      auth_url    => $keystone_admin_uri,
      auth_type   => 'password',
    }
    class { '::manila::network::neutron':
      cafile      => $api_endpoint_ca_file,
      region_name => $region_name,
      username    => 'neutron',
      password    => $pass_neutron_authtoken,
      auth_url    => $keystone_admin_uri,
      auth_type   => 'password',
    }
    class { '::manila::volume::cinder':
      cafile      => $api_endpoint_ca_file,
      region_name => $region_name,
      username    => 'cinder',
      password    => $pass_cinder_authtoken,
      auth_url    => $keystone_admin_uri,
      auth_type   => 'password',
    }
    class { '::manila::scheduler': }
    # manila-share cannot be setup on the controllers,
    # because it needs access to OVS.
    #class { '::manila::share': }
    class { '::manila::backends':
      enabled_share_backends => ['generic'],
    }
    # Deploy the generic driver using driver_handles_share_servers=True (ie: in VM shares...)
    ::manila::backend::generic { 'generic':
      driver_handles_share_servers => true,
      share_backend_name           => 'GENERIC',
    }
    ::manila::service_instance { 'generic':
      service_instance_user           => 'debian',
      service_instance_password       => 'do-not-use-passwords',
      service_image_name              => 'manila-service-image',
      create_service_image            => false,
      manila_service_keypair_name     => 'manila-ssh-key',
      path_to_public_key              => '/etc/manila/.ssh/id_rsa.pub',
      path_to_private_key             => '/etc/manila/.ssh/id_rsa',
      service_instance_flavor_id      => '100',
      service_instance_security_group => 'manila-security-group',
      service_network_name            => 'manila-service-network',
      service_network_cidr            => $manila_network_cidr,
      service_network_division_mask   => $manila_network_division_mask,
    }

    manila_config {
      "DEFAULT/my_ip": value => $machine_ip;
    }
  }

  ##############################################
  ### Manage the cron jobs of the controller ###
  ##############################################
  # First, provision the ssh keys for Glance and Keystone
  if $has_subrole_glance {
    ::oci::oci_ssh_keypair { 'glance-keypair':
      path    => '/var/lib/glance',
      type    => 'ssh-rsa',
      user    => 'glance',
      group   => 'glance',
      pubkey  => $pass_glance_ssh_pub,
      privkey => base64('decode', $pass_glance_ssh_priv),
      require => Package['glance-api'],
    }
  }

  if $has_subrole_api_keystone {
    ::oci::oci_ssh_keypair { 'keystone-keypair':
      path    => '/var/lib/keystone',
      type    => 'ssh-rsa',
      user    => 'keystone',
      group   => 'keystone',
      pubkey  => $pass_keystone_ssh_pub,
      privkey => base64('decode', $pass_keystone_ssh_priv),
      require => Package['keystone'],
    }
  }

  if $is_first_master {
    if $has_subrole_glance {
      cron { 'oci-glance-images-rsync':
        command => '/usr/bin/oci-glance-image-rsync',
        user    => 'glance',
        minute  => 0,
        require => Anchor['glance::install::end'],
      }
    }
    if $has_subrole_api_keystone {
      cron { 'oci-fernet-keys-rotate':
        command => '/usr/bin/oci-fernet-keys-rotate',
        user    => 'keystone',
        minute  => 40,
        hour    => 6,
        weekday => 2,
        require => Anchor['keystone::install::end'],
      }
    }
  }else{
    file { "/var/spool/cron/crontabs/root":
      ensure                  => absent,
    }
  }
}
