/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright (c) 2022 NVIDIA Corporation & Affiliates
 */

#include <rte_flow.h>

#include <mlx5_malloc.h>
#include "mlx5_defs.h"
#include "mlx5_flow.h"
#include "mlx5_rx.h"

#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
#include "mlx5_hws_cnt.h"

/* The maximum actions support in the flow. */
#define MLX5_HW_MAX_ACTS 16

/*
 * The default ipool threshold value indicates which per_core_cache
 * value to set.
 */
#define MLX5_HW_IPOOL_SIZE_THRESHOLD (1 << 19)
/* The default min local cache size. */
#define MLX5_HW_IPOOL_CACHE_MIN (1 << 9)

/* Default push burst threshold. */
#define BURST_THR 32u

/* Default queue to flush the flows. */
#define MLX5_DEFAULT_FLUSH_QUEUE 0

/* Maximum number of rules in control flow tables. */
#define MLX5_HW_CTRL_FLOW_NB_RULES (4096)

/* Lowest flow group usable by an application if group translation is done. */
#define MLX5_HW_LOWEST_USABLE_GROUP (1)

/* Maximum group index usable by user applications for transfer flows. */
#define MLX5_HW_MAX_TRANSFER_GROUP (UINT32_MAX - 1)

/* Maximum group index usable by user applications for egress flows. */
#define MLX5_HW_MAX_EGRESS_GROUP (UINT32_MAX - 1)

/* Lowest priority for HW root table. */
#define MLX5_HW_LOWEST_PRIO_ROOT 15

/* Lowest priority for HW non-root table. */
#define MLX5_HW_LOWEST_PRIO_NON_ROOT (UINT32_MAX)

/* Priorities for Rx control flow rules. */
#define MLX5_HW_CTRL_RX_PRIO_L2 (MLX5_HW_LOWEST_PRIO_ROOT)
#define MLX5_HW_CTRL_RX_PRIO_L3 (MLX5_HW_LOWEST_PRIO_ROOT - 1)
#define MLX5_HW_CTRL_RX_PRIO_L4 (MLX5_HW_LOWEST_PRIO_ROOT - 2)

#define MLX5_HW_VLAN_PUSH_TYPE_IDX 0
#define MLX5_HW_VLAN_PUSH_VID_IDX 1
#define MLX5_HW_VLAN_PUSH_PCP_IDX 2

static int flow_hw_flush_all_ctrl_flows(struct rte_eth_dev *dev);
static int flow_hw_translate_group(struct rte_eth_dev *dev,
				   const struct mlx5_flow_template_table_cfg *cfg,
				   uint32_t group,
				   uint32_t *table_group,
				   struct rte_flow_error *error);
static __rte_always_inline int
flow_hw_set_vlan_vid_construct(struct rte_eth_dev *dev,
			       struct mlx5_hw_q_job *job,
			       struct mlx5_action_construct_data *act_data,
			       const struct mlx5_hw_actions *hw_acts,
			       const struct rte_flow_action *action);

static __rte_always_inline uint32_t flow_hw_tx_tag_regc_mask(struct rte_eth_dev *dev);
static __rte_always_inline uint32_t flow_hw_tx_tag_regc_value(struct rte_eth_dev *dev);

const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops;

/* DR action flags with different table. */
static uint32_t mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_MAX]
				[MLX5DR_TABLE_TYPE_MAX] = {
	{
		MLX5DR_ACTION_FLAG_ROOT_RX,
		MLX5DR_ACTION_FLAG_ROOT_TX,
		MLX5DR_ACTION_FLAG_ROOT_FDB,
	},
	{
		MLX5DR_ACTION_FLAG_HWS_RX,
		MLX5DR_ACTION_FLAG_HWS_TX,
		MLX5DR_ACTION_FLAG_HWS_FDB,
	},
};

/* Ethernet item spec for promiscuous mode. */
static const struct rte_flow_item_eth ctrl_rx_eth_promisc_spec = {
	.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.type = 0,
};
/* Ethernet item mask for promiscuous mode. */
static const struct rte_flow_item_eth ctrl_rx_eth_promisc_mask = {
	.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.type = 0,
};

/* Ethernet item spec for all multicast mode. */
static const struct rte_flow_item_eth ctrl_rx_eth_mcast_spec = {
	.dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
	.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.type = 0,
};
/* Ethernet item mask for all multicast mode. */
static const struct rte_flow_item_eth ctrl_rx_eth_mcast_mask = {
	.dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
	.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.type = 0,
};

/* Ethernet item spec for IPv4 multicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_ipv4_mcast_spec = {
	.dst.addr_bytes = "\x01\x00\x5e\x00\x00\x00",
	.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.type = 0,
};
/* Ethernet item mask for IPv4 multicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_ipv4_mcast_mask = {
	.dst.addr_bytes = "\xff\xff\xff\x00\x00\x00",
	.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.type = 0,
};

/* Ethernet item spec for IPv6 multicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_ipv6_mcast_spec = {
	.dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
	.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.type = 0,
};
/* Ethernet item mask for IPv6 multicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_ipv6_mcast_mask = {
	.dst.addr_bytes = "\xff\xff\x00\x00\x00\x00",
	.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.type = 0,
};

/* Ethernet item mask for unicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_dmac_mask = {
	.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
	.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.type = 0,
};

/* Ethernet item spec for broadcast. */
static const struct rte_flow_item_eth ctrl_rx_eth_bcast_spec = {
	.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
	.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	.type = 0,
};

/**
 * Set the hash fields according to the @p rss_desc information.
 *
 * @param[in] rss_desc
 *   Pointer to the mlx5_flow_rss_desc.
 * @param[out] hash_fields
 *   Pointer to the RSS hash fields.
 */
static void
flow_hw_hashfields_set(struct mlx5_flow_rss_desc *rss_desc,
		       uint64_t *hash_fields)
{
	uint64_t fields = 0;
	int rss_inner = 0;
	uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);

#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
	if (rss_desc->level >= 2)
		rss_inner = 1;
#endif
	if (rss_types & MLX5_IPV4_LAYER_TYPES) {
		if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
			fields |= IBV_RX_HASH_SRC_IPV4;
		else if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
			fields |= IBV_RX_HASH_DST_IPV4;
		else
			fields |= MLX5_IPV4_IBV_RX_HASH;
	} else if (rss_types & MLX5_IPV6_LAYER_TYPES) {
		if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
			fields |= IBV_RX_HASH_SRC_IPV6;
		else if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
			fields |= IBV_RX_HASH_DST_IPV6;
		else
			fields |= MLX5_IPV6_IBV_RX_HASH;
	}
	if (rss_types & RTE_ETH_RSS_UDP) {
		if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
			fields |= IBV_RX_HASH_SRC_PORT_UDP;
		else if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
			fields |= IBV_RX_HASH_DST_PORT_UDP;
		else
			fields |= MLX5_UDP_IBV_RX_HASH;
	} else if (rss_types & RTE_ETH_RSS_TCP) {
		if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
			fields |= IBV_RX_HASH_SRC_PORT_TCP;
		else if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
			fields |= IBV_RX_HASH_DST_PORT_TCP;
		else
			fields |= MLX5_TCP_IBV_RX_HASH;
	}
	if (rss_types & RTE_ETH_RSS_ESP)
		fields |= IBV_RX_HASH_IPSEC_SPI;
	if (rss_inner)
		fields |= IBV_RX_HASH_INNER;
	*hash_fields = fields;
}

/**
 * Generate the pattern item flags.
 * Will be used for shared RSS action.
 *
 * @param[in] items
 *   Pointer to the list of items.
 *
 * @return
 *   Item flags.
 */
static uint64_t
flow_hw_rss_item_flags_get(const struct rte_flow_item items[])
{
	uint64_t item_flags = 0;
	uint64_t last_item = 0;

	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
		int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
		int item_type = items->type;

		switch (item_type) {
		case RTE_FLOW_ITEM_TYPE_IPV4:
			last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
					     MLX5_FLOW_LAYER_OUTER_L3_IPV4;
			break;
		case RTE_FLOW_ITEM_TYPE_IPV6:
			last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
					     MLX5_FLOW_LAYER_OUTER_L3_IPV6;
			break;
		case RTE_FLOW_ITEM_TYPE_TCP:
			last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
					     MLX5_FLOW_LAYER_OUTER_L4_TCP;
			break;
		case RTE_FLOW_ITEM_TYPE_UDP:
			last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
					     MLX5_FLOW_LAYER_OUTER_L4_UDP;
			break;
		case RTE_FLOW_ITEM_TYPE_GRE:
			last_item = MLX5_FLOW_LAYER_GRE;
			break;
		case RTE_FLOW_ITEM_TYPE_NVGRE:
			last_item = MLX5_FLOW_LAYER_GRE;
			break;
		case RTE_FLOW_ITEM_TYPE_VXLAN:
			last_item = MLX5_FLOW_LAYER_VXLAN;
			break;
		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
			last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
			break;
		case RTE_FLOW_ITEM_TYPE_GENEVE:
			last_item = MLX5_FLOW_LAYER_GENEVE;
			break;
		case RTE_FLOW_ITEM_TYPE_MPLS:
			last_item = MLX5_FLOW_LAYER_MPLS;
			break;
		case RTE_FLOW_ITEM_TYPE_GTP:
			last_item = MLX5_FLOW_LAYER_GTP;
			break;
		default:
			break;
		}
		item_flags |= last_item;
	}
	return item_flags;
}

/**
 * Register destination table DR jump action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] table_attr
 *   Pointer to the flow attributes.
 * @param[in] dest_group
 *   The destination group ID.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    Table on success, NULL otherwise and rte_errno is set.
 */
static struct mlx5_hw_jump_action *
flow_hw_jump_action_register(struct rte_eth_dev *dev,
			     const struct mlx5_flow_template_table_cfg *cfg,
			     uint32_t dest_group,
			     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_attr jattr = cfg->attr.flow_attr;
	struct mlx5_flow_group *grp;
	struct mlx5_flow_cb_ctx ctx = {
		.dev = dev,
		.error = error,
		.data = &jattr,
	};
	struct mlx5_list_entry *ge;
	uint32_t target_group;

	target_group = dest_group;
	if (flow_hw_translate_group(dev, cfg, dest_group, &target_group, error))
		return NULL;
	jattr.group = target_group;
	ge = mlx5_hlist_register(priv->sh->flow_tbls, target_group, &ctx);
	if (!ge)
		return NULL;
	grp = container_of(ge, struct mlx5_flow_group, entry);
	return &grp->jump;
}

/**
 * Release jump action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] jump
 *   Pointer to the jump action.
 */

static void
flow_hw_jump_release(struct rte_eth_dev *dev, struct mlx5_hw_jump_action *jump)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_flow_group *grp;

	grp = container_of
		(jump, struct mlx5_flow_group, jump);
	mlx5_hlist_unregister(priv->sh->flow_tbls, &grp->entry);
}

/**
 * Register queue/RSS action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] hws_flags
 *   DR action flags.
 * @param[in] action
 *   rte flow action.
 *
 * @return
 *    Table on success, NULL otherwise and rte_errno is set.
 */
static inline struct mlx5_hrxq*
flow_hw_tir_action_register(struct rte_eth_dev *dev,
			    uint32_t hws_flags,
			    const struct rte_flow_action *action)
{
	struct mlx5_flow_rss_desc rss_desc = {
		.hws_flags = hws_flags,
	};
	struct mlx5_hrxq *hrxq;

	if (action->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
		const struct rte_flow_action_queue *queue = action->conf;

		rss_desc.const_q = &queue->index;
		rss_desc.queue_num = 1;
	} else {
		const struct rte_flow_action_rss *rss = action->conf;

		rss_desc.queue_num = rss->queue_num;
		rss_desc.const_q = rss->queue;
		memcpy(rss_desc.key,
		       !rss->key ? rss_hash_default_key : rss->key,
		       MLX5_RSS_HASH_KEY_LEN);
		rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
		rss_desc.types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
		flow_hw_hashfields_set(&rss_desc, &rss_desc.hash_fields);
		flow_dv_action_rss_l34_hash_adjust(rss->types,
						   &rss_desc.hash_fields);
		if (rss->level > 1) {
			rss_desc.hash_fields |= IBV_RX_HASH_INNER;
			rss_desc.tunnel = 1;
		}
	}
	hrxq = mlx5_hrxq_get(dev, &rss_desc);
	return hrxq;
}

static __rte_always_inline int
flow_hw_ct_compile(struct rte_eth_dev *dev,
		   uint32_t queue, uint32_t idx,
		   struct mlx5dr_rule_action *rule_act)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_ct_action *ct;

	ct = mlx5_ipool_get(priv->hws_ctpool->cts, MLX5_ACTION_CTX_CT_GET_IDX(idx));
	if (!ct || mlx5_aso_ct_available(priv->sh, queue, ct))
		return -1;
	rule_act->action = priv->hws_ctpool->dr_action;
	rule_act->aso_ct.offset = ct->offset;
	rule_act->aso_ct.direction = ct->is_original ?
		MLX5DR_ACTION_ASO_CT_DIRECTION_INITIATOR :
		MLX5DR_ACTION_ASO_CT_DIRECTION_RESPONDER;
	return 0;
}

/**
 * Destroy DR actions created by action template.
 *
 * For DR actions created during table creation's action translate.
 * Need to destroy the DR action when destroying the table.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 */
static void
__flow_hw_action_template_destroy(struct rte_eth_dev *dev,
				 struct mlx5_hw_actions *acts)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_action_construct_data *data;

	while (!LIST_EMPTY(&acts->act_list)) {
		data = LIST_FIRST(&acts->act_list);
		LIST_REMOVE(data, next);
		mlx5_ipool_free(priv->acts_ipool, data->idx);
	}

	if (acts->mark)
		if (!__atomic_sub_fetch(&priv->hws_mark_refcnt, 1, __ATOMIC_RELAXED))
			flow_hw_rxq_flag_set(dev, false);

	if (acts->jump) {
		struct mlx5_flow_group *grp;

		grp = container_of
			(acts->jump, struct mlx5_flow_group, jump);
		mlx5_hlist_unregister(priv->sh->flow_tbls, &grp->entry);
		acts->jump = NULL;
	}
	if (acts->tir) {
		mlx5_hrxq_release(dev, acts->tir->idx);
		acts->tir = NULL;
	}
	if (acts->encap_decap) {
		if (acts->encap_decap->action)
			mlx5dr_action_destroy(acts->encap_decap->action);
		mlx5_free(acts->encap_decap);
		acts->encap_decap = NULL;
	}
	if (acts->mhdr) {
		if (acts->mhdr->action)
			mlx5dr_action_destroy(acts->mhdr->action);
		mlx5_free(acts->mhdr);
		acts->mhdr = NULL;
	}
	if (mlx5_hws_cnt_id_valid(acts->cnt_id)) {
		mlx5_hws_cnt_shared_put(priv->hws_cpool, &acts->cnt_id);
		acts->cnt_id = 0;
	}
	if (acts->mtr_id) {
		mlx5_ipool_free(priv->hws_mpool->idx_pool, acts->mtr_id);
		acts->mtr_id = 0;
	}
}

/**
 * Append dynamic action to the dynamic action list.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline struct mlx5_action_construct_data *
__flow_hw_act_data_alloc(struct mlx5_priv *priv,
			 enum rte_flow_action_type type,
			 uint16_t action_src,
			 uint16_t action_dst)
{
	struct mlx5_action_construct_data *act_data;
	uint32_t idx = 0;

	act_data = mlx5_ipool_zmalloc(priv->acts_ipool, &idx);
	if (!act_data)
		return NULL;
	act_data->idx = idx;
	act_data->type = type;
	act_data->action_src = action_src;
	act_data->action_dst = action_dst;
	return act_data;
}

/**
 * Append dynamic action to the dynamic action list.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
__flow_hw_act_data_general_append(struct mlx5_priv *priv,
				  struct mlx5_hw_actions *acts,
				  enum rte_flow_action_type type,
				  uint16_t action_src,
				  uint16_t action_dst)
{
	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}

/**
 * Append dynamic encap action to the dynamic action list.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 * @param[in] len
 *   Length of the data to be updated.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
__flow_hw_act_data_encap_append(struct mlx5_priv *priv,
				struct mlx5_hw_actions *acts,
				enum rte_flow_action_type type,
				uint16_t action_src,
				uint16_t action_dst,
				uint16_t len)
{
	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	act_data->encap.len = len;
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}

static __rte_always_inline int
__flow_hw_act_data_hdr_modify_append(struct mlx5_priv *priv,
				     struct mlx5_hw_actions *acts,
				     enum rte_flow_action_type type,
				     uint16_t action_src,
				     uint16_t action_dst,
				     uint16_t mhdr_cmds_off,
				     uint16_t mhdr_cmds_end,
				     bool shared,
				     struct field_modify_info *field,
				     struct field_modify_info *dcopy,
				     uint32_t *mask)
{
	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	act_data->modify_header.mhdr_cmds_off = mhdr_cmds_off;
	act_data->modify_header.mhdr_cmds_end = mhdr_cmds_end;
	act_data->modify_header.shared = shared;
	rte_memcpy(act_data->modify_header.field, field,
		   sizeof(*field) * MLX5_ACT_MAX_MOD_FIELDS);
	rte_memcpy(act_data->modify_header.dcopy, dcopy,
		   sizeof(*dcopy) * MLX5_ACT_MAX_MOD_FIELDS);
	rte_memcpy(act_data->modify_header.mask, mask,
		   sizeof(*mask) * MLX5_ACT_MAX_MOD_FIELDS);
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}

/**
 * Append shared RSS action to the dynamic action list.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 * @param[in] idx
 *   Shared RSS index.
 * @param[in] rss
 *   Pointer to the shared RSS info.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
__flow_hw_act_data_shared_rss_append(struct mlx5_priv *priv,
				     struct mlx5_hw_actions *acts,
				     enum rte_flow_action_type type,
				     uint16_t action_src,
				     uint16_t action_dst,
				     uint32_t idx,
				     struct mlx5_shared_action_rss *rss)
{
	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	act_data->shared_rss.level = rss->origin.level;
	act_data->shared_rss.types = !rss->origin.types ? RTE_ETH_RSS_IP :
				     rss->origin.types;
	act_data->shared_rss.idx = idx;
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}

/**
 * Append shared counter action to the dynamic action list.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 * @param[in] cnt_id
 *   Shared counter id.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
__flow_hw_act_data_shared_cnt_append(struct mlx5_priv *priv,
				     struct mlx5_hw_actions *acts,
				     enum rte_flow_action_type type,
				     uint16_t action_src,
				     uint16_t action_dst,
				     cnt_id_t cnt_id)
{
	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	act_data->type = type;
	act_data->shared_counter.id = cnt_id;
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}

/**
 * Append shared meter_mark action to the dynamic action list.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] type
 *   Action type.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 * @param[in] mtr_id
 *   Shared meter id.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
__flow_hw_act_data_shared_mtr_append(struct mlx5_priv *priv,
				     struct mlx5_hw_actions *acts,
				     enum rte_flow_action_type type,
				     uint16_t action_src,
				     uint16_t action_dst,
				     cnt_id_t mtr_id)
{	struct mlx5_action_construct_data *act_data;

	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
	if (!act_data)
		return -1;
	act_data->type = type;
	act_data->shared_meter.id = mtr_id;
	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
	return 0;
}

/**
 * Translate shared indirect action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev data structure.
 * @param[in] action
 *   Pointer to the shared indirect rte_flow action.
 * @param[in] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] action_src
 *   Offset of source rte flow action.
 * @param[in] action_dst
 *   Offset of destination DR action.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
flow_hw_shared_action_translate(struct rte_eth_dev *dev,
				const struct rte_flow_action *action,
				struct mlx5_hw_actions *acts,
				uint16_t action_src,
				uint16_t action_dst)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_shared_action_rss *shared_rss;
	uint32_t act_idx = (uint32_t)(uintptr_t)action->conf;
	uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
	uint32_t idx = act_idx &
		       ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);

	switch (type) {
	case MLX5_INDIRECT_ACTION_TYPE_RSS:
		shared_rss = mlx5_ipool_get
		  (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
		if (!shared_rss || __flow_hw_act_data_shared_rss_append
		    (priv, acts,
		    (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_RSS,
		    action_src, action_dst, idx, shared_rss))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_COUNT:
		if (__flow_hw_act_data_shared_cnt_append(priv, acts,
			(enum rte_flow_action_type)
			MLX5_RTE_FLOW_ACTION_TYPE_COUNT,
			action_src, action_dst, act_idx))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_AGE:
		/* Not supported, prevent by validate function. */
		MLX5_ASSERT(0);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_CT:
		if (flow_hw_ct_compile(dev, MLX5_HW_INV_QUEUE,
				       idx, &acts->rule_acts[action_dst]))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
		if (__flow_hw_act_data_shared_mtr_append(priv, acts,
			(enum rte_flow_action_type)
			MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK,
			action_src, action_dst, idx))
			return -1;
		break;
	default:
		DRV_LOG(WARNING, "Unsupported shared action type:%d", type);
		break;
	}
	return 0;
}

static __rte_always_inline bool
flow_hw_action_modify_field_is_shared(const struct rte_flow_action *action,
				      const struct rte_flow_action *mask)
{
	const struct rte_flow_action_modify_field *v = action->conf;
	const struct rte_flow_action_modify_field *m = mask->conf;

	if (v->src.field == RTE_FLOW_FIELD_VALUE) {
		uint32_t j;

		if (m == NULL)
			return false;
		for (j = 0; j < RTE_DIM(m->src.value); ++j) {
			/*
			 * Immediate value is considered to be masked
			 * (and thus shared by all flow rules), if mask
			 * is non-zero. Partial mask over immediate value
			 * is not allowed.
			 */
			if (m->src.value[j])
				return true;
		}
		return false;
	}
	if (v->src.field == RTE_FLOW_FIELD_POINTER)
		return m->src.pvalue != NULL;
	/*
	 * Source field types other than VALUE and
	 * POINTER are always shared.
	 */
	return true;
}

static __rte_always_inline bool
flow_hw_should_insert_nop(const struct mlx5_hw_modify_header_action *mhdr,
			  const struct mlx5_modification_cmd *cmd)
{
	struct mlx5_modification_cmd last_cmd = { { 0 } };
	struct mlx5_modification_cmd new_cmd = { { 0 } };
	const uint32_t cmds_num = mhdr->mhdr_cmds_num;
	unsigned int last_type;
	bool should_insert = false;

	if (cmds_num == 0)
		return false;
	last_cmd = *(&mhdr->mhdr_cmds[cmds_num - 1]);
	last_cmd.data0 = rte_be_to_cpu_32(last_cmd.data0);
	last_cmd.data1 = rte_be_to_cpu_32(last_cmd.data1);
	last_type = last_cmd.action_type;
	new_cmd = *cmd;
	new_cmd.data0 = rte_be_to_cpu_32(new_cmd.data0);
	new_cmd.data1 = rte_be_to_cpu_32(new_cmd.data1);
	switch (new_cmd.action_type) {
	case MLX5_MODIFICATION_TYPE_SET:
	case MLX5_MODIFICATION_TYPE_ADD:
		if (last_type == MLX5_MODIFICATION_TYPE_SET ||
		    last_type == MLX5_MODIFICATION_TYPE_ADD)
			should_insert = new_cmd.field == last_cmd.field;
		else if (last_type == MLX5_MODIFICATION_TYPE_COPY)
			should_insert = new_cmd.field == last_cmd.dst_field;
		else if (last_type == MLX5_MODIFICATION_TYPE_NOP)
			should_insert = false;
		else
			MLX5_ASSERT(false); /* Other types are not supported. */
		break;
	case MLX5_MODIFICATION_TYPE_COPY:
		if (last_type == MLX5_MODIFICATION_TYPE_SET ||
		    last_type == MLX5_MODIFICATION_TYPE_ADD)
			should_insert = (new_cmd.field == last_cmd.field ||
					 new_cmd.dst_field == last_cmd.field);
		else if (last_type == MLX5_MODIFICATION_TYPE_COPY)
			should_insert = (new_cmd.field == last_cmd.dst_field ||
					 new_cmd.dst_field == last_cmd.dst_field);
		else if (last_type == MLX5_MODIFICATION_TYPE_NOP)
			should_insert = false;
		else
			MLX5_ASSERT(false); /* Other types are not supported. */
		break;
	default:
		/* Other action types should be rejected on AT validation. */
		MLX5_ASSERT(false);
		break;
	}
	return should_insert;
}

static __rte_always_inline int
flow_hw_mhdr_cmd_nop_append(struct mlx5_hw_modify_header_action *mhdr)
{
	struct mlx5_modification_cmd *nop;
	uint32_t num = mhdr->mhdr_cmds_num;

	if (num + 1 >= MLX5_MHDR_MAX_CMD)
		return -ENOMEM;
	nop = mhdr->mhdr_cmds + num;
	nop->data0 = 0;
	nop->action_type = MLX5_MODIFICATION_TYPE_NOP;
	nop->data0 = rte_cpu_to_be_32(nop->data0);
	nop->data1 = 0;
	mhdr->mhdr_cmds_num = num + 1;
	return 0;
}

static __rte_always_inline int
flow_hw_mhdr_cmd_append(struct mlx5_hw_modify_header_action *mhdr,
			struct mlx5_modification_cmd *cmd)
{
	uint32_t num = mhdr->mhdr_cmds_num;

	if (num + 1 >= MLX5_MHDR_MAX_CMD)
		return -ENOMEM;
	mhdr->mhdr_cmds[num] = *cmd;
	mhdr->mhdr_cmds_num = num + 1;
	return 0;
}

static __rte_always_inline int
flow_hw_converted_mhdr_cmds_append(struct mlx5_hw_modify_header_action *mhdr,
				   struct mlx5_flow_dv_modify_hdr_resource *resource)
{
	uint32_t idx;
	int ret;

	for (idx = 0; idx < resource->actions_num; ++idx) {
		struct mlx5_modification_cmd *src = &resource->actions[idx];

		if (flow_hw_should_insert_nop(mhdr, src)) {
			ret = flow_hw_mhdr_cmd_nop_append(mhdr);
			if (ret)
				return ret;
		}
		ret = flow_hw_mhdr_cmd_append(mhdr, src);
		if (ret)
			return ret;
	}
	return 0;
}

static __rte_always_inline void
flow_hw_modify_field_init(struct mlx5_hw_modify_header_action *mhdr,
			  struct rte_flow_actions_template *at)
{
	memset(mhdr, 0, sizeof(*mhdr));
	/* Modify header action without any commands is shared by default. */
	mhdr->shared = true;
	mhdr->pos = at->mhdr_off;
}

static __rte_always_inline int
flow_hw_modify_field_compile(struct rte_eth_dev *dev,
			     const struct rte_flow_attr *attr,
			     const struct rte_flow_action *action_start, /* Start of AT actions. */
			     const struct rte_flow_action *action, /* Current action from AT. */
			     const struct rte_flow_action *action_mask, /* Current mask from AT. */
			     struct mlx5_hw_actions *acts,
			     struct mlx5_hw_modify_header_action *mhdr,
			     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_modify_field *conf = action->conf;
	union {
		struct mlx5_flow_dv_modify_hdr_resource resource;
		uint8_t data[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
			     sizeof(struct mlx5_modification_cmd) * MLX5_MHDR_MAX_CMD];
	} dummy;
	struct mlx5_flow_dv_modify_hdr_resource *resource;
	struct rte_flow_item item = {
		.spec = NULL,
		.mask = NULL
	};
	struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
						{0, 0, MLX5_MODI_OUT_NONE} };
	struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
						{0, 0, MLX5_MODI_OUT_NONE} };
	uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = { 0 };
	uint32_t type, value = 0;
	uint16_t cmds_start, cmds_end;
	bool shared;
	int ret;

	/*
	 * Modify header action is shared if previous modify_field actions
	 * are shared and currently compiled action is shared.
	 */
	shared = flow_hw_action_modify_field_is_shared(action, action_mask);
	mhdr->shared &= shared;
	if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
	    conf->src.field == RTE_FLOW_FIELD_VALUE) {
		type = conf->operation == RTE_FLOW_MODIFY_SET ? MLX5_MODIFICATION_TYPE_SET :
								MLX5_MODIFICATION_TYPE_ADD;
		/* For SET/ADD fill the destination field (field) first. */
		mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
						  conf->width, dev,
						  attr, error);
		item.spec = conf->src.field == RTE_FLOW_FIELD_POINTER ?
				(void *)(uintptr_t)conf->src.pvalue :
				(void *)(uintptr_t)&conf->src.value;
		if (conf->dst.field == RTE_FLOW_FIELD_META ||
		    conf->dst.field == RTE_FLOW_FIELD_TAG ||
		    conf->dst.field == RTE_FLOW_FIELD_METER_COLOR ||
		    conf->dst.field == (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) {
			value = *(const unaligned_uint32_t *)item.spec;
			value = rte_cpu_to_be_32(value);
			item.spec = &value;
		} else if (conf->dst.field == RTE_FLOW_FIELD_GTP_PSC_QFI) {
			/*
			 * QFI is passed as an uint8_t integer, but it is accessed through
			 * a 2nd least significant byte of a 32-bit field in modify header command.
			 */
			value = *(const uint8_t *)item.spec;
			value = rte_cpu_to_be_32(value << 8);
			item.spec = &value;
		}
	} else {
		type = MLX5_MODIFICATION_TYPE_COPY;
		/* For COPY fill the destination field (dcopy) without mask. */
		mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
						  conf->width, dev,
						  attr, error);
		/* Then construct the source field (field) with mask. */
		mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
						  conf->width, dev,
						  attr, error);
	}
	item.mask = &mask;
	memset(&dummy, 0, sizeof(dummy));
	resource = &dummy.resource;
	ret = flow_dv_convert_modify_action(&item, field, dcopy, resource, type, error);
	if (ret)
		return ret;
	MLX5_ASSERT(resource->actions_num > 0);
	/*
	 * If previous modify field action collide with this one, then insert NOP command.
	 * This NOP command will not be a part of action's command range used to update commands
	 * on rule creation.
	 */
	if (flow_hw_should_insert_nop(mhdr, &resource->actions[0])) {
		ret = flow_hw_mhdr_cmd_nop_append(mhdr);
		if (ret)
			return rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
						  NULL, "too many modify field operations specified");
	}
	cmds_start = mhdr->mhdr_cmds_num;
	ret = flow_hw_converted_mhdr_cmds_append(mhdr, resource);
	if (ret)
		return rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "too many modify field operations specified");

	cmds_end = mhdr->mhdr_cmds_num;
	if (shared)
		return 0;
	ret = __flow_hw_act_data_hdr_modify_append(priv, acts, RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
						   action - action_start, mhdr->pos,
						   cmds_start, cmds_end, shared,
						   field, dcopy, mask);
	if (ret)
		return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "not enough memory to store modify field metadata");
	return 0;
}

static int
flow_hw_represented_port_compile(struct rte_eth_dev *dev,
				 const struct rte_flow_attr *attr,
				 const struct rte_flow_action *action_start,
				 const struct rte_flow_action *action,
				 const struct rte_flow_action *action_mask,
				 struct mlx5_hw_actions *acts,
				 uint16_t action_dst,
				 struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_ethdev *v = action->conf;
	const struct rte_flow_action_ethdev *m = action_mask->conf;
	int ret;

	if (!attr->group)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ATTR, NULL,
					  "represented_port action cannot"
					  " be used on group 0");
	if (!attr->transfer)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
					  NULL,
					  "represented_port action requires"
					  " transfer attribute");
	if (attr->ingress || attr->egress)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ATTR, NULL,
					  "represented_port action cannot"
					  " be used with direction attributes");
	if (!priv->master)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					  "represented_port acton must"
					  " be used on proxy port");
	if (m && !!m->port_id) {
		struct mlx5_priv *port_priv;

		if (!v)
			return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
						  action, "port index was not provided");
		port_priv = mlx5_port_to_eswitch_info(v->port_id, false);
		if (port_priv == NULL)
			return rte_flow_error_set
					(error, EINVAL,
					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					 "port does not exist or unable to"
					 " obtain E-Switch info for port");
		MLX5_ASSERT(priv->hw_vport != NULL);
		if (priv->hw_vport[v->port_id]) {
			acts->rule_acts[action_dst].action =
					priv->hw_vport[v->port_id];
		} else {
			return rte_flow_error_set
					(error, EINVAL,
					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					 "cannot use represented_port action"
					 " with this port");
		}
	} else {
		ret = __flow_hw_act_data_general_append
				(priv, acts, action->type,
				 action - action_start, action_dst);
		if (ret)
			return rte_flow_error_set
					(error, ENOMEM,
					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					 "not enough memory to store"
					 " vport action");
	}
	return 0;
}

static __rte_always_inline int
flow_hw_meter_compile(struct rte_eth_dev *dev,
		      const struct mlx5_flow_template_table_cfg *cfg,
		      uint16_t aso_mtr_pos,
		      uint16_t jump_pos,
		      const struct rte_flow_action *action,
		      struct mlx5_hw_actions *acts,
		      struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr *aso_mtr;
	const struct rte_flow_action_meter *meter = action->conf;
	uint32_t group = cfg->attr.flow_attr.group;

	aso_mtr = mlx5_aso_meter_by_idx(priv, meter->mtr_id);
	acts->rule_acts[aso_mtr_pos].action = priv->mtr_bulk.action;
	acts->rule_acts[aso_mtr_pos].aso_meter.offset = aso_mtr->offset;
	acts->jump = flow_hw_jump_action_register
		(dev, cfg, aso_mtr->fm.group, error);
	if (!acts->jump)
		return -ENOMEM;
	acts->rule_acts[jump_pos].action = (!!group) ?
				    acts->jump->hws_action :
				    acts->jump->root_action;
	if (mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr))
		return -ENOMEM;
	return 0;
}

static __rte_always_inline int
flow_hw_cnt_compile(struct rte_eth_dev *dev, uint32_t  start_pos,
		      struct mlx5_hw_actions *acts)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint32_t pos = start_pos;
	cnt_id_t cnt_id;
	int ret;

	ret = mlx5_hws_cnt_shared_get(priv->hws_cpool, &cnt_id, 0);
	if (ret != 0)
		return ret;
	ret = mlx5_hws_cnt_pool_get_action_offset
				(priv->hws_cpool,
				 cnt_id,
				 &acts->rule_acts[pos].action,
				 &acts->rule_acts[pos].counter.offset);
	if (ret != 0)
		return ret;
	acts->cnt_id = cnt_id;
	return 0;
}

static __rte_always_inline bool
is_of_vlan_pcp_present(const struct rte_flow_action *actions)
{
	/*
	 * Order of RTE VLAN push actions is
	 * OF_PUSH_VLAN / OF_SET_VLAN_VID [ / OF_SET_VLAN_PCP ]
	 */
	return actions[MLX5_HW_VLAN_PUSH_PCP_IDX].type ==
		RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP;
}

static __rte_always_inline bool
is_template_masked_push_vlan(const struct rte_flow_action_of_push_vlan *mask)
{
	/*
	 * In masked push VLAN template all RTE push actions are masked.
	 */
	return mask && mask->ethertype != 0;
}

static rte_be32_t vlan_hdr_to_be32(const struct rte_flow_action *actions)
{
/*
 * OpenFlow Switch Specification defines 801.1q VID as 12+1 bits.
 */
	rte_be32_t type, vid, pcp;
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
	rte_be32_t vid_lo, vid_hi;
#endif

	type = ((const struct rte_flow_action_of_push_vlan *)
		actions[MLX5_HW_VLAN_PUSH_TYPE_IDX].conf)->ethertype;
	vid = ((const struct rte_flow_action_of_set_vlan_vid *)
		actions[MLX5_HW_VLAN_PUSH_VID_IDX].conf)->vlan_vid;
	pcp = is_of_vlan_pcp_present(actions) ?
	      ((const struct rte_flow_action_of_set_vlan_pcp *)
		      actions[MLX5_HW_VLAN_PUSH_PCP_IDX].conf)->vlan_pcp : 0;
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
	vid_hi = vid & 0xff;
	vid_lo = vid >> 8;
	return (((vid_lo << 8) | (pcp << 5) | vid_hi) << 16) | type;
#else
	return (type << 16) | (pcp << 13) | vid;
#endif
}

static __rte_always_inline struct mlx5_aso_mtr *
flow_hw_meter_mark_alloc(struct rte_eth_dev *dev, uint32_t queue,
			 const struct rte_flow_action *action,
			 void *user_data, bool push)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	const struct rte_flow_action_meter_mark *meter_mark = action->conf;
	struct mlx5_aso_mtr *aso_mtr;
	struct mlx5_flow_meter_info *fm;
	uint32_t mtr_id;

	aso_mtr = mlx5_ipool_malloc(priv->hws_mpool->idx_pool, &mtr_id);
	if (!aso_mtr)
		return NULL;
	/* Fill the flow meter parameters. */
	aso_mtr->type = ASO_METER_INDIRECT;
	fm = &aso_mtr->fm;
	fm->meter_id = mtr_id;
	fm->profile = (struct mlx5_flow_meter_profile *)(meter_mark->profile);
	fm->is_enable = meter_mark->state;
	fm->color_aware = meter_mark->color_mode;
	aso_mtr->pool = pool;
	aso_mtr->state = (queue == MLX5_HW_INV_QUEUE) ?
			  ASO_METER_WAIT : ASO_METER_WAIT_ASYNC;
	aso_mtr->offset = mtr_id - 1;
	aso_mtr->init_color = (meter_mark->color_mode) ?
		meter_mark->init_color : RTE_COLOR_GREEN;
	/* Update ASO flow meter by wqe. */
	if (mlx5_aso_meter_update_by_wqe(priv->sh, queue, aso_mtr,
					 &priv->mtr_bulk, user_data, push)) {
		mlx5_ipool_free(pool->idx_pool, mtr_id);
		return NULL;
	}
	/* Wait for ASO object completion. */
	if (queue == MLX5_HW_INV_QUEUE &&
	    mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) {
		mlx5_ipool_free(pool->idx_pool, mtr_id);
		return NULL;
	}
	return aso_mtr;
}

static __rte_always_inline int
flow_hw_meter_mark_compile(struct rte_eth_dev *dev,
			   uint16_t aso_mtr_pos,
			   const struct rte_flow_action *action,
			   struct mlx5dr_rule_action *acts,
			   uint32_t *index,
			   uint32_t queue)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	struct mlx5_aso_mtr *aso_mtr;

	aso_mtr = flow_hw_meter_mark_alloc(dev, queue, action, NULL, true);
	if (!aso_mtr)
		return -1;

	/* Compile METER_MARK action */
	acts[aso_mtr_pos].action = pool->action;
	acts[aso_mtr_pos].aso_meter.offset = aso_mtr->offset;
	acts[aso_mtr_pos].aso_meter.init_color =
		(enum mlx5dr_action_aso_meter_color)
		rte_col_2_mlx5_col(aso_mtr->init_color);
	*index = aso_mtr->fm.meter_id;
	return 0;
}

/**
 * Translate rte_flow actions to DR action.
 *
 * As the action template has already indicated the actions. Translate
 * the rte_flow actions to DR action if possbile. So in flow create
 * stage we will save cycles from handing the actions' organizing.
 * For the actions with limited information, need to add these to a
 * list.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] cfg
 *   Pointer to the table configuration.
 * @param[in/out] acts
 *   Pointer to the template HW steering DR actions.
 * @param[in] at
 *   Action template.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno otherwise and rte_errno is set.
 */
static int
__flow_hw_actions_translate(struct rte_eth_dev *dev,
			    const struct mlx5_flow_template_table_cfg *cfg,
			    struct mlx5_hw_actions *acts,
			    struct rte_flow_actions_template *at,
			    struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_template_table_attr *table_attr = &cfg->attr;
	const struct rte_flow_attr *attr = &table_attr->flow_attr;
	struct rte_flow_action *actions = at->actions;
	struct rte_flow_action *action_start = actions;
	struct rte_flow_action *masks = at->masks;
	enum mlx5dr_action_reformat_type refmt_type = 0;
	const struct rte_flow_action_raw_encap *raw_encap_data;
	const struct rte_flow_item *enc_item = NULL, *enc_item_m = NULL;
	uint16_t reformat_src = 0;
	uint8_t *encap_data = NULL, *encap_data_m = NULL;
	size_t data_size = 0;
	struct mlx5_hw_modify_header_action mhdr = { 0 };
	bool actions_end = false;
	uint32_t type;
	bool reformat_used = false;
	unsigned int of_vlan_offset;
	uint16_t action_pos;
	uint16_t jump_pos;
	uint32_t ct_idx;
	int err;
	uint32_t target_grp = 0;

	flow_hw_modify_field_init(&mhdr, at);
	if (attr->transfer)
		type = MLX5DR_TABLE_TYPE_FDB;
	else if (attr->egress)
		type = MLX5DR_TABLE_TYPE_NIC_TX;
	else
		type = MLX5DR_TABLE_TYPE_NIC_RX;
	for (; !actions_end; actions++, masks++) {
		switch (actions->type) {
		case RTE_FLOW_ACTION_TYPE_INDIRECT:
			action_pos = at->actions_off[actions - at->actions];
			if (!attr->group) {
				DRV_LOG(ERR, "Indirect action is not supported in root table.");
				goto err;
			}
			if (actions->conf && masks->conf) {
				if (flow_hw_shared_action_translate
				(dev, actions, acts, actions - action_start, action_pos))
					goto err;
			} else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 actions - action_start, action_pos)){
				goto err;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_VOID:
			break;
		case RTE_FLOW_ACTION_TYPE_DROP:
			action_pos = at->actions_off[actions - at->actions];
			acts->rule_acts[action_pos].action =
				priv->hw_drop[!!attr->group];
			break;
		case RTE_FLOW_ACTION_TYPE_MARK:
			action_pos = at->actions_off[actions - at->actions];
			acts->mark = true;
			if (masks->conf &&
			    ((const struct rte_flow_action_mark *)
			     masks->conf)->id)
				acts->rule_acts[action_pos].tag.value =
					mlx5_flow_mark_set
					(((const struct rte_flow_action_mark *)
					(actions->conf))->id);
			else if (__flow_hw_act_data_general_append(priv, acts,
				actions->type, actions - action_start, action_pos))
				goto err;
			acts->rule_acts[action_pos].action =
				priv->hw_tag[!!attr->group];
			__atomic_add_fetch(&priv->hws_mark_refcnt, 1, __ATOMIC_RELAXED);
			flow_hw_rxq_flag_set(dev, true);
			break;
		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
			action_pos = at->actions_off[actions - at->actions];
			acts->rule_acts[action_pos].action =
				priv->hw_push_vlan[type];
			if (is_template_masked_push_vlan(masks->conf))
				acts->rule_acts[action_pos].push_vlan.vlan_hdr =
					vlan_hdr_to_be32(actions);
			else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 actions - action_start, action_pos))
				goto err;
			of_vlan_offset = is_of_vlan_pcp_present(actions) ?
					MLX5_HW_VLAN_PUSH_PCP_IDX :
					MLX5_HW_VLAN_PUSH_VID_IDX;
			actions += of_vlan_offset;
			masks += of_vlan_offset;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
			action_pos = at->actions_off[actions - at->actions];
			acts->rule_acts[action_pos].action =
				priv->hw_pop_vlan[type];
			break;
		case RTE_FLOW_ACTION_TYPE_JUMP:
			action_pos = at->actions_off[actions - at->actions];
			if (masks->conf &&
			    ((const struct rte_flow_action_jump *)
			     masks->conf)->group) {
				uint32_t jump_group =
					((const struct rte_flow_action_jump *)
					actions->conf)->group;
				acts->jump = flow_hw_jump_action_register
						(dev, cfg, jump_group, error);
				if (!acts->jump)
					goto err;
				acts->rule_acts[action_pos].action = (!!attr->group) ?
						acts->jump->hws_action :
						acts->jump->root_action;
			} else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 actions - action_start, action_pos)){
				goto err;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_QUEUE:
			action_pos = at->actions_off[actions - at->actions];
			if (masks->conf &&
			    ((const struct rte_flow_action_queue *)
			     masks->conf)->index) {
				acts->tir = flow_hw_tir_action_register
				(dev,
				 mlx5_hw_act_flag[!!attr->group][type],
				 actions);
				if (!acts->tir)
					goto err;
				acts->rule_acts[action_pos].action =
					acts->tir->action;
			} else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 actions - action_start, action_pos)) {
				goto err;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_RSS:
			action_pos = at->actions_off[actions - at->actions];
			if (actions->conf && masks->conf) {
				acts->tir = flow_hw_tir_action_register
				(dev,
				 mlx5_hw_act_flag[!!attr->group][type],
				 actions);
				if (!acts->tir)
					goto err;
				acts->rule_acts[action_pos].action =
					acts->tir->action;
			} else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 actions - action_start, action_pos)) {
				goto err;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
			MLX5_ASSERT(!reformat_used);
			enc_item = ((const struct rte_flow_action_vxlan_encap *)
				   actions->conf)->definition;
			if (masks->conf)
				enc_item_m = ((const struct rte_flow_action_vxlan_encap *)
					     masks->conf)->definition;
			reformat_used = true;
			reformat_src = actions - action_start;
			refmt_type = MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L2;
			break;
		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
			MLX5_ASSERT(!reformat_used);
			enc_item = ((const struct rte_flow_action_nvgre_encap *)
				   actions->conf)->definition;
			if (masks->conf)
				enc_item_m = ((const struct rte_flow_action_nvgre_encap *)
					     masks->conf)->definition;
			reformat_used = true;
			reformat_src = actions - action_start;
			refmt_type = MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L2;
			break;
		case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
		case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
			MLX5_ASSERT(!reformat_used);
			reformat_used = true;
			refmt_type = MLX5DR_ACTION_REFORMAT_TYPE_TNL_L2_TO_L2;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
			raw_encap_data =
				(const struct rte_flow_action_raw_encap *)
				 masks->conf;
			if (raw_encap_data)
				encap_data_m = raw_encap_data->data;
			raw_encap_data =
				(const struct rte_flow_action_raw_encap *)
				 actions->conf;
			encap_data = raw_encap_data->data;
			data_size = raw_encap_data->size;
			if (reformat_used) {
				refmt_type = data_size <
				MLX5_ENCAPSULATION_DECISION_SIZE ?
				MLX5DR_ACTION_REFORMAT_TYPE_TNL_L3_TO_L2 :
				MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L3;
			} else {
				reformat_used = true;
				refmt_type =
				MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L2;
			}
			reformat_src = actions - action_start;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
			reformat_used = true;
			refmt_type = MLX5DR_ACTION_REFORMAT_TYPE_TNL_L2_TO_L2;
			break;
		case RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL:
			DRV_LOG(ERR, "send to kernel action is not supported in HW steering.");
			goto err;
		case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
			err = flow_hw_modify_field_compile(dev, attr, action_start,
							   actions, masks, acts, &mhdr,
							   error);
			if (err)
				goto err;
			/*
			 * Adjust the action source position for the following.
			 * ... / MODIFY_FIELD: rx_cpy_pos / (QUEUE|RSS) / ...
			 * The next action will be Q/RSS, there will not be
			 * another adjustment and the real source position of
			 * the following actions will be decreased by 1.
			 * No change of the total actions in the new template.
			 */
			if ((actions - action_start) == at->rx_cpy_pos)
				action_start += 1;
			break;
		case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
			action_pos = at->actions_off[actions - at->actions];
			if (flow_hw_represented_port_compile
					(dev, attr, action_start, actions,
					 masks, acts, action_pos, error))
				goto err;
			break;
		case RTE_FLOW_ACTION_TYPE_METER:
			/*
			 * METER action is compiled to 2 DR actions - ASO_METER and FT.
			 * Calculated DR offset is stored only for ASO_METER and FT
			 * is assumed to be the next action.
			 */
			action_pos = at->actions_off[actions - at->actions];
			jump_pos = action_pos + 1;
			if (actions->conf && masks->conf &&
			    ((const struct rte_flow_action_meter *)
			     masks->conf)->mtr_id) {
				err = flow_hw_meter_compile(dev, cfg,
						action_pos, jump_pos, actions, acts, error);
				if (err)
					goto err;
			} else if (__flow_hw_act_data_general_append(priv, acts,
							actions->type,
							actions - action_start,
							action_pos))
				goto err;
			break;
		case RTE_FLOW_ACTION_TYPE_AGE:
			flow_hw_translate_group(dev, cfg, attr->group,
						&target_grp, error);
			if (target_grp == 0) {
				__flow_hw_action_template_destroy(dev, acts);
				return rte_flow_error_set(error, ENOTSUP,
						RTE_FLOW_ERROR_TYPE_ACTION,
						NULL,
						"Age action on root table is not supported in HW steering mode");
			}
			action_pos = at->actions_off[actions - at->actions];
			if (__flow_hw_act_data_general_append(priv, acts,
							 actions->type,
							 actions - action_start,
							 action_pos))
				goto err;
			break;
		case RTE_FLOW_ACTION_TYPE_COUNT:
			flow_hw_translate_group(dev, cfg, attr->group,
						&target_grp, error);
			if (target_grp == 0) {
				__flow_hw_action_template_destroy(dev, acts);
				return rte_flow_error_set(error, ENOTSUP,
						RTE_FLOW_ERROR_TYPE_ACTION,
						NULL,
						"Counter action on root table is not supported in HW steering mode");
			}
			if ((at->action_flags & MLX5_FLOW_ACTION_AGE) ||
			    (at->action_flags & MLX5_FLOW_ACTION_INDIRECT_AGE))
				/*
				 * When both COUNT and AGE are requested, it is
				 * saved as AGE action which creates also the
				 * counter.
				 */
				break;
			action_pos = at->actions_off[actions - at->actions];
			if (masks->conf &&
			    ((const struct rte_flow_action_count *)
			     masks->conf)->id) {
				err = flow_hw_cnt_compile(dev, action_pos, acts);
				if (err)
					goto err;
			} else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 actions - action_start, action_pos)) {
				goto err;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_CONNTRACK:
			action_pos = at->actions_off[actions - at->actions];
			if (masks->conf) {
				ct_idx = MLX5_ACTION_CTX_CT_GET_IDX
					 ((uint32_t)(uintptr_t)actions->conf);
				if (flow_hw_ct_compile(dev, MLX5_HW_INV_QUEUE, ct_idx,
						       &acts->rule_acts[action_pos]))
					goto err;
			} else if (__flow_hw_act_data_general_append
					(priv, acts, actions->type,
					 actions - action_start, action_pos)) {
				goto err;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_METER_MARK:
			action_pos = at->actions_off[actions - at->actions];
			if (actions->conf && masks->conf &&
			    ((const struct rte_flow_action_meter_mark *)
			     masks->conf)->profile) {
				err = flow_hw_meter_mark_compile(dev,
							action_pos, actions,
							acts->rule_acts,
							&acts->mtr_id,
							MLX5_HW_INV_QUEUE);
				if (err)
					goto err;
			} else if (__flow_hw_act_data_general_append(priv, acts,
							actions->type,
							actions - action_start,
							action_pos))
				goto err;
			break;
		case RTE_FLOW_ACTION_TYPE_END:
			actions_end = true;
			break;
		default:
			break;
		}
	}
	if (mhdr.pos != UINT16_MAX) {
		uint32_t flags;
		uint32_t bulk_size;
		size_t mhdr_len;

		acts->mhdr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*acts->mhdr),
					 0, SOCKET_ID_ANY);
		if (!acts->mhdr)
			goto err;
		rte_memcpy(acts->mhdr, &mhdr, sizeof(*acts->mhdr));
		mhdr_len = sizeof(struct mlx5_modification_cmd) * acts->mhdr->mhdr_cmds_num;
		flags = mlx5_hw_act_flag[!!attr->group][type];
		if (acts->mhdr->shared) {
			flags |= MLX5DR_ACTION_FLAG_SHARED;
			bulk_size = 0;
		} else {
			bulk_size = rte_log2_u32(table_attr->nb_flows);
		}
		acts->mhdr->action = mlx5dr_action_create_modify_header
				(priv->dr_ctx, mhdr_len, (__be64 *)acts->mhdr->mhdr_cmds,
				 bulk_size, flags);
		if (!acts->mhdr->action)
			goto err;
		acts->rule_acts[acts->mhdr->pos].action = acts->mhdr->action;
	}
	if (reformat_used) {
		uint8_t buf[MLX5_ENCAP_MAX_LEN];
		bool shared_rfmt = true;

		MLX5_ASSERT(at->reformat_off != UINT16_MAX);
		if (enc_item) {
			MLX5_ASSERT(!encap_data);
			if (flow_dv_convert_encap_data(enc_item, buf, &data_size, error))
				goto err;
			encap_data = buf;
			if (!enc_item_m)
				shared_rfmt = false;
		} else if (encap_data && !encap_data_m) {
			shared_rfmt = false;
		}
		acts->encap_decap = mlx5_malloc(MLX5_MEM_ZERO,
				    sizeof(*acts->encap_decap) + data_size,
				    0, SOCKET_ID_ANY);
		if (!acts->encap_decap)
			goto err;
		if (data_size) {
			acts->encap_decap->data_size = data_size;
			memcpy(acts->encap_decap->data, encap_data, data_size);
		}
		acts->encap_decap->action = mlx5dr_action_create_reformat
				(priv->dr_ctx, refmt_type,
				 data_size, encap_data,
				 shared_rfmt ? 0 : rte_log2_u32(table_attr->nb_flows),
				 mlx5_hw_act_flag[!!attr->group][type] |
				 (shared_rfmt ? MLX5DR_ACTION_FLAG_SHARED : 0));
		if (!acts->encap_decap->action)
			goto err;
		acts->rule_acts[at->reformat_off].action = acts->encap_decap->action;
		acts->rule_acts[at->reformat_off].reformat.data = acts->encap_decap->data;
		if (shared_rfmt)
			acts->rule_acts[at->reformat_off].reformat.offset = 0;
		else if (__flow_hw_act_data_encap_append(priv, acts,
				 (action_start + reformat_src)->type,
				 reformat_src, at->reformat_off, data_size))
			goto err;
		acts->encap_decap->shared = shared_rfmt;
		acts->encap_decap_pos = at->reformat_off;
	}
	return 0;
err:
	err = rte_errno;
	__flow_hw_action_template_destroy(dev, acts);
	return rte_flow_error_set(error, err,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				  "fail to create rte table");
}

/**
 * Translate rte_flow actions to DR action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] tbl
 *   Pointer to the flow template table.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_actions_translate(struct rte_eth_dev *dev,
			  struct rte_flow_template_table *tbl,
			  struct rte_flow_error *error)
{
	uint32_t i;

	for (i = 0; i < tbl->nb_action_templates; i++) {
		if (__flow_hw_actions_translate(dev, &tbl->cfg,
						&tbl->ats[i].acts,
						tbl->ats[i].action_template,
						error))
			goto err;
	}
	return 0;
err:
	while (i--)
		__flow_hw_action_template_destroy(dev, &tbl->ats[i].acts);
	return -1;
}

/**
 * Get shared indirect action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev data structure.
 * @param[in] act_data
 *   Pointer to the recorded action construct data.
 * @param[in] item_flags
 *   The matcher itme_flags used for RSS lookup.
 * @param[in] rule_act
 *   Pointer to the shared action's destination rule DR action.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
flow_hw_shared_action_get(struct rte_eth_dev *dev,
			  struct mlx5_action_construct_data *act_data,
			  const uint64_t item_flags,
			  struct mlx5dr_rule_action *rule_act)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_flow_rss_desc rss_desc = { 0 };
	uint64_t hash_fields = 0;
	uint32_t hrxq_idx = 0;
	struct mlx5_hrxq *hrxq = NULL;
	int act_type = act_data->type;

	switch (act_type) {
	case MLX5_RTE_FLOW_ACTION_TYPE_RSS:
		rss_desc.level = act_data->shared_rss.level;
		rss_desc.types = act_data->shared_rss.types;
		flow_dv_hashfields_set(item_flags, &rss_desc, &hash_fields);
		hrxq_idx = flow_dv_action_rss_hrxq_lookup
			(dev, act_data->shared_rss.idx, hash_fields);
		if (hrxq_idx)
			hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
					      hrxq_idx);
		if (hrxq) {
			rule_act->action = hrxq->action;
			return 0;
		}
		break;
	default:
		DRV_LOG(WARNING, "Unsupported shared action type:%d",
			act_data->type);
		break;
	}
	return -1;
}

/**
 * Construct shared indirect action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev data structure.
 * @param[in] queue
 *   The flow creation queue index.
 * @param[in] action
 *   Pointer to the shared indirect rte_flow action.
 * @param[in] table
 *   Pointer to the flow table.
 * @param[in] it_idx
 *   Item template index the action template refer to.
 * @param[in] action_flags
 *   Actions bit-map detected in this template.
 * @param[in, out] flow
 *   Pointer to the flow containing the counter.
 * @param[in] rule_act
 *   Pointer to the shared action's destination rule DR action.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
flow_hw_shared_action_construct(struct rte_eth_dev *dev, uint32_t queue,
				const struct rte_flow_action *action,
				struct rte_flow_template_table *table,
				const uint8_t it_idx, uint64_t action_flags,
				struct rte_flow_hw *flow,
				struct mlx5dr_rule_action *rule_act)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	struct mlx5_action_construct_data act_data;
	struct mlx5_shared_action_rss *shared_rss;
	struct mlx5_aso_mtr *aso_mtr;
	struct mlx5_age_info *age_info;
	struct mlx5_hws_age_param *param;
	uint32_t act_idx = (uint32_t)(uintptr_t)action->conf;
	uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
	uint32_t idx = act_idx &
		       ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
	uint64_t item_flags;
	cnt_id_t age_cnt;

	memset(&act_data, 0, sizeof(act_data));
	switch (type) {
	case MLX5_INDIRECT_ACTION_TYPE_RSS:
		act_data.type = MLX5_RTE_FLOW_ACTION_TYPE_RSS;
		shared_rss = mlx5_ipool_get
			(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
		if (!shared_rss)
			return -1;
		act_data.shared_rss.idx = idx;
		act_data.shared_rss.level = shared_rss->origin.level;
		act_data.shared_rss.types = !shared_rss->origin.types ?
					    RTE_ETH_RSS_IP :
					    shared_rss->origin.types;
		item_flags = table->its[it_idx]->item_flags;
		if (flow_hw_shared_action_get
				(dev, &act_data, item_flags, rule_act))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_COUNT:
		if (mlx5_hws_cnt_pool_get_action_offset(priv->hws_cpool,
				act_idx,
				&rule_act->action,
				&rule_act->counter.offset))
			return -1;
		flow->cnt_id = act_idx;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_AGE:
		/*
		 * Save the index with the indirect type, to recognize
		 * it in flow destroy.
		 */
		flow->age_idx = act_idx;
		if (action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT)
			/*
			 * The mutual update for idirect AGE & COUNT will be
			 * performed later after we have ID for both of them.
			 */
			break;
		age_info = GET_PORT_AGE_INFO(priv);
		param = mlx5_ipool_get(age_info->ages_ipool, idx);
		if (param == NULL)
			return -1;
		if (action_flags & MLX5_FLOW_ACTION_COUNT) {
			if (mlx5_hws_cnt_pool_get(priv->hws_cpool,
						  &param->queue_id, &age_cnt,
						  idx) < 0)
				return -1;
			flow->cnt_id = age_cnt;
			param->nb_cnts++;
		} else {
			/*
			 * Get the counter of this indirect AGE or create one
			 * if doesn't exist.
			 */
			age_cnt = mlx5_hws_age_cnt_get(priv, param, idx);
			if (age_cnt == 0)
				return -1;
		}
		if (mlx5_hws_cnt_pool_get_action_offset(priv->hws_cpool,
						     age_cnt, &rule_act->action,
						     &rule_act->counter.offset))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_CT:
		if (flow_hw_ct_compile(dev, queue, idx, rule_act))
			return -1;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
		/* Find ASO object. */
		aso_mtr = mlx5_ipool_get(pool->idx_pool, idx);
		if (!aso_mtr)
			return -1;
		rule_act->action = pool->action;
		rule_act->aso_meter.offset = aso_mtr->offset;
		rule_act->aso_meter.init_color =
			(enum mlx5dr_action_aso_meter_color)
			rte_col_2_mlx5_col(aso_mtr->init_color);
		break;
	default:
		DRV_LOG(WARNING, "Unsupported shared action type:%d", type);
		break;
	}
	return 0;
}

static __rte_always_inline int
flow_hw_mhdr_cmd_is_nop(const struct mlx5_modification_cmd *cmd)
{
	struct mlx5_modification_cmd cmd_he = {
		.data0 = rte_be_to_cpu_32(cmd->data0),
		.data1 = 0,
	};

	return cmd_he.action_type == MLX5_MODIFICATION_TYPE_NOP;
}

/**
 * Construct flow action array.
 *
 * For action template contains dynamic actions, these actions need to
 * be updated according to the rte_flow action during flow creation.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] job
 *   Pointer to job descriptor.
 * @param[in] hw_acts
 *   Pointer to translated actions from template.
 * @param[in] it_idx
 *   Item template index the action template refer to.
 * @param[in] actions
 *   Array of rte_flow action need to be checked.
 * @param[in] rule_acts
 *   Array of DR rule actions to be used during flow creation..
 * @param[in] acts_num
 *   Pointer to the real acts_num flow has.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
flow_hw_modify_field_construct(struct mlx5_hw_q_job *job,
			       struct mlx5_action_construct_data *act_data,
			       const struct mlx5_hw_actions *hw_acts,
			       const struct rte_flow_action *action)
{
	const struct rte_flow_action_modify_field *mhdr_action = action->conf;
	uint8_t values[16] = { 0 };
	unaligned_uint32_t *value_p;
	uint32_t i;
	struct field_modify_info *field;

	if (!hw_acts->mhdr)
		return -1;
	if (hw_acts->mhdr->shared || act_data->modify_header.shared)
		return 0;
	MLX5_ASSERT(mhdr_action->operation == RTE_FLOW_MODIFY_SET ||
		    mhdr_action->operation == RTE_FLOW_MODIFY_ADD);
	if (mhdr_action->src.field != RTE_FLOW_FIELD_VALUE &&
	    mhdr_action->src.field != RTE_FLOW_FIELD_POINTER)
		return 0;
	if (mhdr_action->src.field == RTE_FLOW_FIELD_VALUE)
		rte_memcpy(values, &mhdr_action->src.value, sizeof(values));
	else
		rte_memcpy(values, mhdr_action->src.pvalue, sizeof(values));
	if (mhdr_action->dst.field == RTE_FLOW_FIELD_META ||
	    mhdr_action->dst.field == RTE_FLOW_FIELD_TAG ||
	    mhdr_action->dst.field == RTE_FLOW_FIELD_METER_COLOR ||
	    mhdr_action->dst.field == (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) {
		value_p = (unaligned_uint32_t *)values;
		*value_p = rte_cpu_to_be_32(*value_p);
	} else if (mhdr_action->dst.field == RTE_FLOW_FIELD_GTP_PSC_QFI) {
		uint32_t tmp;

		/*
		 * QFI is passed as an uint8_t integer, but it is accessed through
		 * a 2nd least significant byte of a 32-bit field in modify header command.
		 */
		tmp = values[0];
		value_p = (unaligned_uint32_t *)values;
		*value_p = rte_cpu_to_be_32(tmp << 8);
	}
	i = act_data->modify_header.mhdr_cmds_off;
	field = act_data->modify_header.field;
	do {
		uint32_t off_b;
		uint32_t mask;
		uint32_t data;
		const uint8_t *mask_src;

		if (i >= act_data->modify_header.mhdr_cmds_end)
			return -1;
		if (flow_hw_mhdr_cmd_is_nop(&job->mhdr_cmd[i])) {
			++i;
			continue;
		}
		mask_src = (const uint8_t *)act_data->modify_header.mask;
		mask = flow_dv_fetch_field(mask_src + field->offset, field->size);
		if (!mask) {
			++field;
			continue;
		}
		off_b = rte_bsf32(mask);
		data = flow_dv_fetch_field(values + field->offset, field->size);
		data = (data & mask) >> off_b;
		job->mhdr_cmd[i++].data1 = rte_cpu_to_be_32(data);
		++field;
	} while (field->size);
	return 0;
}

/**
 * Construct flow action array.
 *
 * For action template contains dynamic actions, these actions need to
 * be updated according to the rte_flow action during flow creation.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] job
 *   Pointer to job descriptor.
 * @param[in] hw_acts
 *   Pointer to translated actions from template.
 * @param[in] it_idx
 *   Item template index the action template refer to.
 * @param[in] actions
 *   Array of rte_flow action need to be checked.
 * @param[in] rule_acts
 *   Array of DR rule actions to be used during flow creation..
 * @param[in] acts_num
 *   Pointer to the real acts_num flow has.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static __rte_always_inline int
flow_hw_actions_construct(struct rte_eth_dev *dev,
			  struct mlx5_hw_q_job *job,
			  const struct mlx5_hw_action_template *hw_at,
			  const uint8_t it_idx,
			  const struct rte_flow_action actions[],
			  struct mlx5dr_rule_action *rule_acts,
			  uint32_t queue,
			  struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	struct rte_flow_template_table *table = job->flow->table;
	struct mlx5_action_construct_data *act_data;
	const struct rte_flow_actions_template *at = hw_at->action_template;
	const struct mlx5_hw_actions *hw_acts = &hw_at->acts;
	const struct rte_flow_action *action;
	const struct rte_flow_action_raw_encap *raw_encap_data;
	const struct rte_flow_item *enc_item = NULL;
	const struct rte_flow_action_ethdev *port_action = NULL;
	const struct rte_flow_action_meter *meter = NULL;
	const struct rte_flow_action_age *age = NULL;
	uint8_t *buf = job->encap_data;
	struct rte_flow_attr attr = {
			.ingress = 1,
	};
	uint32_t ft_flag;
	size_t encap_len = 0;
	int ret;
	uint32_t age_idx = 0;
	struct mlx5_aso_mtr *aso_mtr;

	rte_memcpy(rule_acts, hw_acts->rule_acts, sizeof(*rule_acts) * at->dr_actions_num);
	attr.group = table->grp->group_id;
	ft_flag = mlx5_hw_act_flag[!!table->grp->group_id][table->type];
	if (table->type == MLX5DR_TABLE_TYPE_FDB) {
		attr.transfer = 1;
		attr.ingress = 1;
	} else if (table->type == MLX5DR_TABLE_TYPE_NIC_TX) {
		attr.egress = 1;
		attr.ingress = 0;
	} else {
		attr.ingress = 1;
	}
	if (hw_acts->mhdr && hw_acts->mhdr->mhdr_cmds_num > 0) {
		uint16_t pos = hw_acts->mhdr->pos;

		if (!hw_acts->mhdr->shared) {
			rule_acts[pos].modify_header.offset =
						job->flow->idx - 1;
			rule_acts[pos].modify_header.data =
						(uint8_t *)job->mhdr_cmd;
			rte_memcpy(job->mhdr_cmd, hw_acts->mhdr->mhdr_cmds,
				   sizeof(*job->mhdr_cmd) * hw_acts->mhdr->mhdr_cmds_num);
		}
	}
	LIST_FOREACH(act_data, &hw_acts->act_list, next) {
		uint32_t jump_group;
		uint32_t tag;
		uint64_t item_flags;
		struct mlx5_hw_jump_action *jump;
		struct mlx5_hrxq *hrxq;
		uint32_t ct_idx;
		cnt_id_t cnt_id;
		uint32_t mtr_id;

		action = &actions[act_data->action_src];
		/*
		 * action template construction replaces
		 * OF_SET_VLAN_VID with MODIFY_FIELD
		 */
		if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)
			MLX5_ASSERT(act_data->type ==
				    RTE_FLOW_ACTION_TYPE_MODIFY_FIELD);
		else
			MLX5_ASSERT(action->type ==
				    RTE_FLOW_ACTION_TYPE_INDIRECT ||
				    (int)action->type == act_data->type);
		switch (act_data->type) {
		case RTE_FLOW_ACTION_TYPE_INDIRECT:
			if (flow_hw_shared_action_construct
					(dev, queue, action, table, it_idx,
					 at->action_flags, job->flow,
					 &rule_acts[act_data->action_dst]))
				return -1;
			break;
		case RTE_FLOW_ACTION_TYPE_VOID:
			break;
		case RTE_FLOW_ACTION_TYPE_MARK:
			tag = mlx5_flow_mark_set
			      (((const struct rte_flow_action_mark *)
			      (action->conf))->id);
			rule_acts[act_data->action_dst].tag.value = tag;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
			rule_acts[act_data->action_dst].push_vlan.vlan_hdr =
				vlan_hdr_to_be32(action);
			break;
		case RTE_FLOW_ACTION_TYPE_JUMP:
			jump_group = ((const struct rte_flow_action_jump *)
						action->conf)->group;
			jump = flow_hw_jump_action_register
				(dev, &table->cfg, jump_group, NULL);
			if (!jump)
				return -1;
			rule_acts[act_data->action_dst].action =
			(!!attr.group) ? jump->hws_action : jump->root_action;
			job->flow->jump = jump;
			job->flow->fate_type = MLX5_FLOW_FATE_JUMP;
			break;
		case RTE_FLOW_ACTION_TYPE_RSS:
		case RTE_FLOW_ACTION_TYPE_QUEUE:
			hrxq = flow_hw_tir_action_register(dev,
					ft_flag,
					action);
			if (!hrxq)
				return -1;
			rule_acts[act_data->action_dst].action = hrxq->action;
			job->flow->hrxq = hrxq;
			job->flow->fate_type = MLX5_FLOW_FATE_QUEUE;
			break;
		case MLX5_RTE_FLOW_ACTION_TYPE_RSS:
			item_flags = table->its[it_idx]->item_flags;
			if (flow_hw_shared_action_get
				(dev, act_data, item_flags,
				 &rule_acts[act_data->action_dst]))
				return -1;
			break;
		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
			enc_item = ((const struct rte_flow_action_vxlan_encap *)
				   action->conf)->definition;
			if (flow_dv_convert_encap_data(enc_item, buf, &encap_len, NULL))
				return -1;
			break;
		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
			enc_item = ((const struct rte_flow_action_nvgre_encap *)
				   action->conf)->definition;
			if (flow_dv_convert_encap_data(enc_item, buf, &encap_len, NULL))
				return -1;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
			raw_encap_data =
				(const struct rte_flow_action_raw_encap *)
				 action->conf;
			rte_memcpy((void *)buf, raw_encap_data->data, act_data->encap.len);
			MLX5_ASSERT(raw_encap_data->size ==
				    act_data->encap.len);
			break;
		case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
			if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)
				ret = flow_hw_set_vlan_vid_construct(dev, job,
								     act_data,
								     hw_acts,
								     action);
			else
				ret = flow_hw_modify_field_construct(job,
								     act_data,
								     hw_acts,
								     action);
			if (ret)
				return -1;
			break;
		case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
			port_action = action->conf;
			if (!priv->hw_vport[port_action->port_id])
				return -1;
			rule_acts[act_data->action_dst].action =
					priv->hw_vport[port_action->port_id];
			break;
		case RTE_FLOW_ACTION_TYPE_METER:
			meter = action->conf;
			mtr_id = meter->mtr_id;
			aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_id);
			rule_acts[act_data->action_dst].action =
				priv->mtr_bulk.action;
			rule_acts[act_data->action_dst].aso_meter.offset =
								aso_mtr->offset;
			jump = flow_hw_jump_action_register
				(dev, &table->cfg, aso_mtr->fm.group, NULL);
			if (!jump)
				return -1;
			MLX5_ASSERT
				(!rule_acts[act_data->action_dst + 1].action);
			rule_acts[act_data->action_dst + 1].action =
					(!!attr.group) ? jump->hws_action :
							 jump->root_action;
			job->flow->jump = jump;
			job->flow->fate_type = MLX5_FLOW_FATE_JUMP;
			if (mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr))
				return -1;
			break;
		case RTE_FLOW_ACTION_TYPE_AGE:
			age = action->conf;
			/*
			 * First, create the AGE parameter, then create its
			 * counter later:
			 * Regular counter - in next case.
			 * Indirect counter - update it after the loop.
			 */
			age_idx = mlx5_hws_age_action_create(priv, queue, 0,
							     age,
							     job->flow->idx,
							     error);
			if (age_idx == 0)
				return -rte_errno;
			job->flow->age_idx = age_idx;
			if (at->action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT)
				/*
				 * When AGE uses indirect counter, no need to
				 * create counter but need to update it with the
				 * AGE parameter, will be done after the loop.
				 */
				break;
			/* Fall-through. */
		case RTE_FLOW_ACTION_TYPE_COUNT:
			ret = mlx5_hws_cnt_pool_get(priv->hws_cpool, &queue,
						    &cnt_id, age_idx);
			if (ret != 0)
				return ret;
			ret = mlx5_hws_cnt_pool_get_action_offset
				(priv->hws_cpool,
				 cnt_id,
				 &rule_acts[act_data->action_dst].action,
				 &rule_acts[act_data->action_dst].counter.offset
				 );
			if (ret != 0)
				return ret;
			job->flow->cnt_id = cnt_id;
			break;
		case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
			ret = mlx5_hws_cnt_pool_get_action_offset
				(priv->hws_cpool,
				 act_data->shared_counter.id,
				 &rule_acts[act_data->action_dst].action,
				 &rule_acts[act_data->action_dst].counter.offset
				 );
			if (ret != 0)
				return ret;
			job->flow->cnt_id = act_data->shared_counter.id;
			break;
		case RTE_FLOW_ACTION_TYPE_CONNTRACK:
			ct_idx = MLX5_ACTION_CTX_CT_GET_IDX
				 ((uint32_t)(uintptr_t)action->conf);
			if (flow_hw_ct_compile(dev, queue, ct_idx,
					       &rule_acts[act_data->action_dst]))
				return -1;
			break;
		case MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK:
			mtr_id = act_data->shared_meter.id &
				((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
			/* Find ASO object. */
			aso_mtr = mlx5_ipool_get(pool->idx_pool, mtr_id);
			if (!aso_mtr)
				return -1;
			rule_acts[act_data->action_dst].action =
							pool->action;
			rule_acts[act_data->action_dst].aso_meter.offset =
							aso_mtr->offset;
			rule_acts[act_data->action_dst].aso_meter.init_color =
				(enum mlx5dr_action_aso_meter_color)
				rte_col_2_mlx5_col(aso_mtr->init_color);
			break;
		case RTE_FLOW_ACTION_TYPE_METER_MARK:
			/*
			 * Allocate meter directly will slow down flow
			 * insertion rate.
			 */
			ret = flow_hw_meter_mark_compile(dev,
				act_data->action_dst, action,
				rule_acts, &job->flow->mtr_id, MLX5_HW_INV_QUEUE);
			if (ret != 0)
				return ret;
			break;
		default:
			break;
		}
	}
	if (at->action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT) {
		if (at->action_flags & MLX5_FLOW_ACTION_INDIRECT_AGE) {
			age_idx = job->flow->age_idx & MLX5_HWS_AGE_IDX_MASK;
			if (mlx5_hws_cnt_age_get(priv->hws_cpool,
						 job->flow->cnt_id) != age_idx)
				/*
				 * This is first use of this indirect counter
				 * for this indirect AGE, need to increase the
				 * number of counters.
				 */
				mlx5_hws_age_nb_cnt_increase(priv, age_idx);
		}
		/*
		 * Update this indirect counter the indirect/direct AGE in which
		 * using it.
		 */
		mlx5_hws_cnt_age_set(priv->hws_cpool, job->flow->cnt_id,
				     age_idx);
	}
	if (hw_acts->encap_decap && !hw_acts->encap_decap->shared) {
		rule_acts[hw_acts->encap_decap_pos].reformat.offset =
				job->flow->idx - 1;
		rule_acts[hw_acts->encap_decap_pos].reformat.data = buf;
	}
	if (mlx5_hws_cnt_id_valid(hw_acts->cnt_id))
		job->flow->cnt_id = hw_acts->cnt_id;
	return 0;
}

static const struct rte_flow_item *
flow_hw_get_rule_items(struct rte_eth_dev *dev,
		       struct rte_flow_template_table *table,
		       const struct rte_flow_item items[],
		       uint8_t pattern_template_index,
		       struct mlx5_hw_q_job *job)
{
	struct rte_flow_pattern_template *pt = table->its[pattern_template_index];

	/* Only one implicit item can be added to flow rule pattern. */
	MLX5_ASSERT(!pt->implicit_port || !pt->implicit_tag);
	/* At least one item was allocated in job descriptor for items. */
	MLX5_ASSERT(MLX5_HW_MAX_ITEMS >= 1);
	if (pt->implicit_port) {
		if (pt->orig_item_nb + 1 > MLX5_HW_MAX_ITEMS) {
			rte_errno = ENOMEM;
			return NULL;
		}
		/* Set up represented port item in job descriptor. */
		job->port_spec = (struct rte_flow_item_ethdev){
			.port_id = dev->data->port_id,
		};
		job->items[0] = (struct rte_flow_item){
			.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
			.spec = &job->port_spec,
		};
		rte_memcpy(&job->items[1], items, sizeof(*items) * pt->orig_item_nb);
		return job->items;
	} else if (pt->implicit_tag) {
		if (pt->orig_item_nb + 1 > MLX5_HW_MAX_ITEMS) {
			rte_errno = ENOMEM;
			return NULL;
		}
		/* Set up tag item in job descriptor. */
		job->tag_spec = (struct rte_flow_item_tag){
			.data = flow_hw_tx_tag_regc_value(dev),
		};
		job->items[0] = (struct rte_flow_item){
			.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG,
			.spec = &job->tag_spec,
		};
		rte_memcpy(&job->items[1], items, sizeof(*items) * pt->orig_item_nb);
		return job->items;
	} else {
		return items;
	}
}

/**
 * Enqueue HW steering flow creation.
 *
 * The flow will be applied to the HW only if the postpone bit is not set or
 * the extra push function is called.
 * The flow creation status should be checked from dequeue result.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   The queue to create the flow.
 * @param[in] attr
 *   Pointer to the flow operation attributes.
 * @param[in] items
 *   Items with flow spec value.
 * @param[in] pattern_template_index
 *   The item pattern flow follows from the table.
 * @param[in] actions
 *   Action with flow spec value.
 * @param[in] action_template_index
 *   The action pattern flow follows from the table.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    Flow pointer on success, NULL otherwise and rte_errno is set.
 */
static struct rte_flow *
flow_hw_async_flow_create(struct rte_eth_dev *dev,
			  uint32_t queue,
			  const struct rte_flow_op_attr *attr,
			  struct rte_flow_template_table *table,
			  const struct rte_flow_item items[],
			  uint8_t pattern_template_index,
			  const struct rte_flow_action actions[],
			  uint8_t action_template_index,
			  void *user_data,
			  struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5dr_rule_attr rule_attr = {
		.queue_id = queue,
		.user_data = user_data,
		.burst = attr->postpone,
	};
	struct mlx5dr_rule_action rule_acts[MLX5_HW_MAX_ACTS];
	struct rte_flow_hw *flow;
	struct mlx5_hw_q_job *job;
	const struct rte_flow_item *rule_items;
	uint32_t flow_idx;
	int ret;

	if (unlikely((!dev->data->dev_started))) {
		rte_errno = EINVAL;
		goto error;
	}
	if (unlikely(!priv->hw_q[queue].job_idx)) {
		rte_errno = ENOMEM;
		goto error;
	}
	flow = mlx5_ipool_zmalloc(table->flow, &flow_idx);
	if (!flow)
		goto error;
	/*
	 * Set the table here in order to know the destination table
	 * when free the flow afterwards.
	 */
	flow->table = table;
	flow->idx = flow_idx;
	job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
	/*
	 * Set the job type here in order to know if the flow memory
	 * should be freed or not when get the result from dequeue.
	 */
	job->type = MLX5_HW_Q_JOB_TYPE_CREATE;
	job->flow = flow;
	job->user_data = user_data;
	rule_attr.user_data = job;
	/*
	 * Indexed pool returns 1-based indices, but mlx5dr expects 0-based indices for rule
	 * insertion hints.
	 */
	MLX5_ASSERT(flow_idx > 0);
	rule_attr.rule_idx = flow_idx - 1;
	/*
	 * Construct the flow actions based on the input actions.
	 * The implicitly appended action is always fixed, like metadata
	 * copy action from FDB to NIC Rx.
	 * No need to copy and contrust a new "actions" list based on the
	 * user's input, in order to save the cost.
	 */
	if (flow_hw_actions_construct(dev, job,
				      &table->ats[action_template_index],
				      pattern_template_index, actions,
				      rule_acts, queue, error)) {
		rte_errno = EINVAL;
		goto free;
	}
	rule_items = flow_hw_get_rule_items(dev, table, items,
					    pattern_template_index, job);
	if (!rule_items)
		goto free;
	ret = mlx5dr_rule_create(table->matcher,
				 pattern_template_index, rule_items,
				 action_template_index, rule_acts,
				 &rule_attr, (struct mlx5dr_rule *)flow->rule);
	if (likely(!ret))
		return (struct rte_flow *)flow;
free:
	/* Flow created fail, return the descriptor and flow memory. */
	mlx5_ipool_free(table->flow, flow_idx);
	priv->hw_q[queue].job_idx++;
error:
	rte_flow_error_set(error, rte_errno,
			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
			   "fail to create rte flow");
	return NULL;
}

/**
 * Enqueue HW steering flow destruction.
 *
 * The flow will be applied to the HW only if the postpone bit is not set or
 * the extra push function is called.
 * The flow destruction status should be checked from dequeue result.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   The queue to destroy the flow.
 * @param[in] attr
 *   Pointer to the flow operation attributes.
 * @param[in] flow
 *   Pointer to the flow to be destroyed.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_async_flow_destroy(struct rte_eth_dev *dev,
			   uint32_t queue,
			   const struct rte_flow_op_attr *attr,
			   struct rte_flow *flow,
			   void *user_data,
			   struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5dr_rule_attr rule_attr = {
		.queue_id = queue,
		.user_data = user_data,
		.burst = attr->postpone,
	};
	struct rte_flow_hw *fh = (struct rte_flow_hw *)flow;
	struct mlx5_hw_q_job *job;
	int ret;

	if (unlikely(!priv->hw_q[queue].job_idx)) {
		rte_errno = ENOMEM;
		goto error;
	}
	job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
	job->type = MLX5_HW_Q_JOB_TYPE_DESTROY;
	job->user_data = user_data;
	job->flow = fh;
	rule_attr.user_data = job;
	ret = mlx5dr_rule_destroy((struct mlx5dr_rule *)fh->rule, &rule_attr);
	if (likely(!ret))
		return 0;
	priv->hw_q[queue].job_idx++;
error:
	return rte_flow_error_set(error, rte_errno,
			RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
			"fail to destroy rte flow");
}

/**
 * Release the AGE and counter for given flow.
 *
 * @param[in] priv
 *   Pointer to the port private data structure.
 * @param[in] queue
 *   The queue to release the counter.
 * @param[in, out] flow
 *   Pointer to the flow containing the counter.
 * @param[out] error
 *   Pointer to error structure.
 */
static void
flow_hw_age_count_release(struct mlx5_priv *priv, uint32_t queue,
			  struct rte_flow_hw *flow,
			  struct rte_flow_error *error)
{
	if (mlx5_hws_cnt_is_shared(priv->hws_cpool, flow->cnt_id)) {
		if (flow->age_idx && !mlx5_hws_age_is_indirect(flow->age_idx)) {
			/* Remove this AGE parameter from indirect counter. */
			mlx5_hws_cnt_age_set(priv->hws_cpool, flow->cnt_id, 0);
			/* Release the AGE parameter. */
			mlx5_hws_age_action_destroy(priv, flow->age_idx, error);
			flow->age_idx = 0;
		}
		return;
	}
	/* Put the counter first to reduce the race risk in BG thread. */
	mlx5_hws_cnt_pool_put(priv->hws_cpool, &queue, &flow->cnt_id);
	flow->cnt_id = 0;
	if (flow->age_idx) {
		if (mlx5_hws_age_is_indirect(flow->age_idx)) {
			uint32_t idx = flow->age_idx & MLX5_HWS_AGE_IDX_MASK;

			mlx5_hws_age_nb_cnt_decrease(priv, idx);
		} else {
			/* Release the AGE parameter. */
			mlx5_hws_age_action_destroy(priv, flow->age_idx, error);
		}
		flow->age_idx = 0;
	}
}

static inline int
__flow_hw_pull_indir_action_comp(struct rte_eth_dev *dev,
				 uint32_t queue,
				 struct rte_flow_op_result res[],
				 uint16_t n_res)

{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_ring *r = priv->hw_q[queue].indir_cq;
	struct mlx5_hw_q_job *job;
	void *user_data = NULL;
	uint32_t type, idx;
	struct mlx5_aso_mtr *aso_mtr;
	struct mlx5_aso_ct_action *aso_ct;
	int ret_comp, i;

	ret_comp = (int)rte_ring_count(r);
	if (ret_comp > n_res)
		ret_comp = n_res;
	for (i = 0; i < ret_comp; i++) {
		rte_ring_dequeue(r, &user_data);
		res[i].user_data = user_data;
		res[i].status = RTE_FLOW_OP_SUCCESS;
	}
	if (ret_comp < n_res && priv->hws_mpool)
		ret_comp += mlx5_aso_pull_completion(&priv->hws_mpool->sq[queue],
				&res[ret_comp], n_res - ret_comp);
	if (ret_comp < n_res && priv->hws_ctpool)
		ret_comp += mlx5_aso_pull_completion(&priv->ct_mng->aso_sqs[queue],
				&res[ret_comp], n_res - ret_comp);
	for (i = 0; i <  ret_comp; i++) {
		job = (struct mlx5_hw_q_job *)res[i].user_data;
		/* Restore user data. */
		res[i].user_data = job->user_data;
		if (job->type == MLX5_HW_Q_JOB_TYPE_DESTROY) {
			type = MLX5_INDIRECT_ACTION_TYPE_GET(job->action);
			if (type == MLX5_INDIRECT_ACTION_TYPE_METER_MARK) {
				idx = MLX5_INDIRECT_ACTION_IDX_GET(job->action);
				mlx5_ipool_free(priv->hws_mpool->idx_pool, idx);
			}
		} else if (job->type == MLX5_HW_Q_JOB_TYPE_CREATE) {
			type = MLX5_INDIRECT_ACTION_TYPE_GET(job->action);
			if (type == MLX5_INDIRECT_ACTION_TYPE_METER_MARK) {
				idx = MLX5_INDIRECT_ACTION_IDX_GET(job->action);
				aso_mtr = mlx5_ipool_get(priv->hws_mpool->idx_pool, idx);
				aso_mtr->state = ASO_METER_READY;
			} else if (type == MLX5_INDIRECT_ACTION_TYPE_CT) {
				idx = MLX5_ACTION_CTX_CT_GET_IDX
					((uint32_t)(uintptr_t)job->action);
				aso_ct = mlx5_ipool_get(priv->hws_ctpool->cts, idx);
				aso_ct->state = ASO_CONNTRACK_READY;
			}
		} else if (job->type == MLX5_HW_Q_JOB_TYPE_QUERY) {
			type = MLX5_INDIRECT_ACTION_TYPE_GET(job->action);
			if (type == MLX5_INDIRECT_ACTION_TYPE_CT) {
				idx = MLX5_ACTION_CTX_CT_GET_IDX
					((uint32_t)(uintptr_t)job->action);
				aso_ct = mlx5_ipool_get(priv->hws_ctpool->cts, idx);
				mlx5_aso_ct_obj_analyze(job->profile,
							job->out_data);
				aso_ct->state = ASO_CONNTRACK_READY;
			}
		}
		priv->hw_q[queue].job[priv->hw_q[queue].job_idx++] = job;
	}
	return ret_comp;
}

/**
 * Pull the enqueued flows.
 *
 * For flows enqueued from creation/destruction, the status should be
 * checked from the dequeue result.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   The queue to pull the result.
 * @param[in/out] res
 *   Array to save the results.
 * @param[in] n_res
 *   Available result with the array.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    Result number on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_pull(struct rte_eth_dev *dev,
	     uint32_t queue,
	     struct rte_flow_op_result res[],
	     uint16_t n_res,
	     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	struct mlx5_hw_q_job *job;
	int ret, i;

	/* 1. Pull the flow completion. */
	ret = mlx5dr_send_queue_poll(priv->dr_ctx, queue, res, n_res);
	if (ret < 0)
		return rte_flow_error_set(error, rte_errno,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				"fail to query flow queue");
	for (i = 0; i <  ret; i++) {
		job = (struct mlx5_hw_q_job *)res[i].user_data;
		/* Restore user data. */
		res[i].user_data = job->user_data;
		if (job->type == MLX5_HW_Q_JOB_TYPE_DESTROY) {
			if (job->flow->fate_type == MLX5_FLOW_FATE_JUMP)
				flow_hw_jump_release(dev, job->flow->jump);
			else if (job->flow->fate_type == MLX5_FLOW_FATE_QUEUE)
				mlx5_hrxq_obj_release(dev, job->flow->hrxq);
			if (mlx5_hws_cnt_id_valid(job->flow->cnt_id))
				flow_hw_age_count_release(priv, queue,
							  job->flow, error);
			if (job->flow->mtr_id) {
				mlx5_ipool_free(pool->idx_pool,	job->flow->mtr_id);
				job->flow->mtr_id = 0;
			}
			mlx5_ipool_free(job->flow->table->flow, job->flow->idx);
		}
		priv->hw_q[queue].job[priv->hw_q[queue].job_idx++] = job;
	}
	/* 2. Pull indirect action comp. */
	if (ret < n_res)
		ret += __flow_hw_pull_indir_action_comp(dev, queue, &res[ret],
							n_res - ret);
	return ret;
}

static inline void
__flow_hw_push_action(struct rte_eth_dev *dev,
		    uint32_t queue)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_ring *iq = priv->hw_q[queue].indir_iq;
	struct rte_ring *cq = priv->hw_q[queue].indir_cq;
	void *job = NULL;
	uint32_t ret, i;

	ret = rte_ring_count(iq);
	for (i = 0; i < ret; i++) {
		rte_ring_dequeue(iq, &job);
		rte_ring_enqueue(cq, job);
	}
	if (priv->hws_ctpool)
		mlx5_aso_push_wqe(priv->sh, &priv->ct_mng->aso_sqs[queue]);
	if (priv->hws_mpool)
		mlx5_aso_push_wqe(priv->sh, &priv->hws_mpool->sq[queue]);
}

/**
 * Push the enqueued flows to HW.
 *
 * Force apply all the enqueued flows to the HW.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   The queue to push the flow.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_push(struct rte_eth_dev *dev,
	     uint32_t queue,
	     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	int ret;

	__flow_hw_push_action(dev, queue);
	ret = mlx5dr_send_queue_action(priv->dr_ctx, queue,
				       MLX5DR_SEND_QUEUE_ACTION_DRAIN);
	if (ret) {
		rte_flow_error_set(error, rte_errno,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				   "fail to push flows");
		return ret;
	}
	return 0;
}

/**
 * Drain the enqueued flows' completion.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   The queue to pull the flow.
 * @param[in] pending_rules
 *   The pending flow number.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
static int
__flow_hw_pull_comp(struct rte_eth_dev *dev,
		    uint32_t queue,
		    uint32_t pending_rules,
		    struct rte_flow_error *error)
{
	struct rte_flow_op_result comp[BURST_THR];
	int ret, i, empty_loop = 0;

	ret = flow_hw_push(dev, queue, error);
	if (ret < 0)
		return ret;
	while (pending_rules) {
		ret = flow_hw_pull(dev, queue, comp, BURST_THR, error);
		if (ret < 0)
			return -1;
		if (!ret) {
			rte_delay_us_sleep(20000);
			if (++empty_loop > 5) {
				DRV_LOG(WARNING, "No available dequeue, quit.");
				break;
			}
			continue;
		}
		for (i = 0; i < ret; i++) {
			if (comp[i].status == RTE_FLOW_OP_ERROR)
				DRV_LOG(WARNING, "Flow flush get error CQE.");
		}
		if ((uint32_t)ret > pending_rules) {
			DRV_LOG(WARNING, "Flow flush get extra CQE.");
			return rte_flow_error_set(error, ERANGE,
					RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					"get extra CQE");
		}
		pending_rules -= ret;
		empty_loop = 0;
	}
	return 0;
}

/**
 * Flush created flows.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
int
flow_hw_q_flow_flush(struct rte_eth_dev *dev,
		     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hw_q *hw_q;
	struct rte_flow_template_table *tbl;
	struct rte_flow_hw *flow;
	struct rte_flow_op_attr attr = {
		.postpone = 0,
	};
	uint32_t pending_rules = 0;
	uint32_t queue;
	uint32_t fidx;

	/*
	 * Ensure to push and dequeue all the enqueued flow
	 * creation/destruction jobs in case user forgot to
	 * dequeue. Or the enqueued created flows will be
	 * leaked. The forgotten dequeues would also cause
	 * flow flush get extra CQEs as expected and pending_rules
	 * be minus value.
	 */
	for (queue = 0; queue < priv->nb_queue; queue++) {
		hw_q = &priv->hw_q[queue];
		if (__flow_hw_pull_comp(dev, queue, hw_q->size - hw_q->job_idx,
					error))
			return -1;
	}
	/* Flush flow per-table from MLX5_DEFAULT_FLUSH_QUEUE. */
	hw_q = &priv->hw_q[MLX5_DEFAULT_FLUSH_QUEUE];
	LIST_FOREACH(tbl, &priv->flow_hw_tbl, next) {
		if (!tbl->cfg.external)
			continue;
		MLX5_IPOOL_FOREACH(tbl->flow, fidx, flow) {
			if (flow_hw_async_flow_destroy(dev,
						MLX5_DEFAULT_FLUSH_QUEUE,
						&attr,
						(struct rte_flow *)flow,
						NULL,
						error))
				return -1;
			pending_rules++;
			/* Drain completion with queue size. */
			if (pending_rules >= hw_q->size) {
				if (__flow_hw_pull_comp(dev,
						MLX5_DEFAULT_FLUSH_QUEUE,
						pending_rules, error))
					return -1;
				pending_rules = 0;
			}
		}
	}
	/* Drain left completion. */
	if (pending_rules &&
	    __flow_hw_pull_comp(dev, MLX5_DEFAULT_FLUSH_QUEUE, pending_rules,
				error))
		return -1;
	return 0;
}

/**
 * Create flow table.
 *
 * The input item and action templates will be binded to the table.
 * Flow memory will also be allocated. Matcher will be created based
 * on the item template. Action will be translated to the dedicated
 * DR action if possible.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] table_cfg
 *   Pointer to the table configuration.
 * @param[in] item_templates
 *   Item template array to be binded to the table.
 * @param[in] nb_item_templates
 *   Number of item template.
 * @param[in] action_templates
 *   Action template array to be binded to the table.
 * @param[in] nb_action_templates
 *   Number of action template.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    Table on success, NULL otherwise and rte_errno is set.
 */
static struct rte_flow_template_table *
flow_hw_table_create(struct rte_eth_dev *dev,
		     const struct mlx5_flow_template_table_cfg *table_cfg,
		     struct rte_flow_pattern_template *item_templates[],
		     uint8_t nb_item_templates,
		     struct rte_flow_actions_template *action_templates[],
		     uint8_t nb_action_templates,
		     struct rte_flow_error *error)
{
	struct rte_flow_error sub_error = {
		.type = RTE_FLOW_ERROR_TYPE_NONE,
		.cause = NULL,
		.message = NULL,
	};
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5dr_matcher_attr matcher_attr = {0};
	struct rte_flow_template_table *tbl = NULL;
	struct mlx5_flow_group *grp;
	struct mlx5dr_match_template *mt[MLX5_HW_TBL_MAX_ITEM_TEMPLATE];
	struct mlx5dr_action_template *at[MLX5_HW_TBL_MAX_ACTION_TEMPLATE];
	const struct rte_flow_template_table_attr *attr = &table_cfg->attr;
	struct rte_flow_attr flow_attr = attr->flow_attr;
	struct mlx5_flow_cb_ctx ctx = {
		.dev = dev,
		.error = &sub_error,
		.data = &flow_attr,
	};
	struct mlx5_indexed_pool_config cfg = {
		.size = sizeof(struct rte_flow_hw) + mlx5dr_rule_get_handle_size(),
		.trunk_size = 1 << 12,
		.per_core_cache = 1 << 13,
		.need_lock = 1,
		.release_mem_en = !!priv->sh->config.reclaim_mode,
		.malloc = mlx5_malloc,
		.free = mlx5_free,
		.type = "mlx5_hw_table_flow",
	};
	struct mlx5_list_entry *ge;
	uint32_t i, max_tpl = MLX5_HW_TBL_MAX_ITEM_TEMPLATE;
	uint32_t nb_flows = rte_align32pow2(attr->nb_flows);
	bool port_started = !!dev->data->dev_started;
	int err;

	/* HWS layer accepts only 1 item template with root table. */
	if (!attr->flow_attr.group)
		max_tpl = 1;
	cfg.max_idx = nb_flows;
	/* For table has very limited flows, disable cache. */
	if (nb_flows < cfg.trunk_size) {
		cfg.per_core_cache = 0;
		cfg.trunk_size = nb_flows;
	} else if (nb_flows <= MLX5_HW_IPOOL_SIZE_THRESHOLD) {
		cfg.per_core_cache = MLX5_HW_IPOOL_CACHE_MIN;
	}
	/* Check if we requires too many templates. */
	if (nb_item_templates > max_tpl ||
	    nb_action_templates > MLX5_HW_TBL_MAX_ACTION_TEMPLATE) {
		rte_errno = EINVAL;
		goto error;
	}
	/* Allocate the table memory. */
	tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl), 0, rte_socket_id());
	if (!tbl)
		goto error;
	tbl->cfg = *table_cfg;
	/* Allocate flow indexed pool. */
	tbl->flow = mlx5_ipool_create(&cfg);
	if (!tbl->flow)
		goto error;
	/* Register the flow group. */
	ge = mlx5_hlist_register(priv->sh->groups, attr->flow_attr.group, &ctx);
	if (!ge)
		goto error;
	grp = container_of(ge, struct mlx5_flow_group, entry);
	tbl->grp = grp;
	/* Prepare matcher information. */
	matcher_attr.priority = attr->flow_attr.priority;
	matcher_attr.optimize_using_rule_idx = true;
	matcher_attr.mode = MLX5DR_MATCHER_RESOURCE_MODE_RULE;
	matcher_attr.rule.num_log = rte_log2_u32(nb_flows);
	/* Build the item template. */
	for (i = 0; i < nb_item_templates; i++) {
		uint32_t ret;

		if ((flow_attr.ingress && !item_templates[i]->attr.ingress) ||
		    (flow_attr.egress && !item_templates[i]->attr.egress) ||
		    (flow_attr.transfer && !item_templates[i]->attr.transfer)) {
			DRV_LOG(ERR, "pattern template and template table attribute mismatch");
			rte_errno = EINVAL;
			goto it_error;
		}
		ret = __atomic_add_fetch(&item_templates[i]->refcnt, 1,
					 __ATOMIC_RELAXED);
		if (ret <= 1) {
			rte_errno = EINVAL;
			goto it_error;
		}
		mt[i] = item_templates[i]->mt;
		tbl->its[i] = item_templates[i];
	}
	tbl->nb_item_templates = nb_item_templates;
	/* Build the action template. */
	for (i = 0; i < nb_action_templates; i++) {
		uint32_t ret;

		ret = __atomic_add_fetch(&action_templates[i]->refcnt, 1,
					 __ATOMIC_RELAXED);
		if (ret <= 1) {
			rte_errno = EINVAL;
			goto at_error;
		}
		at[i] = action_templates[i]->tmpl;
		tbl->ats[i].action_template = action_templates[i];
		LIST_INIT(&tbl->ats[i].acts.act_list);
		if (!port_started)
			continue;
		err = __flow_hw_actions_translate(dev, &tbl->cfg,
						  &tbl->ats[i].acts,
						  action_templates[i], &sub_error);
		if (err) {
			i++;
			goto at_error;
		}
	}
	tbl->nb_action_templates = nb_action_templates;
	tbl->matcher = mlx5dr_matcher_create
		(tbl->grp->tbl, mt, nb_item_templates, at, nb_action_templates, &matcher_attr);
	if (!tbl->matcher)
		goto at_error;
	tbl->type = attr->flow_attr.transfer ? MLX5DR_TABLE_TYPE_FDB :
		    (attr->flow_attr.egress ? MLX5DR_TABLE_TYPE_NIC_TX :
		    MLX5DR_TABLE_TYPE_NIC_RX);
	if (port_started)
		LIST_INSERT_HEAD(&priv->flow_hw_tbl, tbl, next);
	else
		LIST_INSERT_HEAD(&priv->flow_hw_tbl_ongo, tbl, next);
	return tbl;
at_error:
	while (i--) {
		__flow_hw_action_template_destroy(dev, &tbl->ats[i].acts);
		__atomic_sub_fetch(&action_templates[i]->refcnt,
				   1, __ATOMIC_RELAXED);
	}
	i = nb_item_templates;
it_error:
	while (i--)
		__atomic_sub_fetch(&item_templates[i]->refcnt,
				   1, __ATOMIC_RELAXED);
error:
	err = rte_errno;
	if (tbl) {
		if (tbl->grp)
			mlx5_hlist_unregister(priv->sh->groups,
					      &tbl->grp->entry);
		if (tbl->flow)
			mlx5_ipool_destroy(tbl->flow);
		mlx5_free(tbl);
	}
	if (error != NULL) {
		if (sub_error.type == RTE_FLOW_ERROR_TYPE_NONE)
			rte_flow_error_set(error, err, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					   "Failed to create template table");
		else
			rte_memcpy(error, &sub_error, sizeof(sub_error));
	}
	return NULL;
}

/**
 * Update flow template table.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *    0 on success, negative value otherwise and rte_errno is set.
 */
int
flow_hw_table_update(struct rte_eth_dev *dev,
		     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_template_table *tbl;

	while ((tbl = LIST_FIRST(&priv->flow_hw_tbl_ongo)) != NULL) {
		if (flow_hw_actions_translate(dev, tbl, error))
			return -1;
		LIST_REMOVE(tbl, next);
		LIST_INSERT_HEAD(&priv->flow_hw_tbl, tbl, next);
	}
	return 0;
}

/**
 * Translates group index specified by the user in @p attr to internal
 * group index.
 *
 * Translation is done by incrementing group index, so group n becomes n + 1.
 *
 * @param[in] dev
 *   Pointer to Ethernet device.
 * @param[in] cfg
 *   Pointer to the template table configuration.
 * @param[in] group
 *   Currently used group index (table group or jump destination).
 * @param[out] table_group
 *   Pointer to output group index.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success. Otherwise, returns negative error code, rte_errno is set
 *   and error structure is filled.
 */
static int
flow_hw_translate_group(struct rte_eth_dev *dev,
			const struct mlx5_flow_template_table_cfg *cfg,
			uint32_t group,
			uint32_t *table_group,
			struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_sh_config *config = &priv->sh->config;
	const struct rte_flow_attr *flow_attr = &cfg->attr.flow_attr;

	if (config->dv_esw_en &&
	    priv->fdb_def_rule &&
	    cfg->external &&
	    flow_attr->transfer) {
		if (group > MLX5_HW_MAX_TRANSFER_GROUP)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
						  NULL,
						  "group index not supported");
		*table_group = group + 1;
	} else if (config->dv_esw_en &&
		   !(config->repr_matching && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) &&
		   cfg->external &&
		   flow_attr->egress) {
		/*
		 * On E-Switch setups, egress group translation is not done if and only if
		 * representor matching is disabled and legacy metadata mode is selected.
		 * In all other cases, egree group 0 is reserved for representor tagging flows
		 * and metadata copy flows.
		 */
		if (group > MLX5_HW_MAX_EGRESS_GROUP)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
						  NULL,
						  "group index not supported");
		*table_group = group + 1;
	} else {
		*table_group = group;
	}
	return 0;
}

/**
 * Create flow table.
 *
 * This function is a wrapper over @ref flow_hw_table_create(), which translates parameters
 * provided by user to proper internal values.
 *
 * @param[in] dev
 *   Pointer to Ethernet device.
 * @param[in] attr
 *   Pointer to the table attributes.
 * @param[in] item_templates
 *   Item template array to be binded to the table.
 * @param[in] nb_item_templates
 *   Number of item templates.
 * @param[in] action_templates
 *   Action template array to be binded to the table.
 * @param[in] nb_action_templates
 *   Number of action templates.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   Table on success, Otherwise, returns negative error code, rte_errno is set
 *   and error structure is filled.
 */
static struct rte_flow_template_table *
flow_hw_template_table_create(struct rte_eth_dev *dev,
			      const struct rte_flow_template_table_attr *attr,
			      struct rte_flow_pattern_template *item_templates[],
			      uint8_t nb_item_templates,
			      struct rte_flow_actions_template *action_templates[],
			      uint8_t nb_action_templates,
			      struct rte_flow_error *error)
{
	struct mlx5_flow_template_table_cfg cfg = {
		.attr = *attr,
		.external = true,
	};
	uint32_t group = attr->flow_attr.group;

	if (flow_hw_translate_group(dev, &cfg, group, &cfg.attr.flow_attr.group, error))
		return NULL;
	return flow_hw_table_create(dev, &cfg, item_templates, nb_item_templates,
				    action_templates, nb_action_templates, error);
}

/**
 * Destroy flow table.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] table
 *   Pointer to the table to be destroyed.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_table_destroy(struct rte_eth_dev *dev,
		      struct rte_flow_template_table *table,
		      struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	int i;
	uint32_t fidx = 1;

	/* Build ipool allocated object bitmap. */
	mlx5_ipool_flush_cache(table->flow);
	/* Check if ipool has allocated objects. */
	if (table->refcnt || mlx5_ipool_get_next(table->flow, &fidx)) {
		DRV_LOG(WARNING, "Table %p is still in using.", (void *)table);
		return rte_flow_error_set(error, EBUSY,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "table in using");
	}
	LIST_REMOVE(table, next);
	for (i = 0; i < table->nb_item_templates; i++)
		__atomic_sub_fetch(&table->its[i]->refcnt,
				   1, __ATOMIC_RELAXED);
	for (i = 0; i < table->nb_action_templates; i++) {
		__flow_hw_action_template_destroy(dev, &table->ats[i].acts);
		__atomic_sub_fetch(&table->ats[i].action_template->refcnt,
				   1, __ATOMIC_RELAXED);
	}
	mlx5dr_matcher_destroy(table->matcher);
	mlx5_hlist_unregister(priv->sh->groups, &table->grp->entry);
	mlx5_ipool_destroy(table->flow);
	mlx5_free(table);
	return 0;
}

static bool
flow_hw_modify_field_is_used(const struct rte_flow_action_modify_field *action,
			     enum rte_flow_field_id field)
{
	return action->src.field == field || action->dst.field == field;
}

static int
flow_hw_validate_action_modify_field(const struct rte_flow_action *action,
				     const struct rte_flow_action *mask,
				     struct rte_flow_error *error)
{
	const struct rte_flow_action_modify_field *action_conf =
		action->conf;
	const struct rte_flow_action_modify_field *mask_conf =
		mask->conf;

	if (action_conf->operation != mask_conf->operation)
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"modify_field operation mask and template are not equal");
	if (action_conf->dst.field != mask_conf->dst.field)
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"destination field mask and template are not equal");
	if (action_conf->dst.field == RTE_FLOW_FIELD_POINTER ||
	    action_conf->dst.field == RTE_FLOW_FIELD_VALUE)
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"immediate value and pointer cannot be used as destination");
	if (mask_conf->dst.level != UINT32_MAX)
		return rte_flow_error_set(error, EINVAL,
			RTE_FLOW_ERROR_TYPE_ACTION, action,
			"destination encapsulation level must be fully masked");
	if (mask_conf->dst.offset != UINT32_MAX)
		return rte_flow_error_set(error, EINVAL,
			RTE_FLOW_ERROR_TYPE_ACTION, action,
			"destination offset level must be fully masked");
	if (action_conf->src.field != mask_conf->src.field)
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"destination field mask and template are not equal");
	if (action_conf->src.field != RTE_FLOW_FIELD_POINTER &&
	    action_conf->src.field != RTE_FLOW_FIELD_VALUE) {
		if (mask_conf->src.level != UINT32_MAX)
			return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"source encapsulation level must be fully masked");
		if (mask_conf->src.offset != UINT32_MAX)
			return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"source offset level must be fully masked");
	}
	if (mask_conf->width != UINT32_MAX)
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"modify_field width field must be fully masked");
	if (flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_START))
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"modifying arbitrary place in a packet is not supported");
	if (flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_VLAN_TYPE))
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"modifying vlan_type is not supported");
	if (flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_GENEVE_VNI))
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_ACTION, action,
				"modifying Geneve VNI is not supported");
	return 0;
}

static int
flow_hw_validate_action_represented_port(struct rte_eth_dev *dev,
					 const struct rte_flow_action *action,
					 const struct rte_flow_action *mask,
					 struct rte_flow_error *error)
{
	const struct rte_flow_action_ethdev *action_conf = action->conf;
	const struct rte_flow_action_ethdev *mask_conf = mask->conf;
	struct mlx5_priv *priv = dev->data->dev_private;

	if (!priv->sh->config.dv_esw_en)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					  "cannot use represented_port actions"
					  " without an E-Switch");
	if (mask_conf && mask_conf->port_id) {
		struct mlx5_priv *port_priv;
		struct mlx5_priv *dev_priv;

		if (!action_conf)
			return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
						  action, "port index was not provided");
		port_priv = mlx5_port_to_eswitch_info(action_conf->port_id, false);
		if (!port_priv)
			return rte_flow_error_set(error, rte_errno,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  action,
						  "failed to obtain E-Switch"
						  " info for port");
		dev_priv = mlx5_dev_to_eswitch_info(dev);
		if (!dev_priv)
			return rte_flow_error_set(error, rte_errno,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  action,
						  "failed to obtain E-Switch"
						  " info for transfer proxy");
		if (port_priv->domain_id != dev_priv->domain_id)
			return rte_flow_error_set(error, rte_errno,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  action,
						  "cannot forward to port from"
						  " a different E-Switch");
	}
	return 0;
}

/**
 * Validate AGE action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the indirect action.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[in] fixed_cnt
 *   Indicator if this list has a fixed COUNT action.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_validate_action_age(struct rte_eth_dev *dev,
			    const struct rte_flow_action *action,
			    uint64_t action_flags, bool fixed_cnt,
			    struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_age_info *age_info = GET_PORT_AGE_INFO(priv);

	if (!priv->sh->cdev->config.devx)
		return rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "AGE action not supported");
	if (age_info->ages_ipool == NULL)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "aging pool not initialized");
	if ((action_flags & MLX5_FLOW_ACTION_AGE) ||
	    (action_flags & MLX5_FLOW_ACTION_INDIRECT_AGE))
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "duplicate AGE actions set");
	if (fixed_cnt)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "AGE and fixed COUNT combination is not supported");
	return 0;
}

/**
 * Validate count action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the indirect action.
 * @param[in] mask
 *   Pointer to the indirect action mask.
 * @param[in] action_flags
 *   Holds the actions detected until now.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_validate_action_count(struct rte_eth_dev *dev,
			      const struct rte_flow_action *action,
			      const struct rte_flow_action *mask,
			      uint64_t action_flags,
			      struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_count *count = mask->conf;

	if (!priv->sh->cdev->config.devx)
		return rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "count action not supported");
	if (!priv->hws_cpool)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "counters pool not initialized");
	if ((action_flags & MLX5_FLOW_ACTION_COUNT) ||
	    (action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT))
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "duplicate count actions set");
	if (count && count->id && (action_flags & MLX5_FLOW_ACTION_AGE))
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, mask,
					  "AGE and COUNT action shared by mask combination is not supported");
	return 0;
}

/**
 * Validate meter_mark action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the indirect action.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_validate_action_meter_mark(struct rte_eth_dev *dev,
			      const struct rte_flow_action *action,
			      struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	RTE_SET_USED(action);

	if (!priv->sh->cdev->config.devx)
		return rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "meter_mark action not supported");
	if (!priv->hws_mpool)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "meter_mark pool not initialized");
	return 0;
}

/**
 * Validate indirect action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the indirect action.
 * @param[in] mask
 *   Pointer to the indirect action mask.
 * @param[in, out] action_flags
 *   Holds the actions detected until now.
 * @param[in, out] fixed_cnt
 *   Pointer to indicator if this list has a fixed COUNT action.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_validate_action_indirect(struct rte_eth_dev *dev,
				 const struct rte_flow_action *action,
				 const struct rte_flow_action *mask,
				 uint64_t *action_flags, bool *fixed_cnt,
				 struct rte_flow_error *error)
{
	uint32_t type;
	int ret;

	if (!mask)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
					  "Unable to determine indirect action type without a mask specified");
	type = mask->type;
	switch (type) {
	case RTE_FLOW_ACTION_TYPE_METER_MARK:
		ret = flow_hw_validate_action_meter_mark(dev, mask, error);
		if (ret < 0)
			return ret;
		*action_flags |= MLX5_FLOW_ACTION_METER;
		break;
	case RTE_FLOW_ACTION_TYPE_RSS:
		/* TODO: Validation logic (same as flow_hw_actions_validate) */
		*action_flags |= MLX5_FLOW_ACTION_RSS;
		break;
	case RTE_FLOW_ACTION_TYPE_CONNTRACK:
		/* TODO: Validation logic (same as flow_hw_actions_validate) */
		*action_flags |= MLX5_FLOW_ACTION_CT;
		break;
	case RTE_FLOW_ACTION_TYPE_COUNT:
		if (action->conf && mask->conf) {
			if ((*action_flags & MLX5_FLOW_ACTION_AGE) ||
			    (*action_flags & MLX5_FLOW_ACTION_INDIRECT_AGE))
				/*
				 * AGE cannot use indirect counter which is
				 * shared with enother flow rules.
				 */
				return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  NULL,
						  "AGE and fixed COUNT combination is not supported");
			*fixed_cnt = true;
		}
		ret = flow_hw_validate_action_count(dev, action, mask,
						    *action_flags, error);
		if (ret < 0)
			return ret;
		*action_flags |= MLX5_FLOW_ACTION_INDIRECT_COUNT;
		break;
	case RTE_FLOW_ACTION_TYPE_AGE:
		ret = flow_hw_validate_action_age(dev, action, *action_flags,
						  *fixed_cnt, error);
		if (ret < 0)
			return ret;
		*action_flags |= MLX5_FLOW_ACTION_INDIRECT_AGE;
		break;
	default:
		DRV_LOG(WARNING, "Unsupported shared action type: %d", type);
		return rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, mask,
					  "Unsupported indirect action type");
	}
	return 0;
}

/**
 * Validate raw_encap action.
 *
 * @param[in] dev
 *   Pointer to rte_eth_dev structure.
 * @param[in] action
 *   Pointer to the indirect action.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_validate_action_raw_encap(struct rte_eth_dev *dev __rte_unused,
				  const struct rte_flow_action *action,
				  struct rte_flow_error *error)
{
	const struct rte_flow_action_raw_encap *raw_encap_data = action->conf;

	if (!raw_encap_data || !raw_encap_data->size || !raw_encap_data->data)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "invalid raw_encap_data");
	return 0;
}

static inline uint16_t
flow_hw_template_expand_modify_field(const struct rte_flow_action actions[],
				     const struct rte_flow_action masks[],
				     const struct rte_flow_action *mf_action,
				     const struct rte_flow_action *mf_mask,
				     struct rte_flow_action *new_actions,
				     struct rte_flow_action *new_masks,
				     uint64_t flags, uint32_t act_num)
{
	uint32_t i, tail;

	MLX5_ASSERT(actions && masks);
	MLX5_ASSERT(new_actions && new_masks);
	MLX5_ASSERT(mf_action && mf_mask);
	if (flags & MLX5_FLOW_ACTION_MODIFY_FIELD) {
		/*
		 * Application action template already has Modify Field.
		 * It's location will be used in DR.
		 * Expanded MF action can be added before the END.
		 */
		i = act_num - 1;
		goto insert;
	}
	/**
	 * Locate the first action positioned BEFORE the new MF.
	 *
	 * Search for a place to insert modify header
	 * from the END action backwards:
	 * 1. END is always present in actions array
	 * 2. END location is always at action[act_num - 1]
	 * 3. END always positioned AFTER modify field location
	 *
	 * Relative actions order is the same for RX, TX and FDB.
	 *
	 * Current actions order (draft-3)
	 * @see action_order_arr[]
	 */
	for (i = act_num - 2; (int)i >= 0; i--) {
		enum rte_flow_action_type type = actions[i].type;

		if (type == RTE_FLOW_ACTION_TYPE_INDIRECT)
			type = masks[i].type;
		switch (type) {
		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
		case RTE_FLOW_ACTION_TYPE_DROP:
		case RTE_FLOW_ACTION_TYPE_JUMP:
		case RTE_FLOW_ACTION_TYPE_QUEUE:
		case RTE_FLOW_ACTION_TYPE_RSS:
		case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
		case RTE_FLOW_ACTION_TYPE_VOID:
		case RTE_FLOW_ACTION_TYPE_END:
			break;
		default:
			i++; /* new MF inserted AFTER actions[i] */
			goto insert;
			break;
		}
	}
	i = 0;
insert:
	tail = act_num - i; /* num action to move */
	memcpy(new_actions, actions, sizeof(actions[0]) * i);
	new_actions[i] = *mf_action;
	memcpy(new_actions + i + 1, actions + i, sizeof(actions[0]) * tail);
	memcpy(new_masks, masks, sizeof(masks[0]) * i);
	new_masks[i] = *mf_mask;
	memcpy(new_masks + i + 1, masks + i, sizeof(masks[0]) * tail);
	return i;
}

static int
flow_hw_validate_action_push_vlan(struct rte_eth_dev *dev,
				  const
				  struct rte_flow_actions_template_attr *attr,
				  const struct rte_flow_action *action,
				  const struct rte_flow_action *mask,
				  struct rte_flow_error *error)
{
#define X_FIELD(ptr, t, f) (((ptr)->conf) && ((t *)((ptr)->conf))->f)

	const bool masked_push =
		X_FIELD(mask + MLX5_HW_VLAN_PUSH_TYPE_IDX,
			const struct rte_flow_action_of_push_vlan, ethertype);
	bool masked_param;

	/*
	 * Mandatory actions order:
	 * OF_PUSH_VLAN / OF_SET_VLAN_VID [ / OF_SET_VLAN_PCP ]
	 */
	RTE_SET_USED(dev);
	RTE_SET_USED(attr);
	/* Check that mark matches OF_PUSH_VLAN */
	if (mask[MLX5_HW_VLAN_PUSH_TYPE_IDX].type !=
	    RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION,
					  action, "OF_PUSH_VLAN: mask does not match");
	/* Check that the second template and mask items are SET_VLAN_VID */
	if (action[MLX5_HW_VLAN_PUSH_VID_IDX].type !=
	    RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID ||
	    mask[MLX5_HW_VLAN_PUSH_VID_IDX].type !=
	    RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION,
					  action, "OF_PUSH_VLAN: invalid actions order");
	masked_param = X_FIELD(mask + MLX5_HW_VLAN_PUSH_VID_IDX,
			       const struct rte_flow_action_of_set_vlan_vid,
			       vlan_vid);
	/*
	 * PMD requires OF_SET_VLAN_VID mask to must match OF_PUSH_VLAN
	 */
	if (masked_push ^ masked_param)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_ACTION, action,
					  "OF_SET_VLAN_VID: mask does not match OF_PUSH_VLAN");
	if (is_of_vlan_pcp_present(action)) {
		if (mask[MLX5_HW_VLAN_PUSH_PCP_IDX].type !=
		     RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  action, "OF_SET_VLAN_PCP: missing mask configuration");
		masked_param = X_FIELD(mask + MLX5_HW_VLAN_PUSH_PCP_IDX,
				       const struct
				       rte_flow_action_of_set_vlan_pcp,
				       vlan_pcp);
		/*
		 * PMD requires OF_SET_VLAN_PCP mask to must match OF_PUSH_VLAN
		 */
		if (masked_push ^ masked_param)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION, action,
						  "OF_SET_VLAN_PCP: mask does not match OF_PUSH_VLAN");
	}
	return 0;
#undef X_FIELD
}

static int
mlx5_flow_hw_actions_validate(struct rte_eth_dev *dev,
			      const struct rte_flow_actions_template_attr *attr,
			      const struct rte_flow_action actions[],
			      const struct rte_flow_action masks[],
			      uint64_t *act_flags,
			      struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_count *count_mask = NULL;
	bool fixed_cnt = false;
	uint64_t action_flags = 0;
	uint16_t i;
	bool actions_end = false;
	int ret;

	/* FDB actions are only valid to proxy port. */
	if (attr->transfer && (!priv->sh->config.dv_esw_en || !priv->master))
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL,
					  "transfer actions are only valid to proxy port");
	for (i = 0; !actions_end; ++i) {
		const struct rte_flow_action *action = &actions[i];
		const struct rte_flow_action *mask = &masks[i];

		MLX5_ASSERT(i < MLX5_HW_MAX_ACTS);
		if (action->type != RTE_FLOW_ACTION_TYPE_INDIRECT &&
		    action->type != mask->type)
			return rte_flow_error_set(error, ENOTSUP,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  action,
						  "mask type does not match action type");
		switch (action->type) {
		case RTE_FLOW_ACTION_TYPE_VOID:
			break;
		case RTE_FLOW_ACTION_TYPE_INDIRECT:
			ret = flow_hw_validate_action_indirect(dev, action,
							       mask,
							       &action_flags,
							       &fixed_cnt,
							       error);
			if (ret < 0)
				return ret;
			break;
		case RTE_FLOW_ACTION_TYPE_MARK:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_MARK;
			break;
		case RTE_FLOW_ACTION_TYPE_DROP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_DROP;
			break;
		case RTE_FLOW_ACTION_TYPE_JUMP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_JUMP;
			break;
		case RTE_FLOW_ACTION_TYPE_QUEUE:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_QUEUE;
			break;
		case RTE_FLOW_ACTION_TYPE_RSS:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_RSS;
			break;
		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_ENCAP;
			break;
		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_ENCAP;
			break;
		case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_DECAP;
			break;
		case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_DECAP;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
			ret = flow_hw_validate_action_raw_encap(dev, action, error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_ENCAP;
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_DECAP;
			break;
		case RTE_FLOW_ACTION_TYPE_METER:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_METER;
			break;
		case RTE_FLOW_ACTION_TYPE_METER_MARK:
			ret = flow_hw_validate_action_meter_mark(dev, action,
								 error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_METER;
			break;
		case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
			ret = flow_hw_validate_action_modify_field(action,
									mask,
									error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
			break;
		case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
			ret = flow_hw_validate_action_represented_port
					(dev, action, mask, error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_PORT_ID;
			break;
		case RTE_FLOW_ACTION_TYPE_AGE:
			if (count_mask && count_mask->id)
				fixed_cnt = true;
			ret = flow_hw_validate_action_age(dev, action,
							  action_flags,
							  fixed_cnt, error);
			if (ret < 0)
				return ret;
			action_flags |= MLX5_FLOW_ACTION_AGE;
			break;
		case RTE_FLOW_ACTION_TYPE_COUNT:
			ret = flow_hw_validate_action_count(dev, action, mask,
							    action_flags,
							    error);
			if (ret < 0)
				return ret;
			count_mask = mask->conf;
			action_flags |= MLX5_FLOW_ACTION_COUNT;
			break;
		case RTE_FLOW_ACTION_TYPE_CONNTRACK:
			/* TODO: Validation logic */
			action_flags |= MLX5_FLOW_ACTION_CT;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
			action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
			action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
			ret = flow_hw_validate_action_push_vlan
					(dev, attr, action, mask, error);
			if (ret != 0)
				return ret;
			i += is_of_vlan_pcp_present(action) ?
				MLX5_HW_VLAN_PUSH_PCP_IDX :
				MLX5_HW_VLAN_PUSH_VID_IDX;
			action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
			break;
		case RTE_FLOW_ACTION_TYPE_END:
			actions_end = true;
			break;
		default:
			return rte_flow_error_set(error, ENOTSUP,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  action,
						  "action not supported in template API");
		}
	}
	if (act_flags != NULL)
		*act_flags = action_flags;
	return 0;
}

static int
flow_hw_actions_validate(struct rte_eth_dev *dev,
			 const struct rte_flow_actions_template_attr *attr,
			 const struct rte_flow_action actions[],
			 const struct rte_flow_action masks[],
			 struct rte_flow_error *error)
{
	return mlx5_flow_hw_actions_validate(dev, attr, actions, masks, NULL,
					     error);
}


static enum mlx5dr_action_type mlx5_hw_dr_action_types[] = {
	[RTE_FLOW_ACTION_TYPE_MARK] = MLX5DR_ACTION_TYP_TAG,
	[RTE_FLOW_ACTION_TYPE_DROP] = MLX5DR_ACTION_TYP_DROP,
	[RTE_FLOW_ACTION_TYPE_JUMP] = MLX5DR_ACTION_TYP_FT,
	[RTE_FLOW_ACTION_TYPE_QUEUE] = MLX5DR_ACTION_TYP_TIR,
	[RTE_FLOW_ACTION_TYPE_RSS] = MLX5DR_ACTION_TYP_TIR,
	[RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP] = MLX5DR_ACTION_TYP_L2_TO_TNL_L2,
	[RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP] = MLX5DR_ACTION_TYP_L2_TO_TNL_L2,
	[RTE_FLOW_ACTION_TYPE_VXLAN_DECAP] = MLX5DR_ACTION_TYP_TNL_L2_TO_L2,
	[RTE_FLOW_ACTION_TYPE_NVGRE_DECAP] = MLX5DR_ACTION_TYP_TNL_L2_TO_L2,
	[RTE_FLOW_ACTION_TYPE_MODIFY_FIELD] = MLX5DR_ACTION_TYP_MODIFY_HDR,
	[RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT] = MLX5DR_ACTION_TYP_VPORT,
	[RTE_FLOW_ACTION_TYPE_CONNTRACK] = MLX5DR_ACTION_TYP_ASO_CT,
	[RTE_FLOW_ACTION_TYPE_OF_POP_VLAN] = MLX5DR_ACTION_TYP_POP_VLAN,
	[RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN] = MLX5DR_ACTION_TYP_PUSH_VLAN,
};

static int
flow_hw_dr_actions_template_handle_shared(const struct rte_flow_action *mask,
					  unsigned int action_src,
					  enum mlx5dr_action_type *action_types,
					  uint16_t *curr_off, uint16_t *cnt_off,
					  struct rte_flow_actions_template *at)
{
	uint32_t type;

	if (!mask) {
		DRV_LOG(WARNING, "Unable to determine indirect action type "
			"without a mask specified");
		return -EINVAL;
	}
	type = mask->type;
	switch (type) {
	case RTE_FLOW_ACTION_TYPE_RSS:
		at->actions_off[action_src] = *curr_off;
		action_types[*curr_off] = MLX5DR_ACTION_TYP_TIR;
		*curr_off = *curr_off + 1;
		break;
	case RTE_FLOW_ACTION_TYPE_AGE:
	case RTE_FLOW_ACTION_TYPE_COUNT:
		/*
		 * Both AGE and COUNT action need counter, the first one fills
		 * the action_types array, and the second only saves the offset.
		 */
		if (*cnt_off == UINT16_MAX) {
			*cnt_off = *curr_off;
			action_types[*cnt_off] = MLX5DR_ACTION_TYP_CTR;
			*curr_off = *curr_off + 1;
		}
		at->actions_off[action_src] = *cnt_off;
		break;
	case RTE_FLOW_ACTION_TYPE_CONNTRACK:
		at->actions_off[action_src] = *curr_off;
		action_types[*curr_off] = MLX5DR_ACTION_TYP_ASO_CT;
		*curr_off = *curr_off + 1;
		break;
	case RTE_FLOW_ACTION_TYPE_METER_MARK:
		at->actions_off[action_src] = *curr_off;
		action_types[*curr_off] = MLX5DR_ACTION_TYP_ASO_METER;
		*curr_off = *curr_off + 1;
		break;
	default:
		DRV_LOG(WARNING, "Unsupported shared action type: %d", type);
		return -EINVAL;
	}
	return 0;
}

/**
 * Create DR action template based on a provided sequence of flow actions.
 *
 * @param[in] at
 *   Pointer to flow actions template to be updated.
 *
 * @return
 *   DR action template pointer on success and action offsets in @p at are updated.
 *   NULL otherwise.
 */
static struct mlx5dr_action_template *
flow_hw_dr_actions_template_create(struct rte_flow_actions_template *at)
{
	struct mlx5dr_action_template *dr_template;
	enum mlx5dr_action_type action_types[MLX5_HW_MAX_ACTS] = { MLX5DR_ACTION_TYP_LAST };
	unsigned int i;
	uint16_t curr_off;
	enum mlx5dr_action_type reformat_act_type = MLX5DR_ACTION_TYP_TNL_L2_TO_L2;
	uint16_t reformat_off = UINT16_MAX;
	uint16_t mhdr_off = UINT16_MAX;
	uint16_t cnt_off = UINT16_MAX;
	int ret;
	for (i = 0, curr_off = 0; at->actions[i].type != RTE_FLOW_ACTION_TYPE_END; ++i) {
		const struct rte_flow_action_raw_encap *raw_encap_data;
		size_t data_size;
		enum mlx5dr_action_type type;

		if (curr_off >= MLX5_HW_MAX_ACTS)
			goto err_actions_num;
		switch (at->actions[i].type) {
		case RTE_FLOW_ACTION_TYPE_VOID:
			break;
		case RTE_FLOW_ACTION_TYPE_INDIRECT:
			ret = flow_hw_dr_actions_template_handle_shared
								 (&at->masks[i],
								  i,
								  action_types,
								  &curr_off,
								  &cnt_off, at);
			if (ret)
				return NULL;
			break;
		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
		case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
		case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
			MLX5_ASSERT(reformat_off == UINT16_MAX);
			reformat_off = curr_off++;
			reformat_act_type = mlx5_hw_dr_action_types[at->actions[i].type];
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
			raw_encap_data = at->actions[i].conf;
			data_size = raw_encap_data->size;
			if (reformat_off != UINT16_MAX) {
				reformat_act_type = data_size < MLX5_ENCAPSULATION_DECISION_SIZE ?
					MLX5DR_ACTION_TYP_TNL_L3_TO_L2 :
					MLX5DR_ACTION_TYP_L2_TO_TNL_L3;
			} else {
				reformat_off = curr_off++;
				reformat_act_type = MLX5DR_ACTION_TYP_L2_TO_TNL_L2;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
			reformat_off = curr_off++;
			reformat_act_type = MLX5DR_ACTION_TYP_TNL_L2_TO_L2;
			break;
		case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
			if (mhdr_off == UINT16_MAX) {
				mhdr_off = curr_off++;
				type = mlx5_hw_dr_action_types[at->actions[i].type];
				action_types[mhdr_off] = type;
			}
			break;
		case RTE_FLOW_ACTION_TYPE_METER:
			at->actions_off[i] = curr_off;
			action_types[curr_off++] = MLX5DR_ACTION_TYP_ASO_METER;
			if (curr_off >= MLX5_HW_MAX_ACTS)
				goto err_actions_num;
			action_types[curr_off++] = MLX5DR_ACTION_TYP_FT;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
			type = mlx5_hw_dr_action_types[at->actions[i].type];
			at->actions_off[i] = curr_off;
			action_types[curr_off++] = type;
			i += is_of_vlan_pcp_present(at->actions + i) ?
				MLX5_HW_VLAN_PUSH_PCP_IDX :
				MLX5_HW_VLAN_PUSH_VID_IDX;
			break;
		case RTE_FLOW_ACTION_TYPE_METER_MARK:
			at->actions_off[i] = curr_off;
			action_types[curr_off++] = MLX5DR_ACTION_TYP_ASO_METER;
			if (curr_off >= MLX5_HW_MAX_ACTS)
				goto err_actions_num;
			break;
		case RTE_FLOW_ACTION_TYPE_AGE:
		case RTE_FLOW_ACTION_TYPE_COUNT:
			/*
			 * Both AGE and COUNT action need counter, the first
			 * one fills the action_types array, and the second only
			 * saves the offset.
			 */
			if (cnt_off == UINT16_MAX) {
				cnt_off = curr_off++;
				action_types[cnt_off] = MLX5DR_ACTION_TYP_CTR;
			}
			at->actions_off[i] = cnt_off;
			break;
		default:
			type = mlx5_hw_dr_action_types[at->actions[i].type];
			at->actions_off[i] = curr_off;
			action_types[curr_off++] = type;
			break;
		}
	}
	if (curr_off >= MLX5_HW_MAX_ACTS)
		goto err_actions_num;
	if (mhdr_off != UINT16_MAX)
		at->mhdr_off = mhdr_off;
	if (reformat_off != UINT16_MAX) {
		at->reformat_off = reformat_off;
		action_types[reformat_off] = reformat_act_type;
	}
	dr_template = mlx5dr_action_template_create(action_types);
	if (dr_template)
		at->dr_actions_num = curr_off;
	else
		DRV_LOG(ERR, "Failed to create DR action template: %d", rte_errno);
	return dr_template;
err_actions_num:
	DRV_LOG(ERR, "Number of HW actions (%u) exceeded maximum (%u) allowed in template",
		curr_off, MLX5_HW_MAX_ACTS);
	return NULL;
}

static void
flow_hw_set_vlan_vid(struct rte_eth_dev *dev,
		     struct rte_flow_action *ra,
		     struct rte_flow_action *rm,
		     struct rte_flow_action_modify_field *spec,
		     struct rte_flow_action_modify_field *mask,
		     int set_vlan_vid_ix)
{
	struct rte_flow_error error;
	const bool masked = rm[set_vlan_vid_ix].conf &&
		(((const struct rte_flow_action_of_set_vlan_vid *)
			rm[set_vlan_vid_ix].conf)->vlan_vid != 0);
	const struct rte_flow_action_of_set_vlan_vid *conf =
		ra[set_vlan_vid_ix].conf;
	rte_be16_t vid = masked ? conf->vlan_vid : 0;
	int width = mlx5_flow_item_field_width(dev, RTE_FLOW_FIELD_VLAN_ID, 0,
					       NULL, &error);
	*spec = (typeof(*spec)) {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = RTE_FLOW_FIELD_VLAN_ID,
			.level = 0, .offset = 0,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
			.level = vid,
			.offset = 0,
		},
		.width = width,
	};
	*mask = (typeof(*mask)) {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = RTE_FLOW_FIELD_VLAN_ID,
			.level = 0xffffffff, .offset = 0xffffffff,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
			.level = masked ? (1U << width) - 1 : 0,
			.offset = 0,
		},
		.width = 0xffffffff,
	};
	ra[set_vlan_vid_ix].type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD;
	ra[set_vlan_vid_ix].conf = spec;
	rm[set_vlan_vid_ix].type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD;
	rm[set_vlan_vid_ix].conf = mask;
}

static __rte_always_inline int
flow_hw_set_vlan_vid_construct(struct rte_eth_dev *dev,
			       struct mlx5_hw_q_job *job,
			       struct mlx5_action_construct_data *act_data,
			       const struct mlx5_hw_actions *hw_acts,
			       const struct rte_flow_action *action)
{
	struct rte_flow_error error;
	rte_be16_t vid = ((const struct rte_flow_action_of_set_vlan_vid *)
			   action->conf)->vlan_vid;
	int width = mlx5_flow_item_field_width(dev, RTE_FLOW_FIELD_VLAN_ID, 0,
					       NULL, &error);
	struct rte_flow_action_modify_field conf = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = RTE_FLOW_FIELD_VLAN_ID,
			.level = 0, .offset = 0,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
			.level = vid,
			.offset = 0,
		},
		.width = width,
	};
	struct rte_flow_action modify_action = {
		.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
		.conf = &conf
	};

	return flow_hw_modify_field_construct(job, act_data, hw_acts,
					      &modify_action);
}

/**
 * Create flow action template.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] attr
 *   Pointer to the action template attributes.
 * @param[in] actions
 *   Associated actions (list terminated by the END action).
 * @param[in] masks
 *   List of actions that marks which of the action's member is constant.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   Action template pointer on success, NULL otherwise and rte_errno is set.
 */
static struct rte_flow_actions_template *
flow_hw_actions_template_create(struct rte_eth_dev *dev,
			const struct rte_flow_actions_template_attr *attr,
			const struct rte_flow_action actions[],
			const struct rte_flow_action masks[],
			struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	int len, act_len, mask_len;
	unsigned int act_num;
	unsigned int i;
	struct rte_flow_actions_template *at = NULL;
	uint16_t pos = UINT16_MAX;
	uint64_t action_flags = 0;
	struct rte_flow_action tmp_action[MLX5_HW_MAX_ACTS];
	struct rte_flow_action tmp_mask[MLX5_HW_MAX_ACTS];
	struct rte_flow_action *ra = (void *)(uintptr_t)actions;
	struct rte_flow_action *rm = (void *)(uintptr_t)masks;
	int set_vlan_vid_ix = -1;
	struct rte_flow_action_modify_field set_vlan_vid_spec = {0, };
	struct rte_flow_action_modify_field set_vlan_vid_mask = {0, };
	const struct rte_flow_action_modify_field rx_mreg = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = REG_B,
		},
		.src = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = REG_C_1,
		},
		.width = 32,
	};
	const struct rte_flow_action_modify_field rx_mreg_mask = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT32_MAX,
			.offset = UINT32_MAX,
		},
		.src = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT32_MAX,
			.offset = UINT32_MAX,
		},
		.width = UINT32_MAX,
	};
	const struct rte_flow_action rx_cpy = {
		.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
		.conf = &rx_mreg,
	};
	const struct rte_flow_action rx_cpy_mask = {
		.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
		.conf = &rx_mreg_mask,
	};

	if (mlx5_flow_hw_actions_validate(dev, attr, actions, masks,
					  &action_flags, error))
		return NULL;
	for (i = 0; ra[i].type != RTE_FLOW_ACTION_TYPE_END; ++i) {
		switch (ra[i].type) {
		/* OF_PUSH_VLAN *MUST* come before OF_SET_VLAN_VID */
		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
			i += is_of_vlan_pcp_present(ra + i) ?
				MLX5_HW_VLAN_PUSH_PCP_IDX :
				MLX5_HW_VLAN_PUSH_VID_IDX;
			break;
		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
			set_vlan_vid_ix = i;
			break;
		default:
			break;
		}
	}
	/*
	 * Count flow actions to allocate required space for storing DR offsets and to check
	 * if temporary buffer would not be overrun.
	 */
	act_num = i + 1;
	if (act_num >= MLX5_HW_MAX_ACTS) {
		rte_flow_error_set(error, EINVAL,
				   RTE_FLOW_ERROR_TYPE_ACTION, NULL, "Too many actions");
		return NULL;
	}
	if (priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS &&
	    priv->sh->config.dv_esw_en &&
	    (action_flags & (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS))) {
		/* Insert META copy */
		if (act_num + 1 > MLX5_HW_MAX_ACTS) {
			rte_flow_error_set(error, E2BIG,
					   RTE_FLOW_ERROR_TYPE_ACTION,
					   NULL, "cannot expand: too many actions");
			return NULL;
		}
		/* Application should make sure only one Q/RSS exist in one rule. */
		pos = flow_hw_template_expand_modify_field(actions, masks,
							   &rx_cpy,
							   &rx_cpy_mask,
							   tmp_action, tmp_mask,
							   action_flags,
							   act_num);
		ra = tmp_action;
		rm = tmp_mask;
		act_num++;
		action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
	}
	if (set_vlan_vid_ix != -1) {
		/* If temporary action buffer was not used, copy template actions to it */
		if (ra == actions && rm == masks) {
			for (i = 0; i < act_num; ++i) {
				tmp_action[i] = actions[i];
				tmp_mask[i] = masks[i];
				if (actions[i].type == RTE_FLOW_ACTION_TYPE_END)
					break;
			}
			ra = tmp_action;
			rm = tmp_mask;
		}
		flow_hw_set_vlan_vid(dev, ra, rm,
				     &set_vlan_vid_spec, &set_vlan_vid_mask,
				     set_vlan_vid_ix);
	}
	act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, ra, error);
	if (act_len <= 0)
		return NULL;
	len = RTE_ALIGN(act_len, 16);
	mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, rm, error);
	if (mask_len <= 0)
		return NULL;
	len += RTE_ALIGN(mask_len, 16);
	len += RTE_ALIGN(act_num * sizeof(*at->actions_off), 16);
	at = mlx5_malloc(MLX5_MEM_ZERO, len + sizeof(*at),
			 RTE_CACHE_LINE_SIZE, rte_socket_id());
	if (!at) {
		rte_flow_error_set(error, ENOMEM,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "cannot allocate action template");
		return NULL;
	}
	/* Actions part is in the first part. */
	at->attr = *attr;
	at->actions = (struct rte_flow_action *)(at + 1);
	act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->actions,
				len, ra, error);
	if (act_len <= 0)
		goto error;
	/* Masks part is in the second part. */
	at->masks = (struct rte_flow_action *)(((uint8_t *)at->actions) + act_len);
	mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->masks,
				 len - act_len, rm, error);
	if (mask_len <= 0)
		goto error;
	/* DR actions offsets in the third part. */
	at->actions_off = (uint16_t *)((uint8_t *)at->masks + mask_len);
	at->actions_num = act_num;
	for (i = 0; i < at->actions_num; ++i)
		at->actions_off[i] = UINT16_MAX;
	at->reformat_off = UINT16_MAX;
	at->mhdr_off = UINT16_MAX;
	at->rx_cpy_pos = pos;
	/*
	 * mlx5 PMD hacks indirect action index directly to the action conf.
	 * The rte_flow_conv() function copies the content from conf pointer.
	 * Need to restore the indirect action index from action conf here.
	 */
	for (i = 0; actions->type != RTE_FLOW_ACTION_TYPE_END;
	     actions++, masks++, i++) {
		if (actions->type == RTE_FLOW_ACTION_TYPE_INDIRECT) {
			at->actions[i].conf = actions->conf;
			at->masks[i].conf = masks->conf;
		}
	}
	at->tmpl = flow_hw_dr_actions_template_create(at);
	if (!at->tmpl)
		goto error;
	at->action_flags = action_flags;
	__atomic_fetch_add(&at->refcnt, 1, __ATOMIC_RELAXED);
	LIST_INSERT_HEAD(&priv->flow_hw_at, at, next);
	return at;
error:
	if (at) {
		if (at->tmpl)
			mlx5dr_action_template_destroy(at->tmpl);
		mlx5_free(at);
	}
	return NULL;
}

/**
 * Destroy flow action template.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] template
 *   Pointer to the action template to be destroyed.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_actions_template_destroy(struct rte_eth_dev *dev __rte_unused,
				 struct rte_flow_actions_template *template,
				 struct rte_flow_error *error __rte_unused)
{
	if (__atomic_load_n(&template->refcnt, __ATOMIC_RELAXED) > 1) {
		DRV_LOG(WARNING, "Action template %p is still in use.",
			(void *)template);
		return rte_flow_error_set(error, EBUSY,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "action template in using");
	}
	LIST_REMOVE(template, next);
	if (template->tmpl)
		mlx5dr_action_template_destroy(template->tmpl);
	mlx5_free(template);
	return 0;
}

static uint32_t
flow_hw_count_items(const struct rte_flow_item *items)
{
	const struct rte_flow_item *curr_item;
	uint32_t nb_items;

	nb_items = 0;
	for (curr_item = items; curr_item->type != RTE_FLOW_ITEM_TYPE_END; ++curr_item)
		++nb_items;
	return ++nb_items;
}

static struct rte_flow_item *
flow_hw_prepend_item(const struct rte_flow_item *items,
		     const uint32_t nb_items,
		     const struct rte_flow_item *new_item,
		     struct rte_flow_error *error)
{
	struct rte_flow_item *copied_items;
	size_t size;

	/* Allocate new array of items. */
	size = sizeof(*copied_items) * (nb_items + 1);
	copied_items = mlx5_malloc(MLX5_MEM_ZERO, size, 0, rte_socket_id());
	if (!copied_items) {
		rte_flow_error_set(error, ENOMEM,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "cannot allocate item template");
		return NULL;
	}
	/* Put new item at the beginning and copy the rest. */
	copied_items[0] = *new_item;
	rte_memcpy(&copied_items[1], items, sizeof(*items) * nb_items);
	return copied_items;
}

static int
flow_hw_pattern_validate(struct rte_eth_dev *dev,
			 const struct rte_flow_pattern_template_attr *attr,
			 const struct rte_flow_item items[],
			 struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	int i;
	bool items_end = false;

	if (!attr->ingress && !attr->egress && !attr->transfer)
		return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR, NULL,
					  "at least one of the direction attributes"
					  " must be specified");
	if (priv->sh->config.dv_esw_en) {
		MLX5_ASSERT(priv->master || priv->representor);
		if (priv->master) {
			if ((attr->ingress && attr->egress) ||
			    (attr->ingress && attr->transfer) ||
			    (attr->egress && attr->transfer))
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_ATTR, NULL,
							  "only one direction attribute at once"
							  " can be used on transfer proxy port");
		} else {
			if (attr->transfer)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, NULL,
							  "transfer attribute cannot be used with"
							  " port representors");
			if (attr->ingress && attr->egress)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_ATTR, NULL,
							  "ingress and egress direction attributes"
							  " cannot be used at the same time on"
							  " port representors");
		}
	} else {
		if (attr->transfer)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, NULL,
						  "transfer attribute cannot be used when"
						  " E-Switch is disabled");
	}
	for (i = 0; !items_end; i++) {
		int type = items[i].type;

		switch (type) {
		case RTE_FLOW_ITEM_TYPE_TAG:
		{
			int reg;
			const struct rte_flow_item_tag *tag =
				(const struct rte_flow_item_tag *)items[i].spec;

			reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_TAG, tag->index);
			if (reg == REG_NON)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
							  NULL,
							  "Unsupported tag index");
			break;
		}
		case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
		{
			const struct rte_flow_item_tag *tag =
				(const struct rte_flow_item_tag *)items[i].spec;
			uint8_t regcs = (uint8_t)priv->sh->cdev->config.hca_attr.set_reg_c;

			if (!((1 << (tag->index - REG_C_0)) & regcs))
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
							  NULL,
							  "Unsupported internal tag index");
			break;
		}
		case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT:
			if (attr->ingress && priv->sh->config.repr_matching)
				return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
						  "represented port item cannot be used"
						  " when ingress attribute is set");
			if (attr->egress)
				return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
						  "represented port item cannot be used"
						  " when egress attribute is set");
			break;
		case RTE_FLOW_ITEM_TYPE_META:
			if (!priv->sh->config.dv_esw_en ||
			    priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) {
				if (attr->ingress)
					return rte_flow_error_set(error, EINVAL,
								  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
								  "META item is not supported"
								  " on current FW with ingress"
								  " attribute");
			}
			break;
		case RTE_FLOW_ITEM_TYPE_METER_COLOR:
		{
			int reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_METER_COLOR, 0);
			if (reg == REG_NON)
				return rte_flow_error_set(error, EINVAL,
							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
							  NULL,
							  "Unsupported meter color register");
			break;
		}
		case RTE_FLOW_ITEM_TYPE_VOID:
		case RTE_FLOW_ITEM_TYPE_ETH:
		case RTE_FLOW_ITEM_TYPE_VLAN:
		case RTE_FLOW_ITEM_TYPE_IPV4:
		case RTE_FLOW_ITEM_TYPE_IPV6:
		case RTE_FLOW_ITEM_TYPE_UDP:
		case RTE_FLOW_ITEM_TYPE_TCP:
		case RTE_FLOW_ITEM_TYPE_GTP:
		case RTE_FLOW_ITEM_TYPE_GTP_PSC:
		case RTE_FLOW_ITEM_TYPE_VXLAN:
		case MLX5_RTE_FLOW_ITEM_TYPE_SQ:
		case RTE_FLOW_ITEM_TYPE_GRE:
		case RTE_FLOW_ITEM_TYPE_GRE_KEY:
		case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
		case RTE_FLOW_ITEM_TYPE_ICMP:
		case RTE_FLOW_ITEM_TYPE_ICMP6:
		case RTE_FLOW_ITEM_TYPE_CONNTRACK:
			break;
		case RTE_FLOW_ITEM_TYPE_INTEGRITY:
			/*
			 * Integrity flow item validation require access to
			 * both item mask and spec.
			 * Current HWS model allows item mask in pattern
			 * template and item spec in flow rule.
			 */
			break;
		case RTE_FLOW_ITEM_TYPE_END:
			items_end = true;
			break;
		default:
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
						  NULL,
						  "Unsupported item type");
		}
	}
	return 0;
}

static bool
flow_hw_pattern_has_sq_match(const struct rte_flow_item *items)
{
	unsigned int i;

	for (i = 0; items[i].type != RTE_FLOW_ITEM_TYPE_END; ++i)
		if (items[i].type == (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ)
			return true;
	return false;
}

/**
 * Create flow item template.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] attr
 *   Pointer to the item template attributes.
 * @param[in] items
 *   The template item pattern.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *  Item template pointer on success, NULL otherwise and rte_errno is set.
 */
static struct rte_flow_pattern_template *
flow_hw_pattern_template_create(struct rte_eth_dev *dev,
			     const struct rte_flow_pattern_template_attr *attr,
			     const struct rte_flow_item items[],
			     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_pattern_template *it;
	struct rte_flow_item *copied_items = NULL;
	const struct rte_flow_item *tmpl_items;
	uint64_t orig_item_nb;
	struct rte_flow_item port = {
		.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
		.mask = &rte_flow_item_ethdev_mask,
	};
	struct rte_flow_item_tag tag_v = {
		.data = 0,
		.index = REG_C_0,
	};
	struct rte_flow_item_tag tag_m = {
		.data = flow_hw_tx_tag_regc_mask(dev),
		.index = 0xff,
	};
	struct rte_flow_item tag = {
		.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG,
		.spec = &tag_v,
		.mask = &tag_m,
		.last = NULL
	};

	if (flow_hw_pattern_validate(dev, attr, items, error))
		return NULL;
	orig_item_nb = flow_hw_count_items(items);
	if (priv->sh->config.dv_esw_en &&
	    priv->sh->config.repr_matching &&
	    attr->ingress && !attr->egress && !attr->transfer) {
		copied_items = flow_hw_prepend_item(items, orig_item_nb, &port, error);
		if (!copied_items)
			return NULL;
		tmpl_items = copied_items;
	} else if (priv->sh->config.dv_esw_en &&
		   priv->sh->config.repr_matching &&
		   !attr->ingress && attr->egress && !attr->transfer) {
		if (flow_hw_pattern_has_sq_match(items)) {
			DRV_LOG(DEBUG, "Port %u omitting implicit REG_C_0 match for egress "
				       "pattern template", dev->data->port_id);
			tmpl_items = items;
			goto setup_pattern_template;
		}
		copied_items = flow_hw_prepend_item(items, orig_item_nb, &tag, error);
		if (!copied_items)
			return NULL;
		tmpl_items = copied_items;
	} else {
		tmpl_items = items;
	}
setup_pattern_template:
	it = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*it), 0, rte_socket_id());
	if (!it) {
		if (copied_items)
			mlx5_free(copied_items);
		rte_flow_error_set(error, ENOMEM,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "cannot allocate item template");
		return NULL;
	}
	it->attr = *attr;
	it->orig_item_nb = orig_item_nb;
	it->mt = mlx5dr_match_template_create(tmpl_items, attr->relaxed_matching);
	if (!it->mt) {
		if (copied_items)
			mlx5_free(copied_items);
		mlx5_free(it);
		rte_flow_error_set(error, rte_errno,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "cannot create match template");
		return NULL;
	}
	it->item_flags = flow_hw_rss_item_flags_get(tmpl_items);
	if (copied_items) {
		if (attr->ingress)
			it->implicit_port = true;
		else if (attr->egress)
			it->implicit_tag = true;
		mlx5_free(copied_items);
	}
	__atomic_fetch_add(&it->refcnt, 1, __ATOMIC_RELAXED);
	LIST_INSERT_HEAD(&priv->flow_hw_itt, it, next);
	return it;
}

/**
 * Destroy flow item template.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] template
 *   Pointer to the item template to be destroyed.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_pattern_template_destroy(struct rte_eth_dev *dev __rte_unused,
			      struct rte_flow_pattern_template *template,
			      struct rte_flow_error *error __rte_unused)
{
	if (__atomic_load_n(&template->refcnt, __ATOMIC_RELAXED) > 1) {
		DRV_LOG(WARNING, "Item template %p is still in use.",
			(void *)template);
		return rte_flow_error_set(error, EBUSY,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "item template in using");
	}
	LIST_REMOVE(template, next);
	claim_zero(mlx5dr_match_template_destroy(template->mt));
	mlx5_free(template);
	return 0;
}

/*
 * Get information about HWS pre-configurable resources.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[out] port_info
 *   Pointer to port information.
 * @param[out] queue_info
 *   Pointer to queue information.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_info_get(struct rte_eth_dev *dev,
		 struct rte_flow_port_info *port_info,
		 struct rte_flow_queue_info *queue_info,
		 struct rte_flow_error *error __rte_unused)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint16_t port_id = dev->data->port_id;
	struct rte_mtr_capabilities mtr_cap;
	int ret;

	memset(port_info, 0, sizeof(*port_info));
	/* Queue size is unlimited from low-level. */
	port_info->max_nb_queues = UINT32_MAX;
	queue_info->max_size = UINT32_MAX;

	memset(&mtr_cap, 0, sizeof(struct rte_mtr_capabilities));
	ret = rte_mtr_capabilities_get(port_id, &mtr_cap, NULL);
	if (!ret)
		port_info->max_nb_meters = mtr_cap.n_max;
	port_info->max_nb_counters = priv->sh->hws_max_nb_counters;
	port_info->max_nb_aging_objects = port_info->max_nb_counters;
	return 0;
}

/**
 * Create group callback.
 *
 * @param[in] tool_ctx
 *   Pointer to the hash list related context.
 * @param[in] cb_ctx
 *   Pointer to the group creation context.
 *
 * @return
 *   Group entry on success, NULL otherwise and rte_errno is set.
 */
struct mlx5_list_entry *
flow_hw_grp_create_cb(void *tool_ctx, void *cb_ctx)
{
	struct mlx5_dev_ctx_shared *sh = tool_ctx;
	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
	struct rte_eth_dev *dev = ctx->dev;
	struct rte_flow_attr *attr = (struct rte_flow_attr *)ctx->data;
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5dr_table_attr dr_tbl_attr = {0};
	struct rte_flow_error *error = ctx->error;
	struct mlx5_flow_group *grp_data;
	struct mlx5dr_table *tbl = NULL;
	struct mlx5dr_action *jump;
	uint32_t idx = 0;

	grp_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_HW_GRP], &idx);
	if (!grp_data) {
		rte_flow_error_set(error, ENOMEM,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "cannot allocate flow table data entry");
		return NULL;
	}
	dr_tbl_attr.level = attr->group;
	if (attr->transfer)
		dr_tbl_attr.type = MLX5DR_TABLE_TYPE_FDB;
	else if (attr->egress)
		dr_tbl_attr.type = MLX5DR_TABLE_TYPE_NIC_TX;
	else
		dr_tbl_attr.type = MLX5DR_TABLE_TYPE_NIC_RX;
	tbl = mlx5dr_table_create(priv->dr_ctx, &dr_tbl_attr);
	if (!tbl)
		goto error;
	grp_data->tbl = tbl;
	if (attr->group) {
		/* Jump action be used by non-root table. */
		jump = mlx5dr_action_create_dest_table
			(priv->dr_ctx, tbl,
			 mlx5_hw_act_flag[!!attr->group][dr_tbl_attr.type]);
		if (!jump)
			goto error;
		grp_data->jump.hws_action = jump;
		/* Jump action be used by root table.  */
		jump = mlx5dr_action_create_dest_table
			(priv->dr_ctx, tbl,
			 mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_ROOT]
					 [dr_tbl_attr.type]);
		if (!jump)
			goto error;
		grp_data->jump.root_action = jump;
	}
	grp_data->dev = dev;
	grp_data->idx = idx;
	grp_data->group_id = attr->group;
	grp_data->type = dr_tbl_attr.type;
	return &grp_data->entry;
error:
	if (grp_data->jump.root_action)
		mlx5dr_action_destroy(grp_data->jump.root_action);
	if (grp_data->jump.hws_action)
		mlx5dr_action_destroy(grp_data->jump.hws_action);
	if (tbl)
		mlx5dr_table_destroy(tbl);
	if (idx)
		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], idx);
	rte_flow_error_set(error, ENOMEM,
			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
			   NULL,
			   "cannot allocate flow dr table");
	return NULL;
}

/**
 * Remove group callback.
 *
 * @param[in] tool_ctx
 *   Pointer to the hash list related context.
 * @param[in] entry
 *   Pointer to the entry to be removed.
 */
void
flow_hw_grp_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
{
	struct mlx5_dev_ctx_shared *sh = tool_ctx;
	struct mlx5_flow_group *grp_data =
		    container_of(entry, struct mlx5_flow_group, entry);

	MLX5_ASSERT(entry && sh);
	/* To use the wrapper glue functions instead. */
	if (grp_data->jump.hws_action)
		mlx5dr_action_destroy(grp_data->jump.hws_action);
	if (grp_data->jump.root_action)
		mlx5dr_action_destroy(grp_data->jump.root_action);
	mlx5dr_table_destroy(grp_data->tbl);
	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], grp_data->idx);
}

/**
 * Match group callback.
 *
 * @param[in] tool_ctx
 *   Pointer to the hash list related context.
 * @param[in] entry
 *   Pointer to the group to be matched.
 * @param[in] cb_ctx
 *   Pointer to the group matching context.
 *
 * @return
 *   0 on matched, 1 on miss matched.
 */
int
flow_hw_grp_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
		     void *cb_ctx)
{
	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
	struct mlx5_flow_group *grp_data =
		container_of(entry, struct mlx5_flow_group, entry);
	struct rte_flow_attr *attr =
			(struct rte_flow_attr *)ctx->data;

	return (grp_data->dev != ctx->dev) ||
		(grp_data->group_id != attr->group) ||
		((grp_data->type != MLX5DR_TABLE_TYPE_FDB) &&
		attr->transfer) ||
		((grp_data->type != MLX5DR_TABLE_TYPE_NIC_TX) &&
		attr->egress) ||
		((grp_data->type != MLX5DR_TABLE_TYPE_NIC_RX) &&
		attr->ingress);
}

/**
 * Clone group entry callback.
 *
 * @param[in] tool_ctx
 *   Pointer to the hash list related context.
 * @param[in] entry
 *   Pointer to the group to be matched.
 * @param[in] cb_ctx
 *   Pointer to the group matching context.
 *
 * @return
 *   0 on matched, 1 on miss matched.
 */
struct mlx5_list_entry *
flow_hw_grp_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
		     void *cb_ctx)
{
	struct mlx5_dev_ctx_shared *sh = tool_ctx;
	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
	struct mlx5_flow_group *grp_data;
	struct rte_flow_error *error = ctx->error;
	uint32_t idx = 0;

	grp_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_HW_GRP], &idx);
	if (!grp_data) {
		rte_flow_error_set(error, ENOMEM,
				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				   NULL,
				   "cannot allocate flow table data entry");
		return NULL;
	}
	memcpy(grp_data, oentry, sizeof(*grp_data));
	grp_data->idx = idx;
	return &grp_data->entry;
}

/**
 * Free cloned group entry callback.
 *
 * @param[in] tool_ctx
 *   Pointer to the hash list related context.
 * @param[in] entry
 *   Pointer to the group to be freed.
 */
void
flow_hw_grp_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
{
	struct mlx5_dev_ctx_shared *sh = tool_ctx;
	struct mlx5_flow_group *grp_data =
		    container_of(entry, struct mlx5_flow_group, entry);

	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], grp_data->idx);
}

/**
 * Create and cache a vport action for given @p dev port. vport actions
 * cache is used in HWS with FDB flows.
 *
 * This function does not create any function if proxy port for @p dev port
 * was not configured for HW Steering.
 *
 * This function assumes that E-Switch is enabled and PMD is running with
 * HW Steering configured.
 *
 * @param dev
 *   Pointer to Ethernet device which will be the action destination.
 *
 * @return
 *   0 on success, positive value otherwise.
 */
int
flow_hw_create_vport_action(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_eth_dev *proxy_dev;
	struct mlx5_priv *proxy_priv;
	uint16_t port_id = dev->data->port_id;
	uint16_t proxy_port_id = port_id;
	int ret;

	ret = mlx5_flow_pick_transfer_proxy(dev, &proxy_port_id, NULL);
	if (ret)
		return ret;
	proxy_dev = &rte_eth_devices[proxy_port_id];
	proxy_priv = proxy_dev->data->dev_private;
	if (!proxy_priv->hw_vport)
		return 0;
	if (proxy_priv->hw_vport[port_id]) {
		DRV_LOG(ERR, "port %u HWS vport action already created",
			port_id);
		return -EINVAL;
	}
	proxy_priv->hw_vport[port_id] = mlx5dr_action_create_dest_vport
			(proxy_priv->dr_ctx, priv->dev_port,
			 MLX5DR_ACTION_FLAG_HWS_FDB);
	if (!proxy_priv->hw_vport[port_id]) {
		DRV_LOG(ERR, "port %u unable to create HWS vport action",
			port_id);
		return -EINVAL;
	}
	return 0;
}

/**
 * Destroys the vport action associated with @p dev device
 * from actions' cache.
 *
 * This function does not destroy any action if there is no action cached
 * for @p dev or proxy port was not configured for HW Steering.
 *
 * This function assumes that E-Switch is enabled and PMD is running with
 * HW Steering configured.
 *
 * @param dev
 *   Pointer to Ethernet device which will be the action destination.
 */
void
flow_hw_destroy_vport_action(struct rte_eth_dev *dev)
{
	struct rte_eth_dev *proxy_dev;
	struct mlx5_priv *proxy_priv;
	uint16_t port_id = dev->data->port_id;
	uint16_t proxy_port_id = port_id;

	if (mlx5_flow_pick_transfer_proxy(dev, &proxy_port_id, NULL))
		return;
	proxy_dev = &rte_eth_devices[proxy_port_id];
	proxy_priv = proxy_dev->data->dev_private;
	if (!proxy_priv->hw_vport || !proxy_priv->hw_vport[port_id])
		return;
	mlx5dr_action_destroy(proxy_priv->hw_vport[port_id]);
	proxy_priv->hw_vport[port_id] = NULL;
}

static int
flow_hw_create_vport_actions(struct mlx5_priv *priv)
{
	uint16_t port_id;

	MLX5_ASSERT(!priv->hw_vport);
	priv->hw_vport = mlx5_malloc(MLX5_MEM_ZERO,
				     sizeof(*priv->hw_vport) * RTE_MAX_ETHPORTS,
				     0, SOCKET_ID_ANY);
	if (!priv->hw_vport)
		return -ENOMEM;
	DRV_LOG(DEBUG, "port %u :: creating vport actions", priv->dev_data->port_id);
	DRV_LOG(DEBUG, "port %u ::    domain_id=%u", priv->dev_data->port_id, priv->domain_id);
	MLX5_ETH_FOREACH_DEV(port_id, NULL) {
		struct mlx5_priv *port_priv = rte_eth_devices[port_id].data->dev_private;

		if (!port_priv ||
		    port_priv->domain_id != priv->domain_id)
			continue;
		DRV_LOG(DEBUG, "port %u :: for port_id=%u, calling mlx5dr_action_create_dest_vport() with ibport=%u",
			priv->dev_data->port_id, port_id, port_priv->dev_port);
		priv->hw_vport[port_id] = mlx5dr_action_create_dest_vport
				(priv->dr_ctx, port_priv->dev_port,
				 MLX5DR_ACTION_FLAG_HWS_FDB);
		DRV_LOG(DEBUG, "port %u :: priv->hw_vport[%u]=%p",
			priv->dev_data->port_id, port_id, (void *)priv->hw_vport[port_id]);
		if (!priv->hw_vport[port_id])
			return -EINVAL;
	}
	return 0;
}

static void
flow_hw_free_vport_actions(struct mlx5_priv *priv)
{
	uint16_t port_id;

	if (!priv->hw_vport)
		return;
	for (port_id = 0; port_id < RTE_MAX_ETHPORTS; ++port_id)
		if (priv->hw_vport[port_id])
			mlx5dr_action_destroy(priv->hw_vport[port_id]);
	mlx5_free(priv->hw_vport);
	priv->hw_vport = NULL;
}

/**
 * Create an egress pattern template matching on source SQ.
 *
 * @param dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   Pointer to pattern template on success. NULL otherwise, and rte_errno is set.
 */
static struct rte_flow_pattern_template *
flow_hw_create_tx_repr_sq_pattern_tmpl(struct rte_eth_dev *dev)
{
	struct rte_flow_pattern_template_attr attr = {
		.relaxed_matching = 0,
		.egress = 1,
	};
	struct mlx5_rte_flow_item_sq sq_mask = {
		.queue = UINT32_MAX,
	};
	struct rte_flow_item items[] = {
		{
			.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
			.mask = &sq_mask,
		},
		{
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};

	return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}

static __rte_always_inline uint32_t
flow_hw_tx_tag_regc_mask(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint32_t mask = priv->sh->dv_regc0_mask;

	/* Mask is verified during device initialization. Sanity checking here. */
	MLX5_ASSERT(mask != 0);
	/*
	 * Availability of sufficient number of bits in REG_C_0 is verified on initialization.
	 * Sanity checking here.
	 */
	MLX5_ASSERT(__builtin_popcount(mask) >= __builtin_popcount(priv->vport_meta_mask));
	return mask;
}

static __rte_always_inline uint32_t
flow_hw_tx_tag_regc_value(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint32_t tag;

	/* Mask is verified during device initialization. Sanity checking here. */
	MLX5_ASSERT(priv->vport_meta_mask != 0);
	tag = priv->vport_meta_tag >> (rte_bsf32(priv->vport_meta_mask));
	/*
	 * Availability of sufficient number of bits in REG_C_0 is verified on initialization.
	 * Sanity checking here.
	 */
	MLX5_ASSERT((tag & priv->sh->dv_regc0_mask) == tag);
	return tag;
}

static void
flow_hw_update_action_mask(struct rte_flow_action *action,
			   struct rte_flow_action *mask,
			   enum rte_flow_action_type type,
			   void *conf_v,
			   void *conf_m)
{
	action->type = type;
	action->conf = conf_v;
	mask->type = type;
	mask->conf = conf_m;
}

/**
 * Create an egress actions template with MODIFY_FIELD action for setting unused REG_C_0 bits
 * to vport tag and JUMP action to group 1.
 *
 * If extended metadata mode is enabled, then MODIFY_FIELD action for copying software metadata
 * to REG_C_1 is added as well.
 *
 * @param dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   Pointer to actions template on success. NULL otherwise, and rte_errno is set.
 */
static struct rte_flow_actions_template *
flow_hw_create_tx_repr_tag_jump_acts_tmpl(struct rte_eth_dev *dev)
{
	uint32_t tag_mask = flow_hw_tx_tag_regc_mask(dev);
	uint32_t tag_value = flow_hw_tx_tag_regc_value(dev);
	struct rte_flow_actions_template_attr attr = {
		.egress = 1,
	};
	struct rte_flow_action_modify_field set_tag_v = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = REG_C_0,
			.offset = rte_bsf32(tag_mask),
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
		},
		.width = __builtin_popcount(tag_mask),
	};
	struct rte_flow_action_modify_field set_tag_m = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT32_MAX,
			.offset = UINT32_MAX,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
		},
		.width = UINT32_MAX,
	};
	struct rte_flow_action_modify_field copy_metadata_v = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = REG_C_1,
		},
		.src = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = REG_A,
		},
		.width = 32,
	};
	struct rte_flow_action_modify_field copy_metadata_m = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT32_MAX,
			.offset = UINT32_MAX,
		},
		.src = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT32_MAX,
			.offset = UINT32_MAX,
		},
		.width = UINT32_MAX,
	};
	struct rte_flow_action_jump jump_v = {
		.group = MLX5_HW_LOWEST_USABLE_GROUP,
	};
	struct rte_flow_action_jump jump_m = {
		.group = UINT32_MAX,
	};
	struct rte_flow_action actions_v[4] = { { 0 } };
	struct rte_flow_action actions_m[4] = { { 0 } };
	unsigned int idx = 0;

	rte_memcpy(set_tag_v.src.value, &tag_value, sizeof(tag_value));
	rte_memcpy(set_tag_m.src.value, &tag_mask, sizeof(tag_mask));
	flow_hw_update_action_mask(&actions_v[idx], &actions_m[idx],
				   RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
				   &set_tag_v, &set_tag_m);
	idx++;
	if (MLX5_SH(dev)->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) {
		flow_hw_update_action_mask(&actions_v[idx], &actions_m[idx],
					   RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
					   &copy_metadata_v, &copy_metadata_m);
		idx++;
	}
	flow_hw_update_action_mask(&actions_v[idx], &actions_m[idx], RTE_FLOW_ACTION_TYPE_JUMP,
				   &jump_v, &jump_m);
	idx++;
	flow_hw_update_action_mask(&actions_v[idx], &actions_m[idx], RTE_FLOW_ACTION_TYPE_END,
				   NULL, NULL);
	idx++;
	MLX5_ASSERT(idx <= RTE_DIM(actions_v));
	return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, NULL);
}

static void
flow_hw_cleanup_tx_repr_tagging(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	if (priv->hw_tx_repr_tagging_tbl) {
		flow_hw_table_destroy(dev, priv->hw_tx_repr_tagging_tbl, NULL);
		priv->hw_tx_repr_tagging_tbl = NULL;
	}
	if (priv->hw_tx_repr_tagging_at) {
		flow_hw_actions_template_destroy(dev, priv->hw_tx_repr_tagging_at, NULL);
		priv->hw_tx_repr_tagging_at = NULL;
	}
	if (priv->hw_tx_repr_tagging_pt) {
		flow_hw_pattern_template_destroy(dev, priv->hw_tx_repr_tagging_pt, NULL);
		priv->hw_tx_repr_tagging_pt = NULL;
	}
}

/**
 * Setup templates and table used to create default Tx flow rules. These default rules
 * allow for matching Tx representor traffic using a vport tag placed in unused bits of
 * REG_C_0 register.
 *
 * @param dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   0 on success, negative errno value otherwise.
 */
static int
flow_hw_setup_tx_repr_tagging(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_template_table_attr attr = {
		.flow_attr = {
			.group = 0,
			.priority = MLX5_HW_LOWEST_PRIO_ROOT,
			.egress = 1,
		},
		.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
	};
	struct mlx5_flow_template_table_cfg cfg = {
		.attr = attr,
		.external = false,
	};

	MLX5_ASSERT(priv->sh->config.dv_esw_en);
	MLX5_ASSERT(priv->sh->config.repr_matching);
	priv->hw_tx_repr_tagging_pt = flow_hw_create_tx_repr_sq_pattern_tmpl(dev);
	if (!priv->hw_tx_repr_tagging_pt)
		goto error;
	priv->hw_tx_repr_tagging_at = flow_hw_create_tx_repr_tag_jump_acts_tmpl(dev);
	if (!priv->hw_tx_repr_tagging_at)
		goto error;
	priv->hw_tx_repr_tagging_tbl = flow_hw_table_create(dev, &cfg,
							    &priv->hw_tx_repr_tagging_pt, 1,
							    &priv->hw_tx_repr_tagging_at, 1,
							    NULL);
	if (!priv->hw_tx_repr_tagging_tbl)
		goto error;
	return 0;
error:
	flow_hw_cleanup_tx_repr_tagging(dev);
	return -rte_errno;
}

static uint32_t
flow_hw_esw_mgr_regc_marker_mask(struct rte_eth_dev *dev)
{
	uint32_t mask = MLX5_SH(dev)->dv_regc0_mask;

	/* Mask is verified during device initialization. */
	MLX5_ASSERT(mask != 0);
	return mask;
}

static uint32_t
flow_hw_esw_mgr_regc_marker(struct rte_eth_dev *dev)
{
	uint32_t mask = MLX5_SH(dev)->dv_regc0_mask;

	/* Mask is verified during device initialization. */
	MLX5_ASSERT(mask != 0);
	return RTE_BIT32(rte_bsf32(mask));
}

/**
 * Creates a flow pattern template used to match on E-Switch Manager.
 * This template is used to set up a table for SQ miss default flow.
 *
 * @param dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   Pointer to flow pattern template on success, NULL otherwise.
 */
static struct rte_flow_pattern_template *
flow_hw_create_ctrl_esw_mgr_pattern_template(struct rte_eth_dev *dev)
{
	struct rte_flow_pattern_template_attr attr = {
		.relaxed_matching = 0,
		.transfer = 1,
	};
	struct rte_flow_item_ethdev port_spec = {
		.port_id = MLX5_REPRESENTED_PORT_ESW_MGR,
	};
	struct rte_flow_item_ethdev port_mask = {
		.port_id = UINT16_MAX,
	};
	struct mlx5_rte_flow_item_sq sq_mask = {
		.queue = UINT32_MAX,
	};
	struct rte_flow_item items[] = {
		{
			.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
			.spec = &port_spec,
			.mask = &port_mask,
		},
		{
			.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
			.mask = &sq_mask,
		},
		{
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};

	return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}

/**
 * Creates a flow pattern template used to match REG_C_0 and a SQ.
 * Matching on REG_C_0 is set up to match on all bits usable by user-space.
 * If traffic was sent from E-Switch Manager, then all usable bits will be set to 0,
 * except the least significant bit, which will be set to 1.
 *
 * This template is used to set up a table for SQ miss default flow.
 *
 * @param dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   Pointer to flow pattern template on success, NULL otherwise.
 */
static struct rte_flow_pattern_template *
flow_hw_create_ctrl_regc_sq_pattern_template(struct rte_eth_dev *dev)
{
	struct rte_flow_pattern_template_attr attr = {
		.relaxed_matching = 0,
		.transfer = 1,
	};
	struct rte_flow_item_tag reg_c0_spec = {
		.index = (uint8_t)REG_C_0,
	};
	struct rte_flow_item_tag reg_c0_mask = {
		.index = 0xff,
		.data = flow_hw_esw_mgr_regc_marker_mask(dev),
	};
	struct mlx5_rte_flow_item_sq queue_mask = {
		.queue = UINT32_MAX,
	};
	struct rte_flow_item items[] = {
		{
			.type = (enum rte_flow_item_type)
				MLX5_RTE_FLOW_ITEM_TYPE_TAG,
			.spec = &reg_c0_spec,
			.mask = &reg_c0_mask,
		},
		{
			.type = (enum rte_flow_item_type)
				MLX5_RTE_FLOW_ITEM_TYPE_SQ,
			.mask = &queue_mask,
		},
		{
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};

	return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}

/**
 * Creates a flow pattern template with unmasked represented port matching.
 * This template is used to set up a table for default transfer flows
 * directing packets to group 1.
 *
 * @param dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   Pointer to flow pattern template on success, NULL otherwise.
 */
static struct rte_flow_pattern_template *
flow_hw_create_ctrl_port_pattern_template(struct rte_eth_dev *dev)
{
	struct rte_flow_pattern_template_attr attr = {
		.relaxed_matching = 0,
		.transfer = 1,
	};
	struct rte_flow_item_ethdev port_mask = {
		.port_id = UINT16_MAX,
	};
	struct rte_flow_item items[] = {
		{
			.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
			.mask = &port_mask,
		},
		{
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};

	return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}

/*
 * Creating a flow pattern template with all ETH packets matching.
 * This template is used to set up a table for default Tx copy (Tx metadata
 * to REG_C_1) flow rule usage.
 *
 * @param dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   Pointer to flow pattern template on success, NULL otherwise.
 */
static struct rte_flow_pattern_template *
flow_hw_create_tx_default_mreg_copy_pattern_template(struct rte_eth_dev *dev)
{
	struct rte_flow_pattern_template_attr tx_pa_attr = {
		.relaxed_matching = 0,
		.egress = 1,
	};
	struct rte_flow_item_eth promisc = {
		.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
		.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
		.type = 0,
	};
	struct rte_flow_item eth_all[] = {
		[0] = {
			.type = RTE_FLOW_ITEM_TYPE_ETH,
			.spec = &promisc,
			.mask = &promisc,
		},
		[1] = {
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};
	struct rte_flow_error drop_err;

	RTE_SET_USED(drop_err);
	return flow_hw_pattern_template_create(dev, &tx_pa_attr, eth_all, &drop_err);
}

/**
 * Creates a flow actions template with modify field action and masked jump action.
 * Modify field action sets the least significant bit of REG_C_0 (usable by user-space)
 * to 1, meaning that packet was originated from E-Switch Manager. Jump action
 * transfers steering to group 1.
 *
 * @param dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   Pointer to flow actions template on success, NULL otherwise.
 */
static struct rte_flow_actions_template *
flow_hw_create_ctrl_regc_jump_actions_template(struct rte_eth_dev *dev)
{
	uint32_t marker_mask = flow_hw_esw_mgr_regc_marker_mask(dev);
	uint32_t marker_bits = flow_hw_esw_mgr_regc_marker(dev);
	struct rte_flow_actions_template_attr attr = {
		.transfer = 1,
	};
	struct rte_flow_action_modify_field set_reg_v = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = REG_C_0,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
		},
		.width = __builtin_popcount(marker_mask),
	};
	struct rte_flow_action_modify_field set_reg_m = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT32_MAX,
			.offset = UINT32_MAX,
		},
		.src = {
			.field = RTE_FLOW_FIELD_VALUE,
		},
		.width = UINT32_MAX,
	};
	struct rte_flow_action_jump jump_v = {
		.group = MLX5_HW_LOWEST_USABLE_GROUP,
	};
	struct rte_flow_action_jump jump_m = {
		.group = UINT32_MAX,
	};
	struct rte_flow_action actions_v[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
			.conf = &set_reg_v,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump_v,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};
	struct rte_flow_action actions_m[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
			.conf = &set_reg_m,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump_m,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};

	set_reg_v.dst.offset = rte_bsf32(marker_mask);
	rte_memcpy(set_reg_v.src.value, &marker_bits, sizeof(marker_bits));
	rte_memcpy(set_reg_m.src.value, &marker_mask, sizeof(marker_mask));
	return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, NULL);
}

/**
 * Creates a flow actions template with an unmasked JUMP action. Flows
 * based on this template will perform a jump to some group. This template
 * is used to set up tables for control flows.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param group
 *   Destination group for this action template.
 *
 * @return
 *   Pointer to flow actions template on success, NULL otherwise.
 */
static struct rte_flow_actions_template *
flow_hw_create_ctrl_jump_actions_template(struct rte_eth_dev *dev,
					  uint32_t group)
{
	struct rte_flow_actions_template_attr attr = {
		.transfer = 1,
	};
	struct rte_flow_action_jump jump_v = {
		.group = group,
	};
	struct rte_flow_action_jump jump_m = {
		.group = UINT32_MAX,
	};
	struct rte_flow_action actions_v[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump_v,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};
	struct rte_flow_action actions_m[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump_m,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};

	return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m,
					       NULL);
}

/**
 * Creates a flow action template with a unmasked REPRESENTED_PORT action.
 * It is used to create control flow tables.
 *
 * @param dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   Pointer to flow action template on success, NULL otherwise.
 */
static struct rte_flow_actions_template *
flow_hw_create_ctrl_port_actions_template(struct rte_eth_dev *dev)
{
	struct rte_flow_actions_template_attr attr = {
		.transfer = 1,
	};
	struct rte_flow_action_ethdev port_v = {
		.port_id = 0,
	};
	struct rte_flow_action actions_v[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT,
			.conf = &port_v,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};
	struct rte_flow_action_ethdev port_m = {
		.port_id = 0,
	};
	struct rte_flow_action actions_m[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT,
			.conf = &port_m,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};

	return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m,
					       NULL);
}

/*
 * Creating an actions template to use header modify action for register
 * copying. This template is used to set up a table for copy flow.
 *
 * @param dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   Pointer to flow actions template on success, NULL otherwise.
 */
static struct rte_flow_actions_template *
flow_hw_create_tx_default_mreg_copy_actions_template(struct rte_eth_dev *dev)
{
	struct rte_flow_actions_template_attr tx_act_attr = {
		.egress = 1,
	};
	const struct rte_flow_action_modify_field mreg_action = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = REG_C_1,
		},
		.src = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = REG_A,
		},
		.width = 32,
	};
	const struct rte_flow_action_modify_field mreg_mask = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT32_MAX,
			.offset = UINT32_MAX,
		},
		.src = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = UINT32_MAX,
			.offset = UINT32_MAX,
		},
		.width = UINT32_MAX,
	};
	const struct rte_flow_action_jump jump_action = {
		.group = 1,
	};
	const struct rte_flow_action_jump jump_mask = {
		.group = UINT32_MAX,
	};
	const struct rte_flow_action actions[] = {
		[0] = {
			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
			.conf = &mreg_action,
		},
		[1] = {
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump_action,
		},
		[2] = {
			.type = RTE_FLOW_ACTION_TYPE_END,
		},
	};
	const struct rte_flow_action masks[] = {
		[0] = {
			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
			.conf = &mreg_mask,
		},
		[1] = {
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump_mask,
		},
		[2] = {
			.type = RTE_FLOW_ACTION_TYPE_END,
		},
	};
	struct rte_flow_error drop_err;

	RTE_SET_USED(drop_err);
	return flow_hw_actions_template_create(dev, &tx_act_attr, actions,
					       masks, &drop_err);
}

/**
 * Creates a control flow table used to transfer traffic from E-Switch Manager
 * and TX queues from group 0 to group 1.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param it
 *   Pointer to flow pattern template.
 * @param at
 *   Pointer to flow actions template.
 *
 * @return
 *   Pointer to flow table on success, NULL otherwise.
 */
static struct rte_flow_template_table*
flow_hw_create_ctrl_sq_miss_root_table(struct rte_eth_dev *dev,
				       struct rte_flow_pattern_template *it,
				       struct rte_flow_actions_template *at)
{
	struct rte_flow_template_table_attr attr = {
		.flow_attr = {
			.group = 0,
			.priority = MLX5_HW_LOWEST_PRIO_ROOT,
			.ingress = 0,
			.egress = 0,
			.transfer = 1,
		},
		.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
	};
	struct mlx5_flow_template_table_cfg cfg = {
		.attr = attr,
		.external = false,
	};

	return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL);
}


/**
 * Creates a control flow table used to transfer traffic from E-Switch Manager
 * and TX queues from group 0 to group 1.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param it
 *   Pointer to flow pattern template.
 * @param at
 *   Pointer to flow actions template.
 *
 * @return
 *   Pointer to flow table on success, NULL otherwise.
 */
static struct rte_flow_template_table*
flow_hw_create_ctrl_sq_miss_table(struct rte_eth_dev *dev,
				  struct rte_flow_pattern_template *it,
				  struct rte_flow_actions_template *at)
{
	struct rte_flow_template_table_attr attr = {
		.flow_attr = {
			.group = 1,
			.priority = MLX5_HW_LOWEST_PRIO_NON_ROOT,
			.ingress = 0,
			.egress = 0,
			.transfer = 1,
		},
		.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
	};
	struct mlx5_flow_template_table_cfg cfg = {
		.attr = attr,
		.external = false,
	};

	return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL);
}

/*
 * Creating the default Tx metadata copy table on NIC Tx group 0.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param pt
 *   Pointer to flow pattern template.
 * @param at
 *   Pointer to flow actions template.
 *
 * @return
 *   Pointer to flow table on success, NULL otherwise.
 */
static struct rte_flow_template_table*
flow_hw_create_tx_default_mreg_copy_table(struct rte_eth_dev *dev,
					  struct rte_flow_pattern_template *pt,
					  struct rte_flow_actions_template *at)
{
	struct rte_flow_template_table_attr tx_tbl_attr = {
		.flow_attr = {
			.group = 0, /* Root */
			.priority = MLX5_HW_LOWEST_PRIO_ROOT,
			.egress = 1,
		},
		.nb_flows = 1, /* One default flow rule for all. */
	};
	struct mlx5_flow_template_table_cfg tx_tbl_cfg = {
		.attr = tx_tbl_attr,
		.external = false,
	};
	struct rte_flow_error drop_err = {
		.type = RTE_FLOW_ERROR_TYPE_NONE,
		.cause = NULL,
		.message = NULL,
	};

	RTE_SET_USED(drop_err);
	return flow_hw_table_create(dev, &tx_tbl_cfg, &pt, 1, &at, 1, &drop_err);
}

/**
 * Creates a control flow table used to transfer traffic
 * from group 0 to group 1.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param it
 *   Pointer to flow pattern template.
 * @param at
 *   Pointer to flow actions template.
 *
 * @return
 *   Pointer to flow table on success, NULL otherwise.
 */
static struct rte_flow_template_table *
flow_hw_create_ctrl_jump_table(struct rte_eth_dev *dev,
			       struct rte_flow_pattern_template *it,
			       struct rte_flow_actions_template *at)
{
	struct rte_flow_template_table_attr attr = {
		.flow_attr = {
			.group = 0,
			.priority = 0,
			.ingress = 0,
			.egress = 0,
			.transfer = 1,
		},
		.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
	};
	struct mlx5_flow_template_table_cfg cfg = {
		.attr = attr,
		.external = false,
	};

	return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL);
}

/**
 * Creates a set of flow tables used to create control flows used
 * when E-Switch is engaged.
 *
 * @param dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   0 on success, EINVAL otherwise
 */
static __rte_unused int
flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_pattern_template *esw_mgr_items_tmpl = NULL;
	struct rte_flow_pattern_template *regc_sq_items_tmpl = NULL;
	struct rte_flow_pattern_template *port_items_tmpl = NULL;
	struct rte_flow_pattern_template *tx_meta_items_tmpl = NULL;
	struct rte_flow_actions_template *regc_jump_actions_tmpl = NULL;
	struct rte_flow_actions_template *port_actions_tmpl = NULL;
	struct rte_flow_actions_template *jump_one_actions_tmpl = NULL;
	struct rte_flow_actions_template *tx_meta_actions_tmpl = NULL;
	uint32_t xmeta = priv->sh->config.dv_xmeta_en;
	uint32_t repr_matching = priv->sh->config.repr_matching;

	/* Create templates and table for default SQ miss flow rules - root table. */
	esw_mgr_items_tmpl = flow_hw_create_ctrl_esw_mgr_pattern_template(dev);
	if (!esw_mgr_items_tmpl) {
		DRV_LOG(ERR, "port %u failed to create E-Switch Manager item"
			" template for control flows", dev->data->port_id);
		goto error;
	}
	regc_jump_actions_tmpl = flow_hw_create_ctrl_regc_jump_actions_template(dev);
	if (!regc_jump_actions_tmpl) {
		DRV_LOG(ERR, "port %u failed to create REG_C set and jump action template"
			" for control flows", dev->data->port_id);
		goto error;
	}
	MLX5_ASSERT(priv->hw_esw_sq_miss_root_tbl == NULL);
	priv->hw_esw_sq_miss_root_tbl = flow_hw_create_ctrl_sq_miss_root_table
			(dev, esw_mgr_items_tmpl, regc_jump_actions_tmpl);
	if (!priv->hw_esw_sq_miss_root_tbl) {
		DRV_LOG(ERR, "port %u failed to create table for default sq miss (root table)"
			" for control flows", dev->data->port_id);
		goto error;
	}
	/* Create templates and table for default SQ miss flow rules - non-root table. */
	regc_sq_items_tmpl = flow_hw_create_ctrl_regc_sq_pattern_template(dev);
	if (!regc_sq_items_tmpl) {
		DRV_LOG(ERR, "port %u failed to create SQ item template for"
			" control flows", dev->data->port_id);
		goto error;
	}
	port_actions_tmpl = flow_hw_create_ctrl_port_actions_template(dev);
	if (!port_actions_tmpl) {
		DRV_LOG(ERR, "port %u failed to create port action template"
			" for control flows", dev->data->port_id);
		goto error;
	}
	MLX5_ASSERT(priv->hw_esw_sq_miss_tbl == NULL);
	priv->hw_esw_sq_miss_tbl = flow_hw_create_ctrl_sq_miss_table(dev, regc_sq_items_tmpl,
								     port_actions_tmpl);
	if (!priv->hw_esw_sq_miss_tbl) {
		DRV_LOG(ERR, "port %u failed to create table for default sq miss (non-root table)"
			" for control flows", dev->data->port_id);
		goto error;
	}
	/* Create templates and table for default FDB jump flow rules. */
	port_items_tmpl = flow_hw_create_ctrl_port_pattern_template(dev);
	if (!port_items_tmpl) {
		DRV_LOG(ERR, "port %u failed to create SQ item template for"
			" control flows", dev->data->port_id);
		goto error;
	}
	jump_one_actions_tmpl = flow_hw_create_ctrl_jump_actions_template
			(dev, MLX5_HW_LOWEST_USABLE_GROUP);
	if (!jump_one_actions_tmpl) {
		DRV_LOG(ERR, "port %u failed to create jump action template"
			" for control flows", dev->data->port_id);
		goto error;
	}
	MLX5_ASSERT(priv->hw_esw_zero_tbl == NULL);
	priv->hw_esw_zero_tbl = flow_hw_create_ctrl_jump_table(dev, port_items_tmpl,
							       jump_one_actions_tmpl);
	if (!priv->hw_esw_zero_tbl) {
		DRV_LOG(ERR, "port %u failed to create table for default jump to group 1"
			" for control flows", dev->data->port_id);
		goto error;
	}
	/* Create templates and table for default Tx metadata copy flow rule. */
	if (!repr_matching && xmeta == MLX5_XMETA_MODE_META32_HWS) {
		tx_meta_items_tmpl = flow_hw_create_tx_default_mreg_copy_pattern_template(dev);
		if (!tx_meta_items_tmpl) {
			DRV_LOG(ERR, "port %u failed to Tx metadata copy pattern"
				" template for control flows", dev->data->port_id);
			goto error;
		}
		tx_meta_actions_tmpl = flow_hw_create_tx_default_mreg_copy_actions_template(dev);
		if (!tx_meta_actions_tmpl) {
			DRV_LOG(ERR, "port %u failed to Tx metadata copy actions"
				" template for control flows", dev->data->port_id);
			goto error;
		}
		MLX5_ASSERT(priv->hw_tx_meta_cpy_tbl == NULL);
		priv->hw_tx_meta_cpy_tbl = flow_hw_create_tx_default_mreg_copy_table(dev,
					tx_meta_items_tmpl, tx_meta_actions_tmpl);
		if (!priv->hw_tx_meta_cpy_tbl) {
			DRV_LOG(ERR, "port %u failed to create table for default"
				" Tx metadata copy flow rule", dev->data->port_id);
			goto error;
		}
	}
	return 0;
error:
	if (priv->hw_esw_zero_tbl) {
		flow_hw_table_destroy(dev, priv->hw_esw_zero_tbl, NULL);
		priv->hw_esw_zero_tbl = NULL;
	}
	if (priv->hw_esw_sq_miss_tbl) {
		flow_hw_table_destroy(dev, priv->hw_esw_sq_miss_tbl, NULL);
		priv->hw_esw_sq_miss_tbl = NULL;
	}
	if (priv->hw_esw_sq_miss_root_tbl) {
		flow_hw_table_destroy(dev, priv->hw_esw_sq_miss_root_tbl, NULL);
		priv->hw_esw_sq_miss_root_tbl = NULL;
	}
	if (tx_meta_actions_tmpl)
		flow_hw_actions_template_destroy(dev, tx_meta_actions_tmpl, NULL);
	if (jump_one_actions_tmpl)
		flow_hw_actions_template_destroy(dev, jump_one_actions_tmpl, NULL);
	if (port_actions_tmpl)
		flow_hw_actions_template_destroy(dev, port_actions_tmpl, NULL);
	if (regc_jump_actions_tmpl)
		flow_hw_actions_template_destroy(dev, regc_jump_actions_tmpl, NULL);
	if (tx_meta_items_tmpl)
		flow_hw_pattern_template_destroy(dev, tx_meta_items_tmpl, NULL);
	if (port_items_tmpl)
		flow_hw_pattern_template_destroy(dev, port_items_tmpl, NULL);
	if (regc_sq_items_tmpl)
		flow_hw_pattern_template_destroy(dev, regc_sq_items_tmpl, NULL);
	if (esw_mgr_items_tmpl)
		flow_hw_pattern_template_destroy(dev, esw_mgr_items_tmpl, NULL);
	return -EINVAL;
}

static void
flow_hw_ct_mng_destroy(struct rte_eth_dev *dev,
		       struct mlx5_aso_ct_pools_mng *ct_mng)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	mlx5_aso_ct_queue_uninit(priv->sh, ct_mng);
	mlx5_free(ct_mng);
}

static void
flow_hw_ct_pool_destroy(struct rte_eth_dev *dev __rte_unused,
			struct mlx5_aso_ct_pool *pool)
{
	if (pool->dr_action)
		mlx5dr_action_destroy(pool->dr_action);
	if (pool->devx_obj)
		claim_zero(mlx5_devx_cmd_destroy(pool->devx_obj));
	if (pool->cts)
		mlx5_ipool_destroy(pool->cts);
	mlx5_free(pool);
}

static struct mlx5_aso_ct_pool *
flow_hw_ct_pool_create(struct rte_eth_dev *dev,
		       const struct rte_flow_port_attr *port_attr)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_ct_pool *pool;
	struct mlx5_devx_obj *obj;
	uint32_t nb_cts = rte_align32pow2(port_attr->nb_conn_tracks);
	uint32_t log_obj_size = rte_log2_u32(nb_cts);
	struct mlx5_indexed_pool_config cfg = {
		.size = sizeof(struct mlx5_aso_ct_action),
		.trunk_size = 1 << 12,
		.per_core_cache = 1 << 13,
		.need_lock = 1,
		.release_mem_en = !!priv->sh->config.reclaim_mode,
		.malloc = mlx5_malloc,
		.free = mlx5_free,
		.type = "mlx5_hw_ct_action",
	};
	int reg_id;
	uint32_t flags;

	pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
	if (!pool) {
		rte_errno = ENOMEM;
		return NULL;
	}
	obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->cdev->ctx,
							  priv->sh->cdev->pdn,
							  log_obj_size);
	if (!obj) {
		rte_errno = ENODATA;
		DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX.");
		goto err;
	}
	pool->devx_obj = obj;
	reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, NULL);
	flags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX;
	if (priv->sh->config.dv_esw_en && priv->master)
		flags |= MLX5DR_ACTION_FLAG_HWS_FDB;
	pool->dr_action = mlx5dr_action_create_aso_ct(priv->dr_ctx,
						      (struct mlx5dr_devx_obj *)obj,
						      reg_id - REG_C_0, flags);
	if (!pool->dr_action)
		goto err;
	/*
	 * No need for local cache if CT number is a small number. Since
	 * flow insertion rate will be very limited in that case. Here let's
	 * set the number to less than default trunk size 4K.
	 */
	if (nb_cts <= cfg.trunk_size) {
		cfg.per_core_cache = 0;
		cfg.trunk_size = nb_cts;
	} else if (nb_cts <= MLX5_HW_IPOOL_SIZE_THRESHOLD) {
		cfg.per_core_cache = MLX5_HW_IPOOL_CACHE_MIN;
	}
	pool->cts = mlx5_ipool_create(&cfg);
	if (!pool->cts)
		goto err;
	pool->sq = priv->ct_mng->aso_sqs;
	/* Assign the last extra ASO SQ as public SQ. */
	pool->shared_sq = &priv->ct_mng->aso_sqs[priv->nb_queue - 1];
	return pool;
err:
	flow_hw_ct_pool_destroy(dev, pool);
	return NULL;
}

static void
flow_hw_destroy_vlan(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	enum mlx5dr_table_type i;

	for (i = MLX5DR_TABLE_TYPE_NIC_RX; i < MLX5DR_TABLE_TYPE_MAX; i++) {
		if (priv->hw_pop_vlan[i]) {
			mlx5dr_action_destroy(priv->hw_pop_vlan[i]);
			priv->hw_pop_vlan[i] = NULL;
		}
		if (priv->hw_push_vlan[i]) {
			mlx5dr_action_destroy(priv->hw_push_vlan[i]);
			priv->hw_push_vlan[i] = NULL;
		}
	}
}

static int
flow_hw_create_vlan(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	enum mlx5dr_table_type i;
	const enum mlx5dr_action_flags flags[MLX5DR_TABLE_TYPE_MAX] = {
		MLX5DR_ACTION_FLAG_HWS_RX,
		MLX5DR_ACTION_FLAG_HWS_TX,
		MLX5DR_ACTION_FLAG_HWS_FDB
	};

	for (i = MLX5DR_TABLE_TYPE_NIC_RX; i <= MLX5DR_TABLE_TYPE_NIC_TX; i++) {
		priv->hw_pop_vlan[i] =
			mlx5dr_action_create_pop_vlan(priv->dr_ctx, flags[i]);
		if (!priv->hw_pop_vlan[i])
			return -ENOENT;
		priv->hw_push_vlan[i] =
			mlx5dr_action_create_push_vlan(priv->dr_ctx, flags[i]);
		if (!priv->hw_pop_vlan[i])
			return -ENOENT;
	}
	if (priv->sh->config.dv_esw_en && priv->master) {
		priv->hw_pop_vlan[MLX5DR_TABLE_TYPE_FDB] =
			mlx5dr_action_create_pop_vlan
				(priv->dr_ctx, MLX5DR_ACTION_FLAG_HWS_FDB);
		if (!priv->hw_pop_vlan[MLX5DR_TABLE_TYPE_FDB])
			return -ENOENT;
		priv->hw_push_vlan[MLX5DR_TABLE_TYPE_FDB] =
			mlx5dr_action_create_push_vlan
				(priv->dr_ctx, MLX5DR_ACTION_FLAG_HWS_FDB);
		if (!priv->hw_pop_vlan[MLX5DR_TABLE_TYPE_FDB])
			return -ENOENT;
	}
	return 0;
}

static void
flow_hw_cleanup_ctrl_rx_tables(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	unsigned int i;
	unsigned int j;

	if (!priv->hw_ctrl_rx)
		return;
	for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX; ++i) {
		for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
			struct rte_flow_template_table *tbl = priv->hw_ctrl_rx->tables[i][j].tbl;
			struct rte_flow_pattern_template *pt = priv->hw_ctrl_rx->tables[i][j].pt;

			if (tbl)
				claim_zero(flow_hw_table_destroy(dev, tbl, NULL));
			if (pt)
				claim_zero(flow_hw_pattern_template_destroy(dev, pt, NULL));
		}
	}
	for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++i) {
		struct rte_flow_actions_template *at = priv->hw_ctrl_rx->rss[i];

		if (at)
			claim_zero(flow_hw_actions_template_destroy(dev, at, NULL));
	}
	mlx5_free(priv->hw_ctrl_rx);
	priv->hw_ctrl_rx = NULL;
}

static uint64_t
flow_hw_ctrl_rx_rss_type_hash_types(const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	switch (rss_type) {
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_NON_IP:
		return 0;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4:
		return RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | RTE_ETH_RSS_NONFRAG_IPV4_OTHER;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_UDP:
		return RTE_ETH_RSS_NONFRAG_IPV4_UDP;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_TCP:
		return RTE_ETH_RSS_NONFRAG_IPV4_TCP;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6:
		return RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | RTE_ETH_RSS_NONFRAG_IPV6_OTHER;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_UDP:
		return RTE_ETH_RSS_NONFRAG_IPV6_UDP | RTE_ETH_RSS_IPV6_UDP_EX;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_TCP:
		return RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_IPV6_TCP_EX;
	default:
		/* Should not reach here. */
		MLX5_ASSERT(false);
		return 0;
	}
}

static struct rte_flow_actions_template *
flow_hw_create_ctrl_rx_rss_template(struct rte_eth_dev *dev,
				    const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_actions_template_attr attr = {
		.ingress = 1,
	};
	uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
	struct rte_flow_action_rss rss_conf = {
		.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
		.level = 0,
		.types = 0,
		.key_len = priv->rss_conf.rss_key_len,
		.key = priv->rss_conf.rss_key,
		.queue_num = priv->reta_idx_n,
		.queue = queue,
	};
	struct rte_flow_action actions[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_RSS,
			.conf = &rss_conf,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};
	struct rte_flow_action masks[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_RSS,
			.conf = &rss_conf,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};
	struct rte_flow_actions_template *at;
	struct rte_flow_error error;
	unsigned int i;

	MLX5_ASSERT(priv->reta_idx_n > 0 && priv->reta_idx);
	/* Select proper RSS hash types and based on that configure the actions template. */
	rss_conf.types = flow_hw_ctrl_rx_rss_type_hash_types(rss_type);
	if (rss_conf.types) {
		for (i = 0; i < priv->reta_idx_n; ++i)
			queue[i] = (*priv->reta_idx)[i];
	} else {
		rss_conf.queue_num = 1;
		queue[0] = (*priv->reta_idx)[0];
	}
	at = flow_hw_actions_template_create(dev, &attr, actions, masks, &error);
	if (!at)
		DRV_LOG(ERR,
			"Failed to create ctrl flow actions template: rte_errno(%d), type(%d): %s",
			rte_errno, error.type,
			error.message ? error.message : "(no stated reason)");
	return at;
}

static uint32_t ctrl_rx_rss_priority_map[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX] = {
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_NON_IP] = MLX5_HW_CTRL_RX_PRIO_L2,
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4] = MLX5_HW_CTRL_RX_PRIO_L3,
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_UDP] = MLX5_HW_CTRL_RX_PRIO_L4,
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_TCP] = MLX5_HW_CTRL_RX_PRIO_L4,
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6] = MLX5_HW_CTRL_RX_PRIO_L3,
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_UDP] = MLX5_HW_CTRL_RX_PRIO_L4,
	[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_TCP] = MLX5_HW_CTRL_RX_PRIO_L4,
};

static uint32_t ctrl_rx_nb_flows_map[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX] = {
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL] = 1,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST] = 1,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST] = 1,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN] = MLX5_MAX_VLAN_IDS,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST] = 1,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN] = MLX5_MAX_VLAN_IDS,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST] = 1,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN] = MLX5_MAX_VLAN_IDS,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC] = MLX5_MAX_UC_MAC_ADDRESSES,
	[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN] =
			MLX5_MAX_UC_MAC_ADDRESSES * MLX5_MAX_VLAN_IDS,
};

static struct rte_flow_template_table_attr
flow_hw_get_ctrl_rx_table_attr(enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type,
			       const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	return (struct rte_flow_template_table_attr){
		.flow_attr = {
			.group = 0,
			.priority = ctrl_rx_rss_priority_map[rss_type],
			.ingress = 1,
		},
		.nb_flows = ctrl_rx_nb_flows_map[eth_pattern_type],
	};
}

static struct rte_flow_item
flow_hw_get_ctrl_rx_eth_item(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type)
{
	struct rte_flow_item item = {
		.type = RTE_FLOW_ITEM_TYPE_ETH,
		.mask = NULL,
	};

	switch (eth_pattern_type) {
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL:
		item.mask = &ctrl_rx_eth_promisc_mask;
		break;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST:
		item.mask = &ctrl_rx_eth_mcast_mask;
		break;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
		item.mask = &ctrl_rx_eth_dmac_mask;
		break;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
		item.mask = &ctrl_rx_eth_ipv4_mcast_mask;
		break;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
		item.mask = &ctrl_rx_eth_ipv6_mcast_mask;
		break;
	default:
		/* Should not reach here - ETH mask must be present. */
		item.type = RTE_FLOW_ITEM_TYPE_END;
		MLX5_ASSERT(false);
		break;
	}
	return item;
}

static struct rte_flow_item
flow_hw_get_ctrl_rx_vlan_item(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type)
{
	struct rte_flow_item item = {
		.type = RTE_FLOW_ITEM_TYPE_VOID,
		.mask = NULL,
	};

	switch (eth_pattern_type) {
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
		item.type = RTE_FLOW_ITEM_TYPE_VLAN;
		item.mask = &rte_flow_item_vlan_mask;
		break;
	default:
		/* Nothing to update. */
		break;
	}
	return item;
}

static struct rte_flow_item
flow_hw_get_ctrl_rx_l3_item(const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct rte_flow_item item = {
		.type = RTE_FLOW_ITEM_TYPE_VOID,
		.mask = NULL,
	};

	switch (rss_type) {
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4:
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_UDP:
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_TCP:
		item.type = RTE_FLOW_ITEM_TYPE_IPV4;
		break;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6:
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_UDP:
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_TCP:
		item.type = RTE_FLOW_ITEM_TYPE_IPV6;
		break;
	default:
		/* Nothing to update. */
		break;
	}
	return item;
}

static struct rte_flow_item
flow_hw_get_ctrl_rx_l4_item(const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct rte_flow_item item = {
		.type = RTE_FLOW_ITEM_TYPE_VOID,
		.mask = NULL,
	};

	switch (rss_type) {
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_UDP:
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_UDP:
		item.type = RTE_FLOW_ITEM_TYPE_UDP;
		break;
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_TCP:
	case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_TCP:
		item.type = RTE_FLOW_ITEM_TYPE_TCP;
		break;
	default:
		/* Nothing to update. */
		break;
	}
	return item;
}

static struct rte_flow_pattern_template *
flow_hw_create_ctrl_rx_pattern_template
		(struct rte_eth_dev *dev,
		 const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type,
		 const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	const struct rte_flow_pattern_template_attr attr = {
		.relaxed_matching = 0,
		.ingress = 1,
	};
	struct rte_flow_item items[] = {
		/* Matching patterns */
		flow_hw_get_ctrl_rx_eth_item(eth_pattern_type),
		flow_hw_get_ctrl_rx_vlan_item(eth_pattern_type),
		flow_hw_get_ctrl_rx_l3_item(rss_type),
		flow_hw_get_ctrl_rx_l4_item(rss_type),
		/* Terminate pattern */
		{ .type = RTE_FLOW_ITEM_TYPE_END }
	};

	return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}

static int
flow_hw_create_ctrl_rx_tables(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	unsigned int i;
	unsigned int j;
	int ret;

	MLX5_ASSERT(!priv->hw_ctrl_rx);
	priv->hw_ctrl_rx = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*priv->hw_ctrl_rx),
				       RTE_CACHE_LINE_SIZE, rte_socket_id());
	if (!priv->hw_ctrl_rx) {
		DRV_LOG(ERR, "Failed to allocate memory for Rx control flow tables");
		rte_errno = ENOMEM;
		return -rte_errno;
	}
	/* Create all pattern template variants. */
	for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX; ++i) {
		enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type = i;

		for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
			const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type = j;
			struct rte_flow_template_table_attr attr;
			struct rte_flow_pattern_template *pt;

			attr = flow_hw_get_ctrl_rx_table_attr(eth_pattern_type, rss_type);
			pt = flow_hw_create_ctrl_rx_pattern_template(dev, eth_pattern_type,
								     rss_type);
			if (!pt)
				goto err;
			priv->hw_ctrl_rx->tables[i][j].attr = attr;
			priv->hw_ctrl_rx->tables[i][j].pt = pt;
		}
	}
	return 0;
err:
	ret = rte_errno;
	flow_hw_cleanup_ctrl_rx_tables(dev);
	rte_errno = ret;
	return -ret;
}

void
mlx5_flow_hw_cleanup_ctrl_rx_templates(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_flow_hw_ctrl_rx *hw_ctrl_rx;
	unsigned int i;
	unsigned int j;

	if (!priv->dr_ctx)
		return;
	if (!priv->hw_ctrl_rx)
		return;
	hw_ctrl_rx = priv->hw_ctrl_rx;
	for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX; ++i) {
		for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
			struct mlx5_flow_hw_ctrl_rx_table *tmpls = &hw_ctrl_rx->tables[i][j];

			if (tmpls->tbl) {
				claim_zero(flow_hw_table_destroy(dev, tmpls->tbl, NULL));
				tmpls->tbl = NULL;
			}
		}
	}
	for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
		if (hw_ctrl_rx->rss[j]) {
			claim_zero(flow_hw_actions_template_destroy(dev, hw_ctrl_rx->rss[j], NULL));
			hw_ctrl_rx->rss[j] = NULL;
		}
	}
}

/**
 * Configure port HWS resources.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] port_attr
 *   Port configuration attributes.
 * @param[in] nb_queue
 *   Number of queue.
 * @param[in] queue_attr
 *   Array that holds attributes for each flow queue.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_configure(struct rte_eth_dev *dev,
		  const struct rte_flow_port_attr *port_attr,
		  uint16_t nb_queue,
		  const struct rte_flow_queue_attr *queue_attr[],
		  struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5dr_context *dr_ctx = NULL;
	struct mlx5dr_context_attr dr_ctx_attr = {0};
	struct mlx5_hw_q *hw_q;
	struct mlx5_hw_q_job *job = NULL;
	uint32_t mem_size, i, j;
	struct mlx5_indexed_pool_config cfg = {
		.size = sizeof(struct mlx5_action_construct_data),
		.trunk_size = 4096,
		.need_lock = 1,
		.release_mem_en = !!priv->sh->config.reclaim_mode,
		.malloc = mlx5_malloc,
		.free = mlx5_free,
		.type = "mlx5_hw_action_construct_data",
	};
	/* Adds one queue to be used by PMD.
	 * The last queue will be used by the PMD.
	 */
	uint16_t nb_q_updated = 0;
	struct rte_flow_queue_attr **_queue_attr = NULL;
	struct rte_flow_queue_attr ctrl_queue_attr = {0};
	bool is_proxy = !!(priv->sh->config.dv_esw_en && priv->master);
	int ret = 0;

	if (!port_attr || !nb_queue || !queue_attr) {
		rte_errno = EINVAL;
		goto err;
	}
	/* In case re-configuring, release existing context at first. */
	if (priv->dr_ctx) {
		/* */
		for (i = 0; i < priv->nb_queue; i++) {
			hw_q = &priv->hw_q[i];
			/* Make sure all queues are empty. */
			if (hw_q->size != hw_q->job_idx) {
				rte_errno = EBUSY;
				goto err;
			}
		}
		flow_hw_resource_release(dev);
	}
	ctrl_queue_attr.size = queue_attr[0]->size;
	nb_q_updated = nb_queue + 1;
	_queue_attr = mlx5_malloc(MLX5_MEM_ZERO,
				  nb_q_updated *
				  sizeof(struct rte_flow_queue_attr *),
				  64, SOCKET_ID_ANY);
	if (!_queue_attr) {
		rte_errno = ENOMEM;
		goto err;
	}

	memcpy(_queue_attr, queue_attr,
	       sizeof(void *) * nb_queue);
	_queue_attr[nb_queue] = &ctrl_queue_attr;
	priv->acts_ipool = mlx5_ipool_create(&cfg);
	if (!priv->acts_ipool)
		goto err;
	/* Allocate the queue job descriptor LIFO. */
	mem_size = sizeof(priv->hw_q[0]) * nb_q_updated;
	for (i = 0; i < nb_q_updated; i++) {
		/*
		 * Check if the queues' size are all the same as the
		 * limitation from HWS layer.
		 */
		if (_queue_attr[i]->size != _queue_attr[0]->size) {
			rte_errno = EINVAL;
			goto err;
		}
		mem_size += (sizeof(struct mlx5_hw_q_job *) +
			    sizeof(struct mlx5_hw_q_job) +
			    sizeof(uint8_t) * MLX5_ENCAP_MAX_LEN +
			    sizeof(struct mlx5_modification_cmd) *
			    MLX5_MHDR_MAX_CMD +
			    sizeof(struct rte_flow_item) *
			    MLX5_HW_MAX_ITEMS) *
			    _queue_attr[i]->size;
	}
	priv->hw_q = mlx5_malloc(MLX5_MEM_ZERO, mem_size,
				 64, SOCKET_ID_ANY);
	if (!priv->hw_q) {
		rte_errno = ENOMEM;
		goto err;
	}
	for (i = 0; i < nb_q_updated; i++) {
		char mz_name[RTE_MEMZONE_NAMESIZE];
		uint8_t *encap = NULL;
		struct mlx5_modification_cmd *mhdr_cmd = NULL;
		struct rte_flow_item *items = NULL;

		priv->hw_q[i].job_idx = _queue_attr[i]->size;
		priv->hw_q[i].size = _queue_attr[i]->size;
		if (i == 0)
			priv->hw_q[i].job = (struct mlx5_hw_q_job **)
					    &priv->hw_q[nb_q_updated];
		else
			priv->hw_q[i].job = (struct mlx5_hw_q_job **)
				&job[_queue_attr[i - 1]->size - 1].items
				 [MLX5_HW_MAX_ITEMS];
		job = (struct mlx5_hw_q_job *)
		      &priv->hw_q[i].job[_queue_attr[i]->size];
		mhdr_cmd = (struct mlx5_modification_cmd *)
			   &job[_queue_attr[i]->size];
		encap = (uint8_t *)
			 &mhdr_cmd[_queue_attr[i]->size * MLX5_MHDR_MAX_CMD];
		items = (struct rte_flow_item *)
			 &encap[_queue_attr[i]->size * MLX5_ENCAP_MAX_LEN];
		for (j = 0; j < _queue_attr[i]->size; j++) {
			job[j].mhdr_cmd = &mhdr_cmd[j * MLX5_MHDR_MAX_CMD];
			job[j].encap_data = &encap[j * MLX5_ENCAP_MAX_LEN];
			job[j].items = &items[j * MLX5_HW_MAX_ITEMS];
			priv->hw_q[i].job[j] = &job[j];
		}
		snprintf(mz_name, sizeof(mz_name), "port_%u_indir_act_cq_%u",
			 dev->data->port_id, i);
		priv->hw_q[i].indir_cq = rte_ring_create(mz_name,
				_queue_attr[i]->size, SOCKET_ID_ANY,
				RING_F_SP_ENQ | RING_F_SC_DEQ |
				RING_F_EXACT_SZ);
		if (!priv->hw_q[i].indir_cq)
			goto err;
		snprintf(mz_name, sizeof(mz_name), "port_%u_indir_act_iq_%u",
			 dev->data->port_id, i);
		priv->hw_q[i].indir_iq = rte_ring_create(mz_name,
				_queue_attr[i]->size, SOCKET_ID_ANY,
				RING_F_SP_ENQ | RING_F_SC_DEQ |
				RING_F_EXACT_SZ);
		if (!priv->hw_q[i].indir_iq)
			goto err;
	}
	dr_ctx_attr.pd = priv->sh->cdev->pd;
	dr_ctx_attr.queues = nb_q_updated;
	/* Queue size should all be the same. Take the first one. */
	dr_ctx_attr.queue_size = _queue_attr[0]->size;
	dr_ctx = mlx5dr_context_open(priv->sh->cdev->ctx, &dr_ctx_attr);
	/* rte_errno has been updated by HWS layer. */
	if (!dr_ctx)
		goto err;
	priv->dr_ctx = dr_ctx;
	priv->nb_queue = nb_q_updated;
	rte_spinlock_init(&priv->hw_ctrl_lock);
	LIST_INIT(&priv->hw_ctrl_flows);
	ret = flow_hw_create_ctrl_rx_tables(dev);
	if (ret) {
		rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				   "Failed to set up Rx control flow templates");
		goto err;
	}
	/* Initialize meter library*/
	if (port_attr->nb_meters)
		if (mlx5_flow_meter_init(dev, port_attr->nb_meters, 1, 1, nb_q_updated))
			goto err;
	/* Add global actions. */
	for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
		uint32_t act_flags = 0;

		act_flags = mlx5_hw_act_flag[i][0] | mlx5_hw_act_flag[i][1];
		if (is_proxy)
			act_flags |= mlx5_hw_act_flag[i][2];
		priv->hw_drop[i] = mlx5dr_action_create_dest_drop(priv->dr_ctx, act_flags);
		if (!priv->hw_drop[i])
			goto err;
		priv->hw_tag[i] = mlx5dr_action_create_tag
			(priv->dr_ctx, mlx5_hw_act_flag[i][0]);
		if (!priv->hw_tag[i])
			goto err;
	}
	if (priv->sh->config.dv_esw_en && priv->sh->config.repr_matching) {
		ret = flow_hw_setup_tx_repr_tagging(dev);
		if (ret) {
			rte_errno = -ret;
			goto err;
		}
	}
	if (is_proxy) {
		ret = flow_hw_create_vport_actions(priv);
		if (ret) {
			rte_errno = -ret;
			goto err;
		}
		ret = flow_hw_create_ctrl_tables(dev);
		if (ret) {
			rte_errno = -ret;
			goto err;
		}
	}
	if (port_attr->nb_conn_tracks) {
		mem_size = sizeof(struct mlx5_aso_sq) * nb_q_updated +
			   sizeof(*priv->ct_mng);
		priv->ct_mng = mlx5_malloc(MLX5_MEM_ZERO, mem_size,
					   RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
		if (!priv->ct_mng)
			goto err;
		if (mlx5_aso_ct_queue_init(priv->sh, priv->ct_mng, nb_q_updated))
			goto err;
		priv->hws_ctpool = flow_hw_ct_pool_create(dev, port_attr);
		if (!priv->hws_ctpool)
			goto err;
		priv->sh->ct_aso_en = 1;
	}
	if (port_attr->nb_counters) {
		priv->hws_cpool = mlx5_hws_cnt_pool_create(dev, port_attr,
							   nb_queue);
		if (priv->hws_cpool == NULL)
			goto err;
	}
	if (port_attr->nb_aging_objects) {
		if (port_attr->nb_counters == 0) {
			/*
			 * Aging management uses counter. Number counters
			 * requesting should take into account a counter for
			 * each flow rules containing AGE without counter.
			 */
			DRV_LOG(ERR, "Port %u AGE objects are requested (%u) "
				"without counters requesting.",
				dev->data->port_id,
				port_attr->nb_aging_objects);
			rte_errno = EINVAL;
			goto err;
		}
		ret = mlx5_hws_age_pool_init(dev, port_attr, nb_queue);
		if (ret < 0)
			goto err;
	}
	ret = flow_hw_create_vlan(dev);
	if (ret)
		goto err;
	if (_queue_attr)
		mlx5_free(_queue_attr);
	if (port_attr->flags & RTE_FLOW_PORT_FLAG_STRICT_QUEUE)
		priv->hws_strict_queue = 1;
	return 0;
err:
	if (priv->hws_ctpool) {
		flow_hw_ct_pool_destroy(dev, priv->hws_ctpool);
		priv->hws_ctpool = NULL;
	}
	if (priv->ct_mng) {
		flow_hw_ct_mng_destroy(dev, priv->ct_mng);
		priv->ct_mng = NULL;
	}
	if (priv->hws_age_req)
		mlx5_hws_age_pool_destroy(priv);
	if (priv->hws_cpool) {
		mlx5_hws_cnt_pool_destroy(priv->sh, priv->hws_cpool);
		priv->hws_cpool = NULL;
	}
	flow_hw_free_vport_actions(priv);
	for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
		if (priv->hw_drop[i])
			mlx5dr_action_destroy(priv->hw_drop[i]);
		if (priv->hw_tag[i])
			mlx5dr_action_destroy(priv->hw_tag[i]);
	}
	flow_hw_destroy_vlan(dev);
	if (dr_ctx)
		claim_zero(mlx5dr_context_close(dr_ctx));
	for (i = 0; i < nb_q_updated; i++) {
		rte_ring_free(priv->hw_q[i].indir_iq);
		rte_ring_free(priv->hw_q[i].indir_cq);
	}
	mlx5_free(priv->hw_q);
	priv->hw_q = NULL;
	if (priv->acts_ipool) {
		mlx5_ipool_destroy(priv->acts_ipool);
		priv->acts_ipool = NULL;
	}
	if (_queue_attr)
		mlx5_free(_queue_attr);
	/* Do not overwrite the internal errno information. */
	if (ret)
		return ret;
	return rte_flow_error_set(error, rte_errno,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				  "fail to configure port");
}

/**
 * Release HWS resources.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 */
void
flow_hw_resource_release(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_template_table *tbl;
	struct rte_flow_pattern_template *it;
	struct rte_flow_actions_template *at;
	uint32_t i;

	if (!priv->dr_ctx)
		return;
	flow_hw_rxq_flag_set(dev, false);
	flow_hw_flush_all_ctrl_flows(dev);
	flow_hw_cleanup_tx_repr_tagging(dev);
	flow_hw_cleanup_ctrl_rx_tables(dev);
	while (!LIST_EMPTY(&priv->flow_hw_tbl_ongo)) {
		tbl = LIST_FIRST(&priv->flow_hw_tbl_ongo);
		flow_hw_table_destroy(dev, tbl, NULL);
	}
	while (!LIST_EMPTY(&priv->flow_hw_tbl)) {
		tbl = LIST_FIRST(&priv->flow_hw_tbl);
		flow_hw_table_destroy(dev, tbl, NULL);
	}
	while (!LIST_EMPTY(&priv->flow_hw_itt)) {
		it = LIST_FIRST(&priv->flow_hw_itt);
		flow_hw_pattern_template_destroy(dev, it, NULL);
	}
	while (!LIST_EMPTY(&priv->flow_hw_at)) {
		at = LIST_FIRST(&priv->flow_hw_at);
		flow_hw_actions_template_destroy(dev, at, NULL);
	}
	for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
		if (priv->hw_drop[i])
			mlx5dr_action_destroy(priv->hw_drop[i]);
		if (priv->hw_tag[i])
			mlx5dr_action_destroy(priv->hw_tag[i]);
	}
	flow_hw_destroy_vlan(dev);
	flow_hw_free_vport_actions(priv);
	if (priv->acts_ipool) {
		mlx5_ipool_destroy(priv->acts_ipool);
		priv->acts_ipool = NULL;
	}
	if (priv->hws_age_req)
		mlx5_hws_age_pool_destroy(priv);
	if (priv->hws_cpool) {
		mlx5_hws_cnt_pool_destroy(priv->sh, priv->hws_cpool);
		priv->hws_cpool = NULL;
	}
	if (priv->hws_ctpool) {
		flow_hw_ct_pool_destroy(dev, priv->hws_ctpool);
		priv->hws_ctpool = NULL;
	}
	if (priv->ct_mng) {
		flow_hw_ct_mng_destroy(dev, priv->ct_mng);
		priv->ct_mng = NULL;
	}
	for (i = 0; i < priv->nb_queue; i++) {
		rte_ring_free(priv->hw_q[i].indir_iq);
		rte_ring_free(priv->hw_q[i].indir_cq);
	}
	mlx5_free(priv->hw_q);
	priv->hw_q = NULL;
	claim_zero(mlx5dr_context_close(priv->dr_ctx));
	priv->dr_ctx = NULL;
	priv->nb_queue = 0;
}

/* Sets vport tag and mask, for given port, used in HWS rules. */
void
flow_hw_set_port_info(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint16_t port_id = dev->data->port_id;
	struct flow_hw_port_info *info;

	MLX5_ASSERT(port_id < RTE_MAX_ETHPORTS);
	info = &mlx5_flow_hw_port_infos[port_id];
	info->regc_mask = priv->vport_meta_mask;
	info->regc_value = priv->vport_meta_tag;
	info->is_wire = priv->master;
}

/* Clears vport tag and mask used for HWS rules. */
void
flow_hw_clear_port_info(struct rte_eth_dev *dev)
{
	uint16_t port_id = dev->data->port_id;
	struct flow_hw_port_info *info;

	MLX5_ASSERT(port_id < RTE_MAX_ETHPORTS);
	info = &mlx5_flow_hw_port_infos[port_id];
	info->regc_mask = 0;
	info->regc_value = 0;
	info->is_wire = 0;
}

/*
 * Initialize the information of available tag registers and an intersection
 * of all the probed devices' REG_C_Xs.
 * PS. No port concept in steering part, right now it cannot be per port level.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 */
void flow_hw_init_tags_set(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint32_t meta_mode = priv->sh->config.dv_xmeta_en;
	uint8_t masks = (uint8_t)priv->sh->cdev->config.hca_attr.set_reg_c;
	uint32_t i, j;
	enum modify_reg copy[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON};
	uint8_t unset = 0;
	uint8_t copy_masks = 0;

	/*
	 * The CAPA is global for common device but only used in net.
	 * It is shared per eswitch domain.
	 */
	if (!!priv->sh->hws_tags)
		return;
	unset |= 1 << (priv->mtr_color_reg - REG_C_0);
	unset |= 1 << (REG_C_6 - REG_C_0);
	if (priv->sh->config.dv_esw_en)
		unset |= 1 << (REG_C_0 - REG_C_0);
	if (meta_mode == MLX5_XMETA_MODE_META32_HWS)
		unset |= 1 << (REG_C_1 - REG_C_0);
	masks &= ~unset;
	if (mlx5_flow_hw_avl_tags_init_cnt) {
		MLX5_ASSERT(mlx5_flow_hw_aso_tag == priv->mtr_color_reg);
		for (i = 0; i < MLX5_FLOW_HW_TAGS_MAX; i++) {
			if (mlx5_flow_hw_avl_tags[i] != REG_NON && !!((1 << i) & masks)) {
				copy[mlx5_flow_hw_avl_tags[i] - REG_C_0] =
						mlx5_flow_hw_avl_tags[i];
				copy_masks |= (1 << (mlx5_flow_hw_avl_tags[i] - REG_C_0));
			}
		}
		if (copy_masks != masks) {
			j = 0;
			for (i = 0; i < MLX5_FLOW_HW_TAGS_MAX; i++)
				if (!!((1 << i) & copy_masks))
					mlx5_flow_hw_avl_tags[j++] = copy[i];
		}
	} else {
		j = 0;
		for (i = 0; i < MLX5_FLOW_HW_TAGS_MAX; i++) {
			if (!!((1 << i) & masks))
				mlx5_flow_hw_avl_tags[j++] =
					(enum modify_reg)(i + (uint32_t)REG_C_0);
		}
	}
	priv->sh->hws_tags = 1;
	mlx5_flow_hw_aso_tag = (enum modify_reg)priv->mtr_color_reg;
	mlx5_flow_hw_avl_tags_init_cnt++;
}

/*
 * Reset the available tag registers information to NONE.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 */
void flow_hw_clear_tags_set(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	if (!priv->sh->hws_tags)
		return;
	priv->sh->hws_tags = 0;
	mlx5_flow_hw_avl_tags_init_cnt--;
	if (!mlx5_flow_hw_avl_tags_init_cnt)
		memset(mlx5_flow_hw_avl_tags, REG_NON,
		       sizeof(enum modify_reg) * MLX5_FLOW_HW_TAGS_MAX);
}

uint32_t mlx5_flow_hw_flow_metadata_config_refcnt;
uint8_t mlx5_flow_hw_flow_metadata_esw_en;
uint8_t mlx5_flow_hw_flow_metadata_xmeta_en;

/**
 * Initializes static configuration of META flow items.
 *
 * As a temporary workaround, META flow item is translated to a register,
 * based on statically saved dv_esw_en and dv_xmeta_en device arguments.
 * It is a workaround for flow_hw_get_reg_id() where port specific information
 * is not available at runtime.
 *
 * Values of dv_esw_en and dv_xmeta_en device arguments are taken from the first opened port.
 * This means that each mlx5 port will use the same configuration for translation
 * of META flow items.
 *
 * @param[in] dev
 *    Pointer to Ethernet device.
 */
void
flow_hw_init_flow_metadata_config(struct rte_eth_dev *dev)
{
	uint32_t refcnt;

	refcnt = __atomic_fetch_add(&mlx5_flow_hw_flow_metadata_config_refcnt, 1,
				    __ATOMIC_RELAXED);
	if (refcnt > 0)
		return;
	mlx5_flow_hw_flow_metadata_esw_en = MLX5_SH(dev)->config.dv_esw_en;
	mlx5_flow_hw_flow_metadata_xmeta_en = MLX5_SH(dev)->config.dv_xmeta_en;
}

/**
 * Clears statically stored configuration related to META flow items.
 */
void
flow_hw_clear_flow_metadata_config(void)
{
	uint32_t refcnt;

	refcnt = __atomic_sub_fetch(&mlx5_flow_hw_flow_metadata_config_refcnt, 1,
				    __ATOMIC_RELAXED);
	if (refcnt > 0)
		return;
	mlx5_flow_hw_flow_metadata_esw_en = 0;
	mlx5_flow_hw_flow_metadata_xmeta_en = 0;
}

static int
flow_hw_conntrack_destroy(struct rte_eth_dev *dev __rte_unused,
			  uint32_t idx,
			  struct rte_flow_error *error)
{
	uint16_t owner = (uint16_t)MLX5_ACTION_CTX_CT_GET_OWNER(idx);
	uint32_t ct_idx = MLX5_ACTION_CTX_CT_GET_IDX(idx);
	struct rte_eth_dev *owndev = &rte_eth_devices[owner];
	struct mlx5_priv *priv = owndev->data->dev_private;
	struct mlx5_aso_ct_pool *pool = priv->hws_ctpool;
	struct mlx5_aso_ct_action *ct;

	ct = mlx5_ipool_get(pool->cts, ct_idx);
	if (!ct) {
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL,
				"Invalid CT destruction index");
	}
	__atomic_store_n(&ct->state, ASO_CONNTRACK_FREE,
				 __ATOMIC_RELAXED);
	mlx5_ipool_free(pool->cts, ct_idx);
	return 0;
}

static int
flow_hw_conntrack_query(struct rte_eth_dev *dev, uint32_t queue, uint32_t idx,
			struct rte_flow_action_conntrack *profile,
			void *user_data, bool push,
			struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_ct_pool *pool = priv->hws_ctpool;
	struct mlx5_aso_ct_action *ct;
	uint16_t owner = (uint16_t)MLX5_ACTION_CTX_CT_GET_OWNER(idx);
	uint32_t ct_idx;

	if (owner != PORT_ID(priv))
		return rte_flow_error_set(error, EACCES,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL,
				"Can't query CT object owned by another port");
	ct_idx = MLX5_ACTION_CTX_CT_GET_IDX(idx);
	ct = mlx5_ipool_get(pool->cts, ct_idx);
	if (!ct) {
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL,
				"Invalid CT query index");
	}
	profile->peer_port = ct->peer;
	profile->is_original_dir = ct->is_original;
	if (mlx5_aso_ct_query_by_wqe(priv->sh, queue, ct, profile, user_data, push))
		return rte_flow_error_set(error, EIO,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL,
				"Failed to query CT context");
	return 0;
}


static int
flow_hw_conntrack_update(struct rte_eth_dev *dev, uint32_t queue,
			 const struct rte_flow_modify_conntrack *action_conf,
			 uint32_t idx, void *user_data, bool push,
			 struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_ct_pool *pool = priv->hws_ctpool;
	struct mlx5_aso_ct_action *ct;
	const struct rte_flow_action_conntrack *new_prf;
	uint16_t owner = (uint16_t)MLX5_ACTION_CTX_CT_GET_OWNER(idx);
	uint32_t ct_idx;
	int ret = 0;

	if (PORT_ID(priv) != owner)
		return rte_flow_error_set(error, EACCES,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL,
					  "Can't update CT object owned by another port");
	ct_idx = MLX5_ACTION_CTX_CT_GET_IDX(idx);
	ct = mlx5_ipool_get(pool->cts, ct_idx);
	if (!ct) {
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL,
				"Invalid CT update index");
	}
	new_prf = &action_conf->new_ct;
	if (action_conf->direction)
		ct->is_original = !!new_prf->is_original_dir;
	if (action_conf->state) {
		/* Only validate the profile when it needs to be updated. */
		ret = mlx5_validate_action_ct(dev, new_prf, error);
		if (ret)
			return ret;
		ret = mlx5_aso_ct_update_by_wqe(priv->sh, queue, ct, new_prf,
						user_data, push);
		if (ret)
			return rte_flow_error_set(error, EIO,
					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					NULL,
					"Failed to send CT context update WQE");
		if (queue != MLX5_HW_INV_QUEUE)
			return 0;
		/* Block until ready or a failure in synchronous mode. */
		ret = mlx5_aso_ct_available(priv->sh, queue, ct);
		if (ret)
			rte_flow_error_set(error, rte_errno,
					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					   NULL,
					   "Timeout to get the CT update");
	}
	return ret;
}

static struct rte_flow_action_handle *
flow_hw_conntrack_create(struct rte_eth_dev *dev, uint32_t queue,
			 const struct rte_flow_action_conntrack *pro,
			 void *user_data, bool push,
			 struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_ct_pool *pool = priv->hws_ctpool;
	struct mlx5_aso_ct_action *ct;
	uint32_t ct_idx = 0;
	int ret;
	bool async = !!(queue != MLX5_HW_INV_QUEUE);

	if (!pool) {
		rte_flow_error_set(error, EINVAL,
				   RTE_FLOW_ERROR_TYPE_ACTION, NULL,
				   "CT is not enabled");
		return 0;
	}
	ct = mlx5_ipool_zmalloc(pool->cts, &ct_idx);
	if (!ct) {
		rte_flow_error_set(error, rte_errno,
				   RTE_FLOW_ERROR_TYPE_ACTION, NULL,
				   "Failed to allocate CT object");
		return 0;
	}
	ct->offset = ct_idx - 1;
	ct->is_original = !!pro->is_original_dir;
	ct->peer = pro->peer_port;
	ct->pool = pool;
	if (mlx5_aso_ct_update_by_wqe(priv->sh, queue, ct, pro, user_data, push)) {
		mlx5_ipool_free(pool->cts, ct_idx);
		rte_flow_error_set(error, EBUSY,
				   RTE_FLOW_ERROR_TYPE_ACTION, NULL,
				   "Failed to update CT");
		return 0;
	}
	if (!async) {
		ret = mlx5_aso_ct_available(priv->sh, queue, ct);
		if (ret) {
			mlx5_ipool_free(pool->cts, ct_idx);
			rte_flow_error_set(error, rte_errno,
					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					   NULL,
					   "Timeout to get the CT update");
			return 0;
		}
	}
	return (struct rte_flow_action_handle *)(uintptr_t)
		MLX5_ACTION_CTX_CT_GEN_IDX(PORT_ID(priv), ct_idx);
}

/**
 * Validate shared action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   Which queue to be used.
 * @param[in] attr
 *   Operation attribute.
 * @param[in] conf
 *   Indirect action configuration.
 * @param[in] action
 *   rte_flow action detail.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, otherwise negative errno value.
 */
static int
flow_hw_action_handle_validate(struct rte_eth_dev *dev, uint32_t queue,
			       const struct rte_flow_op_attr *attr,
			       const struct rte_flow_indir_action_conf *conf,
			       const struct rte_flow_action *action,
			       void *user_data,
			       struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	RTE_SET_USED(attr);
	RTE_SET_USED(queue);
	RTE_SET_USED(user_data);
	switch (action->type) {
	case RTE_FLOW_ACTION_TYPE_AGE:
		if (!priv->hws_age_req)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  NULL,
						  "aging pool not initialized");
		break;
	case RTE_FLOW_ACTION_TYPE_COUNT:
		if (!priv->hws_cpool)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  NULL,
						  "counters pool not initialized");
		break;
	case RTE_FLOW_ACTION_TYPE_CONNTRACK:
		if (priv->hws_ctpool == NULL)
			return rte_flow_error_set(error, EINVAL,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  NULL,
						  "CT pool not initialized");
		return mlx5_validate_action_ct(dev, action->conf, error);
	case RTE_FLOW_ACTION_TYPE_METER_MARK:
		return flow_hw_validate_action_meter_mark(dev, action, error);
	case RTE_FLOW_ACTION_TYPE_RSS:
		return flow_dv_action_validate(dev, conf, action, error);
	default:
		return rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "action type not supported");
	}
	return 0;
}

/**
 * Create shared action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   Which queue to be used.
 * @param[in] attr
 *   Operation attribute.
 * @param[in] conf
 *   Indirect action configuration.
 * @param[in] action
 *   rte_flow action detail.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   Action handle on success, NULL otherwise and rte_errno is set.
 */
static struct rte_flow_action_handle *
flow_hw_action_handle_create(struct rte_eth_dev *dev, uint32_t queue,
			     const struct rte_flow_op_attr *attr,
			     const struct rte_flow_indir_action_conf *conf,
			     const struct rte_flow_action *action,
			     void *user_data,
			     struct rte_flow_error *error)
{
	struct rte_flow_action_handle *handle = NULL;
	struct mlx5_hw_q_job *job = NULL;
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_action_age *age;
	struct mlx5_aso_mtr *aso_mtr;
	cnt_id_t cnt_id;
	uint32_t mtr_id;
	uint32_t age_idx;
	bool push = true;
	bool aso = false;

	if (attr) {
		MLX5_ASSERT(queue != MLX5_HW_INV_QUEUE);
		if (unlikely(!priv->hw_q[queue].job_idx)) {
			rte_flow_error_set(error, ENOMEM,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				"Flow queue full.");
			return NULL;
		}
		job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
		job->type = MLX5_HW_Q_JOB_TYPE_CREATE;
		job->user_data = user_data;
		push = !attr->postpone;
	}
	switch (action->type) {
	case RTE_FLOW_ACTION_TYPE_AGE:
		if (priv->hws_strict_queue) {
			struct mlx5_age_info *info = GET_PORT_AGE_INFO(priv);

			if (queue >= info->hw_q_age->nb_rings) {
				rte_flow_error_set(error, EINVAL,
						   RTE_FLOW_ERROR_TYPE_ACTION,
						   NULL,
						   "Invalid queue ID for indirect AGE.");
				rte_errno = EINVAL;
				return NULL;
			}
		}
		age = action->conf;
		age_idx = mlx5_hws_age_action_create(priv, queue, true, age,
						     0, error);
		if (age_idx == 0) {
			rte_flow_error_set(error, ENODEV,
					   RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					   "AGE are not configured!");
		} else {
			age_idx = (MLX5_INDIRECT_ACTION_TYPE_AGE <<
				   MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx;
			handle =
			    (struct rte_flow_action_handle *)(uintptr_t)age_idx;
		}
		break;
	case RTE_FLOW_ACTION_TYPE_COUNT:
		if (mlx5_hws_cnt_shared_get(priv->hws_cpool, &cnt_id, 0))
			rte_flow_error_set(error, ENODEV,
					RTE_FLOW_ERROR_TYPE_ACTION,
					NULL,
					"counter are not configured!");
		else
			handle = (struct rte_flow_action_handle *)
				 (uintptr_t)cnt_id;
		break;
	case RTE_FLOW_ACTION_TYPE_CONNTRACK:
		aso = true;
		handle = flow_hw_conntrack_create(dev, queue, action->conf, job,
						  push, error);
		break;
	case RTE_FLOW_ACTION_TYPE_METER_MARK:
		aso = true;
		aso_mtr = flow_hw_meter_mark_alloc(dev, queue, action, job, push);
		if (!aso_mtr)
			break;
		mtr_id = (MLX5_INDIRECT_ACTION_TYPE_METER_MARK <<
			MLX5_INDIRECT_ACTION_TYPE_OFFSET) | (aso_mtr->fm.meter_id);
		handle = (struct rte_flow_action_handle *)(uintptr_t)mtr_id;
		break;
	case RTE_FLOW_ACTION_TYPE_RSS:
		handle = flow_dv_action_create(dev, conf, action, error);
		break;
	default:
		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
				   NULL, "action type not supported");
		break;
	}
	if (job) {
		if (!handle) {
			priv->hw_q[queue].job_idx++;
			return NULL;
		}
		job->action = handle;
		if (push)
			__flow_hw_push_action(dev, queue);
		if (aso)
			return handle;
		rte_ring_enqueue(push ? priv->hw_q[queue].indir_cq :
				 priv->hw_q[queue].indir_iq, job);
	}
	return handle;
}

/**
 * Update shared action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   Which queue to be used.
 * @param[in] attr
 *   Operation attribute.
 * @param[in] handle
 *   Action handle to be updated.
 * @param[in] update
 *   Update value.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_action_handle_update(struct rte_eth_dev *dev, uint32_t queue,
			     const struct rte_flow_op_attr *attr,
			     struct rte_flow_action_handle *handle,
			     const void *update,
			     void *user_data,
			     struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	const struct rte_flow_modify_conntrack *ct_conf =
		(const struct rte_flow_modify_conntrack *)update;
	const struct rte_flow_update_meter_mark *upd_meter_mark =
		(const struct rte_flow_update_meter_mark *)update;
	const struct rte_flow_action_meter_mark *meter_mark;
	struct mlx5_hw_q_job *job = NULL;
	struct mlx5_aso_mtr *aso_mtr;
	struct mlx5_flow_meter_info *fm;
	uint32_t act_idx = (uint32_t)(uintptr_t)handle;
	uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
	uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
	int ret = 0;
	bool push = true;
	bool aso = false;

	if (attr) {
		MLX5_ASSERT(queue != MLX5_HW_INV_QUEUE);
		if (unlikely(!priv->hw_q[queue].job_idx))
			return rte_flow_error_set(error, ENOMEM,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				"Action update failed due to queue full.");
		job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
		job->type = MLX5_HW_Q_JOB_TYPE_UPDATE;
		job->user_data = user_data;
		push = !attr->postpone;
	}
	switch (type) {
	case MLX5_INDIRECT_ACTION_TYPE_AGE:
		ret = mlx5_hws_age_action_update(priv, idx, update, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_CT:
		if (ct_conf->state)
			aso = true;
		ret = flow_hw_conntrack_update(dev, queue, update, act_idx,
					       job, push, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
		aso = true;
		meter_mark = &upd_meter_mark->meter_mark;
		/* Find ASO object. */
		aso_mtr = mlx5_ipool_get(pool->idx_pool, idx);
		if (!aso_mtr) {
			ret = -EINVAL;
			rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL, "Invalid meter_mark update index");
			break;
		}
		fm = &aso_mtr->fm;
		if (upd_meter_mark->profile_valid)
			fm->profile = (struct mlx5_flow_meter_profile *)
							(meter_mark->profile);
		if (upd_meter_mark->color_mode_valid)
			fm->color_aware = meter_mark->color_mode;
		if (upd_meter_mark->init_color_valid)
			aso_mtr->init_color = (meter_mark->color_mode) ?
				meter_mark->init_color : RTE_COLOR_GREEN;
		if (upd_meter_mark->state_valid)
			fm->is_enable = meter_mark->state;
		/* Update ASO flow meter by wqe. */
		if (mlx5_aso_meter_update_by_wqe(priv->sh, queue,
						 aso_mtr, &priv->mtr_bulk, job, push)) {
			ret = -EINVAL;
			rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL, "Unable to update ASO meter WQE");
			break;
		}
		/* Wait for ASO object completion. */
		if (queue == MLX5_HW_INV_QUEUE &&
		    mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) {
			ret = -EINVAL;
			rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL, "Unable to wait for ASO meter CQE");
		}
		break;
	case MLX5_INDIRECT_ACTION_TYPE_RSS:
		ret = flow_dv_action_update(dev, handle, update, error);
		break;
	default:
		ret = -ENOTSUP;
		rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "action type not supported");
		break;
	}
	if (job) {
		if (ret) {
			priv->hw_q[queue].job_idx++;
			return ret;
		}
		job->action = handle;
		if (push)
			__flow_hw_push_action(dev, queue);
		if (aso)
			return 0;
		rte_ring_enqueue(push ? priv->hw_q[queue].indir_cq :
				 priv->hw_q[queue].indir_iq, job);
	}
	return ret;
}

/**
 * Destroy shared action.
 *
 * @param[in] dev
 *   Pointer to the rte_eth_dev structure.
 * @param[in] queue
 *   Which queue to be used.
 * @param[in] attr
 *   Operation attribute.
 * @param[in] handle
 *   Action handle to be destroyed.
 * @param[in] user_data
 *   Pointer to the user_data.
 * @param[out] error
 *   Pointer to error structure.
 *
 * @return
 *   0 on success, negative value otherwise and rte_errno is set.
 */
static int
flow_hw_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue,
			      const struct rte_flow_op_attr *attr,
			      struct rte_flow_action_handle *handle,
			      void *user_data,
			      struct rte_flow_error *error)
{
	uint32_t act_idx = (uint32_t)(uintptr_t)handle;
	uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
	uint32_t age_idx = act_idx & MLX5_HWS_AGE_IDX_MASK;
	uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
	struct mlx5_hw_q_job *job = NULL;
	struct mlx5_aso_mtr *aso_mtr;
	struct mlx5_flow_meter_info *fm;
	bool push = true;
	bool aso = false;
	int ret = 0;

	if (attr) {
		MLX5_ASSERT(queue != MLX5_HW_INV_QUEUE);
		if (unlikely(!priv->hw_q[queue].job_idx))
			return rte_flow_error_set(error, ENOMEM,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				"Action destroy failed due to queue full.");
		job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
		job->type = MLX5_HW_Q_JOB_TYPE_DESTROY;
		job->user_data = user_data;
		push = !attr->postpone;
	}
	switch (type) {
	case MLX5_INDIRECT_ACTION_TYPE_AGE:
		ret = mlx5_hws_age_action_destroy(priv, age_idx, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_COUNT:
		age_idx = mlx5_hws_cnt_age_get(priv->hws_cpool, act_idx);
		if (age_idx != 0)
			/*
			 * If this counter belongs to indirect AGE, here is the
			 * time to update the AGE.
			 */
			mlx5_hws_age_nb_cnt_decrease(priv, age_idx);
		mlx5_hws_cnt_shared_put(priv->hws_cpool, &act_idx);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_CT:
		ret = flow_hw_conntrack_destroy(dev, act_idx, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
		aso_mtr = mlx5_ipool_get(pool->idx_pool, idx);
		if (!aso_mtr) {
			ret = -EINVAL;
			rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL, "Invalid meter_mark destroy index");
			break;
		}
		fm = &aso_mtr->fm;
		fm->is_enable = 0;
		/* Update ASO flow meter by wqe. */
		if (mlx5_aso_meter_update_by_wqe(priv->sh, queue, aso_mtr,
						 &priv->mtr_bulk, job, push)) {
			ret = -EINVAL;
			rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL, "Unable to update ASO meter WQE");
			break;
		}
		/* Wait for ASO object completion. */
		if (queue == MLX5_HW_INV_QUEUE &&
		    mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) {
			ret = -EINVAL;
			rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				NULL, "Unable to wait for ASO meter CQE");
			break;
		}
		if (!job)
			mlx5_ipool_free(pool->idx_pool, idx);
		else
			aso = true;
		break;
	case MLX5_INDIRECT_ACTION_TYPE_RSS:
		ret = flow_dv_action_destroy(dev, handle, error);
		break;
	default:
		ret = -ENOTSUP;
		rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "action type not supported");
		break;
	}
	if (job) {
		if (ret) {
			priv->hw_q[queue].job_idx++;
			return ret;
		}
		job->action = handle;
		if (push)
			__flow_hw_push_action(dev, queue);
		if (aso)
			return ret;
		rte_ring_enqueue(push ? priv->hw_q[queue].indir_cq :
				 priv->hw_q[queue].indir_iq, job);
	}
	return ret;
}

static int
flow_hw_query_counter(const struct rte_eth_dev *dev, uint32_t counter,
		      void *data, struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hws_cnt *cnt;
	struct rte_flow_query_count *qc = data;
	uint32_t iidx;
	uint64_t pkts, bytes;

	if (!mlx5_hws_cnt_id_valid(counter))
		return rte_flow_error_set(error, EINVAL,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				"counter are not available");
	iidx = mlx5_hws_cnt_iidx(priv->hws_cpool, counter);
	cnt = &priv->hws_cpool->pool[iidx];
	__hws_cnt_query_raw(priv->hws_cpool, counter, &pkts, &bytes);
	qc->hits_set = 1;
	qc->bytes_set = 1;
	qc->hits = pkts - cnt->reset.hits;
	qc->bytes = bytes - cnt->reset.bytes;
	if (qc->reset) {
		cnt->reset.bytes = bytes;
		cnt->reset.hits = pkts;
	}
	return 0;
}

/**
 * Query a flow rule AGE action for aging information.
 *
 * @param[in] dev
 *   Pointer to Ethernet device.
 * @param[in] age_idx
 *   Index of AGE action parameter.
 * @param[out] data
 *   Data retrieved by the query.
 * @param[out] error
 *   Perform verbose error reporting if not NULL.
 *
 * @return
 *   0 on success, a negative errno value otherwise and rte_errno is set.
 */
static int
flow_hw_query_age(const struct rte_eth_dev *dev, uint32_t age_idx, void *data,
		  struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_age_info *age_info = GET_PORT_AGE_INFO(priv);
	struct mlx5_indexed_pool *ipool = age_info->ages_ipool;
	struct mlx5_hws_age_param *param = mlx5_ipool_get(ipool, age_idx);
	struct rte_flow_query_age *resp = data;

	if (!param || !param->timeout)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "age data not available");
	switch (__atomic_load_n(&param->state, __ATOMIC_RELAXED)) {
	case HWS_AGE_AGED_OUT_REPORTED:
	case HWS_AGE_AGED_OUT_NOT_REPORTED:
		resp->aged = 1;
		break;
	case HWS_AGE_CANDIDATE:
	case HWS_AGE_CANDIDATE_INSIDE_RING:
		resp->aged = 0;
		break;
	case HWS_AGE_FREE:
		/*
		 * When state is FREE the flow itself should be invalid.
		 * Fall-through.
		 */
	default:
		MLX5_ASSERT(0);
		break;
	}
	resp->sec_since_last_hit_valid = !resp->aged;
	if (resp->sec_since_last_hit_valid)
		resp->sec_since_last_hit = __atomic_load_n
				 (&param->sec_since_last_hit, __ATOMIC_RELAXED);
	return 0;
}

static int
flow_hw_query(struct rte_eth_dev *dev, struct rte_flow *flow,
	      const struct rte_flow_action *actions, void *data,
	      struct rte_flow_error *error)
{
	int ret = -EINVAL;
	struct rte_flow_hw *hw_flow = (struct rte_flow_hw *)flow;

	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
		switch (actions->type) {
		case RTE_FLOW_ACTION_TYPE_VOID:
			break;
		case RTE_FLOW_ACTION_TYPE_COUNT:
			ret = flow_hw_query_counter(dev, hw_flow->cnt_id, data,
						    error);
			break;
		case RTE_FLOW_ACTION_TYPE_AGE:
			ret = flow_hw_query_age(dev, hw_flow->age_idx, data,
						error);
			break;
		default:
			return rte_flow_error_set(error, ENOTSUP,
						  RTE_FLOW_ERROR_TYPE_ACTION,
						  actions,
						  "action not supported");
		}
	}
	return ret;
}

/**
 * Validate indirect action.
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] conf
 *   Shared action configuration.
 * @param[in] action
 *   Action specification used to create indirect action.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   0 on success, otherwise negative errno value.
 */
static int
flow_hw_action_validate(struct rte_eth_dev *dev,
			const struct rte_flow_indir_action_conf *conf,
			const struct rte_flow_action *action,
			struct rte_flow_error *err)
{
	return flow_hw_action_handle_validate(dev, MLX5_HW_INV_QUEUE, NULL,
					      conf, action, NULL, err);
}

/**
 * Create indirect action.
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] conf
 *   Shared action configuration.
 * @param[in] action
 *   Action specification used to create indirect action.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   A valid shared action handle in case of success, NULL otherwise and
 *   rte_errno is set.
 */
static struct rte_flow_action_handle *
flow_hw_action_create(struct rte_eth_dev *dev,
		       const struct rte_flow_indir_action_conf *conf,
		       const struct rte_flow_action *action,
		       struct rte_flow_error *err)
{
	return flow_hw_action_handle_create(dev, MLX5_HW_INV_QUEUE,
					    NULL, conf, action, NULL, err);
}

/**
 * Destroy the indirect action.
 * Release action related resources on the NIC and the memory.
 * Lock free, (mutex should be acquired by caller).
 * Dispatcher for action type specific call.
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] handle
 *   The indirect action object handle to be removed.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   0 on success, otherwise negative errno value.
 */
static int
flow_hw_action_destroy(struct rte_eth_dev *dev,
		       struct rte_flow_action_handle *handle,
		       struct rte_flow_error *error)
{
	return flow_hw_action_handle_destroy(dev, MLX5_HW_INV_QUEUE,
			NULL, handle, NULL, error);
}

/**
 * Updates in place shared action configuration.
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] handle
 *   The indirect action object handle to be updated.
 * @param[in] update
 *   Action specification used to modify the action pointed by *handle*.
 *   *update* could be of same type with the action pointed by the *handle*
 *   handle argument, or some other structures like a wrapper, depending on
 *   the indirect action type.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   0 on success, otherwise negative errno value.
 */
static int
flow_hw_action_update(struct rte_eth_dev *dev,
		      struct rte_flow_action_handle *handle,
		      const void *update,
		      struct rte_flow_error *err)
{
	return flow_hw_action_handle_update(dev, MLX5_HW_INV_QUEUE,
			NULL, handle, update, NULL, err);
}

static int
flow_hw_action_handle_query(struct rte_eth_dev *dev, uint32_t queue,
			    const struct rte_flow_op_attr *attr,
			    const struct rte_flow_action_handle *handle,
			    void *data, void *user_data,
			    struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hw_q_job *job = NULL;
	uint32_t act_idx = (uint32_t)(uintptr_t)handle;
	uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
	uint32_t age_idx = act_idx & MLX5_HWS_AGE_IDX_MASK;
	int ret;
	bool push = true;
	bool aso = false;

	if (attr) {
		MLX5_ASSERT(queue != MLX5_HW_INV_QUEUE);
		if (unlikely(!priv->hw_q[queue].job_idx))
			return rte_flow_error_set(error, ENOMEM,
				RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
				"Action destroy failed due to queue full.");
		job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
		job->type = MLX5_HW_Q_JOB_TYPE_QUERY;
		job->user_data = user_data;
		push = !attr->postpone;
	}
	switch (type) {
	case MLX5_INDIRECT_ACTION_TYPE_AGE:
		ret = flow_hw_query_age(dev, age_idx, data, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_COUNT:
		ret = flow_hw_query_counter(dev, act_idx, data, error);
		break;
	case MLX5_INDIRECT_ACTION_TYPE_CT:
		aso = true;
		if (job)
			job->profile = (struct rte_flow_action_conntrack *)data;
		ret = flow_hw_conntrack_query(dev, queue, act_idx, data,
					      job, push, error);
		break;
	default:
		ret = -ENOTSUP;
		rte_flow_error_set(error, ENOTSUP,
					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
					  "action type not supported");
		break;
	}
	if (job) {
		if (ret) {
			priv->hw_q[queue].job_idx++;
			return ret;
		}
		job->action = handle;
		if (push)
			__flow_hw_push_action(dev, queue);
		if (aso)
			return ret;
		rte_ring_enqueue(push ? priv->hw_q[queue].indir_cq :
				 priv->hw_q[queue].indir_iq, job);
	}
	return 0;
}

static int
flow_hw_action_query(struct rte_eth_dev *dev,
		     const struct rte_flow_action_handle *handle, void *data,
		     struct rte_flow_error *error)
{
	return flow_hw_action_handle_query(dev, MLX5_HW_INV_QUEUE, NULL,
			handle, data, NULL, error);
}

/**
 * Get aged-out flows of a given port on the given HWS flow queue.
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] queue_id
 *   Flow queue to query. Ignored when RTE_FLOW_PORT_FLAG_STRICT_QUEUE not set.
 * @param[in, out] contexts
 *   The address of an array of pointers to the aged-out flows contexts.
 * @param[in] nb_contexts
 *   The length of context array pointers.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   if nb_contexts is 0, return the amount of all aged contexts.
 *   if nb_contexts is not 0 , return the amount of aged flows reported
 *   in the context array, otherwise negative errno value.
 */
static int
flow_hw_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id,
			 void **contexts, uint32_t nb_contexts,
			 struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_age_info *age_info = GET_PORT_AGE_INFO(priv);
	struct rte_ring *r;
	int nb_flows = 0;

	if (nb_contexts && !contexts)
		return rte_flow_error_set(error, EINVAL,
					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
					  NULL, "empty context");
	if (priv->hws_strict_queue) {
		if (queue_id >= age_info->hw_q_age->nb_rings)
			return rte_flow_error_set(error, EINVAL,
						RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
						NULL, "invalid queue id");
		r = age_info->hw_q_age->aged_lists[queue_id];
	} else {
		r = age_info->hw_age.aged_list;
		MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
	}
	if (nb_contexts == 0)
		return rte_ring_count(r);
	while ((uint32_t)nb_flows < nb_contexts) {
		uint32_t age_idx;

		if (rte_ring_dequeue_elem(r, &age_idx, sizeof(uint32_t)) < 0)
			break;
		/* get the AGE context if the aged-out index is still valid. */
		contexts[nb_flows] = mlx5_hws_age_context_get(priv, age_idx);
		if (!contexts[nb_flows])
			continue;
		nb_flows++;
	}
	return nb_flows;
}

/**
 * Get aged-out flows.
 *
 * This function is relevant only if RTE_FLOW_PORT_FLAG_STRICT_QUEUE isn't set.
 *
 * @param[in] dev
 *   Pointer to the Ethernet device structure.
 * @param[in] contexts
 *   The address of an array of pointers to the aged-out flows contexts.
 * @param[in] nb_contexts
 *   The length of context array pointers.
 * @param[out] error
 *   Perform verbose error reporting if not NULL. Initialized in case of
 *   error only.
 *
 * @return
 *   how many contexts get in success, otherwise negative errno value.
 *   if nb_contexts is 0, return the amount of all aged contexts.
 *   if nb_contexts is not 0 , return the amount of aged flows reported
 *   in the context array.
 */
static int
flow_hw_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
		       uint32_t nb_contexts, struct rte_flow_error *error)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	if (priv->hws_strict_queue)
		DRV_LOG(WARNING,
			"port %u get aged flows called in strict queue mode.",
			dev->data->port_id);
	return flow_hw_get_q_aged_flows(dev, 0, contexts, nb_contexts, error);
}

const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = {
	.info_get = flow_hw_info_get,
	.configure = flow_hw_configure,
	.pattern_validate = flow_hw_pattern_validate,
	.pattern_template_create = flow_hw_pattern_template_create,
	.pattern_template_destroy = flow_hw_pattern_template_destroy,
	.actions_validate = flow_hw_actions_validate,
	.actions_template_create = flow_hw_actions_template_create,
	.actions_template_destroy = flow_hw_actions_template_destroy,
	.template_table_create = flow_hw_template_table_create,
	.template_table_destroy = flow_hw_table_destroy,
	.async_flow_create = flow_hw_async_flow_create,
	.async_flow_destroy = flow_hw_async_flow_destroy,
	.pull = flow_hw_pull,
	.push = flow_hw_push,
	.async_action_create = flow_hw_action_handle_create,
	.async_action_destroy = flow_hw_action_handle_destroy,
	.async_action_update = flow_hw_action_handle_update,
	.async_action_query = flow_hw_action_handle_query,
	.action_validate = flow_hw_action_validate,
	.action_create = flow_hw_action_create,
	.action_destroy = flow_hw_action_destroy,
	.action_update = flow_hw_action_update,
	.action_query = flow_hw_action_query,
	.query = flow_hw_query,
	.get_aged_flows = flow_hw_get_aged_flows,
	.get_q_aged_flows = flow_hw_get_q_aged_flows,
};

/**
 * Creates a control flow using flow template API on @p proxy_dev device,
 * on behalf of @p owner_dev device.
 *
 * This function uses locks internally to synchronize access to the
 * flow queue.
 *
 * Created flow is stored in private list associated with @p proxy_dev device.
 *
 * @param owner_dev
 *   Pointer to Ethernet device on behalf of which flow is created.
 * @param proxy_dev
 *   Pointer to Ethernet device on which flow is created.
 * @param table
 *   Pointer to flow table.
 * @param items
 *   Pointer to flow rule items.
 * @param item_template_idx
 *   Index of an item template associated with @p table.
 * @param actions
 *   Pointer to flow rule actions.
 * @param action_template_idx
 *   Index of an action template associated with @p table.
 *
 * @return
 *   0 on success, negative errno value otherwise and rte_errno set.
 */
static __rte_unused int
flow_hw_create_ctrl_flow(struct rte_eth_dev *owner_dev,
			 struct rte_eth_dev *proxy_dev,
			 struct rte_flow_template_table *table,
			 struct rte_flow_item items[],
			 uint8_t item_template_idx,
			 struct rte_flow_action actions[],
			 uint8_t action_template_idx)
{
	struct mlx5_priv *priv = proxy_dev->data->dev_private;
	uint32_t queue = CTRL_QUEUE_ID(priv);
	struct rte_flow_op_attr op_attr = {
		.postpone = 0,
	};
	struct rte_flow *flow = NULL;
	struct mlx5_hw_ctrl_flow *entry = NULL;
	int ret;

	rte_spinlock_lock(&priv->hw_ctrl_lock);
	entry = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_SYS, sizeof(*entry),
			    0, SOCKET_ID_ANY);
	if (!entry) {
		DRV_LOG(ERR, "port %u not enough memory to create control flows",
			proxy_dev->data->port_id);
		rte_errno = ENOMEM;
		ret = -rte_errno;
		goto error;
	}
	flow = flow_hw_async_flow_create(proxy_dev, queue, &op_attr, table,
					 items, item_template_idx,
					 actions, action_template_idx,
					 NULL, NULL);
	if (!flow) {
		DRV_LOG(ERR, "port %u failed to enqueue create control"
			" flow operation", proxy_dev->data->port_id);
		ret = -rte_errno;
		goto error;
	}
	ret = flow_hw_push(proxy_dev, queue, NULL);
	if (ret) {
		DRV_LOG(ERR, "port %u failed to drain control flow queue",
			proxy_dev->data->port_id);
		goto error;
	}
	ret = __flow_hw_pull_comp(proxy_dev, queue, 1, NULL);
	if (ret) {
		DRV_LOG(ERR, "port %u failed to insert control flow",
			proxy_dev->data->port_id);
		rte_errno = EINVAL;
		ret = -rte_errno;
		goto error;
	}
	entry->owner_dev = owner_dev;
	entry->flow = flow;
	LIST_INSERT_HEAD(&priv->hw_ctrl_flows, entry, next);
	rte_spinlock_unlock(&priv->hw_ctrl_lock);
	return 0;
error:
	if (entry)
		mlx5_free(entry);
	rte_spinlock_unlock(&priv->hw_ctrl_lock);
	return ret;
}

/**
 * Destroys a control flow @p flow using flow template API on @p dev device.
 *
 * This function uses locks internally to synchronize access to the
 * flow queue.
 *
 * If the @p flow is stored on any private list/pool, then caller must free up
 * the relevant resources.
 *
 * @param dev
 *   Pointer to Ethernet device.
 * @param flow
 *   Pointer to flow rule.
 *
 * @return
 *   0 on success, non-zero value otherwise.
 */
static int
flow_hw_destroy_ctrl_flow(struct rte_eth_dev *dev, struct rte_flow *flow)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	uint32_t queue = CTRL_QUEUE_ID(priv);
	struct rte_flow_op_attr op_attr = {
		.postpone = 0,
	};
	int ret;

	rte_spinlock_lock(&priv->hw_ctrl_lock);
	ret = flow_hw_async_flow_destroy(dev, queue, &op_attr, flow, NULL, NULL);
	if (ret) {
		DRV_LOG(ERR, "port %u failed to enqueue destroy control"
			" flow operation", dev->data->port_id);
		goto exit;
	}
	ret = flow_hw_push(dev, queue, NULL);
	if (ret) {
		DRV_LOG(ERR, "port %u failed to drain control flow queue",
			dev->data->port_id);
		goto exit;
	}
	ret = __flow_hw_pull_comp(dev, queue, 1, NULL);
	if (ret) {
		DRV_LOG(ERR, "port %u failed to destroy control flow",
			dev->data->port_id);
		rte_errno = EINVAL;
		ret = -rte_errno;
		goto exit;
	}
exit:
	rte_spinlock_unlock(&priv->hw_ctrl_lock);
	return ret;
}

/**
 * Destroys control flows created on behalf of @p owner device on @p dev device.
 *
 * @param dev
 *   Pointer to Ethernet device on which control flows were created.
 * @param owner
 *   Pointer to Ethernet device owning control flows.
 *
 * @return
 *   0 on success, otherwise negative error code is returned and
 *   rte_errno is set.
 */
static int
flow_hw_flush_ctrl_flows_owned_by(struct rte_eth_dev *dev, struct rte_eth_dev *owner)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hw_ctrl_flow *cf;
	struct mlx5_hw_ctrl_flow *cf_next;
	int ret;

	cf = LIST_FIRST(&priv->hw_ctrl_flows);
	while (cf != NULL) {
		cf_next = LIST_NEXT(cf, next);
		if (cf->owner_dev == owner) {
			ret = flow_hw_destroy_ctrl_flow(dev, cf->flow);
			if (ret) {
				rte_errno = ret;
				return -ret;
			}
			LIST_REMOVE(cf, next);
			mlx5_free(cf);
		}
		cf = cf_next;
	}
	return 0;
}

/**
 * Destroys control flows created for @p owner_dev device.
 *
 * @param owner_dev
 *   Pointer to Ethernet device owning control flows.
 *
 * @return
 *   0 on success, otherwise negative error code is returned and
 *   rte_errno is set.
 */
int
mlx5_flow_hw_flush_ctrl_flows(struct rte_eth_dev *owner_dev)
{
	struct mlx5_priv *owner_priv = owner_dev->data->dev_private;
	struct rte_eth_dev *proxy_dev;
	uint16_t owner_port_id = owner_dev->data->port_id;
	uint16_t proxy_port_id = owner_dev->data->port_id;
	int ret;

	/* Flush all flows created by this port for itself. */
	ret = flow_hw_flush_ctrl_flows_owned_by(owner_dev, owner_dev);
	if (ret)
		return ret;
	/* Flush all flows created for this port on proxy port. */
	if (owner_priv->sh->config.dv_esw_en) {
		ret = rte_flow_pick_transfer_proxy(owner_port_id, &proxy_port_id, NULL);
		if (ret == -ENODEV) {
			DRV_LOG(DEBUG, "Unable to find transfer proxy port for port %u. It was "
				       "probably closed. Control flows were cleared.",
				       owner_port_id);
			rte_errno = 0;
			return 0;
		} else if (ret) {
			DRV_LOG(ERR, "Unable to find proxy port for port %u (ret = %d)",
				owner_port_id, ret);
			return ret;
		}
		proxy_dev = &rte_eth_devices[proxy_port_id];
	} else {
		proxy_dev = owner_dev;
	}
	return flow_hw_flush_ctrl_flows_owned_by(proxy_dev, owner_dev);
}

/**
 * Destroys all control flows created on @p dev device.
 *
 * @param owner_dev
 *   Pointer to Ethernet device.
 *
 * @return
 *   0 on success, otherwise negative error code is returned and
 *   rte_errno is set.
 */
static int
flow_hw_flush_all_ctrl_flows(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_hw_ctrl_flow *cf;
	struct mlx5_hw_ctrl_flow *cf_next;
	int ret;

	cf = LIST_FIRST(&priv->hw_ctrl_flows);
	while (cf != NULL) {
		cf_next = LIST_NEXT(cf, next);
		ret = flow_hw_destroy_ctrl_flow(dev, cf->flow);
		if (ret) {
			rte_errno = ret;
			return -ret;
		}
		LIST_REMOVE(cf, next);
		mlx5_free(cf);
		cf = cf_next;
	}
	return 0;
}

int
mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn)
{
	uint16_t port_id = dev->data->port_id;
	struct rte_flow_item_ethdev esw_mgr_spec = {
		.port_id = MLX5_REPRESENTED_PORT_ESW_MGR,
	};
	struct rte_flow_item_ethdev esw_mgr_mask = {
		.port_id = MLX5_REPRESENTED_PORT_ESW_MGR,
	};
	struct rte_flow_item_tag reg_c0_spec = {
		.index = (uint8_t)REG_C_0,
		.data = flow_hw_esw_mgr_regc_marker(dev),
	};
	struct rte_flow_item_tag reg_c0_mask = {
		.index = 0xff,
		.data = flow_hw_esw_mgr_regc_marker_mask(dev),
	};
	struct mlx5_rte_flow_item_sq sq_spec = {
		.queue = sqn,
	};
	struct rte_flow_action_ethdev port = {
		.port_id = port_id,
	};
	struct rte_flow_item items[3] = { { 0 } };
	struct rte_flow_action actions[3] = { { 0 } };
	struct rte_eth_dev *proxy_dev;
	struct mlx5_priv *proxy_priv;
	uint16_t proxy_port_id = dev->data->port_id;
	int ret;

	ret = rte_flow_pick_transfer_proxy(port_id, &proxy_port_id, NULL);
	if (ret) {
		DRV_LOG(ERR, "Unable to pick transfer proxy port for port %u. Transfer proxy "
			     "port must be present to create default SQ miss flows.",
			     port_id);
		return ret;
	}
	proxy_dev = &rte_eth_devices[proxy_port_id];
	proxy_priv = proxy_dev->data->dev_private;
	if (!proxy_priv->dr_ctx) {
		DRV_LOG(DEBUG, "Transfer proxy port (port %u) of port %u must be configured "
			       "for HWS to create default SQ miss flows. Default flows will "
			       "not be created.",
			       proxy_port_id, port_id);
		return 0;
	}
	if (!proxy_priv->hw_esw_sq_miss_root_tbl ||
	    !proxy_priv->hw_esw_sq_miss_tbl) {
		DRV_LOG(ERR, "Transfer proxy port (port %u) of port %u was configured, but "
			     "default flow tables were not created.",
			     proxy_port_id, port_id);
		rte_errno = ENOMEM;
		return -rte_errno;
	}
	/*
	 * Create a root SQ miss flow rule - match E-Switch Manager and SQ,
	 * and jump to group 1.
	 */
	items[0] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
		.spec = &esw_mgr_spec,
		.mask = &esw_mgr_mask,
	};
	items[1] = (struct rte_flow_item){
		.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
		.spec = &sq_spec,
	};
	items[2] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_END,
	};
	actions[0] = (struct rte_flow_action){
		.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
	};
	actions[1] = (struct rte_flow_action){
		.type = RTE_FLOW_ACTION_TYPE_JUMP,
	};
	actions[2] = (struct rte_flow_action) {
		.type = RTE_FLOW_ACTION_TYPE_END,
	};
	ret = flow_hw_create_ctrl_flow(dev, proxy_dev, proxy_priv->hw_esw_sq_miss_root_tbl,
				       items, 0, actions, 0);
	if (ret) {
		DRV_LOG(ERR, "Port %u failed to create root SQ miss flow rule for SQ %u, ret %d",
			port_id, sqn, ret);
		return ret;
	}
	/*
	 * Create a non-root SQ miss flow rule - match REG_C_0 marker and SQ,
	 * and forward to port.
	 */
	items[0] = (struct rte_flow_item){
		.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG,
		.spec = &reg_c0_spec,
		.mask = &reg_c0_mask,
	};
	items[1] = (struct rte_flow_item){
		.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
		.spec = &sq_spec,
	};
	items[2] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_END,
	};
	actions[0] = (struct rte_flow_action){
		.type = RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT,
		.conf = &port,
	};
	actions[1] = (struct rte_flow_action){
		.type = RTE_FLOW_ACTION_TYPE_END,
	};
	ret = flow_hw_create_ctrl_flow(dev, proxy_dev, proxy_priv->hw_esw_sq_miss_tbl,
				       items, 0, actions, 0);
	if (ret) {
		DRV_LOG(ERR, "Port %u failed to create HWS SQ miss flow rule for SQ %u, ret %d",
			port_id, sqn, ret);
		return ret;
	}
	return 0;
}

int
mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev)
{
	uint16_t port_id = dev->data->port_id;
	struct rte_flow_item_ethdev port_spec = {
		.port_id = port_id,
	};
	struct rte_flow_item items[] = {
		{
			.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
			.spec = &port_spec,
		},
		{
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};
	struct rte_flow_action_jump jump = {
		.group = 1,
	};
	struct rte_flow_action actions[] = {
		{
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
			.conf = &jump,
		},
		{
			.type = RTE_FLOW_ACTION_TYPE_END,
		}
	};
	struct rte_eth_dev *proxy_dev;
	struct mlx5_priv *proxy_priv;
	uint16_t proxy_port_id = dev->data->port_id;
	int ret;

	ret = rte_flow_pick_transfer_proxy(port_id, &proxy_port_id, NULL);
	if (ret) {
		DRV_LOG(ERR, "Unable to pick transfer proxy port for port %u. Transfer proxy "
			     "port must be present to create default FDB jump rule.",
			     port_id);
		return ret;
	}
	proxy_dev = &rte_eth_devices[proxy_port_id];
	proxy_priv = proxy_dev->data->dev_private;
	if (!proxy_priv->dr_ctx) {
		DRV_LOG(DEBUG, "Transfer proxy port (port %u) of port %u must be configured "
			       "for HWS to create default FDB jump rule. Default rule will "
			       "not be created.",
			       proxy_port_id, port_id);
		return 0;
	}
	if (!proxy_priv->hw_esw_zero_tbl) {
		DRV_LOG(ERR, "Transfer proxy port (port %u) of port %u was configured, but "
			     "default flow tables were not created.",
			     proxy_port_id, port_id);
		rte_errno = EINVAL;
		return -rte_errno;
	}
	return flow_hw_create_ctrl_flow(dev, proxy_dev,
					proxy_priv->hw_esw_zero_tbl,
					items, 0, actions, 0);
}

int
mlx5_flow_hw_create_tx_default_mreg_copy_flow(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_item_eth promisc = {
		.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
		.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
		.type = 0,
	};
	struct rte_flow_item eth_all[] = {
		[0] = {
			.type = RTE_FLOW_ITEM_TYPE_ETH,
			.spec = &promisc,
			.mask = &promisc,
		},
		[1] = {
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};
	struct rte_flow_action_modify_field mreg_action = {
		.operation = RTE_FLOW_MODIFY_SET,
		.dst = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = REG_C_1,
		},
		.src = {
			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
			.level = REG_A,
		},
		.width = 32,
	};
	struct rte_flow_action copy_reg_action[] = {
		[0] = {
			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
			.conf = &mreg_action,
		},
		[1] = {
			.type = RTE_FLOW_ACTION_TYPE_JUMP,
		},
		[2] = {
			.type = RTE_FLOW_ACTION_TYPE_END,
		},
	};

	MLX5_ASSERT(priv->master);
	if (!priv->dr_ctx || !priv->hw_tx_meta_cpy_tbl)
		return 0;
	return flow_hw_create_ctrl_flow(dev, dev,
					priv->hw_tx_meta_cpy_tbl,
					eth_all, 0, copy_reg_action, 0);
}

int
mlx5_flow_hw_tx_repr_matching_flow(struct rte_eth_dev *dev, uint32_t sqn)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_rte_flow_item_sq sq_spec = {
		.queue = sqn,
	};
	struct rte_flow_item items[] = {
		{
			.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
			.spec = &sq_spec,
		},
		{
			.type = RTE_FLOW_ITEM_TYPE_END,
		},
	};
	/*
	 * Allocate actions array suitable for all cases - extended metadata enabled or not.
	 * With extended metadata there will be an additional MODIFY_FIELD action before JUMP.
	 */
	struct rte_flow_action actions[] = {
		{ .type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD },
		{ .type = RTE_FLOW_ACTION_TYPE_JUMP },
		{ .type = RTE_FLOW_ACTION_TYPE_END },
		{ .type = RTE_FLOW_ACTION_TYPE_END },
	};

	/* It is assumed that caller checked for representor matching. */
	MLX5_ASSERT(priv->sh->config.repr_matching);
	if (!priv->dr_ctx) {
		DRV_LOG(DEBUG, "Port %u must be configured for HWS, before creating "
			       "default egress flow rules. Omitting creation.",
			       dev->data->port_id);
		return 0;
	}
	if (!priv->hw_tx_repr_tagging_tbl) {
		DRV_LOG(ERR, "Port %u is configured for HWS, but table for default "
			     "egress flow rules does not exist.",
			     dev->data->port_id);
		rte_errno = EINVAL;
		return -rte_errno;
	}
	/*
	 * If extended metadata mode is enabled, then an additional MODIFY_FIELD action must be
	 * placed before terminating JUMP action.
	 */
	if (priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) {
		actions[1].type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD;
		actions[2].type = RTE_FLOW_ACTION_TYPE_JUMP;
	}
	return flow_hw_create_ctrl_flow(dev, dev, priv->hw_tx_repr_tagging_tbl,
					items, 0, actions, 0);
}

static uint32_t
__calc_pattern_flags(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type)
{
	switch (eth_pattern_type) {
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL:
		return MLX5_CTRL_PROMISCUOUS;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST:
		return MLX5_CTRL_ALL_MULTICAST;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
		return MLX5_CTRL_BROADCAST;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
		return MLX5_CTRL_IPV4_MULTICAST;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
		return MLX5_CTRL_IPV6_MULTICAST;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
		return MLX5_CTRL_DMAC;
	default:
		/* Should not reach here. */
		MLX5_ASSERT(false);
		return 0;
	}
}

static uint32_t
__calc_vlan_flags(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type)
{
	switch (eth_pattern_type) {
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
		return MLX5_CTRL_VLAN_FILTER;
	default:
		return 0;
	}
}

static bool
eth_pattern_type_is_requested(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type,
			      uint32_t flags)
{
	uint32_t pattern_flags = __calc_pattern_flags(eth_pattern_type);
	uint32_t vlan_flags = __calc_vlan_flags(eth_pattern_type);
	bool pattern_requested = !!(pattern_flags & flags);
	bool consider_vlan = vlan_flags || (MLX5_CTRL_VLAN_FILTER & flags);
	bool vlan_requested = !!(vlan_flags & flags);

	if (consider_vlan)
		return pattern_requested && vlan_requested;
	else
		return pattern_requested;
}

static bool
rss_type_is_requested(struct mlx5_priv *priv,
		      const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct rte_flow_actions_template *at = priv->hw_ctrl_rx->rss[rss_type];
	unsigned int i;

	for (i = 0; at->actions[i].type != RTE_FLOW_ACTION_TYPE_END; ++i) {
		if (at->actions[i].type == RTE_FLOW_ACTION_TYPE_RSS) {
			const struct rte_flow_action_rss *rss = at->actions[i].conf;
			uint64_t rss_types = rss->types;

			if ((rss_types & priv->rss_conf.rss_hf) != rss_types)
				return false;
		}
	}
	return true;
}

static const struct rte_flow_item_eth *
__get_eth_spec(const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern)
{
	switch (pattern) {
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL:
		return &ctrl_rx_eth_promisc_spec;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST:
		return &ctrl_rx_eth_mcast_spec;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
		return &ctrl_rx_eth_bcast_spec;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
		return &ctrl_rx_eth_ipv4_mcast_spec;
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
		return &ctrl_rx_eth_ipv6_mcast_spec;
	default:
		/* This case should not be reached. */
		MLX5_ASSERT(false);
		return NULL;
	}
}

static int
__flow_hw_ctrl_flows_single(struct rte_eth_dev *dev,
			    struct rte_flow_template_table *tbl,
			    const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
			    const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	const struct rte_flow_item_eth *eth_spec = __get_eth_spec(pattern_type);
	struct rte_flow_item items[5];
	struct rte_flow_action actions[] = {
		{ .type = RTE_FLOW_ACTION_TYPE_RSS },
		{ .type = RTE_FLOW_ACTION_TYPE_END },
	};

	if (!eth_spec)
		return -EINVAL;
	memset(items, 0, sizeof(items));
	items[0] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_ETH,
		.spec = eth_spec,
	};
	items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_VOID };
	items[2] = flow_hw_get_ctrl_rx_l3_item(rss_type);
	items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type);
	items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END };
	/* Without VLAN filtering, only a single flow rule must be created. */
	return flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0);
}

static int
__flow_hw_ctrl_flows_single_vlan(struct rte_eth_dev *dev,
				 struct rte_flow_template_table *tbl,
				 const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
				 const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	const struct rte_flow_item_eth *eth_spec = __get_eth_spec(pattern_type);
	struct rte_flow_item items[5];
	struct rte_flow_action actions[] = {
		{ .type = RTE_FLOW_ACTION_TYPE_RSS },
		{ .type = RTE_FLOW_ACTION_TYPE_END },
	};
	unsigned int i;

	if (!eth_spec)
		return -EINVAL;
	memset(items, 0, sizeof(items));
	items[0] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_ETH,
		.spec = eth_spec,
	};
	/* Optional VLAN for now will be VOID - will be filled later. */
	items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_VLAN };
	items[2] = flow_hw_get_ctrl_rx_l3_item(rss_type);
	items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type);
	items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END };
	/* Since VLAN filtering is done, create a single flow rule for each registered vid. */
	for (i = 0; i < priv->vlan_filter_n; ++i) {
		uint16_t vlan = priv->vlan_filter[i];
		struct rte_flow_item_vlan vlan_spec = {
			.tci = rte_cpu_to_be_16(vlan),
		};

		items[1].spec = &vlan_spec;
		if (flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0))
			return -rte_errno;
	}
	return 0;
}

static int
__flow_hw_ctrl_flows_unicast(struct rte_eth_dev *dev,
			     struct rte_flow_template_table *tbl,
			     const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
			     const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct rte_flow_item_eth eth_spec;
	struct rte_flow_item items[5];
	struct rte_flow_action actions[] = {
		{ .type = RTE_FLOW_ACTION_TYPE_RSS },
		{ .type = RTE_FLOW_ACTION_TYPE_END },
	};
	const struct rte_ether_addr cmp = {
		.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	};
	unsigned int i;

	RTE_SET_USED(pattern_type);

	memset(&eth_spec, 0, sizeof(eth_spec));
	memset(items, 0, sizeof(items));
	items[0] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_ETH,
		.spec = &eth_spec,
	};
	items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_VOID };
	items[2] = flow_hw_get_ctrl_rx_l3_item(rss_type);
	items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type);
	items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END };
	for (i = 0; i < MLX5_MAX_MAC_ADDRESSES; ++i) {
		struct rte_ether_addr *mac = &dev->data->mac_addrs[i];

		if (!memcmp(mac, &cmp, sizeof(*mac)))
			continue;
		memcpy(&eth_spec.dst.addr_bytes, mac->addr_bytes, RTE_ETHER_ADDR_LEN);
		if (flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0))
			return -rte_errno;
	}
	return 0;
}

static int
__flow_hw_ctrl_flows_unicast_vlan(struct rte_eth_dev *dev,
				  struct rte_flow_template_table *tbl,
				  const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
				  const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct rte_flow_item_eth eth_spec;
	struct rte_flow_item items[5];
	struct rte_flow_action actions[] = {
		{ .type = RTE_FLOW_ACTION_TYPE_RSS },
		{ .type = RTE_FLOW_ACTION_TYPE_END },
	};
	const struct rte_ether_addr cmp = {
		.addr_bytes = "\x00\x00\x00\x00\x00\x00",
	};
	unsigned int i;
	unsigned int j;

	RTE_SET_USED(pattern_type);

	memset(&eth_spec, 0, sizeof(eth_spec));
	memset(items, 0, sizeof(items));
	items[0] = (struct rte_flow_item){
		.type = RTE_FLOW_ITEM_TYPE_ETH,
		.spec = &eth_spec,
	};
	items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_VLAN };
	items[2] = flow_hw_get_ctrl_rx_l3_item(rss_type);
	items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type);
	items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END };
	for (i = 0; i < MLX5_MAX_MAC_ADDRESSES; ++i) {
		struct rte_ether_addr *mac = &dev->data->mac_addrs[i];

		if (!memcmp(mac, &cmp, sizeof(*mac)))
			continue;
		memcpy(&eth_spec.dst.addr_bytes, mac->addr_bytes, RTE_ETHER_ADDR_LEN);
		for (j = 0; j < priv->vlan_filter_n; ++j) {
			uint16_t vlan = priv->vlan_filter[j];
			struct rte_flow_item_vlan vlan_spec = {
				.tci = rte_cpu_to_be_16(vlan),
			};

			items[1].spec = &vlan_spec;
			if (flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0))
				return -rte_errno;
		}
	}
	return 0;
}

static int
__flow_hw_ctrl_flows(struct rte_eth_dev *dev,
		     struct rte_flow_template_table *tbl,
		     const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
		     const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
	switch (pattern_type) {
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST:
		return __flow_hw_ctrl_flows_single(dev, tbl, pattern_type, rss_type);
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
		return __flow_hw_ctrl_flows_single_vlan(dev, tbl, pattern_type, rss_type);
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC:
		return __flow_hw_ctrl_flows_unicast(dev, tbl, pattern_type, rss_type);
	case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
		return __flow_hw_ctrl_flows_unicast_vlan(dev, tbl, pattern_type, rss_type);
	default:
		/* Should not reach here. */
		MLX5_ASSERT(false);
		rte_errno = EINVAL;
		return -EINVAL;
	}
}


int
mlx5_flow_hw_ctrl_flows(struct rte_eth_dev *dev, uint32_t flags)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_flow_hw_ctrl_rx *hw_ctrl_rx;
	unsigned int i;
	unsigned int j;
	int ret = 0;

	RTE_SET_USED(priv);
	RTE_SET_USED(flags);
	if (!priv->dr_ctx) {
		DRV_LOG(DEBUG, "port %u Control flow rules will not be created. "
			       "HWS needs to be configured beforehand.",
			       dev->data->port_id);
		return 0;
	}
	if (!priv->hw_ctrl_rx) {
		DRV_LOG(ERR, "port %u Control flow rules templates were not created.",
			dev->data->port_id);
		rte_errno = EINVAL;
		return -rte_errno;
	}
	hw_ctrl_rx = priv->hw_ctrl_rx;
	for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX; ++i) {
		const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type = i;

		if (!eth_pattern_type_is_requested(eth_pattern_type, flags))
			continue;
		for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
			const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type = j;
			struct rte_flow_actions_template *at;
			struct mlx5_flow_hw_ctrl_rx_table *tmpls = &hw_ctrl_rx->tables[i][j];
			const struct mlx5_flow_template_table_cfg cfg = {
				.attr = tmpls->attr,
				.external = 0,
			};

			if (!hw_ctrl_rx->rss[rss_type]) {
				at = flow_hw_create_ctrl_rx_rss_template(dev, rss_type);
				if (!at)
					return -rte_errno;
				hw_ctrl_rx->rss[rss_type] = at;
			} else {
				at = hw_ctrl_rx->rss[rss_type];
			}
			if (!rss_type_is_requested(priv, rss_type))
				continue;
			if (!tmpls->tbl) {
				tmpls->tbl = flow_hw_table_create(dev, &cfg,
								  &tmpls->pt, 1, &at, 1, NULL);
				if (!tmpls->tbl) {
					DRV_LOG(ERR, "port %u Failed to create template table "
						     "for control flow rules. Unable to create "
						     "control flow rules.",
						     dev->data->port_id);
					return -rte_errno;
				}
			}

			ret = __flow_hw_ctrl_flows(dev, tmpls->tbl, eth_pattern_type, rss_type);
			if (ret) {
				DRV_LOG(ERR, "port %u Failed to create control flow rule.",
					dev->data->port_id);
				return ret;
			}
		}
	}
	return 0;
}

void
mlx5_flow_meter_uninit(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;

	if (priv->mtr_policy_arr) {
		mlx5_free(priv->mtr_policy_arr);
		priv->mtr_policy_arr = NULL;
	}
	if (priv->mtr_profile_arr) {
		mlx5_free(priv->mtr_profile_arr);
		priv->mtr_profile_arr = NULL;
	}
	if (priv->hws_mpool) {
		mlx5_aso_mtr_queue_uninit(priv->sh, priv->hws_mpool, NULL);
		mlx5_ipool_destroy(priv->hws_mpool->idx_pool);
		mlx5_free(priv->hws_mpool);
		priv->hws_mpool = NULL;
	}
	if (priv->mtr_bulk.aso) {
		mlx5_free(priv->mtr_bulk.aso);
		priv->mtr_bulk.aso = NULL;
		priv->mtr_bulk.size = 0;
		mlx5_aso_queue_uninit(priv->sh, ASO_OPC_MOD_POLICER);
	}
	if (priv->mtr_bulk.action) {
		mlx5dr_action_destroy(priv->mtr_bulk.action);
		priv->mtr_bulk.action = NULL;
	}
	if (priv->mtr_bulk.devx_obj) {
		claim_zero(mlx5_devx_cmd_destroy(priv->mtr_bulk.devx_obj));
		priv->mtr_bulk.devx_obj = NULL;
	}
}

int
mlx5_flow_meter_init(struct rte_eth_dev *dev,
		     uint32_t nb_meters,
		     uint32_t nb_meter_profiles,
		     uint32_t nb_meter_policies,
		     uint32_t nb_queues)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_devx_obj *dcs = NULL;
	uint32_t log_obj_size;
	int ret = 0;
	int reg_id;
	struct mlx5_aso_mtr *aso;
	uint32_t i;
	struct rte_flow_error error;
	uint32_t flags;
	uint32_t nb_mtrs = rte_align32pow2(nb_meters);
	struct mlx5_indexed_pool_config cfg = {
		.size = sizeof(struct mlx5_aso_mtr),
		.trunk_size = 1 << 12,
		.per_core_cache = 1 << 13,
		.need_lock = 1,
		.release_mem_en = !!priv->sh->config.reclaim_mode,
		.malloc = mlx5_malloc,
		.max_idx = nb_meters,
		.free = mlx5_free,
		.type = "mlx5_hw_mtr_mark_action",
	};

	if (!nb_meters || !nb_meter_profiles || !nb_meter_policies) {
		ret = ENOTSUP;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter configuration is invalid.");
		goto err;
	}
	if (!priv->mtr_en || !priv->sh->meter_aso_en) {
		ret = ENOTSUP;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter ASO is not supported.");
		goto err;
	}
	priv->mtr_config.nb_meters = nb_meters;
	log_obj_size = rte_log2_u32(nb_meters >> 1);
	dcs = mlx5_devx_cmd_create_flow_meter_aso_obj
		(priv->sh->cdev->ctx, priv->sh->cdev->pdn,
			log_obj_size);
	if (!dcs) {
		ret = ENOMEM;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter ASO object allocation failed.");
		goto err;
	}
	priv->mtr_bulk.devx_obj = dcs;
	reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, NULL);
	if (reg_id < 0) {
		ret = ENOTSUP;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter register is not available.");
		goto err;
	}
	flags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX;
	if (priv->sh->config.dv_esw_en && priv->master)
		flags |= MLX5DR_ACTION_FLAG_HWS_FDB;
	priv->mtr_bulk.action = mlx5dr_action_create_aso_meter
			(priv->dr_ctx, (struct mlx5dr_devx_obj *)dcs,
				reg_id - REG_C_0, flags);
	if (!priv->mtr_bulk.action) {
		ret = ENOMEM;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter action creation failed.");
		goto err;
	}
	priv->mtr_bulk.aso = mlx5_malloc(MLX5_MEM_ZERO,
					 sizeof(struct mlx5_aso_mtr) *
					 nb_meters,
					 RTE_CACHE_LINE_SIZE,
					 SOCKET_ID_ANY);
	if (!priv->mtr_bulk.aso) {
		ret = ENOMEM;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter bulk ASO allocation failed.");
		goto err;
	}
	priv->mtr_bulk.size = nb_meters;
	aso = priv->mtr_bulk.aso;
	for (i = 0; i < priv->mtr_bulk.size; i++) {
		aso->type = ASO_METER_DIRECT;
		aso->state = ASO_METER_WAIT;
		aso->offset = i;
		aso++;
	}
	priv->hws_mpool = mlx5_malloc(MLX5_MEM_ZERO,
				sizeof(struct mlx5_aso_mtr_pool),
				RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
	if (!priv->hws_mpool) {
		ret = ENOMEM;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter ipool allocation failed.");
		goto err;
	}
	priv->hws_mpool->devx_obj = priv->mtr_bulk.devx_obj;
	priv->hws_mpool->action = priv->mtr_bulk.action;
	priv->hws_mpool->nb_sq = nb_queues;
	if (mlx5_aso_mtr_queue_init(priv->sh, priv->hws_mpool,
				    &priv->sh->mtrmng->pools_mng, nb_queues)) {
		ret = ENOMEM;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter ASO queue allocation failed.");
		goto err;
	}
	/*
	 * No need for local cache if Meter number is a small number.
	 * Since flow insertion rate will be very limited in that case.
	 * Here let's set the number to less than default trunk size 4K.
	 */
	if (nb_mtrs <= cfg.trunk_size) {
		cfg.per_core_cache = 0;
		cfg.trunk_size = nb_mtrs;
	} else if (nb_mtrs <= MLX5_HW_IPOOL_SIZE_THRESHOLD) {
		cfg.per_core_cache = MLX5_HW_IPOOL_CACHE_MIN;
	}
	priv->hws_mpool->idx_pool = mlx5_ipool_create(&cfg);
	priv->mtr_config.nb_meter_profiles = nb_meter_profiles;
	priv->mtr_profile_arr =
		mlx5_malloc(MLX5_MEM_ZERO,
			    sizeof(struct mlx5_flow_meter_profile) *
			    nb_meter_profiles,
			    RTE_CACHE_LINE_SIZE,
			    SOCKET_ID_ANY);
	if (!priv->mtr_profile_arr) {
		ret = ENOMEM;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter profile allocation failed.");
		goto err;
	}
	priv->mtr_config.nb_meter_policies = nb_meter_policies;
	priv->mtr_policy_arr =
		mlx5_malloc(MLX5_MEM_ZERO,
			    sizeof(struct mlx5_flow_meter_policy) *
			    nb_meter_policies,
			    RTE_CACHE_LINE_SIZE,
			    SOCKET_ID_ANY);
	if (!priv->mtr_policy_arr) {
		ret = ENOMEM;
		rte_flow_error_set(&error, ENOMEM,
				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
				  NULL, "Meter policy allocation failed.");
		goto err;
	}
	return 0;
err:
	mlx5_flow_meter_uninit(dev);
	return ret;
}

#endif
