Merge tag 'net-next-5.10' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next

Pull networking updates from Jakub Kicinski:

 - Add redirect_neigh() BPF packet redirect helper, allowing to limit
   stack traversal in common container configs and improving TCP
   back-pressure.

   Daniel reports ~10Gbps => ~15Gbps single stream TCP performance gain.

 - Expand netlink policy support and improve policy export to user
   space. (Ge)netlink core performs request validation according to
   declared policies. Expand the expressiveness of those policies
   (min/max length and bitmasks). Allow dumping policies for particular
   commands. This is used for feature discovery by user space (instead
   of kernel version parsing or trial and error).

 - Support IGMPv3/MLDv2 multicast listener discovery protocols in
   bridge.

 - Allow more than 255 IPv4 multicast interfaces.

 - Add support for Type of Service (ToS) reflection in SYN/SYN-ACK
   packets of TCPv6.

 - In Multi-patch TCP (MPTCP) support concurrent transmission of data on
   multiple subflows in a load balancing scenario. Enhance advertising
   addresses via the RM_ADDR/ADD_ADDR options.

 - Support SMC-Dv2 version of SMC, which enables multi-subnet
   deployments.

 - Allow more calls to same peer in RxRPC.

 - Support two new Controller Area Network (CAN) protocols - CAN-FD and
   ISO 15765-2:2016.

 - Add xfrm/IPsec compat layer, solving the 32bit user space on 64bit
   kernel problem.

 - Add TC actions for implementing MPLS L2 VPNs.

 - Improve nexthop code - e.g. handle various corner cases when nexthop
   objects are removed from groups better, skip unnecessary
   notifications and make it easier to offload nexthops into HW by
   converting to a blocking notifier.

 - Support adding and consuming TCP header options by BPF programs,
   opening the doors for easy experimental and deployment-specific TCP
   option use.

 - Reorganize TCP congestion control (CC) initialization to simplify
   life of TCP CC implemented in BPF.

 - Add support for shipping BPF programs with the kernel and loading
   them early on boot via the User Mode Driver mechanism, hence reusing
   all the user space infra we have.

 - Support sleepable BPF programs, initially targeting LSM and tracing.

 - Add bpf_d_path() helper for returning full path for given 'struct
   path'.

 - Make bpf_tail_call compatible with bpf-to-bpf calls.

 - Allow BPF programs to call map_update_elem on sockmaps.

 - Add BPF Type Format (BTF) support for type and enum discovery, as
   well as support for using BTF within the kernel itself (current use
   is for pretty printing structures).

 - Support listing and getting information about bpf_links via the bpf
   syscall.

 - Enhance kernel interfaces around NIC firmware update. Allow
   specifying overwrite mask to control if settings etc. are reset
   during update; report expected max time operation may take to users;
   support firmware activation without machine reboot incl. limits of
   how much impact reset may have (e.g. dropping link or not).

 - Extend ethtool configuration interface to report IEEE-standard
   counters, to limit the need for per-vendor logic in user space.

 - Adopt or extend devlink use for debug, monitoring, fw update in many
   drivers (dsa loop, ice, ionic, sja1105, qed, mlxsw, mv88e6xxx,
   dpaa2-eth).

 - In mlxsw expose critical and emergency SFP module temperature alarms.
   Refactor port buffer handling to make the defaults more suitable and
   support setting these values explicitly via the DCBNL interface.

 - Add XDP support for Intel's igb driver.

 - Support offloading TC flower classification and filtering rules to
   mscc_ocelot switches.

 - Add PTP support for Marvell Octeontx2 and PP2.2 hardware, as well as
   fixed interval period pulse generator and one-step timestamping in
   dpaa-eth.

 - Add support for various auth offloads in WiFi APs, e.g. SAE (WPA3)
   offload.

 - Add Lynx PHY/PCS MDIO module, and convert various drivers which have
   this HW to use it. Convert mvpp2 to split PCS.

 - Support Marvell Prestera 98DX3255 24-port switch ASICs, as well as
   7-port Mediatek MT7531 IP.

 - Add initial support for QCA6390 and IPQ6018 in ath11k WiFi driver,
   and wcn3680 support in wcn36xx.

 - Improve performance for packets which don't require much offloads on
   recent Mellanox NICs by 20% by making multiple packets share a
   descriptor entry.

 - Move chelsio inline crypto drivers (for TLS and IPsec) from the
   crypto subtree to drivers/net. Move MDIO drivers out of the phy
   directory.

 - Clean up a lot of W=1 warnings, reportedly the actively developed
   subsections of networking drivers should now build W=1 warning free.

 - Make sure drivers don't use in_interrupt() to dynamically adapt their
   code. Convert tasklets to use new tasklet_setup API (sadly this
   conversion is not yet complete).

* tag 'net-next-5.10' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next: (2583 commits)
  Revert "bpfilter: Fix build error with CONFIG_BPFILTER_UMH"
  net, sockmap: Don't call bpf_prog_put() on NULL pointer
  bpf, selftest: Fix flaky tcp_hdr_options test when adding addr to lo
  bpf, sockmap: Add locking annotations to iterator
  netfilter: nftables: allow re-computing sctp CRC-32C in 'payload' statements
  net: fix pos incrementment in ipv6_route_seq_next
  net/smc: fix invalid return code in smcd_new_buf_create()
  net/smc: fix valid DMBE buffer sizes
  net/smc: fix use-after-free of delayed events
  bpfilter: Fix build error with CONFIG_BPFILTER_UMH
  cxgb4/ch_ipsec: Replace the module name to ch_ipsec from chcr
  net: sched: Fix suspicious RCU usage while accessing tcf_tunnel_info
  bpf: Fix register equivalence tracking.
  rxrpc: Fix loss of final ack on shutdown
  rxrpc: Fix bundle counting for exclusive connections
  netfilter: restore NF_INET_NUMHOOKS
  ibmveth: Identify ingress large send packets.
  ibmveth: Switch order of ibmveth_helper calls.
  cxgb4: handle 4-tuple PEDIT to NAT mode translation
  selftests: Add VRF route leaking tests
  ...

16 files changed, 0 insertions, 9260 deletions

diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig
index 2984fdf51e85..f886401af13e 100644
--- a/drivers/crypto/chelsio/Kconfig
+++ b/drivers/crypto/chelsio/Kconfig
@@ -21,35 +21,3 @@ config CRYPTO_DEV_CHELSIO
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called chcr.
-
-config CHELSIO_IPSEC_INLINE
-	bool "Chelsio IPSec XFRM Tx crypto offload"
-	depends on CHELSIO_T4
-	depends on CRYPTO_DEV_CHELSIO
-	depends on XFRM_OFFLOAD
-	depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD
-	default n
-	help
-	  Enable support for IPSec Tx Inline.
-
-config CRYPTO_DEV_CHELSIO_TLS
-	tristate "Chelsio Crypto Inline TLS Driver"
-	depends on CHELSIO_T4
-	depends on TLS_TOE
-	select CRYPTO_DEV_CHELSIO
-	help
-	  Support Chelsio Inline TLS with Chelsio crypto accelerator.
-
-	  To compile this driver as a module, choose M here: the module
-	  will be called chtls.
-
-config CHELSIO_TLS_DEVICE
-	bool "Chelsio Inline KTLS Offload"
-	depends on CHELSIO_T4
-	depends on TLS_DEVICE
-	select CRYPTO_DEV_CHELSIO
-	default y
-	help
-	  This flag enables support for kernel tls offload over Chelsio T6
-	  crypto accelerator. CONFIG_CHELSIO_TLS_DEVICE flag can be enabled
-	  only if CONFIG_TLS and CONFIG_TLS_DEVICE flags are enabled.
diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile
index 0e9d035927e9..2e5df484ab01 100644
--- a/drivers/crypto/chelsio/Makefile
+++ b/drivers/crypto/chelsio/Makefile
@@ -3,8 +3,3 @@ ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4
 
 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o
 chcr-objs :=  chcr_core.o chcr_algo.o
-#ifdef CONFIG_CHELSIO_TLS_DEVICE
-chcr-objs += chcr_ktls.o
-#endif
-chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o
-obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
index d4f6e010dc79..507aafe93f21 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -86,39 +86,6 @@
 	 KEY_CONTEXT_OPAD_PRESENT_M)
 #define KEY_CONTEXT_OPAD_PRESENT_F      KEY_CONTEXT_OPAD_PRESENT_V(1U)
 
-#define TLS_KEYCTX_RXFLIT_CNT_S 24
-#define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S)
-
-#define TLS_KEYCTX_RXPROT_VER_S 20
-#define TLS_KEYCTX_RXPROT_VER_M 0xf
-#define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S)
-
-#define TLS_KEYCTX_RXCIPH_MODE_S 16
-#define TLS_KEYCTX_RXCIPH_MODE_M 0xf
-#define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S)
-
-#define TLS_KEYCTX_RXAUTH_MODE_S 12
-#define TLS_KEYCTX_RXAUTH_MODE_M 0xf
-#define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S)
-
-#define TLS_KEYCTX_RXCIAU_CTRL_S 11
-#define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S)
-
-#define TLS_KEYCTX_RX_SEQCTR_S 9
-#define TLS_KEYCTX_RX_SEQCTR_M 0x3
-#define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S)
-
-#define TLS_KEYCTX_RX_VALID_S 8
-#define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S)
-
-#define TLS_KEYCTX_RXCK_SIZE_S 3
-#define TLS_KEYCTX_RXCK_SIZE_M 0x7
-#define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S)
-
-#define TLS_KEYCTX_RXMK_SIZE_S 0
-#define TLS_KEYCTX_RXMK_SIZE_M 0x7
-#define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S)
-
 #define CHCR_HASH_MAX_DIGEST_SIZE 64
 #define CHCR_MAX_SHA_DIGEST_SIZE 64
 
diff --git a/drivers/crypto/chelsio/chcr_common.h b/drivers/crypto/chelsio/chcr_common.h
deleted file mode 100644
index 33f589cbfba1..000000000000
--- a/drivers/crypto/chelsio/chcr_common.h
+++ /dev/null
@@ -1,135 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (C) 2020 Chelsio Communications.  All rights reserved. */
-
-#ifndef __CHCR_COMMON_H__
-#define __CHCR_COMMON_H__
-
-#include "cxgb4.h"
-
-#define CHCR_MAX_SALT                      4
-#define CHCR_KEYCTX_MAC_KEY_SIZE_128       0
-#define CHCR_KEYCTX_CIPHER_KEY_SIZE_128    0
-#define CHCR_SCMD_CIPHER_MODE_AES_GCM      2
-#define CHCR_SCMD_CIPHER_MODE_AES_CTR      3
-#define CHCR_CPL_TX_SEC_PDU_LEN_64BIT      2
-#define CHCR_SCMD_SEQ_NO_CTRL_64BIT        3
-#define CHCR_SCMD_PROTO_VERSION_TLS        0
-#define CHCR_SCMD_PROTO_VERSION_GENERIC    4
-#define CHCR_SCMD_AUTH_MODE_GHASH          4
-#define AES_BLOCK_LEN                      16
-
-enum chcr_state {
-	CHCR_INIT = 0,
-	CHCR_ATTACH,
-	CHCR_DETACH,
-};
-
-struct chcr_dev {
-	spinlock_t lock_chcr_dev; /* chcr dev structure lock */
-	enum chcr_state state;
-	atomic_t inflight;
-	int wqretry;
-	struct delayed_work detach_work;
-	struct completion detach_comp;
-	unsigned char tx_channel_id;
-};
-
-struct uld_ctx {
-	struct list_head entry;
-	struct cxgb4_lld_info lldi;
-	struct chcr_dev dev;
-};
-
-struct ktls_key_ctx {
-	__be32 ctx_hdr;
-	u8 salt[CHCR_MAX_SALT];
-	__be64 iv_to_auth;
-	unsigned char key[TLS_CIPHER_AES_GCM_128_KEY_SIZE +
-			  TLS_CIPHER_AES_GCM_256_TAG_SIZE];
-};
-
-/* Crypto key context */
-#define KEY_CONTEXT_CTX_LEN_S           24
-#define KEY_CONTEXT_CTX_LEN_V(x)        ((x) << KEY_CONTEXT_CTX_LEN_S)
-
-#define KEY_CONTEXT_SALT_PRESENT_S      10
-#define KEY_CONTEXT_SALT_PRESENT_V(x)   ((x) << KEY_CONTEXT_SALT_PRESENT_S)
-#define KEY_CONTEXT_SALT_PRESENT_F      KEY_CONTEXT_SALT_PRESENT_V(1U)
-
-#define KEY_CONTEXT_VALID_S     0
-#define KEY_CONTEXT_VALID_V(x)  ((x) << KEY_CONTEXT_VALID_S)
-#define KEY_CONTEXT_VALID_F     KEY_CONTEXT_VALID_V(1U)
-
-#define KEY_CONTEXT_CK_SIZE_S           6
-#define KEY_CONTEXT_CK_SIZE_V(x)        ((x) << KEY_CONTEXT_CK_SIZE_S)
-
-#define KEY_CONTEXT_MK_SIZE_S           2
-#define KEY_CONTEXT_MK_SIZE_V(x)        ((x) << KEY_CONTEXT_MK_SIZE_S)
-
-#define KEY_CONTEXT_OPAD_PRESENT_S      11
-#define KEY_CONTEXT_OPAD_PRESENT_V(x)   ((x) << KEY_CONTEXT_OPAD_PRESENT_S)
-#define KEY_CONTEXT_OPAD_PRESENT_F      KEY_CONTEXT_OPAD_PRESENT_V(1U)
-
-#define FILL_KEY_CTX_HDR(ck_size, mk_size, ctx_len) \
-		htonl(KEY_CONTEXT_MK_SIZE_V(mk_size) | \
-		      KEY_CONTEXT_CK_SIZE_V(ck_size) | \
-		      KEY_CONTEXT_VALID_F | \
-		      KEY_CONTEXT_SALT_PRESENT_F | \
-		      KEY_CONTEXT_CTX_LEN_V((ctx_len)))
-
-struct uld_ctx *assign_chcr_device(void);
-
-static inline void *chcr_copy_to_txd(const void *src, const struct sge_txq *q,
-				     void *pos, int length)
-{
-	int left = (void *)q->stat - pos;
-	u64 *p;
-
-	if (likely(length <= left)) {
-		memcpy(pos, src, length);
-		pos += length;
-	} else {
-		memcpy(pos, src, left);
-		memcpy(q->desc, src + left, length - left);
-		pos = (void *)q->desc + (length - left);
-	}
-	/* 0-pad to multiple of 16 */
-	p = PTR_ALIGN(pos, 8);
-	if ((uintptr_t)p & 8) {
-		*p = 0;
-		return p + 1;
-	}
-	return p;
-}
-
-static inline unsigned int chcr_txq_avail(const struct sge_txq *q)
-{
-	return q->size - 1 - q->in_use;
-}
-
-static inline void chcr_txq_advance(struct sge_txq *q, unsigned int n)
-{
-	q->in_use += n;
-	q->pidx += n;
-	if (q->pidx >= q->size)
-		q->pidx -= q->size;
-}
-
-static inline void chcr_eth_txq_stop(struct sge_eth_txq *q)
-{
-	netif_tx_stop_queue(q->txq);
-	q->q.stops++;
-}
-
-static inline unsigned int chcr_sgl_len(unsigned int n)
-{
-	n--;
-	return (3 * n) / 2 + (n & 1) + 2;
-}
-
-static inline unsigned int chcr_flits_to_desc(unsigned int n)
-{
-	WARN_ON(n > SGE_MAX_WR_LEN / 8);
-	return DIV_ROUND_UP(n, 8);
-}
-#endif /* __CHCR_COMMON_H__ */
diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c
index ed7989cf151e..f91f9d762a45 100644
--- a/drivers/crypto/chelsio/chcr_core.c
+++ b/drivers/crypto/chelsio/chcr_core.c
@@ -33,23 +33,8 @@ static int cpl_fw6_pld_handler(struct adapter *adap, unsigned char *input);
 static void *chcr_uld_add(const struct cxgb4_lld_info *lld);
 static int chcr_uld_state_change(void *handle, enum cxgb4_state state);
 
-#if defined(CONFIG_CHELSIO_TLS_DEVICE)
-static const struct tlsdev_ops chcr_ktls_ops = {
-	.tls_dev_add = chcr_ktls_dev_add,
-	.tls_dev_del = chcr_ktls_dev_del,
-};
-#endif
-
-#ifdef CONFIG_CHELSIO_IPSEC_INLINE
-static void update_netdev_features(void);
-#endif /* CONFIG_CHELSIO_IPSEC_INLINE */
-
 static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
 	[CPL_FW6_PLD] = cpl_fw6_pld_handler,
-#ifdef CONFIG_CHELSIO_TLS_DEVICE
-	[CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl,
-	[CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl,
-#endif
 };
 
 static struct cxgb4_uld_info chcr_uld_info = {
@@ -60,12 +45,6 @@ static struct cxgb4_uld_info chcr_uld_info = {
 	.add = chcr_uld_add,
 	.state_change = chcr_uld_state_change,
 	.rx_handler = chcr_uld_rx_handler,
-#if defined(CONFIG_CHELSIO_IPSEC_INLINE) || defined(CONFIG_CHELSIO_TLS_DEVICE)
-	.tx_handler = chcr_uld_tx_handler,
-#endif /* CONFIG_CHELSIO_IPSEC_INLINE || CONFIG_CHELSIO_TLS_DEVICE */
-#if defined(CONFIG_CHELSIO_TLS_DEVICE)
-	.tlsdev_ops = &chcr_ktls_ops,
-#endif
 };
 
 static void detach_work_fn(struct work_struct *work)
@@ -241,23 +220,6 @@ int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
 	return 0;
 }
 
-#if defined(CONFIG_CHELSIO_IPSEC_INLINE) || defined(CONFIG_CHELSIO_TLS_DEVICE)
-int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev)
-{
-	/* In case if skb's decrypted bit is set, it's nic tls packet, else it's
-	 * ipsec packet.
-	 */
-#ifdef CONFIG_CHELSIO_TLS_DEVICE
-	if (skb->decrypted)
-		return chcr_ktls_xmit(skb, dev);
-#endif
-#ifdef CONFIG_CHELSIO_IPSEC_INLINE
-	return chcr_ipsec_xmit(skb, dev);
-#endif
-	return 0;
-}
-#endif /* CONFIG_CHELSIO_IPSEC_INLINE || CONFIG_CHELSIO_TLS_DEVICE */
-
 static void chcr_detach_device(struct uld_ctx *u_ctx)
 {
 	struct chcr_dev *dev = &u_ctx->dev;
@@ -305,24 +267,6 @@ static int chcr_uld_state_change(void *handle, enum cxgb4_state state)
 	return ret;
 }
 
-#ifdef CONFIG_CHELSIO_IPSEC_INLINE
-static void update_netdev_features(void)
-{
-	struct uld_ctx *u_ctx, *tmp;
-
-	mutex_lock(&drv_data.drv_mutex);
-	list_for_each_entry_safe(u_ctx, tmp, &drv_data.inact_dev, entry) {
-		if (u_ctx->lldi.crypto & ULP_CRYPTO_IPSEC_INLINE)
-			chcr_add_xfrmops(&u_ctx->lldi);
-	}
-	list_for_each_entry_safe(u_ctx, tmp, &drv_data.act_dev, entry) {
-		if (u_ctx->lldi.crypto & ULP_CRYPTO_IPSEC_INLINE)
-			chcr_add_xfrmops(&u_ctx->lldi);
-	}
-	mutex_unlock(&drv_data.drv_mutex);
-}
-#endif /* CONFIG_CHELSIO_IPSEC_INLINE */
-
 static int __init chcr_crypto_init(void)
 {
 	INIT_LIST_HEAD(&drv_data.act_dev);
@@ -332,12 +276,6 @@ static int __init chcr_crypto_init(void)
 	drv_data.last_dev = NULL;
 	cxgb4_register_uld(CXGB4_ULD_CRYPTO, &chcr_uld_info);
 
-	#ifdef CONFIG_CHELSIO_IPSEC_INLINE
-	rtnl_lock();
-	update_netdev_features();
-	rtnl_unlock();
-	#endif /* CONFIG_CHELSIO_IPSEC_INLINE */
-
 	return 0;
 }
 
diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h
index 67d77abd6775..b02f981e7c32 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -72,54 +72,6 @@ struct _key_ctx {
 	unsigned char key[];
 };
 
-#define KEYCTX_TX_WR_IV_S  55
-#define KEYCTX_TX_WR_IV_M  0x1ffULL
-#define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S)
-#define KEYCTX_TX_WR_IV_G(x) \
-	(((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M)
-
-#define KEYCTX_TX_WR_AAD_S 47
-#define KEYCTX_TX_WR_AAD_M 0xffULL
-#define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S)
-#define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \
-				KEYCTX_TX_WR_AAD_M)
-
-#define KEYCTX_TX_WR_AADST_S 39
-#define KEYCTX_TX_WR_AADST_M 0xffULL
-#define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S)
-#define KEYCTX_TX_WR_AADST_G(x) \
-	(((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M)
-
-#define KEYCTX_TX_WR_CIPHER_S 30
-#define KEYCTX_TX_WR_CIPHER_M 0x1ffULL
-#define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S)
-#define KEYCTX_TX_WR_CIPHER_G(x) \
-	(((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M)
-
-#define KEYCTX_TX_WR_CIPHERST_S 23
-#define KEYCTX_TX_WR_CIPHERST_M 0x7f
-#define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S)
-#define KEYCTX_TX_WR_CIPHERST_G(x) \
-	(((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M)
-
-#define KEYCTX_TX_WR_AUTH_S 14
-#define KEYCTX_TX_WR_AUTH_M 0x1ff
-#define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S)
-#define KEYCTX_TX_WR_AUTH_G(x) \
-	(((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M)
-
-#define KEYCTX_TX_WR_AUTHST_S 7
-#define KEYCTX_TX_WR_AUTHST_M 0x7f
-#define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S)
-#define KEYCTX_TX_WR_AUTHST_G(x) \
-	(((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M)
-
-#define KEYCTX_TX_WR_AUTHIN_S 0
-#define KEYCTX_TX_WR_AUTHIN_M 0x7f
-#define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S)
-#define KEYCTX_TX_WR_AUTHIN_G(x) \
-	(((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTHIN_M)
-
 #define WQ_RETRY	5
 struct chcr_driver_data {
 	struct list_head act_dev;
@@ -157,42 +109,6 @@ struct uld_ctx {
 	struct chcr_dev dev;
 };
 
-struct sge_opaque_hdr {
-	void *dev;
-	dma_addr_t addr[MAX_SKB_FRAGS + 1];
-};
-
-struct chcr_ipsec_req {
-	struct ulp_txpkt ulptx;
-	struct ulptx_idata sc_imm;
-	struct cpl_tx_sec_pdu sec_cpl;
-	struct _key_ctx key_ctx;
-};
-
-struct chcr_ipsec_wr {
-	struct fw_ulptx_wr wreq;
-	struct chcr_ipsec_req req;
-};
-
-#define ESN_IV_INSERT_OFFSET 12
-struct chcr_ipsec_aadiv {
-	__be32 spi;
-	u8 seq_no[8];
-	u8 iv[8];
-};
-
-struct ipsec_sa_entry {
-	int hmac_ctrl;
-	u16 esn;
-	u16 resv;
-	unsigned int enckey_len;
-	unsigned int kctx_len;
-	unsigned int authsize;
-	__be32 key_ctx_hdr;
-	char salt[MAX_SALT];
-	char key[2 * AES_MAX_KEY_SIZE];
-};
-
 /*
  *      sgl_len - calculates the size of an SGL of the given capacity
  *      @n: the number of SGL entries
@@ -221,18 +137,4 @@ int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
 int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev);
 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
 		     int err);
-int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev);
-void chcr_add_xfrmops(const struct cxgb4_lld_info *lld);
-#ifdef CONFIG_CHELSIO_TLS_DEVICE
-int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input);
-int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input);
-int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev);
-extern int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
-			     enum tls_offload_ctx_dir direction,
-			     struct tls_crypto_info *crypto_info,
-			     u32 start_offload_tcp_sn);
-extern void chcr_ktls_dev_del(struct net_device *netdev,
-			      struct tls_context *tls_ctx,
-			      enum tls_offload_ctx_dir direction);
-#endif
 #endif /* __CHCR_CORE_H__ */
diff --git a/drivers/crypto/chelsio/chcr_ipsec.c b/drivers/crypto/chelsio/chcr_ipsec.c
deleted file mode 100644
index 967babd67a51..000000000000
--- a/drivers/crypto/chelsio/chcr_ipsec.c
+++ /dev/null
@@ -1,754 +0,0 @@
-/*
- * This file is part of the Chelsio T6 Crypto driver for Linux.
- *
- * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- * Written and Maintained by:
- *	Atul Gupta (atul.gupta@chelsio.com)
- */
-
-#define pr_fmt(fmt) "chcr:" fmt
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/crypto.h>
-#include <linux/skbuff.h>
-#include <linux/rtnetlink.h>
-#include <linux/highmem.h>
-#include <linux/if_vlan.h>
-#include <linux/ip.h>
-#include <linux/netdevice.h>
-#include <net/esp.h>
-#include <net/xfrm.h>
-#include <crypto/aes.h>
-#include <crypto/algapi.h>
-#include <crypto/hash.h>
-#include <crypto/sha.h>
-#include <crypto/authenc.h>
-#include <crypto/internal/aead.h>
-#include <crypto/null.h>
-#include <crypto/internal/skcipher.h>
-#include <crypto/aead.h>
-#include <crypto/scatterwalk.h>
-#include <crypto/internal/hash.h>
-
-#include "chcr_core.h"
-#include "chcr_algo.h"
-#include "chcr_crypto.h"
-
-/*
- * Max Tx descriptor space we allow for an Ethernet packet to be inlined
- * into a WR.
- */
-#define MAX_IMM_TX_PKT_LEN 256
-#define GCM_ESP_IV_SIZE     8
-
-static int chcr_xfrm_add_state(struct xfrm_state *x);
-static void chcr_xfrm_del_state(struct xfrm_state *x);
-static void chcr_xfrm_free_state(struct xfrm_state *x);
-static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x);
-static void chcr_advance_esn_state(struct xfrm_state *x);
-
-static const struct xfrmdev_ops chcr_xfrmdev_ops = {
-	.xdo_dev_state_add      = chcr_xfrm_add_state,
-	.xdo_dev_state_delete   = chcr_xfrm_del_state,
-	.xdo_dev_state_free     = chcr_xfrm_free_state,
-	.xdo_dev_offload_ok     = chcr_ipsec_offload_ok,
-	.xdo_dev_state_advance_esn = chcr_advance_esn_state,
-};
-
-/* Add offload xfrms to Chelsio Interface */
-void chcr_add_xfrmops(const struct cxgb4_lld_info *lld)
-{
-	struct net_device *netdev = NULL;
-	int i;
-
-	for (i = 0; i < lld->nports; i++) {
-		netdev = lld->ports[i];
-		if (!netdev)
-			continue;
-		netdev->xfrmdev_ops = &chcr_xfrmdev_ops;
-		netdev->hw_enc_features |= NETIF_F_HW_ESP;
-		netdev->features |= NETIF_F_HW_ESP;
-		netdev_change_features(netdev);
-	}
-}
-
-static inline int chcr_ipsec_setauthsize(struct xfrm_state *x,
-					 struct ipsec_sa_entry *sa_entry)
-{
-	int hmac_ctrl;
-	int authsize = x->aead->alg_icv_len / 8;
-
-	sa_entry->authsize = authsize;
-
-	switch (authsize) {
-	case ICV_8:
-		hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
-		break;
-	case ICV_12:
-		hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
-		break;
-	case ICV_16:
-		hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
-		break;
-	default:
-		return -EINVAL;
-	}
-	return hmac_ctrl;
-}
-
-static inline int chcr_ipsec_setkey(struct xfrm_state *x,
-				    struct ipsec_sa_entry *sa_entry)
-{
-	int keylen = (x->aead->alg_key_len + 7) / 8;
-	unsigned char *key = x->aead->alg_key;
-	int ck_size, key_ctx_size = 0;
-	unsigned char ghash_h[AEAD_H_SIZE];
-	struct crypto_aes_ctx aes;
-	int ret = 0;
-
-	if (keylen > 3) {
-		keylen -= 4;  /* nonce/salt is present in the last 4 bytes */
-		memcpy(sa_entry->salt, key + keylen, 4);
-	}
-
-	if (keylen == AES_KEYSIZE_128) {
-		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
-	} else if (keylen == AES_KEYSIZE_192) {
-		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
-	} else if (keylen == AES_KEYSIZE_256) {
-		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
-	} else {
-		pr_err("GCM: Invalid key length %d\n", keylen);
-		ret = -EINVAL;
-		goto out;
-	}
-
-	memcpy(sa_entry->key, key, keylen);
-	sa_entry->enckey_len = keylen;
-	key_ctx_size = sizeof(struct _key_ctx) +
-			      ((DIV_ROUND_UP(keylen, 16)) << 4) +
-			      AEAD_H_SIZE;
-
-	sa_entry->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
-						 CHCR_KEYCTX_MAC_KEY_SIZE_128,
-						 0, 0,
-						 key_ctx_size >> 4);
-
-	/* Calculate the H = CIPH(K, 0 repeated 16 times).
-	 * It will go in key context
-	 */
-	ret = aes_expandkey(&aes, key, keylen);
-	if (ret) {
-		sa_entry->enckey_len = 0;
-		goto out;
-	}
-	memset(ghash_h, 0, AEAD_H_SIZE);
-	aes_encrypt(&aes, ghash_h, ghash_h);
-	memzero_explicit(&aes, sizeof(aes));
-
-	memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) *
-	       16), ghash_h, AEAD_H_SIZE);
-	sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) +
-			      AEAD_H_SIZE;
-out:
-	return ret;
-}
-
-/*
- * chcr_xfrm_add_state
- * returns 0 on success, negative error if failed to send message to FPGA
- * positive error if FPGA returned a bad response
- */
-static int chcr_xfrm_add_state(struct xfrm_state *x)
-{
-	struct ipsec_sa_entry *sa_entry;
-	int res = 0;
-
-	if (x->props.aalgo != SADB_AALG_NONE) {
-		pr_debug("CHCR: Cannot offload authenticated xfrm states\n");
-		return -EINVAL;
-	}
-	if (x->props.calgo != SADB_X_CALG_NONE) {
-		pr_debug("CHCR: Cannot offload compressed xfrm states\n");
-		return -EINVAL;
-	}
-	if (x->props.family != AF_INET &&
-	    x->props.family != AF_INET6) {
-		pr_debug("CHCR: Only IPv4/6 xfrm state offloaded\n");
-		return -EINVAL;
-	}
-	if (x->props.mode != XFRM_MODE_TRANSPORT &&
-	    x->props.mode != XFRM_MODE_TUNNEL) {
-		pr_debug("CHCR: Only transport and tunnel xfrm offload\n");
-		return -EINVAL;
-	}
-	if (x->id.proto != IPPROTO_ESP) {
-		pr_debug("CHCR: Only ESP xfrm state offloaded\n");
-		return -EINVAL;
-	}
-	if (x->encap) {
-		pr_debug("CHCR: Encapsulated xfrm state not offloaded\n");
-		return -EINVAL;
-	}
-	if (!x->aead) {
-		pr_debug("CHCR: Cannot offload xfrm states without aead\n");
-		return -EINVAL;
-	}
-	if (x->aead->alg_icv_len != 128 &&
-	    x->aead->alg_icv_len != 96) {
-		pr_debug("CHCR: Cannot offload xfrm states with AEAD ICV length other than 96b & 128b\n");
-	return -EINVAL;
-	}
-	if ((x->aead->alg_key_len != 128 + 32) &&
-	    (x->aead->alg_key_len != 256 + 32)) {
-		pr_debug("CHCR: Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
-		return -EINVAL;
-	}
-	if (x->tfcpad) {
-		pr_debug("CHCR: Cannot offload xfrm states with tfc padding\n");
-		return -EINVAL;
-	}
-	if (!x->geniv) {
-		pr_debug("CHCR: Cannot offload xfrm states without geniv\n");
-		return -EINVAL;
-	}
-	if (strcmp(x->geniv, "seqiv")) {
-		pr_debug("CHCR: Cannot offload xfrm states with geniv other than seqiv\n");
-		return -EINVAL;
-	}
-
-	sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
-	if (!sa_entry) {
-		res = -ENOMEM;
-		goto out;
-	}
-
-	sa_entry->hmac_ctrl = chcr_ipsec_setauthsize(x, sa_entry);
-	if (x->props.flags & XFRM_STATE_ESN)
-		sa_entry->esn = 1;
-	chcr_ipsec_setkey(x, sa_entry);
-	x->xso.offload_handle = (unsigned long)sa_entry;
-	try_module_get(THIS_MODULE);
-out:
-	return res;
-}
-
-static void chcr_xfrm_del_state(struct xfrm_state *x)
-{
-	/* do nothing */
-	if (!x->xso.offload_handle)
-		return;
-}
-
-static void chcr_xfrm_free_state(struct xfrm_state *x)
-{
-	struct ipsec_sa_entry *sa_entry;
-
-	if (!x->xso.offload_handle)
-		return;
-
-	sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
-	kfree(sa_entry);
-	module_put(THIS_MODULE);
-}
-
-static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
-{
-	if (x->props.family == AF_INET) {
-		/* Offload with IP options is not supported yet */
-		if (ip_hdr(skb)->ihl > 5)
-			return false;
-	} else {
-		/* Offload with IPv6 extension headers is not support yet */
-		if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
-			return false;
-	}
-	return true;
-}
-
-static void chcr_advance_esn_state(struct xfrm_state *x)
-{
-	/* do nothing */
-	if (!x->xso.offload_handle)
-		return;
-}
-
-static inline int is_eth_imm(const struct sk_buff *skb,
-			     struct ipsec_sa_entry *sa_entry)
-{
-	unsigned int kctx_len;
-	int hdrlen;
-
-	kctx_len = sa_entry->kctx_len;
-	hdrlen = sizeof(struct fw_ulptx_wr) +
-		 sizeof(struct chcr_ipsec_req) + kctx_len;
-
-	hdrlen += sizeof(struct cpl_tx_pkt);
-	if (sa_entry->esn)
-		hdrlen += (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16)
-			   << 4);
-	if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)
-		return hdrlen;
-	return 0;
-}
-
-static inline unsigned int calc_tx_sec_flits(const struct sk_buff *skb,
-					     struct ipsec_sa_entry *sa_entry,
-					     bool *immediate)
-{
-	unsigned int kctx_len;
-	unsigned int flits;
-	int aadivlen;
-	int hdrlen;
-
-	kctx_len = sa_entry->kctx_len;
-	hdrlen = is_eth_imm(skb, sa_entry);
-	aadivlen = sa_entry->esn ? DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv),
-						16) : 0;
-	aadivlen <<= 4;
-
-	/* If the skb is small enough, we can pump it out as a work request
-	 * with only immediate data.  In that case we just have to have the
-	 * TX Packet header plus the skb data in the Work Request.
-	 */
-
-	if (hdrlen) {
-		*immediate = true;
-		return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64));
-	}
-
-	flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
-
-	/* Otherwise, we're going to have to construct a Scatter gather list
-	 * of the skb body and fragments.  We also include the flits necessary
-	 * for the TX Packet Work Request and CPL.  We always have a firmware
-	 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
-	 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
-	 * message or, if we're doing a Large Send Offload, an LSO CPL message
-	 * with an embedded TX Packet Write CPL message.
-	 */
-	flits += (sizeof(struct fw_ulptx_wr) +
-		  sizeof(struct chcr_ipsec_req) +
-		  kctx_len +
-		  sizeof(struct cpl_tx_pkt_core) +
-		  aadivlen) / sizeof(__be64);
-	return flits;
-}
-
-inline void *copy_esn_pktxt(struct sk_buff *skb,
-			    struct net_device *dev,
-			    void *pos,
-			    struct ipsec_sa_entry *sa_entry)
-{
-	struct chcr_ipsec_aadiv *aadiv;
-	struct ulptx_idata *sc_imm;
-	struct ip_esp_hdr *esphdr;
-	struct xfrm_offload *xo;
-	struct sge_eth_txq *q;
-	struct adapter *adap;
-	struct port_info *pi;
-	__be64 seqno;
-	u32 qidx;
-	u32 seqlo;
-	u8 *iv;
-	int eoq;
-	int len;
-
-	pi = netdev_priv(dev);
-	adap = pi->adapter;
-	qidx = skb->queue_mapping;
-	q = &adap->sge.ethtxq[qidx + pi->first_qset];
-
-	/* end of queue, reset pos to start of queue */
-	eoq = (void *)q->q.stat - pos;
-	if (!eoq)
-		pos = q->q.desc;
-
-	len = DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16) << 4;
-	memset(pos, 0, len);
-	aadiv = (struct chcr_ipsec_aadiv *)pos;
-	esphdr = (struct ip_esp_hdr *)skb_transport_header(skb);
-	iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr);
-	xo = xfrm_offload(skb);
-
-	aadiv->spi = (esphdr->spi);
-	seqlo = ntohl(esphdr->seq_no);
-	seqno = cpu_to_be64(seqlo + ((u64)xo->seq.hi << 32));
-	memcpy(aadiv->seq_no, &seqno, 8);
-	iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr);
-	memcpy(aadiv->iv, iv, 8);
-
-	if (is_eth_imm(skb, sa_entry) && !skb_is_nonlinear(skb)) {
-		sc_imm = (struct ulptx_idata *)(pos +
-			  (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv),
-					sizeof(__be64)) << 3));
-		sc_imm->cmd_more = FILL_CMD_MORE(0);
-		sc_imm->len = cpu_to_be32(skb->len);
-	}
-	pos += len;
-	return pos;
-}
-
-inline void *copy_cpltx_pktxt(struct sk_buff *skb,
-			      struct net_device *dev,
-			      void *pos,
-			      struct ipsec_sa_entry *sa_entry)
-{
-	struct cpl_tx_pkt_core *cpl;
-	struct sge_eth_txq *q;
-	struct adapter *adap;
-	struct port_info *pi;
-	u32 ctrl0, qidx;
-	u64 cntrl = 0;
-	int left;
-
-	pi = netdev_priv(dev);
-	adap = pi->adapter;
-	qidx = skb->queue_mapping;
-	q = &adap->sge.ethtxq[qidx + pi->first_qset];
-
-	left = (void *)q->q.stat - pos;
-	if (!left)
-		pos = q->q.desc;
-
-	cpl = (struct cpl_tx_pkt_core *)pos;
-
-	cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
-	ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) |
-			       TXPKT_PF_V(adap->pf);
-	if (skb_vlan_tag_present(skb)) {
-		q->vlan_ins++;
-		cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
-	}
-
-	cpl->ctrl0 = htonl(ctrl0);
-	cpl->pack = htons(0);
-	cpl->len = htons(skb->len);
-	cpl->ctrl1 = cpu_to_be64(cntrl);
-
-	pos += sizeof(struct cpl_tx_pkt_core);
-	/* Copy ESN info for HW */
-	if (sa_entry->esn)
-		pos = copy_esn_pktxt(skb, dev, pos, sa_entry);
-	return pos;
-}
-
-inline void *copy_key_cpltx_pktxt(struct sk_buff *skb,
-				struct net_device *dev,
-				void *pos,
-				struct ipsec_sa_entry *sa_entry)
-{
-	struct _key_ctx *key_ctx;
-	int left, eoq, key_len;
-	struct sge_eth_txq *q;
-	struct adapter *adap;
-	struct port_info *pi;
-	unsigned int qidx;
-
-	pi = netdev_priv(dev);
-	adap = pi->adapter;
-	qidx = skb->queue_mapping;
-	q = &adap->sge.ethtxq[qidx + pi->first_qset];
-	key_len = sa_entry->kctx_len;
-
-	/* end of queue, reset pos to start of queue */
-	eoq = (void *)q->q.stat - pos;
-	left = eoq;
-	if (!eoq) {
-		pos = q->q.desc;
-		left = 64 * q->q.size;
-	}
-
-	/* Copy the Key context header */
-	key_ctx = (struct _key_ctx *)pos;
-	key_ctx->ctx_hdr = sa_entry->key_ctx_hdr;
-	memcpy(key_ctx->salt, sa_entry->salt, MAX_SALT);
-	pos += sizeof(struct _key_ctx);
-	left -= sizeof(struct _key_ctx);
-
-	if (likely(key_len <= left)) {
-		memcpy(key_ctx->key, sa_entry->key, key_len);
-		pos += key_len;
-	} else {
-		memcpy(pos, sa_entry->key, left);
-		memcpy(q->q.desc, sa_entry->key + left,
-		       key_len - left);
-		pos = (u8 *)q->q.desc + (key_len - left);
-	}
-	/* Copy CPL TX PKT XT */
-	pos = copy_cpltx_pktxt(skb, dev, pos, sa_entry);
-
-	return pos;
-}
-
-inline void *chcr_crypto_wreq(struct sk_buff *skb,
-			       struct net_device *dev,
-			       void *pos,
-			       int credits,
-			       struct ipsec_sa_entry *sa_entry)
-{
-	struct port_info *pi = netdev_priv(dev);
-	struct adapter *adap = pi->adapter;
-	unsigned int ivsize = GCM_ESP_IV_SIZE;
-	struct chcr_ipsec_wr *wr;
-	bool immediate = false;
-	u16 immdatalen = 0;
-	unsigned int flits;
-	u32 ivinoffset;
-	u32 aadstart;
-	u32 aadstop;
-	u32 ciphstart;
-	u16 sc_more = 0;
-	u32 ivdrop = 0;
-	u32 esnlen = 0;
-	u32 wr_mid;
-	u16 ndesc;
-	int qidx = skb_get_queue_mapping(skb);
-	struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset];
-	unsigned int kctx_len = sa_entry->kctx_len;
-	int qid = q->q.cntxt_id;
-
-	atomic_inc(&adap->chcr_stats.ipsec_cnt);
-
-	flits = calc_tx_sec_flits(skb, sa_entry, &immediate);
-	ndesc = DIV_ROUND_UP(flits, 2);
-	if (sa_entry->esn)
-		ivdrop = 1;
-
-	if (immediate)
-		immdatalen = skb->len;
-
-	if (sa_entry->esn) {
-		esnlen = sizeof(struct chcr_ipsec_aadiv);
-		if (!skb_is_nonlinear(skb))
-			sc_more  = 1;
-	}
-
-	/* WR Header */
-	wr = (struct chcr_ipsec_wr *)pos;
-	wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
-	wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(ndesc);
-
-	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
-		netif_tx_stop_queue(q->txq);
-		q->q.stops++;
-		if (!q->dbqt)
-			wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
-	}
-	wr_mid |= FW_ULPTX_WR_DATA_F;
-	wr->wreq.flowid_len16 = htonl(wr_mid);
-
-	/* ULPTX */
-	wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid);
-	wr->req.ulptx.len = htonl(ndesc - 1);
-
-	/* Sub-command */
-	wr->req.sc_imm.cmd_more = FILL_CMD_MORE(!immdatalen || sc_more);
-	wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
-					 sizeof(wr->req.key_ctx) +
-					 kctx_len +
-					 sizeof(struct cpl_tx_pkt_core) +
-					 esnlen +
-					 (esnlen ? 0 : immdatalen));
-
-	/* CPL_SEC_PDU */
-	ivinoffset = sa_entry->esn ? (ESN_IV_INSERT_OFFSET + 1) :
-				     (skb_transport_offset(skb) +
-				      sizeof(struct ip_esp_hdr) + 1);
-	wr->req.sec_cpl.op_ivinsrtofst = htonl(
-				CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
-				CPL_TX_SEC_PDU_CPLLEN_V(2) |
-				CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
-				CPL_TX_SEC_PDU_IVINSRTOFST_V(
-							     ivinoffset));
-
-	wr->req.sec_cpl.pldlen = htonl(skb->len + esnlen);
-	aadstart = sa_entry->esn ? 1 : (skb_transport_offset(skb) + 1);
-	aadstop = sa_entry->esn ? ESN_IV_INSERT_OFFSET :
-				  (skb_transport_offset(skb) +
-				   sizeof(struct ip_esp_hdr));
-	ciphstart = skb_transport_offset(skb) + sizeof(struct ip_esp_hdr) +
-		    GCM_ESP_IV_SIZE + 1;
-	ciphstart += sa_entry->esn ?  esnlen : 0;
-
-	wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
-							aadstart,
-							aadstop,
-							ciphstart, 0);
-
-	wr->req.sec_cpl.cipherstop_lo_authinsert =
-		FILL_SEC_CPL_AUTHINSERT(0, ciphstart,
-					sa_entry->authsize,
-					 sa_entry->authsize);
-	wr->req.sec_cpl.seqno_numivs =
-		FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1,
-					 CHCR_SCMD_CIPHER_MODE_AES_GCM,
-					 CHCR_SCMD_AUTH_MODE_GHASH,
-					 sa_entry->hmac_ctrl,
-					 ivsize >> 1);
-	wr->req.sec_cpl.ivgen_hdrlen =  FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
-								  0, ivdrop, 0);
-
-	pos += sizeof(struct fw_ulptx_wr) +
-	       sizeof(struct ulp_txpkt) +
-	       sizeof(struct ulptx_idata) +
-	       sizeof(struct cpl_tx_sec_pdu);
-
-	pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry);
-
-	return pos;
-}
-
-/**
- *      flits_to_desc - returns the num of Tx descriptors for the given flits
- *      @n: the number of flits
- *
- *      Returns the number of Tx descriptors needed for the supplied number
- *      of flits.
- */
-static inline unsigned int flits_to_desc(unsigned int n)
-{
-	WARN_ON(n > SGE_MAX_WR_LEN / 8);
-	return DIV_ROUND_UP(n, 8);
-}
-
-static inline unsigned int txq_avail(const struct sge_txq *q)
-{
-	return q->size - 1 - q->in_use;
-}
-
-static void eth_txq_stop(struct sge_eth_txq *q)
-{
-	netif_tx_stop_queue(q->txq);
-	q->q.stops++;
-}
-
-static inline void txq_advance(struct sge_txq *q, unsigned int n)
-{
-	q->in_use += n;
-	q->pidx += n;
-	if (q->pidx >= q->size)
-		q->pidx -= q->size;
-}
-
-/*
- *      chcr_ipsec_xmit called from ULD Tx handler
- */
-int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev)
-{
-	struct xfrm_state *x = xfrm_input_state(skb);
-	unsigned int last_desc, ndesc, flits = 0;
-	struct ipsec_sa_entry *sa_entry;
-	u64 *pos, *end, *before, *sgl;
-	struct tx_sw_desc *sgl_sdesc;
-	int qidx, left, credits;
-	bool immediate = false;
-	struct sge_eth_txq *q;
-	struct adapter *adap;
-	struct port_info *pi;
-	struct sec_path *sp;
-
-	if (!x->xso.offload_handle)
-		return NETDEV_TX_BUSY;
-
-	sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
-
-	sp = skb_sec_path(skb);
-	if (sp->len != 1) {
-out_free:       dev_kfree_skb_any(skb);
-		return NETDEV_TX_OK;
-	}
-
-	pi = netdev_priv(dev);
-	adap = pi->adapter;
-	qidx = skb->queue_mapping;
-	q = &adap->sge.ethtxq[qidx + pi->first_qset];
-
-	cxgb4_reclaim_completed_tx(adap, &q->q, true);
-
-	flits = calc_tx_sec_flits(skb, sa_entry, &immediate);
-	ndesc = flits_to_desc(flits);
-	credits = txq_avail(&q->q) - ndesc;
-
-	if (unlikely(credits < 0)) {
-		eth_txq_stop(q);
-		dev_err(adap->pdev_dev,
-			"%s: Tx ring %u full while queue awake! cred:%d %d %d flits:%d\n",
-			dev->name, qidx, credits, ndesc, txq_avail(&q->q),
-			flits);
-		return NETDEV_TX_BUSY;
-	}
-
-	last_desc = q->q.pidx + ndesc - 1;
-	if (last_desc >= q->q.size)
-		last_desc -= q->q.size;
-	sgl_sdesc = &q->q.sdesc[last_desc];
-
-	if (!immediate &&
-	    unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
-		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
-		q->mapping_err++;
-		goto out_free;
-	}
-
-	pos = (u64 *)&q->q.desc[q->q.pidx];
-	before = (u64 *)pos;
-	end = (u64 *)pos + flits;
-	/* Setup IPSec CPL */
-	pos = (void *)chcr_crypto_wreq(skb, dev, (void *)pos,
-				       credits, sa_entry);
-	if (before > (u64 *)pos) {
-		left = (u8 *)end - (u8 *)q->q.stat;
-		end = (void *)q->q.desc + left;
-	}
-	if (pos == (u64 *)q->q.stat) {
-		left = (u8 *)end - (u8 *)q->q.stat;
-		end = (void *)q->q.desc + left;
-		pos = (void *)q->q.desc;
-	}
-
-	sgl = (void *)pos;
-	if (immediate) {
-		cxgb4_inline_tx_skb(skb, &q->q, sgl);
-		dev_consume_skb_any(skb);
-	} else {
-		cxgb4_write_sgl(skb, &q->q, (void *)sgl, end,
-				0, sgl_sdesc->addr);
-		skb_orphan(skb);
-		sgl_sdesc->skb = skb;
-	}
-	txq_advance(&q->q, ndesc);
-
-	cxgb4_ring_tx_db(adap, &q->q, ndesc);
-	return NETDEV_TX_OK;
-}
diff --git a/drivers/crypto/chelsio/chcr_ktls.c b/drivers/crypto/chelsio/chcr_ktls.c
deleted file mode 100644
index c5cce024886a..000000000000
--- a/drivers/crypto/chelsio/chcr_ktls.c
+++ /dev/null
@@ -1,2017 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (C) 2020 Chelsio Communications.  All rights reserved. */
-
-#ifdef CONFIG_CHELSIO_TLS_DEVICE
-#include <linux/highmem.h>
-#include "chcr_ktls.h"
-#include "clip_tbl.h"
-
-static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
-/*
- * chcr_ktls_save_keys: calculate and save crypto keys.
- * @tx_info - driver specific tls info.
- * @crypto_info - tls crypto information.
- * @direction - TX/RX direction.
- * return - SUCCESS/FAILURE.
- */
-static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
-			       struct tls_crypto_info *crypto_info,
-			       enum tls_offload_ctx_dir direction)
-{
-	int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
-	unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
-	struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
-	struct ktls_key_ctx *kctx = &tx_info->key_ctx;
-	struct crypto_cipher *cipher;
-	unsigned char *key, *salt;
-
-	switch (crypto_info->cipher_type) {
-	case TLS_CIPHER_AES_GCM_128:
-		info_128_gcm =
-			(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
-		keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
-		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
-		tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
-		mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
-		tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
-		tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);
-
-		ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
-		key = info_128_gcm->key;
-		salt = info_128_gcm->salt;
-		tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;
-
-		/* The SCMD fields used when encrypting a full TLS
-		 * record. Its a one time calculation till the
-		 * connection exists.
-		 */
-		tx_info->scmd0_seqno_numivs =
-			SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |
-			SCMD_CIPH_AUTH_SEQ_CTRL_F |
-			SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |
-			SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
-			SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |
-			SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |
-			SCMD_NUM_IVS_V(1);
-
-		/* keys will be sent inline. */
-		tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;
-
-		/* The SCMD fields used when encrypting a partial TLS
-		 * record (no trailer and possibly a truncated payload).
-		 */
-		tx_info->scmd0_short_seqno_numivs =
-			SCMD_CIPH_AUTH_SEQ_CTRL_F |
-			SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |
-			SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |
-			SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);
-
-		tx_info->scmd0_short_ivgen_hdrlen =
-			tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;
-
-		break;
-
-	default:
-		pr_err("GCM: cipher type 0x%x not supported\n",
-		       crypto_info->cipher_type);
-		ret = -EINVAL;
-		goto out;
-	}
-
-	key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
-		       roundup(keylen, 16) + ghash_size;
-	/* Calculate the H = CIPH(K, 0 repeated 16 times).
-	 * It will go in key context
-	 */
-	cipher = crypto_alloc_cipher("aes", 0, 0);
-	if (IS_ERR(cipher)) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	ret = crypto_cipher_setkey(cipher, key, keylen);
-	if (ret)
-		goto out1;
-
-	memset(ghash_h, 0, ghash_size);
-	crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
-
-	/* fill the Key context */
-	if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
-		kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
-						 mac_key_size,
-						 key_ctx_size >> 4);
-	} else {
-		ret = -EINVAL;
-		goto out1;
-	}
-
-	memcpy(kctx->salt, salt, tx_info->salt_size);
-	memcpy(kctx->key, key, keylen);
-	memcpy(kctx->key + keylen, ghash_h, ghash_size);
-	tx_info->key_ctx_len = key_ctx_size;
-
-out1:
-	crypto_free_cipher(cipher);
-out:
-	return ret;
-}
-
-static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info,
-					     int new_state)
-{
-	/* This function can be called from both rx (interrupt context) and tx
-	 * queue contexts.
-	 */
-	spin_lock_bh(&tx_info->lock);
-	switch (tx_info->connection_state) {
-	case KTLS_CONN_CLOSED:
-		tx_info->connection_state = new_state;
-		break;
-
-	case KTLS_CONN_ACT_OPEN_REQ:
-		/* only go forward if state is greater than current state. */
-		if (new_state <= tx_info->connection_state)
-			break;
-		/* update to the next state and also initialize TCB */
-		tx_info->connection_state = new_state;
-		fallthrough;
-	case KTLS_CONN_ACT_OPEN_RPL:
-		/* if we are stuck in this state, means tcb init might not
-		 * received by HW, try sending it again.
-		 */
-		if (!chcr_init_tcb_fields(tx_info))
-			tx_info->connection_state = KTLS_CONN_SET_TCB_REQ;
-		break;
-
-	case KTLS_CONN_SET_TCB_REQ:
-		/* only go forward if state is greater than current state. */
-		if (new_state <= tx_info->connection_state)
-			break;
-		/* update to the next state and check if l2t_state is valid  */
-		tx_info->connection_state = new_state;
-		fallthrough;
-	case KTLS_CONN_SET_TCB_RPL:
-		/* Check if l2t state is valid, then move to ready state. */
-		if (cxgb4_check_l2t_valid(tx_info->l2te)) {
-			tx_info->connection_state = KTLS_CONN_TX_READY;
-			atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ctx);
-		}
-		break;
-
-	case KTLS_CONN_TX_READY:
-		/* nothing to be done here */
-		break;
-
-	default:
-		pr_err("unknown KTLS connection state\n");
-		break;
-	}
-	spin_unlock_bh(&tx_info->lock);
-
-	return tx_info->connection_state;
-}
-/*
- * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
- * @sk - tcp socket.
- * @tx_info - driver specific tls info.
- * @atid - connection active tid.
- * return - send success/failure.
- */
-static int chcr_ktls_act_open_req(struct sock *sk,
-				  struct chcr_ktls_info *tx_info,
-				  int atid)
-{
-	struct inet_sock *inet = inet_sk(sk);
-	struct cpl_t6_act_open_req *cpl6;
-	struct cpl_act_open_req *cpl;
-	struct sk_buff *skb;
-	unsigned int len;
-	int qid_atid;
-	u64 options;
-
-	len = sizeof(*cpl6);
-	skb = alloc_skb(len, GFP_KERNEL);
-	if (unlikely(!skb))
-		return -ENOMEM;
-	/* mark it a control pkt */
-	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
-
-	cpl6 = __skb_put_zero(skb, len);
-	cpl = (struct cpl_act_open_req *)cpl6;
-	INIT_TP_WR(cpl6, 0);
-	qid_atid = TID_QID_V(tx_info->rx_qid) |
-		   TID_TID_V(atid);
-	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
-	cpl->local_port = inet->inet_sport;
-	cpl->peer_port = inet->inet_dport;
-	cpl->local_ip = inet->inet_rcv_saddr;
-	cpl->peer_ip = inet->inet_daddr;
-
-	/* fill first 64 bit option field. */
-	options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
-		  SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
-	cpl->opt0 = cpu_to_be64(options);
-
-	/* next 64 bit option field. */
-	options =
-		TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
-	cpl->opt2 = htonl(options);
-
-	return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
-}
-
-#if IS_ENABLED(CONFIG_IPV6)
-/*
- * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
- * @sk - tcp socket.
- * @tx_info - driver specific tls info.
- * @atid - connection active tid.
- * return - send success/failure.
- */
-static int chcr_ktls_act_open_req6(struct sock *sk,
-				   struct chcr_ktls_info *tx_info,
-				   int atid)
-{
-	struct inet_sock *inet = inet_sk(sk);
-	struct cpl_t6_act_open_req6 *cpl6;
-	struct cpl_act_open_req6 *cpl;
-	struct sk_buff *skb;
-	unsigned int len;
-	int qid_atid;
-	u64 options;
-
-	len = sizeof(*cpl6);
-	skb = alloc_skb(len, GFP_KERNEL);
-	if (unlikely(!skb))
-		return -ENOMEM;
-	/* mark it a control pkt */
-	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
-
-	cpl6 = __skb_put_zero(skb, len);
-	cpl = (struct cpl_act_open_req6 *)cpl6;
-	INIT_TP_WR(cpl6, 0);
-	qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);
-	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));
-	cpl->local_port = inet->inet_sport;
-	cpl->peer_port = inet->inet_dport;
-	cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];
-	cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];
-	cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];
-	cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];
-
-	/* first 64 bit option field. */
-	options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
-		  SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
-	cpl->opt0 = cpu_to_be64(options);
-	/* next 64 bit option field. */
-	options =
-		TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
-	cpl->opt2 = htonl(options);
-
-	return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
-}
-#endif /* #if IS_ENABLED(CONFIG_IPV6) */
-
-/*
- * chcr_setup_connection:  create a TCB entry so that TP will form tcp packets.
- * @sk - tcp socket.
- * @tx_info - driver specific tls info.
- * return: NET_TX_OK/NET_XMIT_DROP
- */
-static int chcr_setup_connection(struct sock *sk,
-				 struct chcr_ktls_info *tx_info)
-{
-	struct tid_info *t = &tx_info->adap->tids;
-	int atid, ret = 0;
-
-	atid = cxgb4_alloc_atid(t, tx_info);
-	if (atid == -1)
-		return -EINVAL;
-
-	tx_info->atid = atid;
-	tx_info->ip_family = sk->sk_family;
-
-	if (sk->sk_family == AF_INET) {
-		tx_info->ip_family = AF_INET;
-		ret = chcr_ktls_act_open_req(sk, tx_info, atid);
-#if IS_ENABLED(CONFIG_IPV6)
-	} else {
-		if (!sk->sk_ipv6only &&
-		    ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
-			tx_info->ip_family = AF_INET;
-			ret = chcr_ktls_act_open_req(sk, tx_info, atid);
-		} else {
-			tx_info->ip_family = AF_INET6;
-			ret = cxgb4_clip_get(tx_info->netdev,
-					     (const u32 *)
-					     &sk->sk_v6_rcv_saddr.s6_addr,
-					     1);
-			if (ret)
-				goto out;
-			ret = chcr_ktls_act_open_req6(sk, tx_info, atid);
-		}
-#endif
-	}
-
-	/* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
-	 * success, if any other return type clear atid and return that failure.
-	 */
-	if (ret) {
-		if (ret == NET_XMIT_CN)
-			ret = 0;
-		else
-			cxgb4_free_atid(t, atid);
-		goto out;
-	}
-
-	/* update the connection state */
-	chcr_ktls_update_connection_state(tx_info, KTLS_CONN_ACT_OPEN_REQ);
-out:
-	return ret;
-}
-
-/*
- * chcr_set_tcb_field: update tcb fields.
- * @tx_info - driver specific tls info.
- * @word - TCB word.
- * @mask - TCB word related mask.
- * @val - TCB word related value.
- * @no_reply - set 1 if not looking for TP response.
- */
-static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
-			      u64 mask, u64 val, int no_reply)
-{
-	struct cpl_set_tcb_field *req;
-	struct sk_buff *skb;
-
-	skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
-	if (!skb)
-		return -ENOMEM;
-
-	req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
-	INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
-	req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
-				NO_REPLY_V(no_reply));
-	req->word_cookie = htons(TCB_WORD_V(word));
-	req->mask = cpu_to_be64(mask);
-	req->val = cpu_to_be64(val);
-
-	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
-	return cxgb4_ofld_send(tx_info->netdev, skb);
-}
-
-/*
- * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE
- * @tx_info - driver specific tls info.
- * return: NET_TX_OK/NET_XMIT_DROP.
- */
-static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info)
-{
-	return chcr_set_tcb_field(tx_info, TCB_T_STATE_W,
-				  TCB_T_STATE_V(TCB_T_STATE_M),
-				  CHCR_TCB_STATE_CLOSED, 1);
-}
-
-/*
- * chcr_ktls_dev_del:  call back for tls_dev_del.
- * Remove the tid and l2t entry and close the connection.
- * it per connection basis.
- * @netdev - net device.
- * @tls_cts - tls context.
- * @direction - TX/RX crypto direction
- */
-void chcr_ktls_dev_del(struct net_device *netdev,
-		       struct tls_context *tls_ctx,
-		       enum tls_offload_ctx_dir direction)
-{
-	struct chcr_ktls_ofld_ctx_tx *tx_ctx =
-				chcr_get_ktls_tx_context(tls_ctx);
-	struct chcr_ktls_info *tx_info = tx_ctx->chcr_info;
-	struct sock *sk;
-
-	if (!tx_info)
-		return;
-	sk = tx_info->sk;
-
-	spin_lock(&tx_info->lock);
-	tx_info->connection_state = KTLS_CONN_CLOSED;
-	spin_unlock(&tx_info->lock);
-
-	/* clear l2t entry */
-	if (tx_info->l2te)
-		cxgb4_l2t_release(tx_info->l2te);
-
-#if IS_ENABLED(CONFIG_IPV6)
-	/* clear clip entry */
-	if (tx_info->ip_family == AF_INET6)
-		cxgb4_clip_release(netdev,
-				   (const u32 *)&sk->sk_v6_daddr.in6_u.u6_addr8,
-				   1);
-#endif
-
-	/* clear tid */
-	if (tx_info->tid != -1) {
-		/* clear tcb state and then release tid */
-		chcr_ktls_mark_tcb_close(tx_info);
-		cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
-				 tx_info->tid, tx_info->ip_family);
-	}
-
-	atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_connection_close);
-	kvfree(tx_info);
-	tx_ctx->chcr_info = NULL;
-	/* release module refcount */
-	module_put(THIS_MODULE);
-}
-
-/*
- * chcr_ktls_dev_add:  call back for tls_dev_add.
- * Create a tcb entry for TP. Also add l2t entry for the connection. And
- * generate keys & save those keys locally.
- * @netdev - net device.
- * @tls_cts - tls context.
- * @direction - TX/RX crypto direction
- * return: SUCCESS/FAILURE.
- */
-int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
-		      enum tls_offload_ctx_dir direction,
-		      struct tls_crypto_info *crypto_info,
-		      u32 start_offload_tcp_sn)
-{
-	struct tls_context *tls_ctx = tls_get_ctx(sk);
-	struct chcr_ktls_ofld_ctx_tx *tx_ctx;
-	struct chcr_ktls_info *tx_info;
-	struct dst_entry *dst;
-	struct adapter *adap;
-	struct port_info *pi;
-	struct neighbour *n;
-	u8 daaddr[16];
-	int ret = -1;
-
-	tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
-
-	pi = netdev_priv(netdev);
-	adap = pi->adapter;
-	if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
-		pr_err("not expecting for RX direction\n");
-		ret = -EINVAL;
-		goto out;
-	}
-	if (tx_ctx->chcr_info) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
-	if (!tx_info) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	spin_lock_init(&tx_info->lock);
-
-	/* clear connection state */
-	spin_lock(&tx_info->lock);
-	tx_info->connection_state = KTLS_CONN_CLOSED;
-	spin_unlock(&tx_info->lock);
-
-	tx_info->sk = sk;
-	/* initialize tid and atid to -1, 0 is a also a valid id. */
-	tx_info->tid = -1;
-	tx_info->atid = -1;
-
-	tx_info->adap = adap;
-	tx_info->netdev = netdev;
-	tx_info->first_qset = pi->first_qset;
-	tx_info->tx_chan = pi->tx_chan;
-	tx_info->smt_idx = pi->smt_idx;
-	tx_info->port_id = pi->port_id;
-
-	tx_info->rx_qid = chcr_get_first_rx_qid(adap);
-	if (unlikely(tx_info->rx_qid < 0))
-		goto out2;
-
-	tx_info->prev_seq = start_offload_tcp_sn;
-	tx_info->tcp_start_seq_number = start_offload_tcp_sn;
-
-	/* save crypto keys */
-	ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
-	if (ret < 0)
-		goto out2;
-
-	/* get peer ip */
-	if (sk->sk_family == AF_INET) {
-		memcpy(daaddr, &sk->sk_daddr, 4);
-#if IS_ENABLED(CONFIG_IPV6)
-	} else {
-		if (!sk->sk_ipv6only &&
-		    ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)
-			memcpy(daaddr, &sk->sk_daddr, 4);
-		else
-			memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);
-#endif
-	}
-
-	/* get the l2t index */
-	dst = sk_dst_get(sk);
-	if (!dst) {
-		pr_err("DST entry not found\n");
-		goto out2;
-	}
-	n = dst_neigh_lookup(dst, daaddr);
-	if (!n || !n->dev) {
-		pr_err("neighbour not found\n");
-		dst_release(dst);
-		goto out2;
-	}
-	tx_info->l2te  = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
-
-	neigh_release(n);
-	dst_release(dst);
-
-	if (!tx_info->l2te) {
-		pr_err("l2t entry not found\n");
-		goto out2;
-	}
-
-	tx_ctx->chcr_info = tx_info;
-
-	/* create a filter and call cxgb4_l2t_send to send the packet out, which
-	 * will take care of updating l2t entry in hw if not already done.
-	 */
-	ret = chcr_setup_connection(sk, tx_info);
-	if (ret)
-		goto out2;
-
-	/* Driver shouldn't be removed until any single connection exists */
-	if (!try_module_get(THIS_MODULE)) {
-		ret = -EINVAL;
-		goto out2;
-	}
-
-	atomic64_inc(&adap->chcr_stats.ktls_tx_connection_open);
-	return 0;
-out2:
-	kvfree(tx_info);
-out:
-	atomic64_inc(&adap->chcr_stats.ktls_tx_connection_fail);
-	return ret;
-}
-
-/*
- * chcr_init_tcb_fields:  Initialize tcb fields to handle TCP seq number
- *			  handling.
- * @tx_info - driver specific tls info.
- * return: NET_TX_OK/NET_XMIT_DROP
- */
-static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
-{
-	int  ret = 0;
-
-	/* set tcb in offload and bypass */
-	ret =
-	chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
-			   TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
-			   TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
-	if (ret)
-		return ret;
-	/* reset snd_una and snd_next fields in tcb */
-	ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
-				 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
-				 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
-				 0, 1);
-	if (ret)
-		return ret;
-
-	/* reset send max */
-	ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
-				 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
-				 0, 1);
-	if (ret)
-		return ret;
-
-	/* update l2t index and request for tp reply to confirm tcb is
-	 * initialised to handle tx traffic.
-	 */
-	ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
-				 TCB_L2T_IX_V(TCB_L2T_IX_M),
-				 TCB_L2T_IX_V(tx_info->l2te->idx), 0);
-	return ret;
-}
-
-/*
- * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
- */
-int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input)
-{
-	const struct cpl_act_open_rpl *p = (void *)input;
-	struct chcr_ktls_info *tx_info = NULL;
-	unsigned int atid, tid, status;
-	struct tid_info *t;
-
-	tid = GET_TID(p);
-	status = AOPEN_STATUS_G(ntohl(p->atid_status));
-	atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
-
-	t = &adap->tids;
-	tx_info = lookup_atid(t, atid);
-
-	if (!tx_info || tx_info->atid != atid) {
-		pr_err("tx_info or atid is not correct\n");
-		return -1;
-	}
-
-	if (!status) {
-		tx_info->tid = tid;
-		cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
-
-		cxgb4_free_atid(t, atid);
-		tx_info->atid = -1;
-		/* update the connection state */
-		chcr_ktls_update_connection_state(tx_info,
-						  KTLS_CONN_ACT_OPEN_RPL);
-	}
-	return 0;
-}
-
-/*
- * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
- */
-int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
-{
-	const struct cpl_set_tcb_rpl *p = (void *)input;
-	struct chcr_ktls_info *tx_info = NULL;
-	struct tid_info *t;
-	u32 tid;
-
-	tid = GET_TID(p);
-
-	t = &adap->tids;
-	tx_info = lookup_tid(t, tid);
-	if (!tx_info || tx_info->tid != tid) {
-		pr_err("tx_info or atid is not correct\n");
-		return -1;
-	}
-	/* update the connection state */
-	chcr_ktls_update_connection_state(tx_info, KTLS_CONN_SET_TCB_RPL);
-	return 0;
-}
-
-static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
-					u32 tid, void *pos, u16 word, u64 mask,
-					u64 val, u32 reply)
-{
-	struct cpl_set_tcb_field_core *cpl;
-	struct ulptx_idata *idata;
-	struct ulp_txpkt *txpkt;
-
-	/* ULP_TXPKT */
-	txpkt = pos;
-	txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0));
-	txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));
-
-	/* ULPTX_IDATA sub-command */
-	idata = (struct ulptx_idata *)(txpkt + 1);
-	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
-	idata->len = htonl(sizeof(*cpl));
-	pos = idata + 1;
-
-	cpl = pos;
-	/* CPL_SET_TCB_FIELD */
-	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
-	cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
-			NO_REPLY_V(!reply));
-	cpl->word_cookie = htons(TCB_WORD_V(word));
-	cpl->mask = cpu_to_be64(mask);
-	cpl->val = cpu_to_be64(val);
-
-	/* ULPTX_NOOP */
-	idata = (struct ulptx_idata *)(cpl + 1);
-	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
-	idata->len = htonl(0);
-	pos = idata + 1;
-
-	return pos;
-}
-
-
-/*
- * chcr_write_cpl_set_tcb_ulp: update tcb values.
- * TCB is responsible to create tcp headers, so all the related values
- * should be correctly updated.
- * @tx_info - driver specific tls info.
- * @q - tx queue on which packet is going out.
- * @tid - TCB identifier.
- * @pos - current index where should we start writing.
- * @word - TCB word.
- * @mask - TCB word related mask.
- * @val - TCB word related value.
- * @reply - set 1 if looking for TP response.
- * return - next position to write.
- */
-static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
-					struct sge_eth_txq *q, u32 tid,
-					void *pos, u16 word, u64 mask,
-					u64 val, u32 reply)
-{
-	int left = (void *)q->q.stat - pos;
-
-	if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {
-		if (!left) {
-			pos = q->q.desc;
-		} else {
-			u8 buf[48] = {0};
-
-			__chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word,
-						     mask, val, reply);
-
-			return chcr_copy_to_txd(buf, &q->q, pos,
-						CHCR_SET_TCB_FIELD_LEN);
-		}
-	}
-
-	pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word,
-					   mask, val, reply);
-
-	/* check again if we are at the end of the queue */
-	if (left == CHCR_SET_TCB_FIELD_LEN)
-		pos = q->q.desc;
-
-	return pos;
-}
-
-/*
- * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
- * with updated values like tcp seq, ack, window etc.
- * @tx_info - driver specific tls info.
- * @q - TX queue.
- * @tcp_seq
- * @tcp_ack
- * @tcp_win
- * return: NETDEV_TX_BUSY/NET_TX_OK.
- */
-static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,
-				   struct sge_eth_txq *q, u64 tcp_seq,
-				   u64 tcp_ack, u64 tcp_win)
-{
-	bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));
-	u32 len, cpl = 0, ndesc, wr_len;
-	struct fw_ulptx_wr *wr;
-	int credits;
-	void *pos;
-
-	wr_len = sizeof(*wr);
-	/* there can be max 4 cpls, check if we have enough credits */
-	len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
-	ndesc = DIV_ROUND_UP(len, 64);
-
-	credits = chcr_txq_avail(&q->q) - ndesc;
-	if (unlikely(credits < 0)) {
-		chcr_eth_txq_stop(q);
-		return NETDEV_TX_BUSY;
-	}
-
-	pos = &q->q.desc[q->q.pidx];
-	/* make space for WR, we'll fill it later when we know all the cpls
-	 * being sent out and have complete length.
-	 */
-	wr = pos;
-	pos += wr_len;
-	/* update tx_max if its a re-transmit or the first wr */
-	if (first_wr || tcp_seq != tx_info->prev_seq) {
-		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
-						 TCB_TX_MAX_W,
-						 TCB_TX_MAX_V(TCB_TX_MAX_M),
-						 TCB_TX_MAX_V(tcp_seq), 0);
-		cpl++;
-	}
-	/* reset snd una if it's a re-transmit pkt */
-	if (tcp_seq != tx_info->prev_seq) {
-		/* reset snd_una */
-		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
-						 TCB_SND_UNA_RAW_W,
-						 TCB_SND_UNA_RAW_V
-						 (TCB_SND_UNA_RAW_M),
-						 TCB_SND_UNA_RAW_V(0), 0);
-		atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ooo);
-		cpl++;
-	}
-	/* update ack */
-	if (first_wr || tx_info->prev_ack != tcp_ack) {
-		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
-						 TCB_RCV_NXT_W,
-						 TCB_RCV_NXT_V(TCB_RCV_NXT_M),
-						 TCB_RCV_NXT_V(tcp_ack), 0);
-		tx_info->prev_ack = tcp_ack;
-		cpl++;
-	}
-	/* update receive window */
-	if (first_wr || tx_info->prev_win != tcp_win) {
-		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
-						 TCB_RCV_WND_W,
-						 TCB_RCV_WND_V(TCB_RCV_WND_M),
-						 TCB_RCV_WND_V(tcp_win), 0);
-		tx_info->prev_win = tcp_win;
-		cpl++;
-	}
-
-	if (cpl) {
-		/* get the actual length */
-		len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
-		/* ULPTX wr */
-		wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
-		wr->cookie = 0;
-		/* fill len in wr field */
-		wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
-
-		ndesc = DIV_ROUND_UP(len, 64);
-		chcr_txq_advance(&q->q, ndesc);
-		cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
-	}
-	return 0;
-}
-
-/*
- * chcr_ktls_skb_copy
- * @nskb - new skb where the frags to be added.
- * @skb - old skb from which frags will be copied.
- */
-static void chcr_ktls_skb_copy(struct sk_buff *skb, struct sk_buff *nskb)
-{
-	int i;
-
-	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
-		skb_shinfo(nskb)->frags[i] = skb_shinfo(skb)->frags[i];
-		__skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
-	}
-
-	skb_shinfo(nskb)->nr_frags = skb_shinfo(skb)->nr_frags;
-	nskb->len += skb->data_len;
-	nskb->data_len = skb->data_len;
-	nskb->truesize += skb->data_len;
-}
-
-/*
- * chcr_ktls_get_tx_flits
- * returns number of flits to be sent out, it includes key context length, WR
- * size and skb fragments.
- */
-static unsigned int
-chcr_ktls_get_tx_flits(const struct sk_buff *skb, unsigned int key_ctx_len)
-{
-	return chcr_sgl_len(skb_shinfo(skb)->nr_frags) +
-	       DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8);
-}
-
-/*
- * chcr_ktls_check_tcp_options: To check if there is any TCP option availbale
- * other than timestamp.
- * @skb - skb contains partial record..
- * return: 1 / 0
- */
-static int
-chcr_ktls_check_tcp_options(struct tcphdr *tcp)
-{
-	int cnt, opt, optlen;
-	u_char *cp;
-
-	cp = (u_char *)(tcp + 1);
-	cnt = (tcp->doff << 2) - sizeof(struct tcphdr);
-	for (; cnt > 0; cnt -= optlen, cp += optlen) {
-		opt = cp[0];
-		if (opt == TCPOPT_EOL)
-			break;
-		if (opt == TCPOPT_NOP) {
-			optlen = 1;
-		} else {
-			if (cnt < 2)
-				break;
-			optlen = cp[1];
-			if (optlen < 2 || optlen > cnt)
-				break;
-		}
-		switch (opt) {
-		case TCPOPT_NOP:
-			break;
-		default:
-			return 1;
-		}
-	}
-	return 0;
-}
-
-/*
- * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to
- * send out separately.
- * @tx_info - driver specific tls info.
- * @skb - skb contains partial record..
- * @q - TX queue.
- * @tx_chan - channel number.
- * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
- */
-static int
-chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb,
-			    struct sge_eth_txq *q, uint32_t tx_chan)
-{
-	struct fw_eth_tx_pkt_wr *wr;
-	struct cpl_tx_pkt_core *cpl;
-	u32 ctrl, iplen, maclen;
-#if IS_ENABLED(CONFIG_IPV6)
-	struct ipv6hdr *ip6;
-#endif
-	unsigned int ndesc;
-	struct tcphdr *tcp;
-	int len16, pktlen;
-	struct iphdr *ip;
-	int credits;
-	u8 buf[150];
-	void *pos;
-
-	iplen = skb_network_header_len(skb);
-	maclen = skb_mac_header_len(skb);
-
-	/* packet length = eth hdr len + ip hdr len + tcp hdr len
-	 * (including options).
-	 */
-	pktlen = skb->len - skb->data_len;
-
-	ctrl = sizeof(*cpl) + pktlen;
-	len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16);
-	/* check how many descriptors needed */
-	ndesc = DIV_ROUND_UP(len16, 4);
-
-	credits = chcr_txq_avail(&q->q) - ndesc;
-	if (unlikely(credits < 0)) {
-		chcr_eth_txq_stop(q);
-		return NETDEV_TX_BUSY;
-	}
-
-	pos = &q->q.desc[q->q.pidx];
-	wr = pos;
-
-	/* Firmware work request header */
-	wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
-			       FW_WR_IMMDLEN_V(ctrl));
-
-	wr->equiq_to_len16 = htonl(FW_WR_LEN16_V(len16));
-	wr->r3 = 0;
-
-	cpl = (void *)(wr + 1);
-
-	/* CPL header */
-	cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) |
-			   TXPKT_PF_V(tx_info->adap->pf));
-	cpl->pack = 0;
-	cpl->len = htons(pktlen);
-	/* checksum offload */
-	cpl->ctrl1 = 0;
-
-	pos = cpl + 1;
-
-	memcpy(buf, skb->data, pktlen);
-	if (tx_info->ip_family == AF_INET) {
-		/* we need to correct ip header len */
-		ip = (struct iphdr *)(buf + maclen);
-		ip->tot_len = htons(pktlen - maclen);
-#if IS_ENABLED(CONFIG_IPV6)
-	} else {
-		ip6 = (struct ipv6hdr *)(buf + maclen);
-		ip6->payload_len = htons(pktlen - maclen - iplen);
-#endif
-	}
-	/* now take care of the tcp header, if fin is not set then clear push
-	 * bit as well, and if fin is set, it will be sent at the last so we
-	 * need to update the tcp sequence number as per the last packet.
-	 */
-	tcp = (struct tcphdr *)(buf + maclen + iplen);
-
-	if (!tcp->fin)
-		tcp->psh = 0;
-	else
-		tcp->seq = htonl(tx_info->prev_seq);
-
-	chcr_copy_to_txd(buf, &q->q, pos, pktlen);
-
-	chcr_txq_advance(&q->q, ndesc);
-	cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
-	return 0;
-}
-
-/* chcr_ktls_skb_shift - Shifts request length paged data from skb to another.
- * @tgt- buffer into which tail data gets added
- * @skb- buffer from which the paged data comes from
- * @shiftlen- shift up to this many bytes
- */
-static int chcr_ktls_skb_shift(struct sk_buff *tgt, struct sk_buff *skb,
-			       int shiftlen)
-{
-	skb_frag_t *fragfrom, *fragto;
-	int from, to, todo;
-
-	WARN_ON(shiftlen > skb->data_len);
-
-	todo = shiftlen;
-	from = 0;
-	to = 0;
-	fragfrom = &skb_shinfo(skb)->frags[from];
-
-	while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
-		fragfrom = &skb_shinfo(skb)->frags[from];
-		fragto = &skb_shinfo(tgt)->frags[to];
-
-		if (todo >= skb_frag_size(fragfrom)) {
-			*fragto = *fragfrom;
-			todo -= skb_frag_size(fragfrom);
-			from++;
-			to++;
-
-		} else {
-			__skb_frag_ref(fragfrom);
-			skb_frag_page_copy(fragto, fragfrom);
-			skb_frag_off_copy(fragto, fragfrom);
-			skb_frag_size_set(fragto, todo);
-
-			skb_frag_off_add(fragfrom, todo);
-			skb_frag_size_sub(fragfrom, todo);
-			todo = 0;
-
-			to++;
-			break;
-		}
-	}
-
-	/* Ready to "commit" this state change to tgt */
-	skb_shinfo(tgt)->nr_frags = to;
-
-	/* Reposition in the original skb */
-	to = 0;
-	while (from < skb_shinfo(skb)->nr_frags)
-		skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
-
-	skb_shinfo(skb)->nr_frags = to;
-
-	WARN_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
-
-	skb->len -= shiftlen;
-	skb->data_len -= shiftlen;
-	skb->truesize -= shiftlen;
-	tgt->len += shiftlen;
-	tgt->data_len += shiftlen;
-	tgt->truesize += shiftlen;
-
-	return shiftlen;
-}
-
-/*
- * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb
- * received has partial end part of the record, send out the complete record, so
- * that crypto block will be able to generate TAG/HASH.
- * @skb - segment which has complete or partial end part.
- * @tx_info - driver specific tls info.
- * @q - TX queue.
- * @tcp_seq
- * @tcp_push - tcp push bit.
- * @mss - segment size.
- * return: NETDEV_TX_BUSY/NET_TX_OK.
- */
-static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb,
-				      struct chcr_ktls_info *tx_info,
-				      struct sge_eth_txq *q, u32 tcp_seq,
-				      bool tcp_push, u32 mss)
-{
-	u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start;
-	struct adapter *adap = tx_info->adap;
-	int credits, left, last_desc;
-	struct tx_sw_desc *sgl_sdesc;
-	struct cpl_tx_data *tx_data;
-	struct cpl_tx_sec_pdu *cpl;
-	struct ulptx_idata *idata;
-	struct ulp_txpkt *ulptx;
-	struct fw_ulptx_wr *wr;
-	void *pos;
-	u64 *end;
-
-	/* get the number of flits required */
-	flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len);
-	/* number of descriptors */
-	ndesc = chcr_flits_to_desc(flits);
-	/* check if enough credits available */
-	credits = chcr_txq_avail(&q->q) - ndesc;
-	if (unlikely(credits < 0)) {
-		chcr_eth_txq_stop(q);
-		return NETDEV_TX_BUSY;
-	}
-
-	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
-		/* Credits are below the threshold vaues, stop the queue after
-		 * injecting the Work Request for this packet.
-		 */
-		chcr_eth_txq_stop(q);
-		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
-	}
-
-	last_desc = q->q.pidx + ndesc - 1;
-	if (last_desc >= q->q.size)
-		last_desc -= q->q.size;
-	sgl_sdesc = &q->q.sdesc[last_desc];
-
-	if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
-		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
-		q->mapping_err++;
-		return NETDEV_TX_BUSY;
-	}
-
-	pos = &q->q.desc[q->q.pidx];
-	end = (u64 *)pos + flits;
-	/* FW_ULPTX_WR */
-	wr = pos;
-	/* WR will need len16 */
-	len16 = DIV_ROUND_UP(flits, 2);
-	wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
-	wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
-	wr->cookie = 0;
-	pos += sizeof(*wr);
-	/* ULP_TXPKT */
-	ulptx = pos;
-	ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
-				ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
-				ULP_TXPKT_FID_V(q->q.cntxt_id) |
-				ULP_TXPKT_RO_F);
-	ulptx->len = htonl(len16 - 1);
-	/* ULPTX_IDATA sub-command */
-	idata = (struct ulptx_idata *)(ulptx + 1);
-	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
-	/* idata length will include cpl_tx_sec_pdu + key context size +
-	 * cpl_tx_data header.
-	 */
-	idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
-			   sizeof(*tx_data));
-	/* SEC CPL */
-	cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
-	cpl->op_ivinsrtofst =
-		htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
-		      CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
-		      CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
-		      CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1));
-	cpl->pldlen = htonl(skb->data_len);
-
-	/* encryption should start after tls header size + iv size */
-	cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1;
-
-	cpl->aadstart_cipherstop_hi =
-		htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |
-		      CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) |
-		      CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
-
-	/* authentication will also start after tls header + iv size */
-	cpl->cipherstop_lo_authinsert =
-	htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) |
-	      CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) |
-	      CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE));
-
-	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
-	cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs);
-	cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen);
-	cpl->scmd1 = cpu_to_be64(tx_info->record_no);
-
-	pos = cpl + 1;
-	/* check if space left to fill the keys */
-	left = (void *)q->q.stat - pos;
-	if (!left) {
-		left = (void *)end - (void *)q->q.stat;
-		pos = q->q.desc;
-		end = pos + left;
-	}
-
-	pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
-			       tx_info->key_ctx_len);
-	left = (void *)q->q.stat - pos;
-
-	if (!left) {
-		left = (void *)end - (void *)q->q.stat;
-		pos = q->q.desc;
-		end = pos + left;
-	}
-	/* CPL_TX_DATA */
-	tx_data = (void *)pos;
-	OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
-	tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(skb->data_len));
-
-	tx_data->rsvd = htonl(tcp_seq);
-
-	tx_data->flags = htonl(TX_BYPASS_F);
-	if (tcp_push)
-		tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
-
-	/* check left again, it might go beyond queue limit */
-	pos = tx_data + 1;
-	left = (void *)q->q.stat - pos;
-
-	/* check the position again */
-	if (!left) {
-		left = (void *)end - (void *)q->q.stat;
-		pos = q->q.desc;
-		end = pos + left;
-	}
-
-	/* send the complete packet except the header */
-	cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len,
-			sgl_sdesc->addr);
-	sgl_sdesc->skb = skb;
-
-	chcr_txq_advance(&q->q, ndesc);
-	cxgb4_ring_tx_db(adap, &q->q, ndesc);
-	atomic64_inc(&adap->chcr_stats.ktls_tx_send_records);
-
-	return 0;
-}
-
-/*
- * chcr_ktls_xmit_wr_short: This is to send out partial records. If its
- * a middle part of a record, fetch the prior data to make it 16 byte aligned
- * and then only send it out.
- *
- * @skb - skb contains partial record..
- * @tx_info - driver specific tls info.
- * @q - TX queue.
- * @tcp_seq
- * @tcp_push - tcp push bit.
- * @mss - segment size.
- * @tls_rec_offset - offset from start of the tls record.
- * @perior_data - data before the current segment, required to make this record
- *		  16 byte aligned.
- * @prior_data_len - prior_data length (less than 16)
- * return: NETDEV_TX_BUSY/NET_TX_OK.
- */
-static int chcr_ktls_xmit_wr_short(struct sk_buff *skb,
-				   struct chcr_ktls_info *tx_info,
-				   struct sge_eth_txq *q,
-				   u32 tcp_seq, bool tcp_push, u32 mss,
-				   u32 tls_rec_offset, u8 *prior_data,
-				   u32 prior_data_len)
-{
-	struct adapter *adap = tx_info->adap;
-	u32 len16, wr_mid = 0, cipher_start;
-	unsigned int flits = 0, ndesc;
-	int credits, left, last_desc;
-	struct tx_sw_desc *sgl_sdesc;
-	struct cpl_tx_data *tx_data;
-	struct cpl_tx_sec_pdu *cpl;
-	struct ulptx_idata *idata;
-	struct ulp_txpkt *ulptx;
-	struct fw_ulptx_wr *wr;
-	__be64 iv_record;
-	void *pos;
-	u64 *end;
-
-	/* get the number of flits required, it's a partial record so 2 flits
-	 * (AES_BLOCK_SIZE) will be added.
-	 */
-	flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len) + 2;
-	/* get the correct 8 byte IV of this record */
-	iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no);
-	/* If it's a middle record and not 16 byte aligned to run AES CTR, need
-	 * to make it 16 byte aligned. So atleadt 2 extra flits of immediate
-	 * data will be added.
-	 */
-	if (prior_data_len)
-		flits += 2;
-	/* number of descriptors */
-	ndesc = chcr_flits_to_desc(flits);
-	/* check if enough credits available */
-	credits = chcr_txq_avail(&q->q) - ndesc;
-	if (unlikely(credits < 0)) {
-		chcr_eth_txq_stop(q);
-		return NETDEV_TX_BUSY;
-	}
-
-	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
-		chcr_eth_txq_stop(q);
-		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
-	}
-
-	last_desc = q->q.pidx + ndesc - 1;
-	if (last_desc >= q->q.size)
-		last_desc -= q->q.size;
-	sgl_sdesc = &q->q.sdesc[last_desc];
-
-	if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
-		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
-		q->mapping_err++;
-		return NETDEV_TX_BUSY;
-	}
-
-	pos = &q->q.desc[q->q.pidx];
-	end = (u64 *)pos + flits;
-	/* FW_ULPTX_WR */
-	wr = pos;
-	/* WR will need len16 */
-	len16 = DIV_ROUND_UP(flits, 2);
-	wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
-	wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
-	wr->cookie = 0;
-	pos += sizeof(*wr);
-	/* ULP_TXPKT */
-	ulptx = pos;
-	ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
-				ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
-				ULP_TXPKT_FID_V(q->q.cntxt_id) |
-				ULP_TXPKT_RO_F);
-	ulptx->len = htonl(len16 - 1);
-	/* ULPTX_IDATA sub-command */
-	idata = (struct ulptx_idata *)(ulptx + 1);
-	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
-	/* idata length will include cpl_tx_sec_pdu + key context size +
-	 * cpl_tx_data header.
-	 */
-	idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
-			   sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len);
-	/* SEC CPL */
-	cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
-	/* cipher start will have tls header + iv size extra if its a header
-	 * part of tls record. else only 16 byte IV will be added.
-	 */
-	cipher_start =
-		AES_BLOCK_LEN + 1 +
-		(!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0);
-
-	cpl->op_ivinsrtofst =
-		htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
-		      CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
-		      CPL_TX_SEC_PDU_IVINSRTOFST_V(1));
-	cpl->pldlen = htonl(skb->data_len + AES_BLOCK_LEN + prior_data_len);
-	cpl->aadstart_cipherstop_hi =
-		htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
-	cpl->cipherstop_lo_authinsert = 0;
-	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
-	cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs);
-	cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen);
-	cpl->scmd1 = 0;
-
-	pos = cpl + 1;
-	/* check if space left to fill the keys */
-	left = (void *)q->q.stat - pos;
-	if (!left) {
-		left = (void *)end - (void *)q->q.stat;
-		pos = q->q.desc;
-		end = pos + left;
-	}
-
-	pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
-			       tx_info->key_ctx_len);
-	left = (void *)q->q.stat - pos;
-
-	if (!left) {
-		left = (void *)end - (void *)q->q.stat;
-		pos = q->q.desc;
-		end = pos + left;
-	}
-	/* CPL_TX_DATA */
-	tx_data = (void *)pos;
-	OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
-	tx_data->len = htonl(TX_DATA_MSS_V(mss) |
-			TX_LENGTH_V(skb->data_len + prior_data_len));
-	tx_data->rsvd = htonl(tcp_seq);
-	tx_data->flags = htonl(TX_BYPASS_F);
-	if (tcp_push)
-		tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
-
-	/* check left again, it might go beyond queue limit */
-	pos = tx_data + 1;
-	left = (void *)q->q.stat - pos;
-
-	/* check the position again */
-	if (!left) {
-		left = (void *)end - (void *)q->q.stat;
-		pos = q->q.desc;
-		end = pos + left;
-	}
-	/* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8
-	 * bytes of actual IV and 4 bytes of 16 byte-sequence.
-	 */
-	memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size);
-	memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size);
-	*(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) =
-		htonl(2 + (tls_rec_offset ? ((tls_rec_offset -
-		(TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0));
-
-	pos += 16;
-	/* Prior_data_len will always be less than 16 bytes, fill the
-	 * prio_data_len after AES_CTRL_BLOCK and clear the remaining length
-	 * to 0.
-	 */
-	if (prior_data_len)
-		pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
-	/* send the complete packet except the header */
-	cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len,
-			sgl_sdesc->addr);
-	sgl_sdesc->skb = skb;
-
-	chcr_txq_advance(&q->q, ndesc);
-	cxgb4_ring_tx_db(adap, &q->q, ndesc);
-
-	return 0;
-}
-
-/*
- * chcr_ktls_tx_plaintxt: This handler will take care of the records which has
- * only plain text (only tls header and iv)
- * @tx_info - driver specific tls info.
- * @skb - skb contains partial record..
- * @tcp_seq
- * @mss - segment size.
- * @tcp_push - tcp push bit.
- * @q - TX queue.
- * @port_id : port number
- * @perior_data - data before the current segment, required to make this record
- *		 16 byte aligned.
- * @prior_data_len - prior_data length (less than 16)
- * return: NETDEV_TX_BUSY/NET_TX_OK.
- */
-static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info,
-				 struct sk_buff *skb, u32 tcp_seq, u32 mss,
-				 bool tcp_push, struct sge_eth_txq *q,
-				 u32 port_id, u8 *prior_data,
-				 u32 prior_data_len)
-{
-	int credits, left, len16, last_desc;
-	unsigned int flits = 0, ndesc;
-	struct tx_sw_desc *sgl_sdesc;
-	struct cpl_tx_data *tx_data;
-	struct ulptx_idata *idata;
-	struct ulp_txpkt *ulptx;
-	struct fw_ulptx_wr *wr;
-	u32 wr_mid = 0;
-	void *pos;
-	u64 *end;
-
-	flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8);
-	flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags);
-	if (prior_data_len)
-		flits += 2;
-	/* WR will need len16 */
-	len16 = DIV_ROUND_UP(flits, 2);
-	/* check how many descriptors needed */
-	ndesc = DIV_ROUND_UP(flits, 8);
-
-	credits = chcr_txq_avail(&q->q) - ndesc;
-	if (unlikely(credits < 0)) {
-		chcr_eth_txq_stop(q);
-		return NETDEV_TX_BUSY;
-	}
-
-	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
-		chcr_eth_txq_stop(q);
-		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
-	}
-
-	last_desc = q->q.pidx + ndesc - 1;
-	if (last_desc >= q->q.size)
-		last_desc -= q->q.size;
-	sgl_sdesc = &q->q.sdesc[last_desc];
-
-	if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
-				   sgl_sdesc->addr) < 0)) {
-		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
-		q->mapping_err++;
-		return NETDEV_TX_BUSY;
-	}
-
-	pos = &q->q.desc[q->q.pidx];
-	end = (u64 *)pos + flits;
-	/* FW_ULPTX_WR */
-	wr = pos;
-	wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
-	wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
-	wr->cookie = 0;
-	pos += sizeof(*wr);
-	/* ULP_TXPKT */
-	ulptx = (struct ulp_txpkt *)(wr + 1);
-	ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
-			ULP_TXPKT_DATAMODIFY_V(0) |
-			ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
-			ULP_TXPKT_DEST_V(0) |
-			ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1));
-	ulptx->len = htonl(len16 - 1);
-	/* ULPTX_IDATA sub-command */
-	idata = (struct ulptx_idata *)(ulptx + 1);
-	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
-	idata->len = htonl(sizeof(*tx_data) + prior_data_len);
-	/* CPL_TX_DATA */
-	tx_data = (struct cpl_tx_data *)(idata + 1);
-	OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
-	tx_data->len = htonl(TX_DATA_MSS_V(mss) |
-			TX_LENGTH_V(skb->data_len + prior_data_len));
-	/* set tcp seq number */
-	tx_data->rsvd = htonl(tcp_seq);
-	tx_data->flags = htonl(TX_BYPASS_F);
-	if (tcp_push)
-		tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
-
-	pos = tx_data + 1;
-	/* apart from prior_data_len, we should set remaining part of 16 bytes
-	 * to be zero.
-	 */
-	if (prior_data_len)
-		pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
-
-	/* check left again, it might go beyond queue limit */
-	left = (void *)q->q.stat - pos;
-
-	/* check the position again */
-	if (!left) {
-		left = (void *)end - (void *)q->q.stat;
-		pos = q->q.desc;
-		end = pos + left;
-	}
-	/* send the complete packet including the header */
-	cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len,
-			sgl_sdesc->addr);
-	sgl_sdesc->skb = skb;
-
-	chcr_txq_advance(&q->q, ndesc);
-	cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
-	return 0;
-}
-
-/*
- * chcr_ktls_copy_record_in_skb
- * @nskb - new skb where the frags to be added.
- * @record - specific record which has complete 16k record in frags.
- */
-static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,
-					 struct tls_record_info *record)
-{
-	int i = 0;
-
-	for (i = 0; i < record->num_frags; i++) {
-		skb_shinfo(nskb)->frags[i] = record->frags[i];
-		/* increase the frag ref count */
-		__skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
-	}
-
-	skb_shinfo(nskb)->nr_frags = record->num_frags;
-	nskb->data_len = record->len;
-	nskb->len += record->len;
-	nskb->truesize += record->len;
-}
-
-/*
- * chcr_ktls_update_snd_una:  Reset the SEND_UNA. It will be done to avoid
- * sending the same segment again. It will discard the segment which is before
- * the current tx max.
- * @tx_info - driver specific tls info.
- * @q - TX queue.
- * return: NET_TX_OK/NET_XMIT_DROP.
- */
-static int chcr_ktls_update_snd_una(struct chcr_ktls_info *tx_info,
-				    struct sge_eth_txq *q)
-{
-	struct fw_ulptx_wr *wr;
-	unsigned int ndesc;
-	int credits;
-	void *pos;
-	u32 len;
-
-	len = sizeof(*wr) + roundup(CHCR_SET_TCB_FIELD_LEN, 16);
-	ndesc = DIV_ROUND_UP(len, 64);
-
-	credits = chcr_txq_avail(&q->q) - ndesc;
-	if (unlikely(credits < 0)) {
-		chcr_eth_txq_stop(q);
-		return NETDEV_TX_BUSY;
-	}
-
-	pos = &q->q.desc[q->q.pidx];
-
-	wr = pos;
-	/* ULPTX wr */
-	wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
-	wr->cookie = 0;
-	/* fill len in wr field */
-	wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
-
-	pos += sizeof(*wr);
-
-	pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
-					 TCB_SND_UNA_RAW_W,
-					 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
-					 TCB_SND_UNA_RAW_V(0), 0);
-
-	chcr_txq_advance(&q->q, ndesc);
-	cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
-
-	return 0;
-}
-
-/*
- * chcr_end_part_handler: This handler will handle the record which
- * is complete or if record's end part is received. T6 adapter has a issue that
- * it can't send out TAG with partial record so if its an end part then we have
- * to send TAG as well and for which we need to fetch the complete record and
- * send it to crypto module.
- * @tx_info - driver specific tls info.
- * @skb - skb contains partial record.
- * @record - complete record of 16K size.
- * @tcp_seq
- * @mss - segment size in which TP needs to chop a packet.
- * @tcp_push_no_fin - tcp push if fin is not set.
- * @q - TX queue.
- * @tls_end_offset - offset from end of the record.
- * @last wr : check if this is the last part of the skb going out.
- * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
- */
-static int chcr_end_part_handler(struct chcr_ktls_info *tx_info,
-				 struct sk_buff *skb,
-				 struct tls_record_info *record,
-				 u32 tcp_seq, int mss, bool tcp_push_no_fin,
-				 struct sge_eth_txq *q,
-				 u32 tls_end_offset, bool last_wr)
-{
-	struct sk_buff *nskb = NULL;
-	/* check if it is a complete record */
-	if (tls_end_offset == record->len) {
-		nskb = skb;
-		atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_complete_pkts);
-	} else {
-		dev_kfree_skb_any(skb);
-
-		nskb = alloc_skb(0, GFP_KERNEL);
-		if (!nskb)
-			return NETDEV_TX_BUSY;
-		/* copy complete record in skb */
-		chcr_ktls_copy_record_in_skb(nskb, record);
-		/* packet is being sent from the beginning, update the tcp_seq
-		 * accordingly.
-		 */
-		tcp_seq = tls_record_start_seq(record);
-		/* reset snd una, so the middle record won't send the already
-		 * sent part.
-		 */
-		if (chcr_ktls_update_snd_una(tx_info, q))
-			goto out;
-		atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_end_pkts);
-	}
-
-	if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq,
-				       (last_wr && tcp_push_no_fin),
-				       mss)) {
-		goto out;
-	}
-	return 0;
-out:
-	dev_kfree_skb_any(nskb);
-	return NETDEV_TX_BUSY;
-}
-
-/*
- * chcr_short_record_handler: This handler will take care of the records which
- * doesn't have end part (1st part or the middle part(/s) of a record). In such
- * cases, AES CTR will be used in place of AES GCM to send out partial packet.
- * This partial record might be the first part of the record, or the middle
- * part. In case of middle record we should fetch the prior data to make it 16
- * byte aligned. If it has a partial tls header or iv then get to the start of
- * tls header. And if it has partial TAG, then remove the complete TAG and send
- * only the payload.
- * There is one more possibility that it gets a partial header, send that
- * portion as a plaintext.
- * @tx_info - driver specific tls info.
- * @skb - skb contains partial record..
- * @record - complete record of 16K size.
- * @tcp_seq
- * @mss - segment size in which TP needs to chop a packet.
- * @tcp_push_no_fin - tcp push if fin is not set.
- * @q - TX queue.
- * @tls_end_offset - offset from end of the record.
- * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
- */
-static int chcr_short_record_handler(struct chcr_ktls_info *tx_info,
-				     struct sk_buff *skb,
-				     struct tls_record_info *record,
-				     u32 tcp_seq, int mss, bool tcp_push_no_fin,
-				     struct sge_eth_txq *q, u32 tls_end_offset)
-{
-	u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record);
-	u8 prior_data[16] = {0};
-	u32 prior_data_len = 0;
-	u32 data_len;
-
-	/* check if the skb is ending in middle of tag/HASH, its a big
-	 * trouble, send the packet before the HASH.
-	 */
-	int remaining_record = tls_end_offset - skb->data_len;
-
-	if (remaining_record > 0 &&
-	    remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) {
-		int trimmed_len = skb->data_len -
-			(TLS_CIPHER_AES_GCM_128_TAG_SIZE - remaining_record);
-		struct sk_buff *tmp_skb = NULL;
-		/* don't process the pkt if it is only a partial tag */
-		if (skb->data_len < TLS_CIPHER_AES_GCM_128_TAG_SIZE)
-			goto out;
-
-		WARN_ON(trimmed_len > skb->data_len);
-
-		/* shift to those many bytes */
-		tmp_skb = alloc_skb(0, GFP_KERNEL);
-		if (unlikely(!tmp_skb))
-			goto out;
-
-		chcr_ktls_skb_shift(tmp_skb, skb, trimmed_len);
-		/* free the last trimmed portion */
-		dev_kfree_skb_any(skb);
-		skb = tmp_skb;
-		atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_trimmed_pkts);
-	}
-	data_len = skb->data_len;
-	/* check if the middle record's start point is 16 byte aligned. CTR
-	 * needs 16 byte aligned start point to start encryption.
-	 */
-	if (tls_rec_offset) {
-		/* there is an offset from start, means its a middle record */
-		int remaining = 0;
-
-		if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) {
-			prior_data_len = tls_rec_offset;
-			tls_rec_offset = 0;
-			remaining = 0;
-		} else {
-			prior_data_len =
-				(tls_rec_offset -
-				(TLS_HEADER_SIZE + tx_info->iv_size))
-				% AES_BLOCK_LEN;
-			remaining = tls_rec_offset - prior_data_len;
-		}
-
-		/* if prior_data_len is not zero, means we need to fetch prior
-		 * data to make this record 16 byte aligned, or we need to reach
-		 * to start offset.
-		 */
-		if (prior_data_len) {
-			int i = 0;
-			u8 *data = NULL;
-			skb_frag_t *f;
-			u8 *vaddr;
-			int frag_size = 0, frag_delta = 0;
-
-			while (remaining > 0) {
-				frag_size = skb_frag_size(&record->frags[i]);
-				if (remaining < frag_size)
-					break;
-
-				remaining -= frag_size;
-				i++;
-			}
-			f = &record->frags[i];
-			vaddr = kmap_atomic(skb_frag_page(f));
-
-			data = vaddr + skb_frag_off(f)  + remaining;
-			frag_delta = skb_frag_size(f) - remaining;
-
-			if (frag_delta >= prior_data_len) {
-				memcpy(prior_data, data, prior_data_len);
-				kunmap_atomic(vaddr);
-			} else {
-				memcpy(prior_data, data, frag_delta);
-				kunmap_atomic(vaddr);
-				/* get the next page */
-				f = &record->frags[i + 1];
-				vaddr = kmap_atomic(skb_frag_page(f));
-				data = vaddr + skb_frag_off(f);
-				memcpy(prior_data + frag_delta,
-				       data, (prior_data_len - frag_delta));
-				kunmap_atomic(vaddr);
-			}
-			/* reset tcp_seq as per the prior_data_required len */
-			tcp_seq -= prior_data_len;
-			/* include prio_data_len for  further calculation.
-			 */
-			data_len += prior_data_len;
-		}
-		/* reset snd una, so the middle record won't send the already
-		 * sent part.
-		 */
-		if (chcr_ktls_update_snd_una(tx_info, q))
-			goto out;
-		atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_middle_pkts);
-	} else {
-		/* Else means, its a partial first part of the record. Check if
-		 * its only the header, don't need to send for encryption then.
-		 */
-		if (data_len <= TLS_HEADER_SIZE + tx_info->iv_size) {
-			if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
-						  tcp_push_no_fin, q,
-						  tx_info->port_id,
-						  prior_data,
-						  prior_data_len)) {
-				goto out;
-			}
-			return 0;
-		}
-		atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_start_pkts);
-	}
-
-	if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin,
-				    mss, tls_rec_offset, prior_data,
-				    prior_data_len)) {
-		goto out;
-	}
-
-	return 0;
-out:
-	dev_kfree_skb_any(skb);
-	return NETDEV_TX_BUSY;
-}
-
-/* nic tls TX handler */
-int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev)
-{
-	struct chcr_ktls_ofld_ctx_tx *tx_ctx;
-	struct tcphdr *th = tcp_hdr(skb);
-	int data_len, qidx, ret = 0, mss;
-	struct tls_record_info *record;
-	struct chcr_stats_debug *stats;
-	struct chcr_ktls_info *tx_info;
-	u32 tls_end_offset, tcp_seq;
-	struct tls_context *tls_ctx;
-	struct sk_buff *local_skb;
-	int new_connection_state;
-	struct sge_eth_txq *q;
-	struct adapter *adap;
-	unsigned long flags;
-
-	tcp_seq = ntohl(th->seq);
-
-	mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : skb->data_len;
-
-	/* check if we haven't set it for ktls offload */
-	if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk))
-		goto out;
-
-	tls_ctx = tls_get_ctx(skb->sk);
-	if (unlikely(tls_ctx->netdev != dev))
-		goto out;
-
-	tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
-	tx_info = tx_ctx->chcr_info;
-
-	if (unlikely(!tx_info))
-		goto out;
-
-	/* check the connection state, we don't need to pass new connection
-	 * state, state machine will check and update the new state if it is
-	 * stuck due to responses not received from HW.
-	 * Start the tx handling only if state is KTLS_CONN_TX_READY.
-	 */
-	new_connection_state = chcr_ktls_update_connection_state(tx_info, 0);
-	if (new_connection_state != KTLS_CONN_TX_READY)
-		goto out;
-
-	/* don't touch the original skb, make a new skb to extract each records
-	 * and send them separately.
-	 */
-	local_skb = alloc_skb(0, GFP_KERNEL);
-
-	if (unlikely(!local_skb))
-		return NETDEV_TX_BUSY;
-
-	adap = tx_info->adap;
-	stats = &adap->chcr_stats;
-
-	qidx = skb->queue_mapping;
-	q = &adap->sge.ethtxq[qidx + tx_info->first_qset];
-	cxgb4_reclaim_completed_tx(adap, &q->q, true);
-	/* if tcp options are set but finish is not send the options first */
-	if (!th->fin && chcr_ktls_check_tcp_options(th)) {
-		ret = chcr_ktls_write_tcp_options(tx_info, skb, q,
-						  tx_info->tx_chan);
-		if (ret)
-			return NETDEV_TX_BUSY;
-	}
-	/* update tcb */
-	ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, ntohl(th->seq),
-				      ntohl(th->ack_seq),
-				      ntohs(th->window));
-	if (ret) {
-		dev_kfree_skb_any(local_skb);
-		return NETDEV_TX_BUSY;
-	}
-
-	/* copy skb contents into local skb */
-	chcr_ktls_skb_copy(skb, local_skb);
-
-	/* go through the skb and send only one record at a time. */
-	data_len = skb->data_len;
-	/* TCP segments can be in received either complete or partial.
-	 * chcr_end_part_handler will handle cases if complete record or end
-	 * part of the record is received. Incase of partial end part of record,
-	 * we will send the complete record again.
-	 */
-
-	do {
-		int i;
-
-		cxgb4_reclaim_completed_tx(adap, &q->q, true);
-		/* lock taken */
-		spin_lock_irqsave(&tx_ctx->base.lock, flags);
-		/* fetch the tls record */
-		record = tls_get_record(&tx_ctx->base, tcp_seq,
-					&tx_info->record_no);
-		/* By the time packet reached to us, ACK is received, and record
-		 * won't be found in that case, handle it gracefully.
-		 */
-		if (unlikely(!record)) {
-			spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
-			atomic64_inc(&stats->ktls_tx_drop_no_sync_data);
-			goto out;
-		}
-
-		if (unlikely(tls_record_is_start_marker(record))) {
-			spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
-			atomic64_inc(&stats->ktls_tx_skip_no_sync_data);
-			goto out;
-		}
-
-		/* increase page reference count of the record, so that there
-		 * won't be any chance of page free in middle if in case stack
-		 * receives ACK and try to delete the record.
-		 */
-		for (i = 0; i < record->num_frags; i++)
-			__skb_frag_ref(&record->frags[i]);
-		/* lock cleared */
-		spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
-
-		tls_end_offset = record->end_seq - tcp_seq;
-
-		pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",
-			 tcp_seq, record->end_seq, tx_info->prev_seq, data_len);
-		/* if a tls record is finishing in this SKB */
-		if (tls_end_offset <= data_len) {
-			struct sk_buff *nskb = NULL;
-
-			if (tls_end_offset < data_len) {
-				nskb = alloc_skb(0, GFP_KERNEL);
-				if (unlikely(!nskb)) {
-					ret = -ENOMEM;
-					goto clear_ref;
-				}
-
-				chcr_ktls_skb_shift(nskb, local_skb,
-						    tls_end_offset);
-			} else {
-				/* its the only record in this skb, directly
-				 * point it.
-				 */
-				nskb = local_skb;
-			}
-			ret = chcr_end_part_handler(tx_info, nskb, record,
-						    tcp_seq, mss,
-						    (!th->fin && th->psh), q,
-						    tls_end_offset,
-						    (nskb == local_skb));
-
-			if (ret && nskb != local_skb)
-				dev_kfree_skb_any(local_skb);
-
-			data_len -= tls_end_offset;
-			/* tcp_seq increment is required to handle next record.
-			 */
-			tcp_seq += tls_end_offset;
-		} else {
-			ret = chcr_short_record_handler(tx_info, local_skb,
-							record, tcp_seq, mss,
-							(!th->fin && th->psh),
-							q, tls_end_offset);
-			data_len = 0;
-		}
-clear_ref:
-		/* clear the frag ref count which increased locally before */
-		for (i = 0; i < record->num_frags; i++) {
-			/* clear the frag ref count */
-			__skb_frag_unref(&record->frags[i]);
-		}
-		/* if any failure, come out from the loop. */
-		if (ret)
-			goto out;
-		/* length should never be less than 0 */
-		WARN_ON(data_len < 0);
-
-	} while (data_len > 0);
-
-	tx_info->prev_seq = ntohl(th->seq) + skb->data_len;
-
-	atomic64_inc(&stats->ktls_tx_encrypted_packets);
-	atomic64_add(skb->data_len, &stats->ktls_tx_encrypted_bytes);
-
-	/* tcp finish is set, send a separate tcp msg including all the options
-	 * as well.
-	 */
-	if (th->fin)
-		chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan);
-
-out:
-	dev_kfree_skb_any(skb);
-	return NETDEV_TX_OK;
-}
-#endif /* CONFIG_CHELSIO_TLS_DEVICE */
diff --git a/drivers/crypto/chelsio/chcr_ktls.h b/drivers/crypto/chelsio/chcr_ktls.h
deleted file mode 100644
index 5cbd84b1da05..000000000000
--- a/drivers/crypto/chelsio/chcr_ktls.h
+++ /dev/null
@@ -1,103 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (C) 2020 Chelsio Communications.  All rights reserved. */
-
-#ifndef __CHCR_KTLS_H__
-#define __CHCR_KTLS_H__
-
-#ifdef CONFIG_CHELSIO_TLS_DEVICE
-#include <net/tls.h>
-#include "cxgb4.h"
-#include "t4_msg.h"
-#include "t4_tcb.h"
-#include "l2t.h"
-#include "chcr_common.h"
-#include "cxgb4_uld.h"
-
-#define CHCR_TCB_STATE_CLOSED	0
-#define CHCR_KTLS_KEY_CTX_LEN	16
-#define CHCR_SET_TCB_FIELD_LEN	sizeof(struct cpl_set_tcb_field)
-#define CHCR_PLAIN_TX_DATA_LEN	(sizeof(struct fw_ulptx_wr) +\
-				 sizeof(struct ulp_txpkt) +\
-				 sizeof(struct ulptx_idata) +\
-				 sizeof(struct cpl_tx_data))
-
-#define CHCR_KTLS_WR_SIZE	(CHCR_PLAIN_TX_DATA_LEN +\
-				 sizeof(struct cpl_tx_sec_pdu))
-
-enum chcr_ktls_conn_state {
-	KTLS_CONN_CLOSED,
-	KTLS_CONN_ACT_OPEN_REQ,
-	KTLS_CONN_ACT_OPEN_RPL,
-	KTLS_CONN_SET_TCB_REQ,
-	KTLS_CONN_SET_TCB_RPL,
-	KTLS_CONN_TX_READY,
-};
-
-struct chcr_ktls_info {
-	struct sock *sk;
-	spinlock_t lock; /* state machine lock */
-	struct ktls_key_ctx key_ctx;
-	struct adapter *adap;
-	struct l2t_entry *l2te;
-	struct net_device *netdev;
-	u64 iv;
-	u64 record_no;
-	int tid;
-	int atid;
-	int rx_qid;
-	u32 iv_size;
-	u32 prev_seq;
-	u32 prev_ack;
-	u32 salt_size;
-	u32 key_ctx_len;
-	u32 scmd0_seqno_numivs;
-	u32 scmd0_ivgen_hdrlen;
-	u32 tcp_start_seq_number;
-	u32 scmd0_short_seqno_numivs;
-	u32 scmd0_short_ivgen_hdrlen;
-	enum chcr_ktls_conn_state connection_state;
-	u16 prev_win;
-	u8 tx_chan;
-	u8 smt_idx;
-	u8 port_id;
-	u8 ip_family;
-	u8 first_qset;
-};
-
-struct chcr_ktls_ofld_ctx_tx {
-	struct tls_offload_context_tx base;
-	struct chcr_ktls_info *chcr_info;
-};
-
-static inline struct chcr_ktls_ofld_ctx_tx *
-chcr_get_ktls_tx_context(struct tls_context *tls_ctx)
-{
-	BUILD_BUG_ON(sizeof(struct chcr_ktls_ofld_ctx_tx) >
-		     TLS_OFFLOAD_CONTEXT_SIZE_TX);
-	return container_of(tls_offload_ctx_tx(tls_ctx),
-			    struct chcr_ktls_ofld_ctx_tx,
-			    base);
-}
-
-static inline int chcr_get_first_rx_qid(struct adapter *adap)
-{
-	/* u_ctx is saved in adap, fetch it */
-	struct uld_ctx *u_ctx = adap->uld[CXGB4_ULD_CRYPTO].handle;
-
-	if (!u_ctx)
-		return -1;
-	return u_ctx->lldi.rxq_ids[0];
-}
-
-int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input);
-int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input);
-int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev);
-int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
-		      enum tls_offload_ctx_dir direction,
-		      struct tls_crypto_info *crypto_info,
-		      u32 start_offload_tcp_sn);
-void chcr_ktls_dev_del(struct net_device *netdev,
-		       struct tls_context *tls_ctx,
-		       enum tls_offload_ctx_dir direction);
-#endif /* CONFIG_CHELSIO_TLS_DEVICE */
-#endif /* __CHCR_KTLS_H__ */
diff --git a/drivers/crypto/chelsio/chtls/Makefile b/drivers/crypto/chelsio/chtls/Makefile
deleted file mode 100644
index bc11495acdb3..000000000000
--- a/drivers/crypto/chelsio/chtls/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4 \
-	     -I $(srctree)/drivers/crypto/chelsio
-
-obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls.o
-chtls-objs := chtls_main.o chtls_cm.o chtls_io.o chtls_hw.o
diff --git a/drivers/crypto/chelsio/chtls/chtls.h b/drivers/crypto/chelsio/chtls/chtls.h
deleted file mode 100644
index 459442704eb1..000000000000
--- a/drivers/crypto/chelsio/chtls/chtls.h
+++ /dev/null
@@ -1,492 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2018 Chelsio Communications, Inc.
- */
-
-#ifndef __CHTLS_H__
-#define __CHTLS_H__
-
-#include <crypto/aes.h>
-#include <crypto/algapi.h>
-#include <crypto/hash.h>
-#include <crypto/sha.h>
-#include <crypto/authenc.h>
-#include <crypto/ctr.h>
-#include <crypto/gf128mul.h>
-#include <crypto/internal/aead.h>
-#include <crypto/null.h>
-#include <crypto/internal/skcipher.h>
-#include <crypto/aead.h>
-#include <crypto/scatterwalk.h>
-#include <crypto/internal/hash.h>
-#include <linux/tls.h>
-#include <net/tls.h>
-#include <net/tls_toe.h>
-
-#include "t4fw_api.h"
-#include "t4_msg.h"
-#include "cxgb4.h"
-#include "cxgb4_uld.h"
-#include "l2t.h"
-#include "chcr_algo.h"
-#include "chcr_core.h"
-#include "chcr_crypto.h"
-
-#define MAX_IVS_PAGE			256
-#define TLS_KEY_CONTEXT_SZ		64
-#define CIPHER_BLOCK_SIZE		16
-#define GCM_TAG_SIZE			16
-#define KEY_ON_MEM_SZ			16
-#define AEAD_EXPLICIT_DATA_SIZE		8
-#define TLS_HEADER_LENGTH		5
-#define SCMD_CIPH_MODE_AES_GCM		2
-/* Any MFS size should work and come from openssl */
-#define TLS_MFS				16384
-
-#define RSS_HDR sizeof(struct rss_header)
-#define TLS_WR_CPL_LEN \
-	(sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo))
-
-enum {
-	CHTLS_KEY_CONTEXT_DSGL,
-	CHTLS_KEY_CONTEXT_IMM,
-	CHTLS_KEY_CONTEXT_DDR,
-};
-
-enum {
-	CHTLS_LISTEN_START,
-	CHTLS_LISTEN_STOP,
-};
-
-/* Flags for return value of CPL message handlers */
-enum {
-	CPL_RET_BUF_DONE =    1,   /* buffer processing done */
-	CPL_RET_BAD_MSG =     2,   /* bad CPL message */
-	CPL_RET_UNKNOWN_TID = 4    /* unexpected unknown TID */
-};
-
-#define LISTEN_INFO_HASH_SIZE 32
-#define RSPQ_HASH_BITS 5
-struct listen_info {
-	struct listen_info *next;  /* Link to next entry */
-	struct sock *sk;           /* The listening socket */
-	unsigned int stid;         /* The server TID */
-};
-
-enum {
-	T4_LISTEN_START_PENDING,
-	T4_LISTEN_STARTED
-};
-
-enum csk_flags {
-	CSK_CALLBACKS_CHKD,	/* socket callbacks have been sanitized */
-	CSK_ABORT_REQ_RCVD,	/* received one ABORT_REQ_RSS message */
-	CSK_TX_MORE_DATA,	/* sending ULP data; don't set SHOVE bit */
-	CSK_TX_WAIT_IDLE,	/* suspend Tx until in-flight data is ACKed */
-	CSK_ABORT_SHUTDOWN,	/* shouldn't send more abort requests */
-	CSK_ABORT_RPL_PENDING,	/* expecting an abort reply */
-	CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
-	CSK_TX_DATA_SENT,	/* sent a TX_DATA WR on this connection */
-	CSK_TX_FAILOVER,	/* Tx traffic failing over */
-	CSK_UPDATE_RCV_WND,	/* Need to update rcv window */
-	CSK_RST_ABORTED,	/* outgoing RST was aborted */
-	CSK_TLS_HANDSHK,	/* TLS Handshake */
-	CSK_CONN_INLINE,	/* Connection on HW */
-};
-
-enum chtls_cdev_state {
-	CHTLS_CDEV_STATE_UP = 1
-};
-
-struct listen_ctx {
-	struct sock *lsk;
-	struct chtls_dev *cdev;
-	struct sk_buff_head synq;
-	u32 state;
-};
-
-struct key_map {
-	unsigned long *addr;
-	unsigned int start;
-	unsigned int available;
-	unsigned int size;
-	spinlock_t lock; /* lock for key id request from map */
-} __packed;
-
-struct tls_scmd {
-	u32 seqno_numivs;
-	u32 ivgen_hdrlen;
-};
-
-struct chtls_dev {
-	struct tls_toe_device tlsdev;
-	struct list_head list;
-	struct cxgb4_lld_info *lldi;
-	struct pci_dev *pdev;
-	struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
-	spinlock_t listen_lock; /* lock for listen list */
-	struct net_device **ports;
-	struct tid_info *tids;
-	unsigned int pfvf;
-	const unsigned short *mtus;
-
-	struct idr hwtid_idr;
-	struct idr stid_idr;
-
-	spinlock_t idr_lock ____cacheline_aligned_in_smp;
-
-	struct net_device *egr_dev[NCHAN * 2];
-	struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
-	struct sk_buff *askb;
-
-	struct sk_buff_head deferq;
-	struct work_struct deferq_task;
-
-	struct list_head list_node;
-	struct list_head rcu_node;
-	struct list_head na_node;
-	unsigned int send_page_order;
-	int max_host_sndbuf;
-	struct key_map kmap;
-	unsigned int cdev_state;
-};
-
-struct chtls_listen {
-	struct chtls_dev *cdev;
-	struct sock *sk;
-};
-
-struct chtls_hws {
-	struct sk_buff_head sk_recv_queue;
-	u8 txqid;
-	u8 ofld;
-	u16 type;
-	u16 rstate;
-	u16 keyrpl;
-	u16 pldlen;
-	u16 rcvpld;
-	u16 compute;
-	u16 expansion;
-	u16 keylen;
-	u16 pdus;
-	u16 adjustlen;
-	u16 ivsize;
-	u16 txleft;
-	u32 mfs;
-	s32 txkey;
-	s32 rxkey;
-	u32 fcplenmax;
-	u32 copied_seq;
-	u64 tx_seq_no;
-	struct tls_scmd scmd;
-	union {
-		struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
-		struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
-	} crypto_info;
-};
-
-struct chtls_sock {
-	struct sock *sk;
-	struct chtls_dev *cdev;
-	struct l2t_entry *l2t_entry;    /* pointer to the L2T entry */
-	struct net_device *egress_dev;  /* TX_CHAN for act open retry */
-
-	struct sk_buff_head txq;
-	struct sk_buff *wr_skb_head;
-	struct sk_buff *wr_skb_tail;
-	struct sk_buff *ctrl_skb_cache;
-	struct sk_buff *txdata_skb_cache; /* abort path messages */
-	struct kref kref;
-	unsigned long flags;
-	u32 opt2;
-	u32 wr_credits;
-	u32 wr_unacked;
-	u32 wr_max_credits;
-	u32 wr_nondata;
-	u32 hwtid;               /* TCP Control Block ID */
-	u32 txq_idx;
-	u32 rss_qid;
-	u32 tid;
-	u32 idr;
-	u32 mss;
-	u32 ulp_mode;
-	u32 tx_chan;
-	u32 rx_chan;
-	u32 sndbuf;
-	u32 txplen_max;
-	u32 mtu_idx;           /* MTU table index */
-	u32 smac_idx;
-	u8 port_id;
-	u8 tos;
-	u16 resv2;
-	u32 delack_mode;
-	u32 delack_seq;
-	u32 snd_win;
-	u32 rcv_win;
-
-	void *passive_reap_next;        /* placeholder for passive */
-	struct chtls_hws tlshws;
-	struct synq {
-		struct sk_buff *next;
-		struct sk_buff *prev;
-	} synq;
-	struct listen_ctx *listen_ctx;
-};
-
-struct tls_hdr {
-	u8  type;
-	u16 version;
-	u16 length;
-} __packed;
-
-struct tlsrx_cmp_hdr {
-	u8  type;
-	u16 version;
-	u16 length;
-
-	u64 tls_seq;
-	u16 reserved1;
-	u8  res_to_mac_error;
-} __packed;
-
-/* res_to_mac_error fields */
-#define TLSRX_HDR_PKT_INT_ERROR_S   4
-#define TLSRX_HDR_PKT_INT_ERROR_M   0x1
-#define TLSRX_HDR_PKT_INT_ERROR_V(x) \
-	((x) << TLSRX_HDR_PKT_INT_ERROR_S)
-#define TLSRX_HDR_PKT_INT_ERROR_G(x) \
-	(((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M)
-#define TLSRX_HDR_PKT_INT_ERROR_F   TLSRX_HDR_PKT_INT_ERROR_V(1U)
-
-#define TLSRX_HDR_PKT_SPP_ERROR_S        3
-#define TLSRX_HDR_PKT_SPP_ERROR_M        0x1
-#define TLSRX_HDR_PKT_SPP_ERROR_V(x)     ((x) << TLSRX_HDR_PKT_SPP_ERROR)
-#define TLSRX_HDR_PKT_SPP_ERROR_G(x)     \
-	(((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M)
-#define TLSRX_HDR_PKT_SPP_ERROR_F        TLSRX_HDR_PKT_SPP_ERROR_V(1U)
-
-#define TLSRX_HDR_PKT_CCDX_ERROR_S       2
-#define TLSRX_HDR_PKT_CCDX_ERROR_M       0x1
-#define TLSRX_HDR_PKT_CCDX_ERROR_V(x)    ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S)
-#define TLSRX_HDR_PKT_CCDX_ERROR_G(x)    \
-	(((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M)
-#define TLSRX_HDR_PKT_CCDX_ERROR_F       TLSRX_HDR_PKT_CCDX_ERROR_V(1U)
-
-#define TLSRX_HDR_PKT_PAD_ERROR_S        1
-#define TLSRX_HDR_PKT_PAD_ERROR_M        0x1
-#define TLSRX_HDR_PKT_PAD_ERROR_V(x)     ((x) << TLSRX_HDR_PKT_PAD_ERROR_S)
-#define TLSRX_HDR_PKT_PAD_ERROR_G(x)     \
-	(((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M)
-#define TLSRX_HDR_PKT_PAD_ERROR_F        TLSRX_HDR_PKT_PAD_ERROR_V(1U)
-
-#define TLSRX_HDR_PKT_MAC_ERROR_S        0
-#define TLSRX_HDR_PKT_MAC_ERROR_M        0x1
-#define TLSRX_HDR_PKT_MAC_ERROR_V(x)     ((x) << TLSRX_HDR_PKT_MAC_ERROR)
-#define TLSRX_HDR_PKT_MAC_ERROR_G(x)     \
-	(((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M)
-#define TLSRX_HDR_PKT_MAC_ERROR_F        TLSRX_HDR_PKT_MAC_ERROR_V(1U)
-
-#define TLSRX_HDR_PKT_ERROR_M           0x1F
-#define CONTENT_TYPE_ERROR		0x7F
-
-struct ulp_mem_rw {
-	__be32 cmd;
-	__be32 len16;             /* command length */
-	__be32 dlen;              /* data length in 32-byte units */
-	__be32 lock_addr;
-};
-
-struct tls_key_wr {
-	__be32 op_to_compl;
-	__be32 flowid_len16;
-	__be32 ftid;
-	u8   reneg_to_write_rx;
-	u8   protocol;
-	__be16 mfs;
-};
-
-struct tls_key_req {
-	struct tls_key_wr wr;
-	struct ulp_mem_rw req;
-	struct ulptx_idata sc_imm;
-};
-
-/*
- * This lives in skb->cb and is used to chain WRs in a linked list.
- */
-struct wr_skb_cb {
-	struct l2t_skb_cb l2t;          /* reserve space for l2t CB */
-	struct sk_buff *next_wr;        /* next write request */
-};
-
-/* Per-skb backlog handler.  Run when a socket's backlog is processed. */
-struct blog_skb_cb {
-	void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb);
-	struct chtls_dev *cdev;
-};
-
-/*
- * Similar to tcp_skb_cb but with ULP elements added to support TLS,
- * etc.
- */
-struct ulp_skb_cb {
-	struct wr_skb_cb wr;		/* reserve space for write request */
-	u16 flags;			/* TCP-like flags */
-	u8 psh;
-	u8 ulp_mode;			/* ULP mode/submode of sk_buff */
-	u32 seq;			/* TCP sequence number */
-	union { /* ULP-specific fields */
-		struct {
-			u8  type;
-			u8  ofld;
-			u8  iv;
-		} tls;
-	} ulp;
-};
-
-#define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0]))
-#define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb)
-
-/*
- * Flags for ulp_skb_cb.flags.
- */
-enum {
-	ULPCB_FLAG_NEED_HDR  = 1 << 0,	/* packet needs a TX_DATA_WR header */
-	ULPCB_FLAG_NO_APPEND = 1 << 1,	/* don't grow this skb */
-	ULPCB_FLAG_BARRIER   = 1 << 2,	/* set TX_WAIT_IDLE after sending */
-	ULPCB_FLAG_HOLD      = 1 << 3,	/* skb not ready for Tx yet */
-	ULPCB_FLAG_COMPL     = 1 << 4,	/* request WR completion */
-	ULPCB_FLAG_URG       = 1 << 5,	/* urgent data */
-	ULPCB_FLAG_TLS_HDR   = 1 << 6,  /* payload with tls hdr */
-	ULPCB_FLAG_NO_HDR    = 1 << 7,  /* not a ofld wr */
-};
-
-/* The ULP mode/submode of an skbuff */
-#define skb_ulp_mode(skb)  (ULP_SKB_CB(skb)->ulp_mode)
-#define TCP_PAGE(sk)   (sk->sk_frag.page)
-#define TCP_OFF(sk)    (sk->sk_frag.offset)
-
-static inline struct chtls_dev *to_chtls_dev(struct tls_toe_device *tlsdev)
-{
-	return container_of(tlsdev, struct chtls_dev, tlsdev);
-}
-
-static inline void csk_set_flag(struct chtls_sock *csk,
-				enum csk_flags flag)
-{
-	__set_bit(flag, &csk->flags);
-}
-
-static inline void csk_reset_flag(struct chtls_sock *csk,
-				  enum csk_flags flag)
-{
-	__clear_bit(flag, &csk->flags);
-}
-
-static inline bool csk_conn_inline(const struct chtls_sock *csk)
-{
-	return test_bit(CSK_CONN_INLINE, &csk->flags);
-}
-
-static inline int csk_flag(const struct sock *sk, enum csk_flags flag)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
-	if (!csk_conn_inline(csk))
-		return 0;
-	return test_bit(flag, &csk->flags);
-}
-
-static inline int csk_flag_nochk(const struct chtls_sock *csk,
-				 enum csk_flags flag)
-{
-	return test_bit(flag, &csk->flags);
-}
-
-static inline void *cplhdr(struct sk_buff *skb)
-{
-	return skb->data;
-}
-
-static inline int is_neg_adv(unsigned int status)
-{
-	return status == CPL_ERR_RTX_NEG_ADVICE ||
-	       status == CPL_ERR_KEEPALV_NEG_ADVICE ||
-	       status == CPL_ERR_PERSIST_NEG_ADVICE;
-}
-
-static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *),
-				   struct sock *sk,
-				   struct sk_buff *skb)
-{
-	skb_reset_mac_header(skb);
-	skb_reset_network_header(skb);
-	skb_reset_transport_header(skb);
-
-	bh_lock_sock(sk);
-	if (unlikely(sock_owned_by_user(sk))) {
-		BLOG_SKB_CB(skb)->backlog_rcv = fn;
-		__sk_add_backlog(sk, skb);
-	} else {
-		fn(sk, skb);
-	}
-	bh_unlock_sock(sk);
-}
-
-static inline void chtls_sock_free(struct kref *ref)
-{
-	struct chtls_sock *csk = container_of(ref, struct chtls_sock,
-					      kref);
-	kfree(csk);
-}
-
-static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk)
-{
-	kref_put(&csk->kref, chtls_sock_free);
-}
-
-static inline void __chtls_sock_get(const char *fn,
-				    struct chtls_sock *csk)
-{
-	kref_get(&csk->kref);
-}
-
-static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp,
-				 struct sk_buff *skb, int through_l2t)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
-	if (through_l2t) {
-		/* send through L2T */
-		cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
-	} else {
-		/* send directly */
-		cxgb4_ofld_send(csk->egress_dev, skb);
-	}
-}
-
-typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *);
-extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS];
-void chtls_install_cpl_ops(struct sock *sk);
-int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi);
-void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk);
-int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk);
-void chtls_close(struct sock *sk, long timeout);
-int chtls_disconnect(struct sock *sk, int flags);
-void chtls_shutdown(struct sock *sk, int how);
-void chtls_destroy_sock(struct sock *sk);
-int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
-int chtls_recvmsg(struct sock *sk, struct msghdr *msg,
-		  size_t len, int nonblock, int flags, int *addr_len);
-int chtls_sendpage(struct sock *sk, struct page *page,
-		   int offset, size_t size, int flags);
-int send_tx_flowc_wr(struct sock *sk, int compl,
-		     u32 snd_nxt, u32 rcv_nxt);
-void chtls_tcp_push(struct sock *sk, int flags);
-int chtls_push_frames(struct chtls_sock *csk, int comp);
-int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val);
-int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode, int cipher_type);
-void skb_entail(struct sock *sk, struct sk_buff *skb, int flags);
-unsigned int keyid_to_addr(int start_addr, int keyid);
-void free_tls_keyid(struct sock *sk);
-#endif
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c b/drivers/crypto/chelsio/chtls/chtls_cm.c
deleted file mode 100644
index 05520dccd906..000000000000
--- a/drivers/crypto/chelsio/chtls/chtls_cm.c
+++ /dev/null
@@ -1,2327 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018 Chelsio Communications, Inc.
- *
- * Written by: Atul Gupta (atul.gupta@chelsio.com)
- */
-
-#include <linux/module.h>
-#include <linux/list.h>
-#include <linux/workqueue.h>
-#include <linux/skbuff.h>
-#include <linux/timer.h>
-#include <linux/notifier.h>
-#include <linux/inetdevice.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/sched/signal.h>
-#include <linux/kallsyms.h>
-#include <linux/kprobes.h>
-#include <linux/if_vlan.h>
-#include <linux/ipv6.h>
-#include <net/ipv6.h>
-#include <net/transp_v6.h>
-#include <net/ip6_route.h>
-#include <net/inet_common.h>
-#include <net/tcp.h>
-#include <net/dst.h>
-#include <net/tls.h>
-#include <net/addrconf.h>
-#include <net/secure_seq.h>
-
-#include "chtls.h"
-#include "chtls_cm.h"
-#include "clip_tbl.h"
-
-/*
- * State transitions and actions for close.  Note that if we are in SYN_SENT
- * we remain in that state as we cannot control a connection while it's in
- * SYN_SENT; such connections are allowed to establish and are then aborted.
- */
-static unsigned char new_state[16] = {
-	/* current state:     new state:      action: */
-	/* (Invalid)       */ TCP_CLOSE,
-	/* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
-	/* TCP_SYN_SENT    */ TCP_SYN_SENT,
-	/* TCP_SYN_RECV    */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
-	/* TCP_FIN_WAIT1   */ TCP_FIN_WAIT1,
-	/* TCP_FIN_WAIT2   */ TCP_FIN_WAIT2,
-	/* TCP_TIME_WAIT   */ TCP_CLOSE,
-	/* TCP_CLOSE       */ TCP_CLOSE,
-	/* TCP_CLOSE_WAIT  */ TCP_LAST_ACK | TCP_ACTION_FIN,
-	/* TCP_LAST_ACK    */ TCP_LAST_ACK,
-	/* TCP_LISTEN      */ TCP_CLOSE,
-	/* TCP_CLOSING     */ TCP_CLOSING,
-};
-
-static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
-{
-	struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
-
-	if (!csk)
-		return NULL;
-
-	csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
-	if (!csk->txdata_skb_cache) {
-		kfree(csk);
-		return NULL;
-	}
-
-	kref_init(&csk->kref);
-	csk->cdev = cdev;
-	skb_queue_head_init(&csk->txq);
-	csk->wr_skb_head = NULL;
-	csk->wr_skb_tail = NULL;
-	csk->mss = MAX_MSS;
-	csk->tlshws.ofld = 1;
-	csk->tlshws.txkey = -1;
-	csk->tlshws.rxkey = -1;
-	csk->tlshws.mfs = TLS_MFS;
-	skb_queue_head_init(&csk->tlshws.sk_recv_queue);
-	return csk;
-}
-
-static void chtls_sock_release(struct kref *ref)
-{
-	struct chtls_sock *csk =
-		container_of(ref, struct chtls_sock, kref);
-
-	kfree(csk);
-}
-
-static struct net_device *chtls_find_netdev(struct chtls_dev *cdev,
-					    struct sock *sk)
-{
-	struct net_device *ndev = cdev->ports[0];
-#if IS_ENABLED(CONFIG_IPV6)
-	struct net_device *temp;
-	int addr_type;
-#endif
-
-	switch (sk->sk_family) {
-	case PF_INET:
-		if (likely(!inet_sk(sk)->inet_rcv_saddr))
-			return ndev;
-		ndev = __ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr, false);
-		break;
-#if IS_ENABLED(CONFIG_IPV6)
-	case PF_INET6:
-		addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
-		if (likely(addr_type == IPV6_ADDR_ANY))
-			return ndev;
-
-		for_each_netdev_rcu(&init_net, temp) {
-			if (ipv6_chk_addr(&init_net, (struct in6_addr *)
-					  &sk->sk_v6_rcv_saddr, temp, 1)) {
-				ndev = temp;
-				break;
-			}
-		}
-	break;
-#endif
-	default:
-		return NULL;
-	}
-
-	if (!ndev)
-		return NULL;
-
-	if (is_vlan_dev(ndev))
-		return vlan_dev_real_dev(ndev);
-	return ndev;
-}
-
-static void assign_rxopt(struct sock *sk, unsigned int opt)
-{
-	const struct chtls_dev *cdev;
-	struct chtls_sock *csk;
-	struct tcp_sock *tp;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	tp = tcp_sk(sk);
-
-	cdev = csk->cdev;
-	tp->tcp_header_len           = sizeof(struct tcphdr);
-	tp->rx_opt.mss_clamp         = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
-	tp->mss_cache                = tp->rx_opt.mss_clamp;
-	tp->rx_opt.tstamp_ok         = TCPOPT_TSTAMP_G(opt);
-	tp->rx_opt.snd_wscale        = TCPOPT_SACK_G(opt);
-	tp->rx_opt.wscale_ok         = TCPOPT_WSCALE_OK_G(opt);
-	SND_WSCALE(tp)               = TCPOPT_SND_WSCALE_G(opt);
-	if (!tp->rx_opt.wscale_ok)
-		tp->rx_opt.rcv_wscale = 0;
-	if (tp->rx_opt.tstamp_ok) {
-		tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
-		tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
-	} else if (csk->opt2 & TSTAMPS_EN_F) {
-		csk->opt2 &= ~TSTAMPS_EN_F;
-		csk->mtu_idx = TCPOPT_MSS_G(opt);
-	}
-}
-
-static void chtls_purge_receive_queue(struct sock *sk)
-{
-	struct sk_buff *skb;
-
-	while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
-		skb_dst_set(skb, (void *)NULL);
-		kfree_skb(skb);
-	}
-}
-
-static void chtls_purge_write_queue(struct sock *sk)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct sk_buff *skb;
-
-	while ((skb = __skb_dequeue(&csk->txq))) {
-		sk->sk_wmem_queued -= skb->truesize;
-		__kfree_skb(skb);
-	}
-}
-
-static void chtls_purge_recv_queue(struct sock *sk)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct chtls_hws *tlsk = &csk->tlshws;
-	struct sk_buff *skb;
-
-	while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) {
-		skb_dst_set(skb, NULL);
-		kfree_skb(skb);
-	}
-}
-
-static void abort_arp_failure(void *handle, struct sk_buff *skb)
-{
-	struct cpl_abort_req *req = cplhdr(skb);
-	struct chtls_dev *cdev;
-
-	cdev = (struct chtls_dev *)handle;
-	req->cmd = CPL_ABORT_NO_RST;
-	cxgb4_ofld_send(cdev->lldi->ports[0], skb);
-}
-
-static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len)
-{
-	if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
-		__skb_trim(skb, 0);
-		refcount_add(2, &skb->users);
-	} else {
-		skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
-	}
-	return skb;
-}
-
-static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb)
-{
-	struct cpl_abort_req *req;
-	struct chtls_sock *csk;
-	struct tcp_sock *tp;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	tp = tcp_sk(sk);
-
-	if (!skb)
-		skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req));
-
-	req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
-	INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid);
-	skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
-	req->rsvd0 = htonl(tp->snd_nxt);
-	req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT);
-	req->cmd = mode;
-	t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure);
-	send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST);
-}
-
-static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
-	if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) ||
-		     !csk->cdev)) {
-		if (sk->sk_state == TCP_SYN_RECV)
-			csk_set_flag(csk, CSK_RST_ABORTED);
-		goto out;
-	}
-
-	if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
-		struct tcp_sock *tp = tcp_sk(sk);
-
-		if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
-			WARN_ONCE(1, "send tx flowc error");
-		csk_set_flag(csk, CSK_TX_DATA_SENT);
-	}
-
-	csk_set_flag(csk, CSK_ABORT_RPL_PENDING);
-	chtls_purge_write_queue(sk);
-
-	csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
-	if (sk->sk_state != TCP_SYN_RECV)
-		chtls_send_abort(sk, mode, skb);
-	else
-		goto out;
-
-	return;
-out:
-	kfree_skb(skb);
-}
-
-static void release_tcp_port(struct sock *sk)
-{
-	if (inet_csk(sk)->icsk_bind_hash)
-		inet_put_port(sk);
-}
-
-static void tcp_uncork(struct sock *sk)
-{
-	struct tcp_sock *tp = tcp_sk(sk);
-
-	if (tp->nonagle & TCP_NAGLE_CORK) {
-		tp->nonagle &= ~TCP_NAGLE_CORK;
-		chtls_tcp_push(sk, 0);
-	}
-}
-
-static void chtls_close_conn(struct sock *sk)
-{
-	struct cpl_close_con_req *req;
-	struct chtls_sock *csk;
-	struct sk_buff *skb;
-	unsigned int tid;
-	unsigned int len;
-
-	len = roundup(sizeof(struct cpl_close_con_req), 16);
-	csk = rcu_dereference_sk_user_data(sk);
-	tid = csk->tid;
-
-	skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
-	req = (struct cpl_close_con_req *)__skb_put(skb, len);
-	memset(req, 0, len);
-	req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) |
-			      FW_WR_IMMDLEN_V(sizeof(*req) -
-					      sizeof(req->wr)));
-	req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) |
-			       FW_WR_FLOWID_V(tid));
-
-	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
-
-	tcp_uncork(sk);
-	skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
-	if (sk->sk_state != TCP_SYN_SENT)
-		chtls_push_frames(csk, 1);
-}
-
-/*
- * Perform a state transition during close and return the actions indicated
- * for the transition.  Do not make this function inline, the main reason
- * it exists at all is to avoid multiple inlining of tcp_set_state.
- */
-static int make_close_transition(struct sock *sk)
-{
-	int next = (int)new_state[sk->sk_state];
-
-	tcp_set_state(sk, next & TCP_STATE_MASK);
-	return next & TCP_ACTION_FIN;
-}
-
-void chtls_close(struct sock *sk, long timeout)
-{
-	int data_lost, prev_state;
-	struct chtls_sock *csk;
-
-	csk = rcu_dereference_sk_user_data(sk);
-
-	lock_sock(sk);
-	sk->sk_shutdown |= SHUTDOWN_MASK;
-
-	data_lost = skb_queue_len(&sk->sk_receive_queue);
-	data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue);
-	chtls_purge_recv_queue(sk);
-	chtls_purge_receive_queue(sk);
-
-	if (sk->sk_state == TCP_CLOSE) {
-		goto wait;
-	} else if (data_lost || sk->sk_state == TCP_SYN_SENT) {
-		chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
-		release_tcp_port(sk);
-		goto unlock;
-	} else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
-		sk->sk_prot->disconnect(sk, 0);
-	} else if (make_close_transition(sk)) {
-		chtls_close_conn(sk);
-	}
-wait:
-	if (timeout)
-		sk_stream_wait_close(sk, timeout);
-
-unlock:
-	prev_state = sk->sk_state;
-	sock_hold(sk);
-	sock_orphan(sk);
-
-	release_sock(sk);
-
-	local_bh_disable();
-	bh_lock_sock(sk);
-
-	if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
-		goto out;
-
-	if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 &&
-	    !csk_flag(sk, CSK_ABORT_SHUTDOWN)) {
-		struct sk_buff *skb;
-
-		skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
-		if (skb)
-			chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb);
-	}
-
-	if (sk->sk_state == TCP_CLOSE)
-		inet_csk_destroy_sock(sk);
-
-out:
-	bh_unlock_sock(sk);
-	local_bh_enable();
-	sock_put(sk);
-}
-
-/*
- * Wait until a socket enters on of the given states.
- */
-static int wait_for_states(struct sock *sk, unsigned int states)
-{
-	DECLARE_WAITQUEUE(wait, current);
-	struct socket_wq _sk_wq;
-	long current_timeo;
-	int err = 0;
-
-	current_timeo = 200;
-
-	/*
-	 * We want this to work even when there's no associated struct socket.
-	 * In that case we provide a temporary wait_queue_head_t.
-	 */
-	if (!sk->sk_wq) {
-		init_waitqueue_head(&_sk_wq.wait);
-		_sk_wq.fasync_list = NULL;
-		init_rcu_head_on_stack(&_sk_wq.rcu);
-		RCU_INIT_POINTER(sk->sk_wq, &_sk_wq);
-	}
-
-	add_wait_queue(sk_sleep(sk), &wait);
-	while (!sk_in_state(sk, states)) {
-		if (!current_timeo) {
-			err = -EBUSY;
-			break;
-		}
-		if (signal_pending(current)) {
-			err = sock_intr_errno(current_timeo);
-			break;
-		}
-		set_current_state(TASK_UNINTERRUPTIBLE);
-		release_sock(sk);
-		if (!sk_in_state(sk, states))
-			current_timeo = schedule_timeout(current_timeo);
-		__set_current_state(TASK_RUNNING);
-		lock_sock(sk);
-	}
-	remove_wait_queue(sk_sleep(sk), &wait);
-
-	if (rcu_dereference(sk->sk_wq) == &_sk_wq)
-		sk->sk_wq = NULL;
-	return err;
-}
-
-int chtls_disconnect(struct sock *sk, int flags)
-{
-	struct tcp_sock *tp;
-	int err;
-
-	tp = tcp_sk(sk);
-	chtls_purge_recv_queue(sk);
-	chtls_purge_receive_queue(sk);
-	chtls_purge_write_queue(sk);
-
-	if (sk->sk_state != TCP_CLOSE) {
-		sk->sk_err = ECONNRESET;
-		chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
-		err = wait_for_states(sk, TCPF_CLOSE);
-		if (err)
-			return err;
-	}
-	chtls_purge_recv_queue(sk);
-	chtls_purge_receive_queue(sk);
-	tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale);
-	return tcp_disconnect(sk, flags);
-}
-
-#define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \
-				 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)
-void chtls_shutdown(struct sock *sk, int how)
-{
-	if ((how & SEND_SHUTDOWN) &&
-	    sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) &&
-	    make_close_transition(sk))
-		chtls_close_conn(sk);
-}
-
-void chtls_destroy_sock(struct sock *sk)
-{
-	struct chtls_sock *csk;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	chtls_purge_recv_queue(sk);
-	csk->ulp_mode = ULP_MODE_NONE;
-	chtls_purge_write_queue(sk);
-	free_tls_keyid(sk);
-	kref_put(&csk->kref, chtls_sock_release);
-	csk->cdev = NULL;
-	if (sk->sk_family == AF_INET)
-		sk->sk_prot = &tcp_prot;
-#if IS_ENABLED(CONFIG_IPV6)
-	else
-		sk->sk_prot = &tcpv6_prot;
-#endif
-	sk->sk_prot->destroy(sk);
-}
-
-static void reset_listen_child(struct sock *child)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(child);
-	struct sk_buff *skb;
-
-	skb = alloc_ctrl_skb(csk->txdata_skb_cache,
-			     sizeof(struct cpl_abort_req));
-
-	chtls_send_reset(child, CPL_ABORT_SEND_RST, skb);
-	sock_orphan(child);
-	INC_ORPHAN_COUNT(child);
-	if (child->sk_state == TCP_CLOSE)
-		inet_csk_destroy_sock(child);
-}
-
-static void chtls_disconnect_acceptq(struct sock *listen_sk)
-{
-	struct request_sock **pprev;
-
-	pprev = ACCEPT_QUEUE(listen_sk);
-	while (*pprev) {
-		struct request_sock *req = *pprev;
-
-		if (req->rsk_ops == &chtls_rsk_ops ||
-		    req->rsk_ops == &chtls_rsk_opsv6) {
-			struct sock *child = req->sk;
-
-			*pprev = req->dl_next;
-			sk_acceptq_removed(listen_sk);
-			reqsk_put(req);
-			sock_hold(child);
-			local_bh_disable();
-			bh_lock_sock(child);
-			release_tcp_port(child);
-			reset_listen_child(child);
-			bh_unlock_sock(child);
-			local_bh_enable();
-			sock_put(child);
-		} else {
-			pprev = &req->dl_next;
-		}
-	}
-}
-
-static int listen_hashfn(const struct sock *sk)
-{
-	return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1);
-}
-
-static struct listen_info *listen_hash_add(struct chtls_dev *cdev,
-					   struct sock *sk,
-					   unsigned int stid)
-{
-	struct listen_info *p = kmalloc(sizeof(*p), GFP_KERNEL);
-
-	if (p) {
-		int key = listen_hashfn(sk);
-
-		p->sk = sk;
-		p->stid = stid;
-		spin_lock(&cdev->listen_lock);
-		p->next = cdev->listen_hash_tab[key];
-		cdev->listen_hash_tab[key] = p;
-		spin_unlock(&cdev->listen_lock);
-	}
-	return p;
-}
-
-static int listen_hash_find(struct chtls_dev *cdev,
-			    struct sock *sk)
-{
-	struct listen_info *p;
-	int stid = -1;
-	int key;
-
-	key = listen_hashfn(sk);
-
-	spin_lock(&cdev->listen_lock);
-	for (p = cdev->listen_hash_tab[key]; p; p = p->next)
-		if (p->sk == sk) {
-			stid = p->stid;
-			break;
-		}
-	spin_unlock(&cdev->listen_lock);
-	return stid;
-}
-
-static int listen_hash_del(struct chtls_dev *cdev,
-			   struct sock *sk)
-{
-	struct listen_info *p, **prev;
-	int stid = -1;
-	int key;
-
-	key = listen_hashfn(sk);
-	prev = &cdev->listen_hash_tab[key];
-
-	spin_lock(&cdev->listen_lock);
-	for (p = *prev; p; prev = &p->next, p = p->next)
-		if (p->sk == sk) {
-			stid = p->stid;
-			*prev = p->next;
-			kfree(p);
-			break;
-		}
-	spin_unlock(&cdev->listen_lock);
-	return stid;
-}
-
-static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent)
-{
-	struct request_sock *req;
-	struct chtls_sock *csk;
-
-	csk = rcu_dereference_sk_user_data(child);
-	req = csk->passive_reap_next;
-
-	reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req);
-	__skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
-	chtls_reqsk_free(req);
-	csk->passive_reap_next = NULL;
-}
-
-static void chtls_reset_synq(struct listen_ctx *listen_ctx)
-{
-	struct sock *listen_sk = listen_ctx->lsk;
-
-	while (!skb_queue_empty(&listen_ctx->synq)) {
-		struct chtls_sock *csk =
-			container_of((struct synq *)__skb_dequeue
-				(&listen_ctx->synq), struct chtls_sock, synq);
-		struct sock *child = csk->sk;
-
-		cleanup_syn_rcv_conn(child, listen_sk);
-		sock_hold(child);
-		local_bh_disable();
-		bh_lock_sock(child);
-		release_tcp_port(child);
-		reset_listen_child(child);
-		bh_unlock_sock(child);
-		local_bh_enable();
-		sock_put(child);
-	}
-}
-
-int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
-{
-	struct net_device *ndev;
-#if IS_ENABLED(CONFIG_IPV6)
-	bool clip_valid = false;
-#endif
-	struct listen_ctx *ctx;
-	struct adapter *adap;
-	struct port_info *pi;
-	int ret = 0;
-	int stid;
-
-	rcu_read_lock();
-	ndev = chtls_find_netdev(cdev, sk);
-	rcu_read_unlock();
-	if (!ndev)
-		return -EBADF;
-
-	pi = netdev_priv(ndev);
-	adap = pi->adapter;
-	if (!(adap->flags & CXGB4_FULL_INIT_DONE))
-		return -EBADF;
-
-	if (listen_hash_find(cdev, sk) >= 0)   /* already have it */
-		return -EADDRINUSE;
-
-	ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
-	if (!ctx)
-		return -ENOMEM;
-
-	__module_get(THIS_MODULE);
-	ctx->lsk = sk;
-	ctx->cdev = cdev;
-	ctx->state = T4_LISTEN_START_PENDING;
-	skb_queue_head_init(&ctx->synq);
-
-	stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx);
-	if (stid < 0)
-		goto free_ctx;
-
-	sock_hold(sk);
-	if (!listen_hash_add(cdev, sk, stid))
-		goto free_stid;
-
-	if (sk->sk_family == PF_INET) {
-		ret = cxgb4_create_server(ndev, stid,
-					  inet_sk(sk)->inet_rcv_saddr,
-					  inet_sk(sk)->inet_sport, 0,
-					  cdev->lldi->rxq_ids[0]);
-#if IS_ENABLED(CONFIG_IPV6)
-	} else {
-		int addr_type;
-
-		addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
-		if (addr_type != IPV6_ADDR_ANY) {
-			ret = cxgb4_clip_get(ndev, (const u32 *)
-					     &sk->sk_v6_rcv_saddr, 1);
-			if (ret)
-				goto del_hash;
-			clip_valid = true;
-		}
-		ret = cxgb4_create_server6(ndev, stid,
-					   &sk->sk_v6_rcv_saddr,
-					   inet_sk(sk)->inet_sport,
-					   cdev->lldi->rxq_ids[0]);
-#endif
-	}
-	if (ret > 0)
-		ret = net_xmit_errno(ret);
-	if (ret)
-		goto del_hash;
-	return 0;
-del_hash:
-#if IS_ENABLED(CONFIG_IPV6)
-	if (clip_valid)
-		cxgb4_clip_release(ndev, (const u32 *)&sk->sk_v6_rcv_saddr, 1);
-#endif
-	listen_hash_del(cdev, sk);
-free_stid:
-	cxgb4_free_stid(cdev->tids, stid, sk->sk_family);
-	sock_put(sk);
-free_ctx:
-	kfree(ctx);
-	module_put(THIS_MODULE);
-	return -EBADF;
-}
-
-void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)
-{
-	struct listen_ctx *listen_ctx;
-	int stid;
-
-	stid = listen_hash_del(cdev, sk);
-	if (stid < 0)
-		return;
-
-	listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
-	chtls_reset_synq(listen_ctx);
-
-	cxgb4_remove_server(cdev->lldi->ports[0], stid,
-			    cdev->lldi->rxq_ids[0], sk->sk_family == PF_INET6);
-
-#if IS_ENABLED(CONFIG_IPV6)
-	if (sk->sk_family == PF_INET6) {
-		struct chtls_sock *csk;
-		int addr_type = 0;
-
-		csk = rcu_dereference_sk_user_data(sk);
-		addr_type = ipv6_addr_type((const struct in6_addr *)
-					  &sk->sk_v6_rcv_saddr);
-		if (addr_type != IPV6_ADDR_ANY)
-			cxgb4_clip_release(csk->egress_dev, (const u32 *)
-					   &sk->sk_v6_rcv_saddr, 1);
-	}
-#endif
-	chtls_disconnect_acceptq(sk);
-}
-
-static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
-{
-	struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR;
-	unsigned int stid = GET_TID(rpl);
-	struct listen_ctx *listen_ctx;
-
-	listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
-	if (!listen_ctx)
-		return CPL_RET_BUF_DONE;
-
-	if (listen_ctx->state == T4_LISTEN_START_PENDING) {
-		listen_ctx->state = T4_LISTEN_STARTED;
-		return CPL_RET_BUF_DONE;
-	}
-
-	if (rpl->status != CPL_ERR_NONE) {
-		pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n",
-			rpl->status, stid);
-		return CPL_RET_BUF_DONE;
-	}
-	cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
-	sock_put(listen_ctx->lsk);
-	kfree(listen_ctx);
-	module_put(THIS_MODULE);
-
-	return 0;
-}
-
-static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
-{
-	struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR;
-	struct listen_ctx *listen_ctx;
-	unsigned int stid;
-	void *data;
-
-	stid = GET_TID(rpl);
-	data = lookup_stid(cdev->tids, stid);
-	listen_ctx = (struct listen_ctx *)data;
-
-	if (rpl->status != CPL_ERR_NONE) {
-		pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n",
-			rpl->status, stid);
-		return CPL_RET_BUF_DONE;
-	}
-
-	cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
-	sock_put(listen_ctx->lsk);
-	kfree(listen_ctx);
-	module_put(THIS_MODULE);
-
-	return 0;
-}
-
-static void chtls_purge_wr_queue(struct sock *sk)
-{
-	struct sk_buff *skb;
-
-	while ((skb = dequeue_wr(sk)) != NULL)
-		kfree_skb(skb);
-}
-
-static void chtls_release_resources(struct sock *sk)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct chtls_dev *cdev = csk->cdev;
-	unsigned int tid = csk->tid;
-	struct tid_info *tids;
-
-	if (!cdev)
-		return;
-
-	tids = cdev->tids;
-	kfree_skb(csk->txdata_skb_cache);
-	csk->txdata_skb_cache = NULL;
-
-	if (csk->wr_credits != csk->wr_max_credits) {
-		chtls_purge_wr_queue(sk);
-		chtls_reset_wr_list(csk);
-	}
-
-	if (csk->l2t_entry) {
-		cxgb4_l2t_release(csk->l2t_entry);
-		csk->l2t_entry = NULL;
-	}
-
-	if (sk->sk_state != TCP_SYN_SENT) {
-		cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family);
-		sock_put(sk);
-	}
-}
-
-static void chtls_conn_done(struct sock *sk)
-{
-	if (sock_flag(sk, SOCK_DEAD))
-		chtls_purge_receive_queue(sk);
-	sk_wakeup_sleepers(sk, 0);
-	tcp_done(sk);
-}
-
-static void do_abort_syn_rcv(struct sock *child, struct sock *parent)
-{
-	/*
-	 * If the server is still open we clean up the child connection,
-	 * otherwise the server already did the clean up as it was purging
-	 * its SYN queue and the skb was just sitting in its backlog.
-	 */
-	if (likely(parent->sk_state == TCP_LISTEN)) {
-		cleanup_syn_rcv_conn(child, parent);
-		/* Without the below call to sock_orphan,
-		 * we leak the socket resource with syn_flood test
-		 * as inet_csk_destroy_sock will not be called
-		 * in tcp_done since SOCK_DEAD flag is not set.
-		 * Kernel handles this differently where new socket is
-		 * created only after 3 way handshake is done.
-		 */
-		sock_orphan(child);
-		percpu_counter_inc((child)->sk_prot->orphan_count);
-		chtls_release_resources(child);
-		chtls_conn_done(child);
-	} else {
-		if (csk_flag(child, CSK_RST_ABORTED)) {
-			chtls_release_resources(child);
-			chtls_conn_done(child);
-		}
-	}
-}
-
-static void pass_open_abort(struct sock *child, struct sock *parent,
-			    struct sk_buff *skb)
-{
-	do_abort_syn_rcv(child, parent);
-	kfree_skb(skb);
-}
-
-static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb)
-{
-	pass_open_abort(skb->sk, lsk, skb);
-}
-
-static void chtls_pass_open_arp_failure(struct sock *sk,
-					struct sk_buff *skb)
-{
-	const struct request_sock *oreq;
-	struct chtls_sock *csk;
-	struct chtls_dev *cdev;
-	struct sock *parent;
-	void *data;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	cdev = csk->cdev;
-
-	/*
-	 * If the connection is being aborted due to the parent listening
-	 * socket going away there's nothing to do, the ABORT_REQ will close
-	 * the connection.
-	 */
-	if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) {
-		kfree_skb(skb);
-		return;
-	}
-
-	oreq = csk->passive_reap_next;
-	data = lookup_stid(cdev->tids, oreq->ts_recent);
-	parent = ((struct listen_ctx *)data)->lsk;
-
-	bh_lock_sock(parent);
-	if (!sock_owned_by_user(parent)) {
-		pass_open_abort(sk, parent, skb);
-	} else {
-		BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort;
-		__sk_add_backlog(parent, skb);
-	}
-	bh_unlock_sock(parent);
-}
-
-static void chtls_accept_rpl_arp_failure(void *handle,
-					 struct sk_buff *skb)
-{
-	struct sock *sk = (struct sock *)handle;
-
-	sock_hold(sk);
-	process_cpl_msg(chtls_pass_open_arp_failure, sk, skb);
-	sock_put(sk);
-}
-
-static unsigned int chtls_select_mss(const struct chtls_sock *csk,
-				     unsigned int pmtu,
-				     struct cpl_pass_accept_req *req)
-{
-	struct chtls_dev *cdev;
-	struct dst_entry *dst;
-	unsigned int tcpoptsz;
-	unsigned int iphdrsz;
-	unsigned int mtu_idx;
-	struct tcp_sock *tp;
-	unsigned int mss;
-	struct sock *sk;
-
-	mss = ntohs(req->tcpopt.mss);
-	sk = csk->sk;
-	dst = __sk_dst_get(sk);
-	cdev = csk->cdev;
-	tp = tcp_sk(sk);
-	tcpoptsz = 0;
-
-#if IS_ENABLED(CONFIG_IPV6)
-	if (sk->sk_family == AF_INET6)
-		iphdrsz = sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
-	else
-#endif
-		iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr);
-	if (req->tcpopt.tstamp)
-		tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4);
-
-	tp->advmss = dst_metric_advmss(dst);
-	if (USER_MSS(tp) && tp->advmss > USER_MSS(tp))
-		tp->advmss = USER_MSS(tp);
-	if (tp->advmss > pmtu - iphdrsz)
-		tp->advmss = pmtu - iphdrsz;
-	if (mss && tp->advmss > mss)
-		tp->advmss = mss;
-
-	tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus,
-					    iphdrsz + tcpoptsz,
-					    tp->advmss - tcpoptsz,
-					    8, &mtu_idx);
-	tp->advmss -= iphdrsz;
-
-	inet_csk(sk)->icsk_pmtu_cookie = pmtu;
-	return mtu_idx;
-}
-
-static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp)
-{
-	int wscale = 0;
-
-	if (space > MAX_RCV_WND)
-		space = MAX_RCV_WND;
-	if (win_clamp && win_clamp < space)
-		space = win_clamp;
-
-	if (wscale_ok) {
-		while (wscale < 14 && (65535 << wscale) < space)
-			wscale++;
-	}
-	return wscale;
-}
-
-static void chtls_pass_accept_rpl(struct sk_buff *skb,
-				  struct cpl_pass_accept_req *req,
-				  unsigned int tid)
-
-{
-	struct cpl_t5_pass_accept_rpl *rpl5;
-	struct cxgb4_lld_info *lldi;
-	const struct tcphdr *tcph;
-	const struct tcp_sock *tp;
-	struct chtls_sock *csk;
-	unsigned int len;
-	struct sock *sk;
-	u32 opt2, hlen;
-	u64 opt0;
-
-	sk = skb->sk;
-	tp = tcp_sk(sk);
-	csk = sk->sk_user_data;
-	csk->tid = tid;
-	lldi = csk->cdev->lldi;
-	len = roundup(sizeof(*rpl5), 16);
-
-	rpl5 = __skb_put_zero(skb, len);
-	INIT_TP_WR(rpl5, tid);
-
-	OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
-						     csk->tid));
-	csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)),
-					req);
-	opt0 = TCAM_BYPASS_F |
-	       WND_SCALE_V(RCV_WSCALE(tp)) |
-	       MSS_IDX_V(csk->mtu_idx) |
-	       L2T_IDX_V(csk->l2t_entry->idx) |
-	       NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) |
-	       TX_CHAN_V(csk->tx_chan) |
-	       SMAC_SEL_V(csk->smac_idx) |
-	       DSCP_V(csk->tos >> 2) |
-	       ULP_MODE_V(ULP_MODE_TLS) |
-	       RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M));
-
-	opt2 = RX_CHANNEL_V(0) |
-		RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);
-
-	if (!is_t5(lldi->adapter_type))
-		opt2 |= RX_FC_DISABLE_F;
-	if (req->tcpopt.tstamp)
-		opt2 |= TSTAMPS_EN_F;
-	if (req->tcpopt.sack)
-		opt2 |= SACK_EN_F;
-	hlen = ntohl(req->hdr_len);
-
-	tcph = (struct tcphdr *)((u8 *)(req + 1) +
-			T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));
-	if (tcph->ece && tcph->cwr)
-		opt2 |= CCTRL_ECN_V(1);
-	opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);
-	opt2 |= T5_ISS_F;
-	opt2 |= T5_OPT_2_VALID_F;
-	opt2 |= WND_SCALE_EN_V(WSCALE_OK(tp));
-	rpl5->opt0 = cpu_to_be64(opt0);
-	rpl5->opt2 = cpu_to_be32(opt2);
-	rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1);
-	set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
-	t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure);
-	cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
-}
-
-static void inet_inherit_port(struct inet_hashinfo *hash_info,
-			      struct sock *lsk, struct sock *newsk)
-{
-	local_bh_disable();
-	__inet_inherit_port(lsk, newsk);
-	local_bh_enable();
-}
-
-static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb)
-{
-	if (skb->protocol) {
-		kfree_skb(skb);
-		return 0;
-	}
-	BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
-	return 0;
-}
-
-static void chtls_set_tcp_window(struct chtls_sock *csk)
-{
-	struct net_device *ndev = csk->egress_dev;
-	struct port_info *pi = netdev_priv(ndev);
-	unsigned int linkspeed;
-	u8 scale;
-
-	linkspeed = pi->link_cfg.speed;
-	scale = linkspeed / SPEED_10000;
-#define CHTLS_10G_RCVWIN (256 * 1024)
-	csk->rcv_win = CHTLS_10G_RCVWIN;
-	if (scale)
-		csk->rcv_win *= scale;
-#define CHTLS_10G_SNDWIN (256 * 1024)
-	csk->snd_win = CHTLS_10G_SNDWIN;
-	if (scale)
-		csk->snd_win *= scale;
-}
-
-static struct sock *chtls_recv_sock(struct sock *lsk,
-				    struct request_sock *oreq,
-				    void *network_hdr,
-				    const struct cpl_pass_accept_req *req,
-				    struct chtls_dev *cdev)
-{
-	struct neighbour *n = NULL;
-	struct inet_sock *newinet;
-	const struct iphdr *iph;
-	struct tls_context *ctx;
-	struct net_device *ndev;
-	struct chtls_sock *csk;
-	struct dst_entry *dst;
-	struct tcp_sock *tp;
-	struct sock *newsk;
-	u16 port_id;
-	int rxq_idx;
-	int step;
-
-	iph = (const struct iphdr *)network_hdr;
-	newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb);
-	if (!newsk)
-		goto free_oreq;
-
-	if (lsk->sk_family == AF_INET) {
-		dst = inet_csk_route_child_sock(lsk, newsk, oreq);
-		if (!dst)
-			goto free_sk;
-
-		n = dst_neigh_lookup(dst, &iph->saddr);
-#if IS_ENABLED(CONFIG_IPV6)
-	} else {
-		const struct ipv6hdr *ip6h;
-		struct flowi6 fl6;
-
-		ip6h = (const struct ipv6hdr *)network_hdr;
-		memset(&fl6, 0, sizeof(fl6));
-		fl6.flowi6_proto = IPPROTO_TCP;
-		fl6.saddr = ip6h->daddr;
-		fl6.daddr = ip6h->saddr;
-		fl6.fl6_dport = inet_rsk(oreq)->ir_rmt_port;
-		fl6.fl6_sport = htons(inet_rsk(oreq)->ir_num);
-		security_req_classify_flow(oreq, flowi6_to_flowi(&fl6));
-		dst = ip6_dst_lookup_flow(sock_net(lsk), lsk, &fl6, NULL);
-		if (IS_ERR(dst))
-			goto free_sk;
-		n = dst_neigh_lookup(dst, &ip6h->saddr);
-#endif
-	}
-	if (!n)
-		goto free_sk;
-
-	ndev = n->dev;
-	if (!ndev)
-		goto free_dst;
-	port_id = cxgb4_port_idx(ndev);
-
-	csk = chtls_sock_create(cdev);
-	if (!csk)
-		goto free_dst;
-
-	csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0);
-	if (!csk->l2t_entry)
-		goto free_csk;
-
-	newsk->sk_user_data = csk;
-	newsk->sk_backlog_rcv = chtls_backlog_rcv;
-
-	tp = tcp_sk(newsk);
-	newinet = inet_sk(newsk);
-
-	if (iph->version == 0x4) {
-		newinet->inet_daddr = iph->saddr;
-		newinet->inet_rcv_saddr = iph->daddr;
-		newinet->inet_saddr = iph->daddr;
-#if IS_ENABLED(CONFIG_IPV6)
-	} else {
-		struct tcp6_sock *newtcp6sk = (struct tcp6_sock *)newsk;
-		struct inet_request_sock *treq = inet_rsk(oreq);
-		struct ipv6_pinfo *newnp = inet6_sk(newsk);
-		struct ipv6_pinfo *np = inet6_sk(lsk);
-
-		inet_sk(newsk)->pinet6 = &newtcp6sk->inet6;
-		memcpy(newnp, np, sizeof(struct ipv6_pinfo));
-		newsk->sk_v6_daddr = treq->ir_v6_rmt_addr;
-		newsk->sk_v6_rcv_saddr = treq->ir_v6_loc_addr;
-		inet6_sk(newsk)->saddr = treq->ir_v6_loc_addr;
-		newnp->ipv6_fl_list = NULL;
-		newnp->pktoptions = NULL;
-		newsk->sk_bound_dev_if = treq->ir_iif;
-		newinet->inet_opt = NULL;
-		newinet->inet_daddr = LOOPBACK4_IPV6;
-		newinet->inet_saddr = LOOPBACK4_IPV6;
-#endif
-	}
-
-	oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid));
-	sk_setup_caps(newsk, dst);
-	ctx = tls_get_ctx(lsk);
-	newsk->sk_destruct = ctx->sk_destruct;
-	csk->sk = newsk;
-	csk->passive_reap_next = oreq;
-	csk->tx_chan = cxgb4_port_chan(ndev);
-	csk->port_id = port_id;
-	csk->egress_dev = ndev;
-	csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));
-	chtls_set_tcp_window(csk);
-	tp->rcv_wnd = csk->rcv_win;
-	csk->sndbuf = csk->snd_win;
-	csk->ulp_mode = ULP_MODE_TLS;
-	step = cdev->lldi->nrxq / cdev->lldi->nchan;
-	csk->rss_qid = cdev->lldi->rxq_ids[port_id * step];
-	rxq_idx = port_id * step;
-	csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx :
-			port_id * step;
-	csk->sndbuf = newsk->sk_sndbuf;
-	csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx;
-	RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk),
-					   sock_net(newsk)->
-						ipv4.sysctl_tcp_window_scaling,
-					   tp->window_clamp);
-	neigh_release(n);
-	inet_inherit_port(&tcp_hashinfo, lsk, newsk);
-	csk_set_flag(csk, CSK_CONN_INLINE);
-	bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */
-
-	return newsk;
-free_csk:
-	chtls_sock_release(&csk->kref);
-free_dst:
-	dst_release(dst);
-free_sk:
-	inet_csk_prepare_forced_close(newsk);
-	tcp_done(newsk);
-free_oreq:
-	chtls_reqsk_free(oreq);
-	return NULL;
-}
-
-/*
- * Populate a TID_RELEASE WR.  The skb must be already propely sized.
- */
-static  void mk_tid_release(struct sk_buff *skb,
-			    unsigned int chan, unsigned int tid)
-{
-	struct cpl_tid_release *req;
-	unsigned int len;
-
-	len = roundup(sizeof(struct cpl_tid_release), 16);
-	req = (struct cpl_tid_release *)__skb_put(skb, len);
-	memset(req, 0, len);
-	set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
-	INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid);
-}
-
-static int chtls_get_module(struct sock *sk)
-{
-	struct inet_connection_sock *icsk = inet_csk(sk);
-
-	if (!try_module_get(icsk->icsk_ulp_ops->owner))
-		return -1;
-
-	return 0;
-}
-
-static void chtls_pass_accept_request(struct sock *sk,
-				      struct sk_buff *skb)
-{
-	struct cpl_t5_pass_accept_rpl *rpl;
-	struct cpl_pass_accept_req *req;
-	struct listen_ctx *listen_ctx;
-	struct vlan_ethhdr *vlan_eh;
-	struct request_sock *oreq;
-	struct sk_buff *reply_skb;
-	struct chtls_sock *csk;
-	struct chtls_dev *cdev;
-	struct ipv6hdr *ip6h;
-	struct tcphdr *tcph;
-	struct sock *newsk;
-	struct ethhdr *eh;
-	struct iphdr *iph;
-	void *network_hdr;
-	unsigned int stid;
-	unsigned int len;
-	unsigned int tid;
-	bool th_ecn, ect;
-	__u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */
-	u16 eth_hdr_len;
-	bool ecn_ok;
-
-	req = cplhdr(skb) + RSS_HDR;
-	tid = GET_TID(req);
-	cdev = BLOG_SKB_CB(skb)->cdev;
-	newsk = lookup_tid(cdev->tids, tid);
-	stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
-	if (newsk) {
-		pr_info("tid (%d) already in use\n", tid);
-		return;
-	}
-
-	len = roundup(sizeof(*rpl), 16);
-	reply_skb = alloc_skb(len, GFP_ATOMIC);
-	if (!reply_skb) {
-		cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family);
-		kfree_skb(skb);
-		return;
-	}
-
-	if (sk->sk_state != TCP_LISTEN)
-		goto reject;
-
-	if (inet_csk_reqsk_queue_is_full(sk))
-		goto reject;
-
-	if (sk_acceptq_is_full(sk))
-		goto reject;
-
-
-	eth_hdr_len = T6_ETH_HDR_LEN_G(ntohl(req->hdr_len));
-	if (eth_hdr_len == ETH_HLEN) {
-		eh = (struct ethhdr *)(req + 1);
-		iph = (struct iphdr *)(eh + 1);
-		ip6h = (struct ipv6hdr *)(eh + 1);
-		network_hdr = (void *)(eh + 1);
-	} else {
-		vlan_eh = (struct vlan_ethhdr *)(req + 1);
-		iph = (struct iphdr *)(vlan_eh + 1);
-		ip6h = (struct ipv6hdr *)(vlan_eh + 1);
-		network_hdr = (void *)(vlan_eh + 1);
-	}
-
-	if (iph->version == 0x4) {
-		tcph = (struct tcphdr *)(iph + 1);
-		skb_set_network_header(skb, (void *)iph - (void *)req);
-		oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true);
-	} else {
-		tcph = (struct tcphdr *)(ip6h + 1);
-		skb_set_network_header(skb, (void *)ip6h - (void *)req);
-		oreq = inet_reqsk_alloc(&chtls_rsk_opsv6, sk, false);
-	}
-
-	if (!oreq)
-		goto reject;
-
-	oreq->rsk_rcv_wnd = 0;
-	oreq->rsk_window_clamp = 0;
-	oreq->syncookie = 0;
-	oreq->mss = 0;
-	oreq->ts_recent = 0;
-
-	tcp_rsk(oreq)->tfo_listener = false;
-	tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq);
-	chtls_set_req_port(oreq, tcph->source, tcph->dest);
-	if (iph->version == 0x4) {
-		chtls_set_req_addr(oreq, iph->daddr, iph->saddr);
-		ip_dsfield = ipv4_get_dsfield(iph);
-#if IS_ENABLED(CONFIG_IPV6)
-	} else {
-		inet_rsk(oreq)->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
-		inet_rsk(oreq)->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
-		ip_dsfield = ipv6_get_dsfield(ipv6_hdr(skb));
-#endif
-	}
-	if (req->tcpopt.wsf <= 14 &&
-	    sock_net(sk)->ipv4.sysctl_tcp_window_scaling) {
-		inet_rsk(oreq)->wscale_ok = 1;
-		inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf;
-	}
-	inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if;
-	th_ecn = tcph->ece && tcph->cwr;
-	if (th_ecn) {
-		ect = !INET_ECN_is_not_ect(ip_dsfield);
-		ecn_ok = sock_net(sk)->ipv4.sysctl_tcp_ecn;
-		if ((!ect && ecn_ok) || tcp_ca_needs_ecn(sk))
-			inet_rsk(oreq)->ecn_ok = 1;
-	}
-
-	newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev);
-	if (!newsk)
-		goto free_oreq;
-
-	if (chtls_get_module(newsk))
-		goto reject;
-	inet_csk_reqsk_queue_added(sk);
-	reply_skb->sk = newsk;
-	chtls_install_cpl_ops(newsk);
-	cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family);
-	csk = rcu_dereference_sk_user_data(newsk);
-	listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
-	csk->listen_ctx = listen_ctx;
-	__skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq);
-	chtls_pass_accept_rpl(reply_skb, req, tid);
-	kfree_skb(skb);
-	return;
-
-free_oreq:
-	chtls_reqsk_free(oreq);
-reject:
-	mk_tid_release(reply_skb, 0, tid);
-	cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
-	kfree_skb(skb);
-}
-
-/*
- * Handle a CPL_PASS_ACCEPT_REQ message.
- */
-static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb)
-{
-	struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR;
-	struct listen_ctx *ctx;
-	unsigned int stid;
-	unsigned int tid;
-	struct sock *lsk;
-	void *data;
-
-	stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
-	tid = GET_TID(req);
-
-	data = lookup_stid(cdev->tids, stid);
-	if (!data)
-		return 1;
-
-	ctx = (struct listen_ctx *)data;
-	lsk = ctx->lsk;
-
-	if (unlikely(tid_out_of_range(cdev->tids, tid))) {
-		pr_info("passive open TID %u too large\n", tid);
-		return 1;
-	}
-
-	BLOG_SKB_CB(skb)->cdev = cdev;
-	process_cpl_msg(chtls_pass_accept_request, lsk, skb);
-	return 0;
-}
-
-/*
- * Completes some final bits of initialization for just established connections
- * and changes their state to TCP_ESTABLISHED.
- *
- * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
- */
-static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt)
-{
-	struct tcp_sock *tp = tcp_sk(sk);
-
-	tp->pushed_seq = snd_isn;
-	tp->write_seq = snd_isn;
-	tp->snd_nxt = snd_isn;
-	tp->snd_una = snd_isn;
-	inet_sk(sk)->inet_id = prandom_u32();
-	assign_rxopt(sk, opt);
-
-	if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10))
-		tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10);
-
-	smp_mb();
-	tcp_set_state(sk, TCP_ESTABLISHED);
-}
-
-static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb)
-{
-	struct sk_buff *abort_skb;
-
-	abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
-	if (abort_skb)
-		chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb);
-}
-
-static struct sock *reap_list;
-static DEFINE_SPINLOCK(reap_list_lock);
-
-/*
- * Process the reap list.
- */
-DECLARE_TASK_FUNC(process_reap_list, task_param)
-{
-	spin_lock_bh(&reap_list_lock);
-	while (reap_list) {
-		struct sock *sk = reap_list;
-		struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
-		reap_list = csk->passive_reap_next;
-		csk->passive_reap_next = NULL;
-		spin_unlock(&reap_list_lock);
-		sock_hold(sk);
-
-		bh_lock_sock(sk);
-		chtls_abort_conn(sk, NULL);
-		sock_orphan(sk);
-		if (sk->sk_state == TCP_CLOSE)
-			inet_csk_destroy_sock(sk);
-		bh_unlock_sock(sk);
-		sock_put(sk);
-		spin_lock(&reap_list_lock);
-	}
-	spin_unlock_bh(&reap_list_lock);
-}
-
-static DECLARE_WORK(reap_task, process_reap_list);
-
-static void add_to_reap_list(struct sock *sk)
-{
-	struct chtls_sock *csk = sk->sk_user_data;
-
-	local_bh_disable();
-	bh_lock_sock(sk);
-	release_tcp_port(sk); /* release the port immediately */
-
-	spin_lock(&reap_list_lock);
-	csk->passive_reap_next = reap_list;
-	reap_list = sk;
-	if (!csk->passive_reap_next)
-		schedule_work(&reap_task);
-	spin_unlock(&reap_list_lock);
-	bh_unlock_sock(sk);
-	local_bh_enable();
-}
-
-static void add_pass_open_to_parent(struct sock *child, struct sock *lsk,
-				    struct chtls_dev *cdev)
-{
-	struct request_sock *oreq;
-	struct chtls_sock *csk;
-
-	if (lsk->sk_state != TCP_LISTEN)
-		return;
-
-	csk = child->sk_user_data;
-	oreq = csk->passive_reap_next;
-	csk->passive_reap_next = NULL;
-
-	reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq);
-	__skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
-
-	if (sk_acceptq_is_full(lsk)) {
-		chtls_reqsk_free(oreq);
-		add_to_reap_list(child);
-	} else {
-		refcount_set(&oreq->rsk_refcnt, 1);
-		inet_csk_reqsk_queue_add(lsk, oreq, child);
-		lsk->sk_data_ready(lsk);
-	}
-}
-
-static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb)
-{
-	struct sock *child = skb->sk;
-
-	skb->sk = NULL;
-	add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev);
-	kfree_skb(skb);
-}
-
-static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb)
-{
-	struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR;
-	struct chtls_sock *csk;
-	struct sock *lsk, *sk;
-	unsigned int hwtid;
-
-	hwtid = GET_TID(req);
-	sk = lookup_tid(cdev->tids, hwtid);
-	if (!sk)
-		return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE);
-
-	bh_lock_sock(sk);
-	if (unlikely(sock_owned_by_user(sk))) {
-		kfree_skb(skb);
-	} else {
-		unsigned int stid;
-		void *data;
-
-		csk = sk->sk_user_data;
-		csk->wr_max_credits = 64;
-		csk->wr_credits = 64;
-		csk->wr_unacked = 0;
-		make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
-		stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
-		sk->sk_state_change(sk);
-		if (unlikely(sk->sk_socket))
-			sk_wake_async(sk, 0, POLL_OUT);
-
-		data = lookup_stid(cdev->tids, stid);
-		lsk = ((struct listen_ctx *)data)->lsk;
-
-		bh_lock_sock(lsk);
-		if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) {
-			/* removed from synq */
-			bh_unlock_sock(lsk);
-			kfree_skb(skb);
-			goto unlock;
-		}
-
-		if (likely(!sock_owned_by_user(lsk))) {
-			kfree_skb(skb);
-			add_pass_open_to_parent(sk, lsk, cdev);
-		} else {
-			skb->sk = sk;
-			BLOG_SKB_CB(skb)->cdev = cdev;
-			BLOG_SKB_CB(skb)->backlog_rcv =
-				bl_add_pass_open_to_parent;
-			__sk_add_backlog(lsk, skb);
-		}
-		bh_unlock_sock(lsk);
-	}
-unlock:
-	bh_unlock_sock(sk);
-	return 0;
-}
-
-/*
- * Handle receipt of an urgent pointer.
- */
-static void handle_urg_ptr(struct sock *sk, u32 urg_seq)
-{
-	struct tcp_sock *tp = tcp_sk(sk);
-
-	urg_seq--;
-	if (tp->urg_data && !after(urg_seq, tp->urg_seq))
-		return;	/* duplicate pointer */
-
-	sk_send_sigurg(sk);
-	if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
-	    !sock_flag(sk, SOCK_URGINLINE) &&
-	    tp->copied_seq != tp->rcv_nxt) {
-		struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
-
-		tp->copied_seq++;
-		if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len)
-			chtls_free_skb(sk, skb);
-	}
-
-	tp->urg_data = TCP_URG_NOTYET;
-	tp->urg_seq = urg_seq;
-}
-
-static void check_sk_callbacks(struct chtls_sock *csk)
-{
-	struct sock *sk = csk->sk;
-
-	if (unlikely(sk->sk_user_data &&
-		     !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD)))
-		csk_set_flag(csk, CSK_CALLBACKS_CHKD);
-}
-
-/*
- * Handles Rx data that arrives in a state where the socket isn't accepting
- * new data.
- */
-static void handle_excess_rx(struct sock *sk, struct sk_buff *skb)
-{
-	if (!csk_flag(sk, CSK_ABORT_SHUTDOWN))
-		chtls_abort_conn(sk, skb);
-
-	kfree_skb(skb);
-}
-
-static void chtls_recv_data(struct sock *sk, struct sk_buff *skb)
-{
-	struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR;
-	struct chtls_sock *csk;
-	struct tcp_sock *tp;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	tp = tcp_sk(sk);
-
-	if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
-		handle_excess_rx(sk, skb);
-		return;
-	}
-
-	ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
-	ULP_SKB_CB(skb)->psh = hdr->psh;
-	skb_ulp_mode(skb) = ULP_MODE_NONE;
-
-	skb_reset_transport_header(skb);
-	__skb_pull(skb, sizeof(*hdr) + RSS_HDR);
-	if (!skb->data_len)
-		__skb_trim(skb, ntohs(hdr->len));
-
-	if (unlikely(hdr->urg))
-		handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg));
-	if (unlikely(tp->urg_data == TCP_URG_NOTYET &&
-		     tp->urg_seq - tp->rcv_nxt < skb->len))
-		tp->urg_data = TCP_URG_VALID |
-			       skb->data[tp->urg_seq - tp->rcv_nxt];
-
-	if (unlikely(hdr->dack_mode != csk->delack_mode)) {
-		csk->delack_mode = hdr->dack_mode;
-		csk->delack_seq = tp->rcv_nxt;
-	}
-
-	tcp_hdr(skb)->fin = 0;
-	tp->rcv_nxt += skb->len;
-
-	__skb_queue_tail(&sk->sk_receive_queue, skb);
-
-	if (!sock_flag(sk, SOCK_DEAD)) {
-		check_sk_callbacks(csk);
-		sk->sk_data_ready(sk);
-	}
-}
-
-static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb)
-{
-	struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR;
-	unsigned int hwtid = GET_TID(req);
-	struct sock *sk;
-
-	sk = lookup_tid(cdev->tids, hwtid);
-	if (unlikely(!sk)) {
-		pr_err("can't find conn. for hwtid %u.\n", hwtid);
-		return -EINVAL;
-	}
-	skb_dst_set(skb, NULL);
-	process_cpl_msg(chtls_recv_data, sk, skb);
-	return 0;
-}
-
-static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb)
-{
-	struct cpl_tls_data *hdr = cplhdr(skb);
-	struct chtls_sock *csk;
-	struct chtls_hws *tlsk;
-	struct tcp_sock *tp;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	tlsk = &csk->tlshws;
-	tp = tcp_sk(sk);
-
-	if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
-		handle_excess_rx(sk, skb);
-		return;
-	}
-
-	ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
-	ULP_SKB_CB(skb)->flags = 0;
-	skb_ulp_mode(skb) = ULP_MODE_TLS;
-
-	skb_reset_transport_header(skb);
-	__skb_pull(skb, sizeof(*hdr));
-	if (!skb->data_len)
-		__skb_trim(skb,
-			   CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)));
-
-	if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq -
-		     tp->rcv_nxt < skb->len))
-		tp->urg_data = TCP_URG_VALID |
-			       skb->data[tp->urg_seq - tp->rcv_nxt];
-
-	tcp_hdr(skb)->fin = 0;
-	tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd));
-	__skb_queue_tail(&tlsk->sk_recv_queue, skb);
-}
-
-static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb)
-{
-	struct cpl_tls_data *req = cplhdr(skb);
-	unsigned int hwtid = GET_TID(req);
-	struct sock *sk;
-
-	sk = lookup_tid(cdev->tids, hwtid);
-	if (unlikely(!sk)) {
-		pr_err("can't find conn. for hwtid %u.\n", hwtid);
-		return -EINVAL;
-	}
-	skb_dst_set(skb, NULL);
-	process_cpl_msg(chtls_recv_pdu, sk, skb);
-	return 0;
-}
-
-static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen)
-{
-	struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb);
-
-	skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length);
-	tls_cmp_hdr->length = ntohs((__force __be16)nlen);
-}
-
-static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
-{
-	struct tlsrx_cmp_hdr *tls_hdr_pkt;
-	struct cpl_rx_tls_cmp *cmp_cpl;
-	struct sk_buff *skb_rec;
-	struct chtls_sock *csk;
-	struct chtls_hws *tlsk;
-	struct tcp_sock *tp;
-
-	cmp_cpl = cplhdr(skb);
-	csk = rcu_dereference_sk_user_data(sk);
-	tlsk = &csk->tlshws;
-	tp = tcp_sk(sk);
-
-	ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq);
-	ULP_SKB_CB(skb)->flags = 0;
-
-	skb_reset_transport_header(skb);
-	__skb_pull(skb, sizeof(*cmp_cpl));
-	tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data;
-	if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M)
-		tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
-	if (!skb->data_len)
-		__skb_trim(skb, TLS_HEADER_LENGTH);
-
-	tp->rcv_nxt +=
-		CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length));
-
-	ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR;
-	skb_rec = __skb_dequeue(&tlsk->sk_recv_queue);
-	if (!skb_rec) {
-		__skb_queue_tail(&sk->sk_receive_queue, skb);
-	} else {
-		chtls_set_hdrlen(skb, tlsk->pldlen);
-		tlsk->pldlen = 0;
-		__skb_queue_tail(&sk->sk_receive_queue, skb);
-		__skb_queue_tail(&sk->sk_receive_queue, skb_rec);
-	}
-
-	if (!sock_flag(sk, SOCK_DEAD)) {
-		check_sk_callbacks(csk);
-		sk->sk_data_ready(sk);
-	}
-}
-
-static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb)
-{
-	struct cpl_rx_tls_cmp *req = cplhdr(skb);
-	unsigned int hwtid = GET_TID(req);
-	struct sock *sk;
-
-	sk = lookup_tid(cdev->tids, hwtid);
-	if (unlikely(!sk)) {
-		pr_err("can't find conn. for hwtid %u.\n", hwtid);
-		return -EINVAL;
-	}
-	skb_dst_set(skb, NULL);
-	process_cpl_msg(chtls_rx_hdr, sk, skb);
-
-	return 0;
-}
-
-static void chtls_timewait(struct sock *sk)
-{
-	struct tcp_sock *tp = tcp_sk(sk);
-
-	tp->rcv_nxt++;
-	tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
-	tp->srtt_us = 0;
-	tcp_time_wait(sk, TCP_TIME_WAIT, 0);
-}
-
-static void chtls_peer_close(struct sock *sk, struct sk_buff *skb)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
-	if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
-		goto out;
-
-	sk->sk_shutdown |= RCV_SHUTDOWN;
-	sock_set_flag(sk, SOCK_DONE);
-
-	switch (sk->sk_state) {
-	case TCP_SYN_RECV:
-	case TCP_ESTABLISHED:
-		tcp_set_state(sk, TCP_CLOSE_WAIT);
-		break;
-	case TCP_FIN_WAIT1:
-		tcp_set_state(sk, TCP_CLOSING);
-		break;
-	case TCP_FIN_WAIT2:
-		chtls_release_resources(sk);
-		if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
-			chtls_conn_done(sk);
-		else
-			chtls_timewait(sk);
-		break;
-	default:
-		pr_info("cpl_peer_close in bad state %d\n", sk->sk_state);
-	}
-
-	if (!sock_flag(sk, SOCK_DEAD)) {
-		sk->sk_state_change(sk);
-		/* Do not send POLL_HUP for half duplex close. */
-
-		if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
-		    sk->sk_state == TCP_CLOSE)
-			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
-		else
-			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
-	}
-out:
-	kfree_skb(skb);
-}
-
-static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb)
-{
-	struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR;
-	struct chtls_sock *csk;
-	struct tcp_sock *tp;
-
-	csk = rcu_dereference_sk_user_data(sk);
-
-	if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
-		goto out;
-
-	tp = tcp_sk(sk);
-
-	tp->snd_una = ntohl(rpl->snd_nxt) - 1;  /* exclude FIN */
-
-	switch (sk->sk_state) {
-	case TCP_CLOSING:
-		chtls_release_resources(sk);
-		if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
-			chtls_conn_done(sk);
-		else
-			chtls_timewait(sk);
-		break;
-	case TCP_LAST_ACK:
-		chtls_release_resources(sk);
-		chtls_conn_done(sk);
-		break;
-	case TCP_FIN_WAIT1:
-		tcp_set_state(sk, TCP_FIN_WAIT2);
-		sk->sk_shutdown |= SEND_SHUTDOWN;
-
-		if (!sock_flag(sk, SOCK_DEAD))
-			sk->sk_state_change(sk);
-		else if (tcp_sk(sk)->linger2 < 0 &&
-			 !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN))
-			chtls_abort_conn(sk, skb);
-		break;
-	default:
-		pr_info("close_con_rpl in bad state %d\n", sk->sk_state);
-	}
-out:
-	kfree_skb(skb);
-}
-
-static struct sk_buff *get_cpl_skb(struct sk_buff *skb,
-				   size_t len, gfp_t gfp)
-{
-	if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) {
-		WARN_ONCE(skb->len < len, "skb alloc error");
-		__skb_trim(skb, len);
-		skb_get(skb);
-	} else {
-		skb = alloc_skb(len, gfp);
-		if (skb)
-			__skb_put(skb, len);
-	}
-	return skb;
-}
-
-static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
-			     int cmd)
-{
-	struct cpl_abort_rpl *rpl = cplhdr(skb);
-
-	INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid);
-	rpl->cmd = cmd;
-}
-
-static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
-{
-	struct cpl_abort_req_rss *req = cplhdr(skb);
-	struct sk_buff *reply_skb;
-
-	reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
-			      GFP_KERNEL | __GFP_NOFAIL);
-	__skb_put(reply_skb, sizeof(struct cpl_abort_rpl));
-	set_abort_rpl_wr(reply_skb, GET_TID(req),
-			 (req->status & CPL_ABORT_NO_RST));
-	set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1);
-	cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
-	kfree_skb(skb);
-}
-
-/*
- * Add an skb to the deferred skb queue for processing from process context.
- */
-static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev,
-			   defer_handler_t handler)
-{
-	DEFERRED_SKB_CB(skb)->handler = handler;
-	spin_lock_bh(&cdev->deferq.lock);
-	__skb_queue_tail(&cdev->deferq, skb);
-	if (skb_queue_len(&cdev->deferq) == 1)
-		schedule_work(&cdev->deferq_task);
-	spin_unlock_bh(&cdev->deferq.lock);
-}
-
-static void send_abort_rpl(struct sock *sk, struct sk_buff *skb,
-			   struct chtls_dev *cdev, int status, int queue)
-{
-	struct cpl_abort_req_rss *req = cplhdr(skb);
-	struct sk_buff *reply_skb;
-	struct chtls_sock *csk;
-
-	csk = rcu_dereference_sk_user_data(sk);
-
-	reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
-			      GFP_KERNEL);
-
-	if (!reply_skb) {
-		req->status = (queue << 1);
-		t4_defer_reply(skb, cdev, send_defer_abort_rpl);
-		return;
-	}
-
-	set_abort_rpl_wr(reply_skb, GET_TID(req), status);
-	kfree_skb(skb);
-
-	set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
-	if (csk_conn_inline(csk)) {
-		struct l2t_entry *e = csk->l2t_entry;
-
-		if (e && sk->sk_state != TCP_SYN_RECV) {
-			cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
-			return;
-		}
-	}
-	cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
-}
-
-static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb,
-				 struct chtls_dev *cdev,
-				 int status, int queue)
-{
-	struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
-	struct sk_buff *reply_skb;
-	struct chtls_sock *csk;
-	unsigned int tid;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	tid = GET_TID(req);
-
-	reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any());
-	if (!reply_skb) {
-		req->status = (queue << 1) | status;
-		t4_defer_reply(skb, cdev, send_defer_abort_rpl);
-		return;
-	}
-
-	set_abort_rpl_wr(reply_skb, tid, status);
-	kfree_skb(skb);
-	set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
-	if (csk_conn_inline(csk)) {
-		struct l2t_entry *e = csk->l2t_entry;
-
-		if (e && sk->sk_state != TCP_SYN_RECV) {
-			cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
-			return;
-		}
-	}
-	cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
-}
-
-/*
- * This is run from a listener's backlog to abort a child connection in
- * SYN_RCV state (i.e., one on the listener's SYN queue).
- */
-static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb)
-{
-	struct chtls_sock *csk;
-	struct sock *child;
-	int queue;
-
-	child = skb->sk;
-	csk = rcu_dereference_sk_user_data(child);
-	queue = csk->txq_idx;
-
-	skb->sk	= NULL;
-	do_abort_syn_rcv(child, lsk);
-	send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev,
-		       CPL_ABORT_NO_RST, queue);
-}
-
-static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb)
-{
-	const struct request_sock *oreq;
-	struct listen_ctx *listen_ctx;
-	struct chtls_sock *csk;
-	struct chtls_dev *cdev;
-	struct sock *psk;
-	void *ctx;
-
-	csk = sk->sk_user_data;
-	oreq = csk->passive_reap_next;
-	cdev = csk->cdev;
-
-	if (!oreq)
-		return -1;
-
-	ctx = lookup_stid(cdev->tids, oreq->ts_recent);
-	if (!ctx)
-		return -1;
-
-	listen_ctx = (struct listen_ctx *)ctx;
-	psk = listen_ctx->lsk;
-
-	bh_lock_sock(psk);
-	if (!sock_owned_by_user(psk)) {
-		int queue = csk->txq_idx;
-
-		do_abort_syn_rcv(sk, psk);
-		send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue);
-	} else {
-		skb->sk = sk;
-		BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv;
-		__sk_add_backlog(psk, skb);
-	}
-	bh_unlock_sock(psk);
-	return 0;
-}
-
-static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb)
-{
-	const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
-	struct chtls_sock *csk = sk->sk_user_data;
-	int rst_status = CPL_ABORT_NO_RST;
-	int queue = csk->txq_idx;
-
-	if (is_neg_adv(req->status)) {
-		if (sk->sk_state == TCP_SYN_RECV)
-			chtls_set_tcb_tflag(sk, 0, 0);
-
-		kfree_skb(skb);
-		return;
-	}
-
-	csk_reset_flag(csk, CSK_ABORT_REQ_RCVD);
-
-	if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) &&
-	    !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
-		struct tcp_sock *tp = tcp_sk(sk);
-
-		if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
-			WARN_ONCE(1, "send_tx_flowc error");
-		csk_set_flag(csk, CSK_TX_DATA_SENT);
-	}
-
-	csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
-
-	if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
-		sk->sk_err = ETIMEDOUT;
-
-		if (!sock_flag(sk, SOCK_DEAD))
-			sk->sk_error_report(sk);
-
-		if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb))
-			return;
-
-		chtls_release_resources(sk);
-		chtls_conn_done(sk);
-	}
-
-	chtls_send_abort_rpl(sk, skb, BLOG_SKB_CB(skb)->cdev,
-			     rst_status, queue);
-}
-
-static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb)
-{
-	struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR;
-	struct chtls_sock *csk;
-	struct chtls_dev *cdev;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	cdev = csk->cdev;
-
-	if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
-		csk_reset_flag(csk, CSK_ABORT_RPL_PENDING);
-		if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) {
-			if (sk->sk_state == TCP_SYN_SENT) {
-				cxgb4_remove_tid(cdev->tids,
-						 csk->port_id,
-						 GET_TID(rpl),
-						 sk->sk_family);
-				sock_put(sk);
-			}
-			chtls_release_resources(sk);
-			chtls_conn_done(sk);
-		}
-	}
-	kfree_skb(skb);
-}
-
-static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb)
-{
-	struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR;
-	void (*fn)(struct sock *sk, struct sk_buff *skb);
-	unsigned int hwtid = GET_TID(req);
-	struct chtls_sock *csk;
-	struct sock *sk;
-	u8 opcode;
-
-	opcode = ((const struct rss_header *)cplhdr(skb))->opcode;
-
-	sk = lookup_tid(cdev->tids, hwtid);
-	if (!sk)
-		goto rel_skb;
-
-	csk = sk->sk_user_data;
-
-	switch (opcode) {
-	case CPL_PEER_CLOSE:
-		fn = chtls_peer_close;
-		break;
-	case CPL_CLOSE_CON_RPL:
-		fn = chtls_close_con_rpl;
-		break;
-	case CPL_ABORT_REQ_RSS:
-		/*
-		 * Save the offload device in the skb, we may process this
-		 * message after the socket has closed.
-		 */
-		BLOG_SKB_CB(skb)->cdev = csk->cdev;
-		fn = chtls_abort_req_rss;
-		break;
-	case CPL_ABORT_RPL_RSS:
-		fn = chtls_abort_rpl_rss;
-		break;
-	default:
-		goto rel_skb;
-	}
-
-	process_cpl_msg(fn, sk, skb);
-	return 0;
-
-rel_skb:
-	kfree_skb(skb);
-	return 0;
-}
-
-static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb)
-{
-	struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR;
-	struct chtls_sock *csk = sk->sk_user_data;
-	struct tcp_sock *tp = tcp_sk(sk);
-	u32 credits = hdr->credits;
-	u32 snd_una;
-
-	snd_una = ntohl(hdr->snd_una);
-	csk->wr_credits += credits;
-
-	if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits)
-		csk->wr_unacked = csk->wr_max_credits - csk->wr_credits;
-
-	while (credits) {
-		struct sk_buff *pskb = csk->wr_skb_head;
-		u32 csum;
-
-		if (unlikely(!pskb)) {
-			if (csk->wr_nondata)
-				csk->wr_nondata -= credits;
-			break;
-		}
-		csum = (__force u32)pskb->csum;
-		if (unlikely(credits < csum)) {
-			pskb->csum = (__force __wsum)(csum - credits);
-			break;
-		}
-		dequeue_wr(sk);
-		credits -= csum;
-		kfree_skb(pskb);
-	}
-	if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {
-		if (unlikely(before(snd_una, tp->snd_una))) {
-			kfree_skb(skb);
-			return;
-		}
-
-		if (tp->snd_una != snd_una) {
-			tp->snd_una = snd_una;
-			tp->rcv_tstamp = tcp_time_stamp(tp);
-			if (tp->snd_una == tp->snd_nxt &&
-			    !csk_flag_nochk(csk, CSK_TX_FAILOVER))
-				csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
-		}
-	}
-
-	if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) {
-		unsigned int fclen16 = roundup(failover_flowc_wr_len, 16);
-
-		csk->wr_credits -= fclen16;
-		csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
-		csk_reset_flag(csk, CSK_TX_FAILOVER);
-	}
-	if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0))
-		sk->sk_write_space(sk);
-
-	kfree_skb(skb);
-}
-
-static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb)
-{
-	struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR;
-	unsigned int hwtid = GET_TID(rpl);
-	struct sock *sk;
-
-	sk = lookup_tid(cdev->tids, hwtid);
-	if (unlikely(!sk)) {
-		pr_err("can't find conn. for hwtid %u.\n", hwtid);
-		return -EINVAL;
-	}
-	process_cpl_msg(chtls_rx_ack, sk, skb);
-
-	return 0;
-}
-
-chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = {
-	[CPL_PASS_OPEN_RPL]     = chtls_pass_open_rpl,
-	[CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl,
-	[CPL_PASS_ACCEPT_REQ]   = chtls_pass_accept_req,
-	[CPL_PASS_ESTABLISH]    = chtls_pass_establish,
-	[CPL_RX_DATA]           = chtls_rx_data,
-	[CPL_TLS_DATA]          = chtls_rx_pdu,
-	[CPL_RX_TLS_CMP]        = chtls_rx_cmp,
-	[CPL_PEER_CLOSE]        = chtls_conn_cpl,
-	[CPL_CLOSE_CON_RPL]     = chtls_conn_cpl,
-	[CPL_ABORT_REQ_RSS]     = chtls_conn_cpl,
-	[CPL_ABORT_RPL_RSS]     = chtls_conn_cpl,
-	[CPL_FW4_ACK]           = chtls_wr_ack,
-};
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.h b/drivers/crypto/chelsio/chtls/chtls_cm.h
deleted file mode 100644
index 47ba81e42f5d..000000000000
--- a/drivers/crypto/chelsio/chtls/chtls_cm.h
+++ /dev/null
@@ -1,222 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2018 Chelsio Communications, Inc.
- */
-
-#ifndef __CHTLS_CM_H__
-#define __CHTLS_CM_H__
-
-/*
- * TCB settings
- */
-/* 3:0 */
-#define TCB_ULP_TYPE_W    0
-#define TCB_ULP_TYPE_S    0
-#define TCB_ULP_TYPE_M    0xfULL
-#define TCB_ULP_TYPE_V(x) ((x) << TCB_ULP_TYPE_S)
-
-/* 11:4 */
-#define TCB_ULP_RAW_W    0
-#define TCB_ULP_RAW_S    4
-#define TCB_ULP_RAW_M    0xffULL
-#define TCB_ULP_RAW_V(x) ((x) << TCB_ULP_RAW_S)
-
-#define TF_TLS_KEY_SIZE_S    7
-#define TF_TLS_KEY_SIZE_V(x) ((x) << TF_TLS_KEY_SIZE_S)
-
-#define TF_TLS_CONTROL_S     2
-#define TF_TLS_CONTROL_V(x) ((x) << TF_TLS_CONTROL_S)
-
-#define TF_TLS_ACTIVE_S      1
-#define TF_TLS_ACTIVE_V(x) ((x) << TF_TLS_ACTIVE_S)
-
-#define TF_TLS_ENABLE_S      0
-#define TF_TLS_ENABLE_V(x) ((x) << TF_TLS_ENABLE_S)
-
-#define TF_RX_QUIESCE_S    15
-#define TF_RX_QUIESCE_V(x) ((x) << TF_RX_QUIESCE_S)
-
-/*
- * Max receive window supported by HW in bytes.  Only a small part of it can
- * be set through option0, the rest needs to be set through RX_DATA_ACK.
- */
-#define MAX_RCV_WND ((1U << 27) - 1)
-#define MAX_MSS     65536
-
-/*
- * Min receive window.  We want it to be large enough to accommodate receive
- * coalescing, handle jumbo frames, and not trigger sender SWS avoidance.
- */
-#define MIN_RCV_WND (24 * 1024U)
-#define LOOPBACK(x)     (((x) & htonl(0xff000000)) == htonl(0x7f000000))
-
-/* ulp_mem_io + ulptx_idata + payload + padding */
-#define MAX_IMM_ULPTX_WR_LEN (32 + 8 + 256 + 8)
-
-/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */
-#define TX_HEADER_LEN \
-	(sizeof(struct fw_ofld_tx_data_wr) + sizeof(struct sge_opaque_hdr))
-#define TX_TLSHDR_LEN \
-	(sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo) + \
-	 sizeof(struct sge_opaque_hdr))
-#define TXDATA_SKB_LEN 128
-
-enum {
-	CPL_TX_TLS_SFO_TYPE_CCS,
-	CPL_TX_TLS_SFO_TYPE_ALERT,
-	CPL_TX_TLS_SFO_TYPE_HANDSHAKE,
-	CPL_TX_TLS_SFO_TYPE_DATA,
-	CPL_TX_TLS_SFO_TYPE_HEARTBEAT,
-};
-
-enum {
-	TLS_HDR_TYPE_CCS = 20,
-	TLS_HDR_TYPE_ALERT,
-	TLS_HDR_TYPE_HANDSHAKE,
-	TLS_HDR_TYPE_RECORD,
-	TLS_HDR_TYPE_HEARTBEAT,
-};
-
-typedef void (*defer_handler_t)(struct chtls_dev *dev, struct sk_buff *skb);
-extern struct request_sock_ops chtls_rsk_ops;
-extern struct request_sock_ops chtls_rsk_opsv6;
-
-struct deferred_skb_cb {
-	defer_handler_t handler;
-	struct chtls_dev *dev;
-};
-
-#define DEFERRED_SKB_CB(skb) ((struct deferred_skb_cb *)(skb)->cb)
-#define failover_flowc_wr_len offsetof(struct fw_flowc_wr, mnemval[3])
-#define WR_SKB_CB(skb) ((struct wr_skb_cb *)(skb)->cb)
-#define ACCEPT_QUEUE(sk) (&inet_csk(sk)->icsk_accept_queue.rskq_accept_head)
-
-#define SND_WSCALE(tp) ((tp)->rx_opt.snd_wscale)
-#define RCV_WSCALE(tp) ((tp)->rx_opt.rcv_wscale)
-#define USER_MSS(tp) ((tp)->rx_opt.user_mss)
-#define TS_RECENT_STAMP(tp) ((tp)->rx_opt.ts_recent_stamp)
-#define WSCALE_OK(tp) ((tp)->rx_opt.wscale_ok)
-#define TSTAMP_OK(tp) ((tp)->rx_opt.tstamp_ok)
-#define SACK_OK(tp) ((tp)->rx_opt.sack_ok)
-#define INC_ORPHAN_COUNT(sk) percpu_counter_inc((sk)->sk_prot->orphan_count)
-
-/* TLS SKB */
-#define skb_ulp_tls_inline(skb)      (ULP_SKB_CB(skb)->ulp.tls.ofld)
-#define skb_ulp_tls_iv_imm(skb)      (ULP_SKB_CB(skb)->ulp.tls.iv)
-
-void chtls_defer_reply(struct sk_buff *skb, struct chtls_dev *dev,
-		       defer_handler_t handler);
-
-/*
- * Returns true if the socket is in one of the supplied states.
- */
-static inline unsigned int sk_in_state(const struct sock *sk,
-				       unsigned int states)
-{
-	return states & (1 << sk->sk_state);
-}
-
-static void chtls_rsk_destructor(struct request_sock *req)
-{
-	/* do nothing */
-}
-
-static inline void chtls_init_rsk_ops(struct proto *chtls_tcp_prot,
-				      struct request_sock_ops *chtls_tcp_ops,
-				      struct proto *tcp_prot, int family)
-{
-	memset(chtls_tcp_ops, 0, sizeof(*chtls_tcp_ops));
-	chtls_tcp_ops->family = family;
-	chtls_tcp_ops->obj_size = sizeof(struct tcp_request_sock);
-	chtls_tcp_ops->destructor = chtls_rsk_destructor;
-	chtls_tcp_ops->slab = tcp_prot->rsk_prot->slab;
-	chtls_tcp_prot->rsk_prot = chtls_tcp_ops;
-}
-
-static inline void chtls_reqsk_free(struct request_sock *req)
-{
-	if (req->rsk_listener)
-		sock_put(req->rsk_listener);
-	kmem_cache_free(req->rsk_ops->slab, req);
-}
-
-#define DECLARE_TASK_FUNC(task, task_param) \
-		static void task(struct work_struct *task_param)
-
-static inline void sk_wakeup_sleepers(struct sock *sk, bool interruptable)
-{
-	struct socket_wq *wq;
-
-	rcu_read_lock();
-	wq = rcu_dereference(sk->sk_wq);
-	if (skwq_has_sleeper(wq)) {
-		if (interruptable)
-			wake_up_interruptible(sk_sleep(sk));
-		else
-			wake_up_all(sk_sleep(sk));
-	}
-	rcu_read_unlock();
-}
-
-static inline void chtls_set_req_port(struct request_sock *oreq,
-				      __be16 source, __be16 dest)
-{
-	inet_rsk(oreq)->ir_rmt_port = source;
-	inet_rsk(oreq)->ir_num = ntohs(dest);
-}
-
-static inline void chtls_set_req_addr(struct request_sock *oreq,
-				      __be32 local_ip, __be32 peer_ip)
-{
-	inet_rsk(oreq)->ir_loc_addr = local_ip;
-	inet_rsk(oreq)->ir_rmt_addr = peer_ip;
-}
-
-static inline void chtls_free_skb(struct sock *sk, struct sk_buff *skb)
-{
-	skb_dst_set(skb, NULL);
-	__skb_unlink(skb, &sk->sk_receive_queue);
-	__kfree_skb(skb);
-}
-
-static inline void chtls_kfree_skb(struct sock *sk, struct sk_buff *skb)
-{
-	skb_dst_set(skb, NULL);
-	__skb_unlink(skb, &sk->sk_receive_queue);
-	kfree_skb(skb);
-}
-
-static inline void chtls_reset_wr_list(struct chtls_sock *csk)
-{
-	csk->wr_skb_head = NULL;
-	csk->wr_skb_tail = NULL;
-}
-
-static inline void enqueue_wr(struct chtls_sock *csk, struct sk_buff *skb)
-{
-	WR_SKB_CB(skb)->next_wr = NULL;
-
-	skb_get(skb);
-
-	if (!csk->wr_skb_head)
-		csk->wr_skb_head = skb;
-	else
-		WR_SKB_CB(csk->wr_skb_tail)->next_wr = skb;
-	csk->wr_skb_tail = skb;
-}
-
-static inline struct sk_buff *dequeue_wr(struct sock *sk)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct sk_buff *skb = NULL;
-
-	skb = csk->wr_skb_head;
-
-	if (likely(skb)) {
-	 /* Don't bother clearing the tail */
-		csk->wr_skb_head = WR_SKB_CB(skb)->next_wr;
-		WR_SKB_CB(skb)->next_wr = NULL;
-	}
-	return skb;
-}
-#endif
diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c b/drivers/crypto/chelsio/chtls/chtls_hw.c
deleted file mode 100644
index f1820aca0d33..000000000000
--- a/drivers/crypto/chelsio/chtls/chtls_hw.c
+++ /dev/null
@@ -1,426 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018 Chelsio Communications, Inc.
- *
- * Written by: Atul Gupta (atul.gupta@chelsio.com)
- */
-
-#include <linux/module.h>
-#include <linux/list.h>
-#include <linux/workqueue.h>
-#include <linux/skbuff.h>
-#include <linux/timer.h>
-#include <linux/notifier.h>
-#include <linux/inetdevice.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/tls.h>
-#include <net/tls.h>
-
-#include "chtls.h"
-#include "chtls_cm.h"
-
-static void __set_tcb_field_direct(struct chtls_sock *csk,
-				   struct cpl_set_tcb_field *req, u16 word,
-				   u64 mask, u64 val, u8 cookie, int no_reply)
-{
-	struct ulptx_idata *sc;
-
-	INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid);
-	req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid));
-	req->reply_ctrl = htons(NO_REPLY_V(no_reply) |
-				QUEUENO_V(csk->rss_qid));
-	req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie));
-	req->mask = cpu_to_be64(mask);
-	req->val = cpu_to_be64(val);
-	sc = (struct ulptx_idata *)(req + 1);
-	sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
-	sc->len = htonl(0);
-}
-
-static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word,
-			    u64 mask, u64 val, u8 cookie, int no_reply)
-{
-	struct cpl_set_tcb_field *req;
-	struct chtls_sock *csk;
-	struct ulptx_idata *sc;
-	unsigned int wrlen;
-
-	wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
-	csk = rcu_dereference_sk_user_data(sk);
-
-	req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen);
-	__set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply);
-	set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
-}
-
-/*
- * Send control message to HW, message go as immediate data and packet
- * is freed immediately.
- */
-static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val)
-{
-	struct cpl_set_tcb_field *req;
-	unsigned int credits_needed;
-	struct chtls_sock *csk;
-	struct ulptx_idata *sc;
-	struct sk_buff *skb;
-	unsigned int wrlen;
-	int ret;
-
-	wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
-
-	skb = alloc_skb(wrlen, GFP_ATOMIC);
-	if (!skb)
-		return -ENOMEM;
-
-	credits_needed = DIV_ROUND_UP(wrlen, 16);
-	csk = rcu_dereference_sk_user_data(sk);
-
-	__set_tcb_field(sk, skb, word, mask, val, 0, 1);
-	skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
-	csk->wr_credits -= credits_needed;
-	csk->wr_unacked += credits_needed;
-	enqueue_wr(csk, skb);
-	ret = cxgb4_ofld_send(csk->egress_dev, skb);
-	if (ret < 0)
-		kfree_skb(skb);
-	return ret < 0 ? ret : 0;
-}
-
-/*
- * Set one of the t_flags bits in the TCB.
- */
-int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val)
-{
-	return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos,
-				   (u64)val << bit_pos);
-}
-
-static int chtls_set_tcb_keyid(struct sock *sk, int keyid)
-{
-	return chtls_set_tcb_field(sk, 31, 0xFFFFFFFFULL, keyid);
-}
-
-static int chtls_set_tcb_seqno(struct sock *sk)
-{
-	return chtls_set_tcb_field(sk, 28, ~0ULL, 0);
-}
-
-static int chtls_set_tcb_quiesce(struct sock *sk, int val)
-{
-	return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S),
-				   TF_RX_QUIESCE_V(val));
-}
-
-/* TLS Key bitmap processing */
-int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi)
-{
-	unsigned int num_key_ctx, bsize;
-	int ksize;
-
-	num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ);
-	bsize = BITS_TO_LONGS(num_key_ctx);
-
-	cdev->kmap.size = num_key_ctx;
-	cdev->kmap.available = bsize;
-	ksize = sizeof(*cdev->kmap.addr) * bsize;
-	cdev->kmap.addr = kvzalloc(ksize, GFP_KERNEL);
-	if (!cdev->kmap.addr)
-		return -ENOMEM;
-
-	cdev->kmap.start = lldi->vr->key.start;
-	spin_lock_init(&cdev->kmap.lock);
-	return 0;
-}
-
-static int get_new_keyid(struct chtls_sock *csk, u32 optname)
-{
-	struct net_device *dev = csk->egress_dev;
-	struct chtls_dev *cdev = csk->cdev;
-	struct chtls_hws *hws;
-	struct adapter *adap;
-	int keyid;
-
-	adap = netdev2adap(dev);
-	hws = &csk->tlshws;
-
-	spin_lock_bh(&cdev->kmap.lock);
-	keyid = find_first_zero_bit(cdev->kmap.addr, cdev->kmap.size);
-	if (keyid < cdev->kmap.size) {
-		__set_bit(keyid, cdev->kmap.addr);
-		if (optname == TLS_RX)
-			hws->rxkey = keyid;
-		else
-			hws->txkey = keyid;
-		atomic_inc(&adap->chcr_stats.tls_key);
-	} else {
-		keyid = -1;
-	}
-	spin_unlock_bh(&cdev->kmap.lock);
-	return keyid;
-}
-
-void free_tls_keyid(struct sock *sk)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct net_device *dev = csk->egress_dev;
-	struct chtls_dev *cdev = csk->cdev;
-	struct chtls_hws *hws;
-	struct adapter *adap;
-
-	if (!cdev->kmap.addr)
-		return;
-
-	adap = netdev2adap(dev);
-	hws = &csk->tlshws;
-
-	spin_lock_bh(&cdev->kmap.lock);
-	if (hws->rxkey >= 0) {
-		__clear_bit(hws->rxkey, cdev->kmap.addr);
-		atomic_dec(&adap->chcr_stats.tls_key);
-		hws->rxkey = -1;
-	}
-	if (hws->txkey >= 0) {
-		__clear_bit(hws->txkey, cdev->kmap.addr);
-		atomic_dec(&adap->chcr_stats.tls_key);
-		hws->txkey = -1;
-	}
-	spin_unlock_bh(&cdev->kmap.lock);
-}
-
-unsigned int keyid_to_addr(int start_addr, int keyid)
-{
-	return (start_addr + (keyid * TLS_KEY_CONTEXT_SZ)) >> 5;
-}
-
-static void chtls_rxkey_ivauth(struct _key_ctx *kctx)
-{
-	kctx->iv_to_auth = cpu_to_be64(KEYCTX_TX_WR_IV_V(6ULL) |
-				  KEYCTX_TX_WR_AAD_V(1ULL) |
-				  KEYCTX_TX_WR_AADST_V(5ULL) |
-				  KEYCTX_TX_WR_CIPHER_V(14ULL) |
-				  KEYCTX_TX_WR_CIPHERST_V(0ULL) |
-				  KEYCTX_TX_WR_AUTH_V(14ULL) |
-				  KEYCTX_TX_WR_AUTHST_V(16ULL) |
-				  KEYCTX_TX_WR_AUTHIN_V(16ULL));
-}
-
-static int chtls_key_info(struct chtls_sock *csk,
-			  struct _key_ctx *kctx,
-			  u32 keylen, u32 optname,
-			  int cipher_type)
-{
-	unsigned char key[AES_MAX_KEY_SIZE];
-	unsigned char *key_p, *salt;
-	unsigned char ghash_h[AEAD_H_SIZE];
-	int ck_size, key_ctx_size, kctx_mackey_size, salt_size;
-	struct crypto_aes_ctx aes;
-	int ret;
-
-	key_ctx_size = sizeof(struct _key_ctx) +
-		       roundup(keylen, 16) + AEAD_H_SIZE;
-
-	/* GCM mode of AES supports 128 and 256 bit encryption, so
-	 * prepare key context base on GCM cipher type
-	 */
-	switch (cipher_type) {
-	case TLS_CIPHER_AES_GCM_128: {
-		struct tls12_crypto_info_aes_gcm_128 *gcm_ctx_128 =
-			(struct tls12_crypto_info_aes_gcm_128 *)
-					&csk->tlshws.crypto_info;
-		memcpy(key, gcm_ctx_128->key, keylen);
-
-		key_p            = gcm_ctx_128->key;
-		salt             = gcm_ctx_128->salt;
-		ck_size          = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
-		salt_size        = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
-		kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
-		break;
-	}
-	case TLS_CIPHER_AES_GCM_256: {
-		struct tls12_crypto_info_aes_gcm_256 *gcm_ctx_256 =
-			(struct tls12_crypto_info_aes_gcm_256 *)
-					&csk->tlshws.crypto_info;
-		memcpy(key, gcm_ctx_256->key, keylen);
-
-		key_p            = gcm_ctx_256->key;
-		salt             = gcm_ctx_256->salt;
-		ck_size          = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
-		salt_size        = TLS_CIPHER_AES_GCM_256_SALT_SIZE;
-		kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
-		break;
-	}
-	default:
-		pr_err("GCM: Invalid key length %d\n", keylen);
-		return -EINVAL;
-	}
-
-	/* Calculate the H = CIPH(K, 0 repeated 16 times).
-	 * It will go in key context
-	 */
-	ret = aes_expandkey(&aes, key, keylen);
-	if (ret)
-		return ret;
-
-	memset(ghash_h, 0, AEAD_H_SIZE);
-	aes_encrypt(&aes, ghash_h, ghash_h);
-	memzero_explicit(&aes, sizeof(aes));
-	csk->tlshws.keylen = key_ctx_size;
-
-	/* Copy the Key context */
-	if (optname == TLS_RX) {
-		int key_ctx;
-
-		key_ctx = ((key_ctx_size >> 4) << 3);
-		kctx->ctx_hdr = FILL_KEY_CRX_HDR(ck_size,
-						 kctx_mackey_size,
-						 0, 0, key_ctx);
-		chtls_rxkey_ivauth(kctx);
-	} else {
-		kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
-						 kctx_mackey_size,
-						 0, 0, key_ctx_size >> 4);
-	}
-
-	memcpy(kctx->salt, salt, salt_size);
-	memcpy(kctx->key, key_p, keylen);
-	memcpy(kctx->key + keylen, ghash_h, AEAD_H_SIZE);
-	/* erase key info from driver */
-	memset(key_p, 0, keylen);
-
-	return 0;
-}
-
-static void chtls_set_scmd(struct chtls_sock *csk)
-{
-	struct chtls_hws *hws = &csk->tlshws;
-
-	hws->scmd.seqno_numivs =
-		SCMD_SEQ_NO_CTRL_V(3) |
-		SCMD_PROTO_VERSION_V(0) |
-		SCMD_ENC_DEC_CTRL_V(0) |
-		SCMD_CIPH_AUTH_SEQ_CTRL_V(1) |
-		SCMD_CIPH_MODE_V(2) |
-		SCMD_AUTH_MODE_V(4) |
-		SCMD_HMAC_CTRL_V(0) |
-		SCMD_IV_SIZE_V(4) |
-		SCMD_NUM_IVS_V(1);
-
-	hws->scmd.ivgen_hdrlen =
-		SCMD_IV_GEN_CTRL_V(1) |
-		SCMD_KEY_CTX_INLINE_V(0) |
-		SCMD_TLS_FRAG_ENABLE_V(1);
-}
-
-int chtls_setkey(struct chtls_sock *csk, u32 keylen,
-		 u32 optname, int cipher_type)
-{
-	struct tls_key_req *kwr;
-	struct chtls_dev *cdev;
-	struct _key_ctx *kctx;
-	int wrlen, klen, len;
-	struct sk_buff *skb;
-	struct sock *sk;
-	int keyid;
-	int kaddr;
-	int ret;
-
-	cdev = csk->cdev;
-	sk = csk->sk;
-
-	klen = roundup((keylen + AEAD_H_SIZE) + sizeof(*kctx), 32);
-	wrlen = roundup(sizeof(*kwr), 16);
-	len = klen + wrlen;
-
-	/* Flush out-standing data before new key takes effect */
-	if (optname == TLS_TX) {
-		lock_sock(sk);
-		if (skb_queue_len(&csk->txq))
-			chtls_push_frames(csk, 0);
-		release_sock(sk);
-	}
-
-	skb = alloc_skb(len, GFP_KERNEL);
-	if (!skb)
-		return -ENOMEM;
-
-	keyid = get_new_keyid(csk, optname);
-	if (keyid < 0) {
-		ret = -ENOSPC;
-		goto out_nokey;
-	}
-
-	kaddr = keyid_to_addr(cdev->kmap.start, keyid);
-	kwr = (struct tls_key_req *)__skb_put_zero(skb, len);
-	kwr->wr.op_to_compl =
-		cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) | FW_WR_COMPL_F |
-		      FW_WR_ATOMIC_V(1U));
-	kwr->wr.flowid_len16 =
-		cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16) |
-			    FW_WR_FLOWID_V(csk->tid)));
-	kwr->wr.protocol = 0;
-	kwr->wr.mfs = htons(TLS_MFS);
-	kwr->wr.reneg_to_write_rx = optname;
-
-	/* ulptx command */
-	kwr->req.cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
-			    T5_ULP_MEMIO_ORDER_V(1) |
-			    T5_ULP_MEMIO_IMM_V(1));
-	kwr->req.len16 = cpu_to_be32((csk->tid << 8) |
-			      DIV_ROUND_UP(len - sizeof(kwr->wr), 16));
-	kwr->req.dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V(klen >> 5));
-	kwr->req.lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(kaddr));
-
-	/* sub command */
-	kwr->sc_imm.cmd_more = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM));
-	kwr->sc_imm.len = cpu_to_be32(klen);
-
-	lock_sock(sk);
-	/* key info */
-	kctx = (struct _key_ctx *)(kwr + 1);
-	ret = chtls_key_info(csk, kctx, keylen, optname, cipher_type);
-	if (ret)
-		goto out_notcb;
-
-	set_wr_txq(skb, CPL_PRIORITY_DATA, csk->tlshws.txqid);
-	csk->wr_credits -= DIV_ROUND_UP(len, 16);
-	csk->wr_unacked += DIV_ROUND_UP(len, 16);
-	enqueue_wr(csk, skb);
-	cxgb4_ofld_send(csk->egress_dev, skb);
-
-	chtls_set_scmd(csk);
-	/* Clear quiesce for Rx key */
-	if (optname == TLS_RX) {
-		ret = chtls_set_tcb_keyid(sk, keyid);
-		if (ret)
-			goto out_notcb;
-		ret = chtls_set_tcb_field(sk, 0,
-					  TCB_ULP_RAW_V(TCB_ULP_RAW_M),
-					  TCB_ULP_RAW_V((TF_TLS_KEY_SIZE_V(1) |
-					  TF_TLS_CONTROL_V(1) |
-					  TF_TLS_ACTIVE_V(1) |
-					  TF_TLS_ENABLE_V(1))));
-		if (ret)
-			goto out_notcb;
-		ret = chtls_set_tcb_seqno(sk);
-		if (ret)
-			goto out_notcb;
-		ret = chtls_set_tcb_quiesce(sk, 0);
-		if (ret)
-			goto out_notcb;
-		csk->tlshws.rxkey = keyid;
-	} else {
-		csk->tlshws.tx_seq_no = 0;
-		csk->tlshws.txkey = keyid;
-	}
-
-	release_sock(sk);
-	return ret;
-out_notcb:
-	release_sock(sk);
-	free_tls_keyid(sk);
-out_nokey:
-	kfree_skb(skb);
-	return ret;
-}
diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c b/drivers/crypto/chelsio/chtls/chtls_io.c
deleted file mode 100644
index 2e9acae1cba3..000000000000
--- a/drivers/crypto/chelsio/chtls/chtls_io.c
+++ /dev/null
@@ -1,1907 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018 Chelsio Communications, Inc.
- *
- * Written by: Atul Gupta (atul.gupta@chelsio.com)
- */
-
-#include <linux/module.h>
-#include <linux/list.h>
-#include <linux/workqueue.h>
-#include <linux/skbuff.h>
-#include <linux/timer.h>
-#include <linux/notifier.h>
-#include <linux/inetdevice.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/sched/signal.h>
-#include <net/tcp.h>
-#include <net/busy_poll.h>
-#include <crypto/aes.h>
-
-#include "chtls.h"
-#include "chtls_cm.h"
-
-static bool is_tls_tx(struct chtls_sock *csk)
-{
-	return csk->tlshws.txkey >= 0;
-}
-
-static bool is_tls_rx(struct chtls_sock *csk)
-{
-	return csk->tlshws.rxkey >= 0;
-}
-
-static int data_sgl_len(const struct sk_buff *skb)
-{
-	unsigned int cnt;
-
-	cnt = skb_shinfo(skb)->nr_frags;
-	return sgl_len(cnt) * 8;
-}
-
-static int nos_ivs(struct sock *sk, unsigned int size)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
-	return DIV_ROUND_UP(size, csk->tlshws.mfs);
-}
-
-static int set_ivs_imm(struct sock *sk, const struct sk_buff *skb)
-{
-	int ivs_size = nos_ivs(sk, skb->len) * CIPHER_BLOCK_SIZE;
-	int hlen = TLS_WR_CPL_LEN + data_sgl_len(skb);
-
-	if ((hlen + KEY_ON_MEM_SZ + ivs_size) <
-	    MAX_IMM_OFLD_TX_DATA_WR_LEN) {
-		ULP_SKB_CB(skb)->ulp.tls.iv = 1;
-		return 1;
-	}
-	ULP_SKB_CB(skb)->ulp.tls.iv = 0;
-	return 0;
-}
-
-static int max_ivs_size(struct sock *sk, int size)
-{
-	return nos_ivs(sk, size) * CIPHER_BLOCK_SIZE;
-}
-
-static int ivs_size(struct sock *sk, const struct sk_buff *skb)
-{
-	return set_ivs_imm(sk, skb) ? (nos_ivs(sk, skb->len) *
-		 CIPHER_BLOCK_SIZE) : 0;
-}
-
-static int flowc_wr_credits(int nparams, int *flowclenp)
-{
-	int flowclen16, flowclen;
-
-	flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
-	flowclen16 = DIV_ROUND_UP(flowclen, 16);
-	flowclen = flowclen16 * 16;
-
-	if (flowclenp)
-		*flowclenp = flowclen;
-
-	return flowclen16;
-}
-
-static struct sk_buff *create_flowc_wr_skb(struct sock *sk,
-					   struct fw_flowc_wr *flowc,
-					   int flowclen)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct sk_buff *skb;
-
-	skb = alloc_skb(flowclen, GFP_ATOMIC);
-	if (!skb)
-		return NULL;
-
-	__skb_put_data(skb, flowc, flowclen);
-	skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
-
-	return skb;
-}
-
-static int send_flowc_wr(struct sock *sk, struct fw_flowc_wr *flowc,
-			 int flowclen)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct tcp_sock *tp = tcp_sk(sk);
-	struct sk_buff *skb;
-	int flowclen16;
-	int ret;
-
-	flowclen16 = flowclen / 16;
-
-	if (csk_flag(sk, CSK_TX_DATA_SENT)) {
-		skb = create_flowc_wr_skb(sk, flowc, flowclen);
-		if (!skb)
-			return -ENOMEM;
-
-		skb_entail(sk, skb,
-			   ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
-		return 0;
-	}
-
-	ret = cxgb4_immdata_send(csk->egress_dev,
-				 csk->txq_idx,
-				 flowc, flowclen);
-	if (!ret)
-		return flowclen16;
-	skb = create_flowc_wr_skb(sk, flowc, flowclen);
-	if (!skb)
-		return -ENOMEM;
-	send_or_defer(sk, tp, skb, 0);
-	return flowclen16;
-}
-
-static u8 tcp_state_to_flowc_state(u8 state)
-{
-	switch (state) {
-	case TCP_ESTABLISHED:
-		return FW_FLOWC_MNEM_TCPSTATE_ESTABLISHED;
-	case TCP_CLOSE_WAIT:
-		return FW_FLOWC_MNEM_TCPSTATE_CLOSEWAIT;
-	case TCP_FIN_WAIT1:
-		return FW_FLOWC_MNEM_TCPSTATE_FINWAIT1;
-	case TCP_CLOSING:
-		return FW_FLOWC_MNEM_TCPSTATE_CLOSING;
-	case TCP_LAST_ACK:
-		return FW_FLOWC_MNEM_TCPSTATE_LASTACK;
-	case TCP_FIN_WAIT2:
-		return FW_FLOWC_MNEM_TCPSTATE_FINWAIT2;
-	}
-
-	return FW_FLOWC_MNEM_TCPSTATE_ESTABLISHED;
-}
-
-int send_tx_flowc_wr(struct sock *sk, int compl,
-		     u32 snd_nxt, u32 rcv_nxt)
-{
-	struct flowc_packed {
-		struct fw_flowc_wr fc;
-		struct fw_flowc_mnemval mnemval[FW_FLOWC_MNEM_MAX];
-	} __packed sflowc;
-	int nparams, paramidx, flowclen16, flowclen;
-	struct fw_flowc_wr *flowc;
-	struct chtls_sock *csk;
-	struct tcp_sock *tp;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	tp = tcp_sk(sk);
-	memset(&sflowc, 0, sizeof(sflowc));
-	flowc = &sflowc.fc;
-
-#define FLOWC_PARAM(__m, __v) \
-	do { \
-		flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \
-		flowc->mnemval[paramidx].val = cpu_to_be32(__v); \
-		paramidx++; \
-	} while (0)
-
-	paramidx = 0;
-
-	FLOWC_PARAM(PFNVFN, FW_PFVF_CMD_PFN_V(csk->cdev->lldi->pf));
-	FLOWC_PARAM(CH, csk->tx_chan);
-	FLOWC_PARAM(PORT, csk->tx_chan);
-	FLOWC_PARAM(IQID, csk->rss_qid);
-	FLOWC_PARAM(SNDNXT, tp->snd_nxt);
-	FLOWC_PARAM(RCVNXT, tp->rcv_nxt);
-	FLOWC_PARAM(SNDBUF, csk->sndbuf);
-	FLOWC_PARAM(MSS, tp->mss_cache);
-	FLOWC_PARAM(TCPSTATE, tcp_state_to_flowc_state(sk->sk_state));
-
-	if (SND_WSCALE(tp))
-		FLOWC_PARAM(RCV_SCALE, SND_WSCALE(tp));
-
-	if (csk->ulp_mode == ULP_MODE_TLS)
-		FLOWC_PARAM(ULD_MODE, ULP_MODE_TLS);
-
-	if (csk->tlshws.fcplenmax)
-		FLOWC_PARAM(TXDATAPLEN_MAX, csk->tlshws.fcplenmax);
-
-	nparams = paramidx;
-#undef FLOWC_PARAM
-
-	flowclen16 = flowc_wr_credits(nparams, &flowclen);
-	flowc->op_to_nparams =
-		cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) |
-			    FW_WR_COMPL_V(compl) |
-			    FW_FLOWC_WR_NPARAMS_V(nparams));
-	flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(flowclen16) |
-					  FW_WR_FLOWID_V(csk->tid));
-
-	return send_flowc_wr(sk, flowc, flowclen);
-}
-
-/* Copy IVs to WR */
-static int tls_copy_ivs(struct sock *sk, struct sk_buff *skb)
-
-{
-	struct chtls_sock *csk;
-	unsigned char *iv_loc;
-	struct chtls_hws *hws;
-	unsigned char *ivs;
-	u16 number_of_ivs;
-	struct page *page;
-	int err = 0;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	hws = &csk->tlshws;
-	number_of_ivs = nos_ivs(sk, skb->len);
-
-	if (number_of_ivs > MAX_IVS_PAGE) {
-		pr_warn("MAX IVs in PAGE exceeded %d\n", number_of_ivs);
-		return -ENOMEM;
-	}
-
-	/* generate the  IVs */
-	ivs = kmalloc_array(CIPHER_BLOCK_SIZE, number_of_ivs, GFP_ATOMIC);
-	if (!ivs)
-		return -ENOMEM;
-	get_random_bytes(ivs, number_of_ivs * CIPHER_BLOCK_SIZE);
-
-	if (skb_ulp_tls_iv_imm(skb)) {
-		/* send the IVs as immediate data in the WR */
-		iv_loc = (unsigned char *)__skb_push(skb, number_of_ivs *
-						CIPHER_BLOCK_SIZE);
-		if (iv_loc)
-			memcpy(iv_loc, ivs, number_of_ivs * CIPHER_BLOCK_SIZE);
-
-		hws->ivsize = number_of_ivs * CIPHER_BLOCK_SIZE;
-	} else {
-		/* Send the IVs as sgls */
-		/* Already accounted IV DSGL for credits */
-		skb_shinfo(skb)->nr_frags--;
-		page = alloc_pages(sk->sk_allocation | __GFP_COMP, 0);
-		if (!page) {
-			pr_info("%s : Page allocation for IVs failed\n",
-				__func__);
-			err = -ENOMEM;
-			goto out;
-		}
-		memcpy(page_address(page), ivs, number_of_ivs *
-		       CIPHER_BLOCK_SIZE);
-		skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, page, 0,
-				   number_of_ivs * CIPHER_BLOCK_SIZE);
-		hws->ivsize = 0;
-	}
-out:
-	kfree(ivs);
-	return err;
-}
-
-/* Copy Key to WR */
-static void tls_copy_tx_key(struct sock *sk, struct sk_buff *skb)
-{
-	struct ulptx_sc_memrd *sc_memrd;
-	struct chtls_sock *csk;
-	struct chtls_dev *cdev;
-	struct ulptx_idata *sc;
-	struct chtls_hws *hws;
-	u32 immdlen;
-	int kaddr;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	hws = &csk->tlshws;
-	cdev = csk->cdev;
-
-	immdlen = sizeof(*sc) + sizeof(*sc_memrd);
-	kaddr = keyid_to_addr(cdev->kmap.start, hws->txkey);
-	sc = (struct ulptx_idata *)__skb_push(skb, immdlen);
-	if (sc) {
-		sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
-		sc->len = htonl(0);
-		sc_memrd = (struct ulptx_sc_memrd *)(sc + 1);
-		sc_memrd->cmd_to_len =
-				htonl(ULPTX_CMD_V(ULP_TX_SC_MEMRD) |
-				ULP_TX_SC_MORE_V(1) |
-				ULPTX_LEN16_V(hws->keylen >> 4));
-		sc_memrd->addr = htonl(kaddr);
-	}
-}
-
-static u64 tlstx_incr_seqnum(struct chtls_hws *hws)
-{
-	return hws->tx_seq_no++;
-}
-
-static bool is_sg_request(const struct sk_buff *skb)
-{
-	return skb->peeked ||
-		(skb->len > MAX_IMM_ULPTX_WR_LEN);
-}
-
-/*
- * Returns true if an sk_buff carries urgent data.
- */
-static bool skb_urgent(struct sk_buff *skb)
-{
-	return ULP_SKB_CB(skb)->flags & ULPCB_FLAG_URG;
-}
-
-/* TLS content type for CPL SFO */
-static unsigned char tls_content_type(unsigned char content_type)
-{
-	switch (content_type) {
-	case TLS_HDR_TYPE_CCS:
-		return CPL_TX_TLS_SFO_TYPE_CCS;
-	case TLS_HDR_TYPE_ALERT:
-		return CPL_TX_TLS_SFO_TYPE_ALERT;
-	case TLS_HDR_TYPE_HANDSHAKE:
-		return CPL_TX_TLS_SFO_TYPE_HANDSHAKE;
-	case TLS_HDR_TYPE_HEARTBEAT:
-		return CPL_TX_TLS_SFO_TYPE_HEARTBEAT;
-	}
-	return CPL_TX_TLS_SFO_TYPE_DATA;
-}
-
-static void tls_tx_data_wr(struct sock *sk, struct sk_buff *skb,
-			   int dlen, int tls_immd, u32 credits,
-			   int expn, int pdus)
-{
-	struct fw_tlstx_data_wr *req_wr;
-	struct cpl_tx_tls_sfo *req_cpl;
-	unsigned int wr_ulp_mode_force;
-	struct tls_scmd *updated_scmd;
-	unsigned char data_type;
-	struct chtls_sock *csk;
-	struct net_device *dev;
-	struct chtls_hws *hws;
-	struct tls_scmd *scmd;
-	struct adapter *adap;
-	unsigned char *req;
-	int immd_len;
-	int iv_imm;
-	int len;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	iv_imm = skb_ulp_tls_iv_imm(skb);
-	dev = csk->egress_dev;
-	adap = netdev2adap(dev);
-	hws = &csk->tlshws;
-	scmd = &hws->scmd;
-	len = dlen + expn;
-
-	dlen = (dlen < hws->mfs) ? dlen : hws->mfs;
-	atomic_inc(&adap->chcr_stats.tls_pdu_tx);
-
-	updated_scmd = scmd;
-	updated_scmd->seqno_numivs &= 0xffffff80;
-	updated_scmd->seqno_numivs |= SCMD_NUM_IVS_V(pdus);
-	hws->scmd = *updated_scmd;
-
-	req = (unsigned char *)__skb_push(skb, sizeof(struct cpl_tx_tls_sfo));
-	req_cpl = (struct cpl_tx_tls_sfo *)req;
-	req = (unsigned char *)__skb_push(skb, (sizeof(struct
-				fw_tlstx_data_wr)));
-
-	req_wr = (struct fw_tlstx_data_wr *)req;
-	immd_len = (tls_immd ? dlen : 0);
-	req_wr->op_to_immdlen =
-		htonl(FW_WR_OP_V(FW_TLSTX_DATA_WR) |
-		FW_TLSTX_DATA_WR_COMPL_V(1) |
-		FW_TLSTX_DATA_WR_IMMDLEN_V(immd_len));
-	req_wr->flowid_len16 = htonl(FW_TLSTX_DATA_WR_FLOWID_V(csk->tid) |
-				     FW_TLSTX_DATA_WR_LEN16_V(credits));
-	wr_ulp_mode_force = TX_ULP_MODE_V(ULP_MODE_TLS);
-
-	if (is_sg_request(skb))
-		wr_ulp_mode_force |= FW_OFLD_TX_DATA_WR_ALIGNPLD_F |
-			((tcp_sk(sk)->nonagle & TCP_NAGLE_OFF) ? 0 :
-			FW_OFLD_TX_DATA_WR_SHOVE_F);
-
-	req_wr->lsodisable_to_flags =
-			htonl(TX_ULP_MODE_V(ULP_MODE_TLS) |
-			      TX_URG_V(skb_urgent(skb)) |
-			      T6_TX_FORCE_F | wr_ulp_mode_force |
-			      TX_SHOVE_V((!csk_flag(sk, CSK_TX_MORE_DATA)) &&
-					 skb_queue_empty(&csk->txq)));
-
-	req_wr->ctxloc_to_exp =
-			htonl(FW_TLSTX_DATA_WR_NUMIVS_V(pdus) |
-			      FW_TLSTX_DATA_WR_EXP_V(expn) |
-			      FW_TLSTX_DATA_WR_CTXLOC_V(CHTLS_KEY_CONTEXT_DDR) |
-			      FW_TLSTX_DATA_WR_IVDSGL_V(!iv_imm) |
-			      FW_TLSTX_DATA_WR_KEYSIZE_V(hws->keylen >> 4));
-
-	/* Fill in the length */
-	req_wr->plen = htonl(len);
-	req_wr->mfs = htons(hws->mfs);
-	req_wr->adjustedplen_pkd =
-		htons(FW_TLSTX_DATA_WR_ADJUSTEDPLEN_V(hws->adjustlen));
-	req_wr->expinplenmax_pkd =
-		htons(FW_TLSTX_DATA_WR_EXPINPLENMAX_V(hws->expansion));
-	req_wr->pdusinplenmax_pkd =
-		FW_TLSTX_DATA_WR_PDUSINPLENMAX_V(hws->pdus);
-	req_wr->r10 = 0;
-
-	data_type = tls_content_type(ULP_SKB_CB(skb)->ulp.tls.type);
-	req_cpl->op_to_seg_len = htonl(CPL_TX_TLS_SFO_OPCODE_V(CPL_TX_TLS_SFO) |
-				       CPL_TX_TLS_SFO_DATA_TYPE_V(data_type) |
-				       CPL_TX_TLS_SFO_CPL_LEN_V(2) |
-				       CPL_TX_TLS_SFO_SEG_LEN_V(dlen));
-	req_cpl->pld_len = htonl(len - expn);
-
-	req_cpl->type_protover = htonl(CPL_TX_TLS_SFO_TYPE_V
-		((data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) ?
-		TLS_HDR_TYPE_HEARTBEAT : 0) |
-		CPL_TX_TLS_SFO_PROTOVER_V(0));
-
-	/* create the s-command */
-	req_cpl->r1_lo = 0;
-	req_cpl->seqno_numivs  = cpu_to_be32(hws->scmd.seqno_numivs);
-	req_cpl->ivgen_hdrlen = cpu_to_be32(hws->scmd.ivgen_hdrlen);
-	req_cpl->scmd1 = cpu_to_be64(tlstx_incr_seqnum(hws));
-}
-
-/*
- * Calculate the TLS data expansion size
- */
-static int chtls_expansion_size(struct sock *sk, int data_len,
-				int fullpdu,
-				unsigned short *pducnt)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct chtls_hws *hws = &csk->tlshws;
-	struct tls_scmd *scmd = &hws->scmd;
-	int fragsize = hws->mfs;
-	int expnsize = 0;
-	int fragleft;
-	int fragcnt;
-	int expppdu;
-
-	if (SCMD_CIPH_MODE_G(scmd->seqno_numivs) ==
-	    SCMD_CIPH_MODE_AES_GCM) {
-		expppdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE +
-			  TLS_HEADER_LENGTH;
-
-		if (fullpdu) {
-			*pducnt = data_len / (expppdu + fragsize);
-			if (*pducnt > 32)
-				*pducnt = 32;
-			else if (!*pducnt)
-				*pducnt = 1;
-			expnsize = (*pducnt) * expppdu;
-			return expnsize;
-		}
-		fragcnt = (data_len / fragsize);
-		expnsize =  fragcnt * expppdu;
-		fragleft = data_len % fragsize;
-		if (fragleft > 0)
-			expnsize += expppdu;
-	}
-	return expnsize;
-}
-
-/* WR with IV, KEY and CPL SFO added */
-static void make_tlstx_data_wr(struct sock *sk, struct sk_buff *skb,
-			       int tls_tx_imm, int tls_len, u32 credits)
-{
-	unsigned short pdus_per_ulp = 0;
-	struct chtls_sock *csk;
-	struct chtls_hws *hws;
-	int expn_sz;
-	int pdus;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	hws = &csk->tlshws;
-	pdus = DIV_ROUND_UP(tls_len, hws->mfs);
-	expn_sz = chtls_expansion_size(sk, tls_len, 0, NULL);
-	if (!hws->compute) {
-		hws->expansion = chtls_expansion_size(sk,
-						      hws->fcplenmax,
-						      1, &pdus_per_ulp);
-		hws->pdus = pdus_per_ulp;
-		hws->adjustlen = hws->pdus *
-			((hws->expansion / hws->pdus) + hws->mfs);
-		hws->compute = 1;
-	}
-	if (tls_copy_ivs(sk, skb))
-		return;
-	tls_copy_tx_key(sk, skb);
-	tls_tx_data_wr(sk, skb, tls_len, tls_tx_imm, credits, expn_sz, pdus);
-	hws->tx_seq_no += (pdus - 1);
-}
-
-static void make_tx_data_wr(struct sock *sk, struct sk_buff *skb,
-			    unsigned int immdlen, int len,
-			    u32 credits, u32 compl)
-{
-	struct fw_ofld_tx_data_wr *req;
-	unsigned int wr_ulp_mode_force;
-	struct chtls_sock *csk;
-	unsigned int opcode;
-
-	csk = rcu_dereference_sk_user_data(sk);
-	opcode = FW_OFLD_TX_DATA_WR;
-
-	req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, sizeof(*req));
-	req->op_to_immdlen = htonl(WR_OP_V(opcode) |
-				FW_WR_COMPL_V(compl) |
-				FW_WR_IMMDLEN_V(immdlen));
-	req->flowid_len16 = htonl(FW_WR_FLOWID_V(csk->tid) |
-				FW_WR_LEN16_V(credits));
-
-	wr_ulp_mode_force = TX_ULP_MODE_V(csk->ulp_mode);
-	if (is_sg_request(skb))
-		wr_ulp_mode_force |= FW_OFLD_TX_DATA_WR_ALIGNPLD_F |
-			((tcp_sk(sk)->nonagle & TCP_NAGLE_OFF) ? 0 :
-				FW_OFLD_TX_DATA_WR_SHOVE_F);
-
-	req->tunnel_to_proxy = htonl(wr_ulp_mode_force |
-			TX_URG_V(skb_urgent(skb)) |
-			TX_SHOVE_V((!csk_flag(sk, CSK_TX_MORE_DATA)) &&
-				   skb_queue_empty(&csk->txq)));
-	req->plen = htonl(len);
-}
-
-static int chtls_wr_size(struct chtls_sock *csk, const struct sk_buff *skb,
-			 bool size)
-{
-	int wr_size;
-
-	wr_size = TLS_WR_CPL_LEN;
-	wr_size += KEY_ON_MEM_SZ;
-	wr_size += ivs_size(csk->sk, skb);
-
-	if (size)
-		return wr_size;
-
-	/* frags counted for IV dsgl */
-	if (!skb_ulp_tls_iv_imm(skb))
-		skb_shinfo(skb)->nr_frags++;
-
-	return wr_size;
-}
-
-static bool is_ofld_imm(struct chtls_sock *csk, const struct sk_buff *skb)
-{
-	int length = skb->len;
-
-	if (skb->peeked || skb->len > MAX_IMM_ULPTX_WR_LEN)
-		return false;
-
-	if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR)) {
-		/* Check TLS header len for Immediate */
-		if (csk->ulp_mode == ULP_MODE_TLS &&
-		    skb_ulp_tls_inline(skb))
-			length += chtls_wr_size(csk, skb, true);
-		else
-			length += sizeof(struct fw_ofld_tx_data_wr);
-
-		return length <= MAX_IMM_OFLD_TX_DATA_WR_LEN;
-	}
-	return true;
-}
-
-static unsigned int calc_tx_flits(const struct sk_buff *skb,
-				  unsigned int immdlen)
-{
-	unsigned int flits, cnt;
-
-	flits = immdlen / 8;   /* headers */
-	cnt = skb_shinfo(skb)->nr_frags;
-	if (skb_tail_pointer(skb) != skb_transport_header(skb))
-		cnt++;
-	return flits + sgl_len(cnt);
-}
-
-static void arp_failure_discard(void *handle, struct sk_buff *skb)
-{
-	kfree_skb(skb);
-}
-
-int chtls_push_frames(struct chtls_sock *csk, int comp)
-{
-	struct chtls_hws *hws = &csk->tlshws;
-	struct tcp_sock *tp;
-	struct sk_buff *skb;
-	int total_size = 0;
-	struct sock *sk;
-	int wr_size;
-
-	wr_size = sizeof(struct fw_ofld_tx_data_wr);
-	sk = csk->sk;
-	tp = tcp_sk(sk);
-
-	if (unlikely(sk_in_state(sk, TCPF_SYN_SENT | TCPF_CLOSE)))
-		return 0;
-
-	if (unlikely(csk_flag(sk, CSK_ABORT_SHUTDOWN)))
-		return 0;
-
-	while (csk->wr_credits && (skb = skb_peek(&csk->txq)) &&
-	       (!(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_HOLD) ||
-		skb_queue_len(&csk->txq) > 1)) {
-		unsigned int credit_len = skb->len;
-		unsigned int credits_needed;
-		unsigned int completion = 0;
-		int tls_len = skb->len;/* TLS data len before IV/key */
-		unsigned int immdlen;
-		int len = skb->len;    /* length [ulp bytes] inserted by hw */
-		int flowclen16 = 0;
-		int tls_tx_imm = 0;
-
-		immdlen = skb->len;
-		if (!is_ofld_imm(csk, skb)) {
-			immdlen = skb_transport_offset(skb);
-			if (skb_ulp_tls_inline(skb))
-				wr_size = chtls_wr_size(csk, skb, false);
-			credit_len = 8 * calc_tx_flits(skb, immdlen);
-		} else {
-			if (skb_ulp_tls_inline(skb)) {
-				wr_size = chtls_wr_size(csk, skb, false);
-				tls_tx_imm = 1;
-			}
-		}
-		if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR))
-			credit_len += wr_size;
-		credits_needed = DIV_ROUND_UP(credit_len, 16);
-		if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
-			flowclen16 = send_tx_flowc_wr(sk, 1, tp->snd_nxt,
-						      tp->rcv_nxt);
-			if (flowclen16 <= 0)
-				break;
-			csk->wr_credits -= flowclen16;
-			csk->wr_unacked += flowclen16;
-			csk->wr_nondata += flowclen16;
-			csk_set_flag(csk, CSK_TX_DATA_SENT);
-		}
-
-		if (csk->wr_credits < credits_needed) {
-			if (skb_ulp_tls_inline(skb) &&
-			    !skb_ulp_tls_iv_imm(skb))
-				skb_shinfo(skb)->nr_frags--;
-			break;
-		}
-
-		__skb_unlink(skb, &csk->txq);
-		skb_set_queue_mapping(skb, (csk->txq_idx << 1) |
-				      CPL_PRIORITY_DATA);
-		if (hws->ofld)
-			hws->txqid = (skb->queue_mapping >> 1);
-		skb->csum = (__force __wsum)(credits_needed + csk->wr_nondata);
-		csk->wr_credits -= credits_needed;
-		csk->wr_unacked += credits_needed;
-		csk->wr_nondata = 0;
-		enqueue_wr(csk, skb);
-
-		if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR)) {
-			if ((comp && csk->wr_unacked == credits_needed) ||
-			    (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_COMPL) ||
-			    csk->wr_unacked >= csk->wr_max_credits / 2) {
-				completion = 1;
-				csk->wr_unacked = 0;
-			}
-			if (skb_ulp_tls_inline(skb))
-				make_tlstx_data_wr(sk, skb, tls_tx_imm,
-						   tls_len, credits_needed);
-			else
-				make_tx_data_wr(sk, skb, immdlen, len,
-						credits_needed, completion);
-			tp->snd_nxt += len;
-			tp->lsndtime = tcp_jiffies32;
-			if (completion)
-				ULP_SKB_CB(skb)->flags &= ~ULPCB_FLAG_NEED_HDR;
-		} else {
-			struct cpl_close_con_req *req = cplhdr(skb);
-			unsigned int cmd  = CPL_OPCODE_G(ntohl
-					     (OPCODE_TID(req)));
-
-			if (cmd == CPL_CLOSE_CON_REQ)
-				csk_set_flag(csk,
-					     CSK_CLOSE_CON_REQUESTED);
-
-			if ((ULP_SKB_CB(skb)->flags & ULPCB_FLAG_COMPL) &&
-			    (csk->wr_unacked >= csk->wr_max_credits / 2)) {
-				req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
-				csk->wr_unacked = 0;
-			}
-		}
-		total_size += skb->truesize;
-		if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_BARRIER)
-			csk_set_flag(csk, CSK_TX_WAIT_IDLE);
-		t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
-		cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
-	}
-	sk->sk_wmem_queued -= total_size;
-	return total_size;
-}
-
-static void mark_urg(struct tcp_sock *tp, int flags,
-		     struct sk_buff *skb)
-{
-	if (unlikely(flags & MSG_OOB)) {
-		tp->snd_up = tp->write_seq;
-		ULP_SKB_CB(skb)->flags = ULPCB_FLAG_URG |
-					 ULPCB_FLAG_BARRIER |
-					 ULPCB_FLAG_NO_APPEND |
-					 ULPCB_FLAG_NEED_HDR;
-	}
-}
-
-/*
- * Returns true if a connection should send more data to TCP engine
- */
-static bool should_push(struct sock *sk)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct chtls_dev *cdev = csk->cdev;
-	struct tcp_sock *tp = tcp_sk(sk);
-
-	/*
-	 * If we've released our offload resources there's nothing to do ...
-	 */
-	if (!cdev)
-		return false;
-
-	/*
-	 * If there aren't any work requests in flight, or there isn't enough
-	 * data in flight, or Nagle is off then send the current TX_DATA
-	 * otherwise hold it and wait to accumulate more data.
-	 */
-	return csk->wr_credits == csk->wr_max_credits ||
-		(tp->nonagle & TCP_NAGLE_OFF);
-}
-
-/*
- * Returns true if a TCP socket is corked.
- */
-static bool corked(const struct tcp_sock *tp, int flags)
-{
-	return (flags & MSG_MORE) || (tp->nonagle & TCP_NAGLE_CORK);
-}
-
-/*
- * Returns true if a send should try to push new data.
- */
-static bool send_should_push(struct sock *sk, int flags)
-{
-	return should_push(sk) && !corked(tcp_sk(sk), flags);
-}
-
-void chtls_tcp_push(struct sock *sk, int flags)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	int qlen = skb_queue_len(&csk->txq);
-
-	if (likely(qlen)) {
-		struct sk_buff *skb = skb_peek_tail(&csk->txq);
-		struct tcp_sock *tp = tcp_sk(sk);
-
-		mark_urg(tp, flags, skb);
-
-		if (!(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) &&
-		    corked(tp, flags)) {
-			ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_HOLD;
-			return;
-		}
-
-		ULP_SKB_CB(skb)->flags &= ~ULPCB_FLAG_HOLD;
-		if (qlen == 1 &&
-		    ((ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) ||
-		     should_push(sk)))
-			chtls_push_frames(csk, 1);
-	}
-}
-
-/*
- * Calculate the size for a new send sk_buff.  It's maximum size so we can
- * pack lots of data into it, unless we plan to send it immediately, in which
- * case we size it more tightly.
- *
- * Note: we don't bother compensating for MSS < PAGE_SIZE because it doesn't
- * arise in normal cases and when it does we are just wasting memory.
- */
-static int select_size(struct sock *sk, int io_len, int flags, int len)
-{
-	const int pgbreak = SKB_MAX_HEAD(len);
-
-	/*
-	 * If the data wouldn't fit in the main body anyway, put only the
-	 * header in the main body so it can use immediate data and place all
-	 * the payload in page fragments.
-	 */
-	if (io_len > pgbreak)
-		return 0;
-
-	/*
-	 * If we will be accumulating payload get a large main body.
-	 */
-	if (!send_should_push(sk, flags))
-		return pgbreak;
-
-	return io_len;
-}
-
-void skb_entail(struct sock *sk, struct sk_buff *skb, int flags)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct tcp_sock *tp = tcp_sk(sk);
-
-	ULP_SKB_CB(skb)->seq = tp->write_seq;
-	ULP_SKB_CB(skb)->flags = flags;
-	__skb_queue_tail(&csk->txq, skb);
-	sk->sk_wmem_queued += skb->truesize;
-
-	if (TCP_PAGE(sk) && TCP_OFF(sk)) {
-		put_page(TCP_PAGE(sk));
-		TCP_PAGE(sk) = NULL;
-		TCP_OFF(sk) = 0;
-	}
-}
-
-static struct sk_buff *get_tx_skb(struct sock *sk, int size)
-{
-	struct sk_buff *skb;
-
-	skb = alloc_skb(size + TX_HEADER_LEN, sk->sk_allocation);
-	if (likely(skb)) {
-		skb_reserve(skb, TX_HEADER_LEN);
-		skb_entail(sk, skb, ULPCB_FLAG_NEED_HDR);
-		skb_reset_transport_header(skb);
-	}
-	return skb;
-}
-
-static struct sk_buff *get_record_skb(struct sock *sk, int size, bool zcopy)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct sk_buff *skb;
-
-	skb = alloc_skb(((zcopy ? 0 : size) + TX_TLSHDR_LEN +
-			KEY_ON_MEM_SZ + max_ivs_size(sk, size)),
-			sk->sk_allocation);
-	if (likely(skb)) {
-		skb_reserve(skb, (TX_TLSHDR_LEN +
-			    KEY_ON_MEM_SZ + max_ivs_size(sk, size)));
-		skb_entail(sk, skb, ULPCB_FLAG_NEED_HDR);
-		skb_reset_transport_header(skb);
-		ULP_SKB_CB(skb)->ulp.tls.ofld = 1;
-		ULP_SKB_CB(skb)->ulp.tls.type = csk->tlshws.type;
-	}
-	return skb;
-}
-
-static void tx_skb_finalize(struct sk_buff *skb)
-{
-	struct ulp_skb_cb *cb = ULP_SKB_CB(skb);
-
-	if (!(cb->flags & ULPCB_FLAG_NO_HDR))
-		cb->flags = ULPCB_FLAG_NEED_HDR;
-	cb->flags |= ULPCB_FLAG_NO_APPEND;
-}
-
-static void push_frames_if_head(struct sock *sk)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
-	if (skb_queue_len(&csk->txq) == 1)
-		chtls_push_frames(csk, 1);
-}
-
-static int chtls_skb_copy_to_page_nocache(struct sock *sk,
-					  struct iov_iter *from,
-					  struct sk_buff *skb,
-					  struct page *page,
-					  int off, int copy)
-{
-	int err;
-
-	err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) +
-				       off, copy, skb->len);
-	if (err)
-		return err;
-
-	skb->len             += copy;
-	skb->data_len        += copy;
-	skb->truesize        += copy;
-	sk->sk_wmem_queued   += copy;
-	return 0;
-}
-
-static int csk_mem_free(struct chtls_dev *cdev, struct sock *sk)
-{
-	return (cdev->max_host_sndbuf - sk->sk_wmem_queued);
-}
-
-static int csk_wait_memory(struct chtls_dev *cdev,
-			   struct sock *sk, long *timeo_p)
-{
-	DEFINE_WAIT_FUNC(wait, woken_wake_function);
-	int err = 0;
-	long current_timeo;
-	long vm_wait = 0;
-	bool noblock;
-
-	current_timeo = *timeo_p;
-	noblock = (*timeo_p ? false : true);
-	if (csk_mem_free(cdev, sk)) {
-		current_timeo = (prandom_u32() % (HZ / 5)) + 2;
-		vm_wait = (prandom_u32() % (HZ / 5)) + 2;
-	}
-
-	add_wait_queue(sk_sleep(sk), &wait);
-	while (1) {
-		sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
-
-		if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
-			goto do_error;
-		if (!*timeo_p) {
-			if (noblock)
-				set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
-			goto do_nonblock;
-		}
-		if (signal_pending(current))
-			goto do_interrupted;
-		sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
-		if (csk_mem_free(cdev, sk) && !vm_wait)
-			break;
-
-		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
-		sk->sk_write_pending++;
-		sk_wait_event(sk, &current_timeo, sk->sk_err ||
-			      (sk->sk_shutdown & SEND_SHUTDOWN) ||
-			      (csk_mem_free(cdev, sk) && !vm_wait), &wait);
-		sk->sk_write_pending--;
-
-		if (vm_wait) {
-			vm_wait -= current_timeo;
-			current_timeo = *timeo_p;
-			if (current_timeo != MAX_SCHEDULE_TIMEOUT) {
-				current_timeo -= vm_wait;
-				if (current_timeo < 0)
-					current_timeo = 0;
-			}
-			vm_wait = 0;
-		}
-		*timeo_p = current_timeo;
-	}
-do_rm_wq:
-	remove_wait_queue(sk_sleep(sk), &wait);
-	return err;
-do_error:
-	err = -EPIPE;
-	goto do_rm_wq;
-do_nonblock:
-	err = -EAGAIN;
-	goto do_rm_wq;
-do_interrupted:
-	err = sock_intr_errno(*timeo_p);
-	goto do_rm_wq;
-}
-
-static int chtls_proccess_cmsg(struct sock *sk, struct msghdr *msg,
-			       unsigned char *record_type)
-{
-	struct cmsghdr *cmsg;
-	int rc = -EINVAL;
-
-	for_each_cmsghdr(cmsg, msg) {
-		if (!CMSG_OK(msg, cmsg))
-			return -EINVAL;
-		if (cmsg->cmsg_level != SOL_TLS)
-			continue;
-
-		switch (cmsg->cmsg_type) {
-		case TLS_SET_RECORD_TYPE:
-			if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type)))
-				return -EINVAL;
-
-			if (msg->msg_flags & MSG_MORE)
-				return -EINVAL;
-
-			*record_type = *(unsigned char *)CMSG_DATA(cmsg);
-			rc = 0;
-			break;
-		default:
-			return -EINVAL;
-		}
-	}
-
-	return rc;
-}
-
-int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct chtls_dev *cdev = csk->cdev;
-	struct tcp_sock *tp = tcp_sk(sk);
-	struct sk_buff *skb;
-	int mss, flags, err;
-	int recordsz = 0;
-	int copied = 0;
-	long timeo;
-
-	lock_sock(sk);
-	flags = msg->msg_flags;
-	timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
-
-	if (!sk_in_state(sk, TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
-		err = sk_stream_wait_connect(sk, &timeo);
-		if (err)
-			goto out_err;
-	}
-
-	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
-	err = -EPIPE;
-	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
-		goto out_err;
-
-	mss = csk->mss;
-	csk_set_flag(csk, CSK_TX_MORE_DATA);
-
-	while (msg_data_left(msg)) {
-		int copy = 0;
-
-		skb = skb_peek_tail(&csk->txq);
-		if (skb) {
-			copy = mss - skb->len;
-			skb->ip_summed = CHECKSUM_UNNECESSARY;
-		}
-		if (!csk_mem_free(cdev, sk))
-			goto wait_for_sndbuf;
-
-		if (is_tls_tx(csk) && !csk->tlshws.txleft) {
-			unsigned char record_type = TLS_RECORD_TYPE_DATA;
-
-			if (unlikely(msg->msg_controllen)) {
-				err = chtls_proccess_cmsg(sk, msg,
-							  &record_type);
-				if (err)
-					goto out_err;
-
-				/* Avoid appending tls handshake, alert to tls data */
-				if (skb)
-					tx_skb_finalize(skb);
-			}
-
-			recordsz = size;
-			csk->tlshws.txleft = recordsz;
-			csk->tlshws.type = record_type;
-		}
-
-		if (!skb || (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) ||
-		    copy <= 0) {
-new_buf:
-			if (skb) {
-				tx_skb_finalize(skb);
-				push_frames_if_head(sk);
-			}
-
-			if (is_tls_tx(csk)) {
-				skb = get_record_skb(sk,
-						     select_size(sk,
-								 recordsz,
-								 flags,
-								 TX_TLSHDR_LEN),
-								 false);
-			} else {
-				skb = get_tx_skb(sk,
-						 select_size(sk, size, flags,
-							     TX_HEADER_LEN));
-			}
-			if (unlikely(!skb))
-				goto wait_for_memory;
-
-			skb->ip_summed = CHECKSUM_UNNECESSARY;
-			copy = mss;
-		}
-		if (copy > size)
-			copy = size;
-
-		if (skb_tailroom(skb) > 0) {
-			copy = min(copy, skb_tailroom(skb));
-			if (is_tls_tx(csk))
-				copy = min_t(int, copy, csk->tlshws.txleft);
-			err = skb_add_data_nocache(sk, skb,
-						   &msg->msg_iter, copy);
-			if (err)
-				goto do_fault;
-		} else {
-			int i = skb_shinfo(skb)->nr_frags;
-			struct page *page = TCP_PAGE(sk);
-			int pg_size = PAGE_SIZE;
-			int off = TCP_OFF(sk);
-			bool merge;
-
-			if (page)
-				pg_size = page_size(page);
-			if (off < pg_size &&
-			    skb_can_coalesce(skb, i, page, off)) {
-				merge = true;
-				goto copy;
-			}
-			merge = false;
-			if (i == (is_tls_tx(csk) ? (MAX_SKB_FRAGS - 1) :
-			    MAX_SKB_FRAGS))
-				goto new_buf;
-
-			if (page && off == pg_size) {
-				put_page(page);
-				TCP_PAGE(sk) = page = NULL;
-				pg_size = PAGE_SIZE;
-			}
-
-			if (!page) {
-				gfp_t gfp = sk->sk_allocation;
-				int order = cdev->send_page_order;
-
-				if (order) {
-					page = alloc_pages(gfp | __GFP_COMP |
-							   __GFP_NOWARN |
-							   __GFP_NORETRY,
-							   order);
-					if (page)
-						pg_size <<= order;
-				}
-				if (!page) {
-					page = alloc_page(gfp);
-					pg_size = PAGE_SIZE;
-				}
-				if (!page)
-					goto wait_for_memory;
-				off = 0;
-			}
-copy:
-			if (copy > pg_size - off)
-				copy = pg_size - off;
-			if (is_tls_tx(csk))
-				copy = min_t(int, copy, csk->tlshws.txleft);
-
-			err = chtls_skb_copy_to_page_nocache(sk, &msg->msg_iter,
-							     skb, page,
-							     off, copy);
-			if (unlikely(err)) {
-				if (!TCP_PAGE(sk)) {
-					TCP_PAGE(sk) = page;
-					TCP_OFF(sk) = 0;
-				}
-				goto do_fault;
-			}
-			/* Update the skb. */
-			if (merge) {
-				skb_frag_size_add(
-						&skb_shinfo(skb)->frags[i - 1],
-						copy);
-			} else {
-				skb_fill_page_desc(skb, i, page, off, copy);
-				if (off + copy < pg_size) {
-					/* space left keep page */
-					get_page(page);
-					TCP_PAGE(sk) = page;
-				} else {
-					TCP_PAGE(sk) = NULL;
-				}
-			}
-			TCP_OFF(sk) = off + copy;
-		}
-		if (unlikely(skb->len == mss))
-			tx_skb_finalize(skb);
-		tp->write_seq += copy;
-		copied += copy;
-		size -= copy;
-
-		if (is_tls_tx(csk))
-			csk->tlshws.txleft -= copy;
-
-		if (corked(tp, flags) &&
-		    (sk_stream_wspace(sk) < sk_stream_min_wspace(sk)))
-			ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_NO_APPEND;
-
-		if (size == 0)
-			goto out;
-
-		if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND)
-			push_frames_if_head(sk);
-		continue;
-wait_for_sndbuf:
-		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
-wait_for_memory:
-		err = csk_wait_memory(cdev, sk, &timeo);
-		if (err)
-			goto do_error;
-	}
-out:
-	csk_reset_flag(csk, CSK_TX_MORE_DATA);
-	if (copied)
-		chtls_tcp_push(sk, flags);
-done:
-	release_sock(sk);
-	return copied;
-do_fault:
-	if (!skb->len) {
-		__skb_unlink(skb, &csk->txq);
-		sk->sk_wmem_queued -= skb->truesize;
-		__kfree_skb(skb);
-	}
-do_error:
-	if (copied)
-		goto out;
-out_err:
-	if (csk_conn_inline(csk))
-		csk_reset_flag(csk, CSK_TX_MORE_DATA);
-	copied = sk_stream_error(sk, flags, err);
-	goto done;
-}
-
-int chtls_sendpage(struct sock *sk, struct page *page,
-		   int offset, size_t size, int flags)
-{
-	struct chtls_sock *csk;
-	struct chtls_dev *cdev;
-	int mss, err, copied;
-	struct tcp_sock *tp;
-	long timeo;
-
-	tp = tcp_sk(sk);
-	copied = 0;
-	csk = rcu_dereference_sk_user_data(sk);
-	cdev = csk->cdev;
-	timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
-
-	err = sk_stream_wait_connect(sk, &timeo);
-	if (!sk_in_state(sk, TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
-	    err != 0)
-		goto out_err;
-
-	mss = csk->mss;
-	csk_set_flag(csk, CSK_TX_MORE_DATA);
-
-	while (size > 0) {
-		struct sk_buff *skb = skb_peek_tail(&csk->txq);
-		int copy, i;
-
-		if (!skb || (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) ||
-		    (copy = mss - skb->len) <= 0) {
-new_buf:
-			if (!csk_mem_free(cdev, sk))
-				goto wait_for_sndbuf;
-
-			if (is_tls_tx(csk)) {
-				skb = get_record_skb(sk,
-						     select_size(sk, size,
-								 flags,
-								 TX_TLSHDR_LEN),
-						     true);
-			} else {
-				skb = get_tx_skb(sk, 0);
-			}
-			if (!skb)
-				goto wait_for_memory;
-			copy = mss;
-		}
-		if (copy > size)
-			copy = size;
-
-		i = skb_shinfo(skb)->nr_frags;
-		if (skb_can_coalesce(skb, i, page, offset)) {
-			skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
-		} else if (i < MAX_SKB_FRAGS) {
-			get_page(page);
-			skb_fill_page_desc(skb, i, page, offset, copy);
-		} else {
-			tx_skb_finalize(skb);
-			push_frames_if_head(sk);
-			goto new_buf;
-		}
-
-		skb->len += copy;
-		if (skb->len == mss)
-			tx_skb_finalize(skb);
-		skb->data_len += copy;
-		skb->truesize += copy;
-		sk->sk_wmem_queued += copy;
-		tp->write_seq += copy;
-		copied += copy;
-		offset += copy;
-		size -= copy;
-
-		if (corked(tp, flags) &&
-		    (sk_stream_wspace(sk) < sk_stream_min_wspace(sk)))
-			ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_NO_APPEND;
-
-		if (!size)
-			break;
-
-		if (unlikely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND))
-			push_frames_if_head(sk);
-		continue;
-wait_for_sndbuf:
-		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
-wait_for_memory:
-		err = csk_wait_memory(cdev, sk, &timeo);
-		if (err)
-			goto do_error;
-	}
-out:
-	csk_reset_flag(csk, CSK_TX_MORE_DATA);
-	if (copied)
-		chtls_tcp_push(sk, flags);
-done:
-	release_sock(sk);
-	return copied;
-
-do_error:
-	if (copied)
-		goto out;
-
-out_err:
-	if (csk_conn_inline(csk))
-		csk_reset_flag(csk, CSK_TX_MORE_DATA);
-	copied = sk_stream_error(sk, flags, err);
-	goto done;
-}
-
-static void chtls_select_window(struct sock *sk)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct tcp_sock *tp = tcp_sk(sk);
-	unsigned int wnd = tp->rcv_wnd;
-
-	wnd = max_t(unsigned int, wnd, tcp_full_space(sk));
-	wnd = max_t(unsigned int, MIN_RCV_WND, wnd);
-
-	if (wnd > MAX_RCV_WND)
-		wnd = MAX_RCV_WND;
-
-/*
- * Check if we need to grow the receive window in response to an increase in
- * the socket's receive buffer size.  Some applications increase the buffer
- * size dynamically and rely on the window to grow accordingly.
- */
-
-	if (wnd > tp->rcv_wnd) {
-		tp->rcv_wup -= wnd - tp->rcv_wnd;
-		tp->rcv_wnd = wnd;
-		/* Mark the receive window as updated */
-		csk_reset_flag(csk, CSK_UPDATE_RCV_WND);
-	}
-}
-
-/*
- * Send RX credits through an RX_DATA_ACK CPL message.  We are permitted
- * to return without sending the message in case we cannot allocate
- * an sk_buff.  Returns the number of credits sent.
- */
-static u32 send_rx_credits(struct chtls_sock *csk, u32 credits)
-{
-	struct cpl_rx_data_ack *req;
-	struct sk_buff *skb;
-
-	skb = alloc_skb(sizeof(*req), GFP_ATOMIC);
-	if (!skb)
-		return 0;
-	__skb_put(skb, sizeof(*req));
-	req = (struct cpl_rx_data_ack *)skb->head;
-
-	set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id);
-	INIT_TP_WR(req, csk->tid);
-	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK,
-						    csk->tid));
-	req->credit_dack = cpu_to_be32(RX_CREDITS_V(credits) |
-				       RX_FORCE_ACK_F);
-	cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
-	return credits;
-}
-
-#define CREDIT_RETURN_STATE (TCPF_ESTABLISHED | \
-			     TCPF_FIN_WAIT1 | \
-			     TCPF_FIN_WAIT2)
-
-/*
- * Called after some received data has been read.  It returns RX credits
- * to the HW for the amount of data processed.
- */
-static void chtls_cleanup_rbuf(struct sock *sk, int copied)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct tcp_sock *tp;
-	int must_send;
-	u32 credits;
-	u32 thres;
-
-	thres = 15 * 1024;
-
-	if (!sk_in_state(sk, CREDIT_RETURN_STATE))
-		return;
-
-	chtls_select_window(sk);
-	tp = tcp_sk(sk);
-	credits = tp->copied_seq - tp->rcv_wup;
-	if (unlikely(!credits))
-		return;
-
-/*
- * For coalescing to work effectively ensure the receive window has
- * at least 16KB left.
- */
-	must_send = credits + 16384 >= tp->rcv_wnd;
-
-	if (must_send || credits >= thres)
-		tp->rcv_wup += send_rx_credits(csk, credits);
-}
-
-static int chtls_pt_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
-			    int nonblock, int flags, int *addr_len)
-{
-	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-	struct chtls_hws *hws = &csk->tlshws;
-	struct net_device *dev = csk->egress_dev;
-	struct adapter *adap = netdev2adap(dev);
-	struct tcp_sock *tp = tcp_sk(sk);
-	unsigned long avail;
-	int buffers_freed;
-	int copied = 0;
-	int target;
-	long timeo;
-
-	buffers_freed = 0;
-
-	timeo = sock_rcvtimeo(sk, nonblock);
-	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
-
-	if (unlikely(csk_flag(sk, CSK_UPDATE_RCV_WND)))
-		chtls_cleanup_rbuf(sk, copied);
-
-	do {
-		struct sk_buff *skb;
-		u32 offset = 0;
-
-		if (unlikely(tp->urg_data &&
-			     tp->urg_seq == tp->copied_seq)) {
-			if (copied)
-				break;
-			if (signal_pending(current)) {
-				copied = timeo ? sock_intr_errno(timeo) :
-					-EAGAIN;
-				break;
-			}
-		}
-		skb = skb_peek(&sk->sk_receive_queue);
-		if (skb)
-			goto found_ok_skb;
-		if (csk->wr_credits &&
-		    skb_queue_len(&csk->txq) &&
-		    chtls_push_frames(csk, csk->wr_credits ==
-				      csk->wr_max_credits))
-			sk->sk_write_space(sk);
-
-		if (copied >= target && !READ_ONCE(sk->sk_backlog.tail))
-			break;
-
-		if (copied) {
-			if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
-			    (sk->sk_shutdown & RCV_SHUTDOWN) ||
-			    signal_pending(current))
-				break;
-
-			if (!timeo)
-				break;
-		} else {
-			if (sock_flag(sk, SOCK_DONE))
-				break;
-			if (sk->sk_err) {
-				copied = sock_error(sk);
-				break;
-			}
-			if (sk->sk_shutdown & RCV_SHUTDOWN)
-				break;
-			if (sk->sk_state == TCP_CLOSE) {
-				copied = -ENOTCONN;
-				break;
-			}
-			if (!timeo) {
-				copied = -EAGAIN;
-				break;
-			}
-			if (signal_pending(current)) {
-				copied = sock_intr_errno(timeo);
-				break;
-			}
-		}
-		if (READ_ONCE(sk->sk_backlog.tail)) {
-			release_sock(sk);
-			lock_sock(sk);
-			chtls_cleanup_rbuf(sk, copied);
-			continue;
-		}
-
-		if (copied >= target)
-			break;
-		chtls_cleanup_rbuf(sk, copied);
-		sk_wait_data(sk, &timeo, NULL);
-		continue;
-found_ok_skb:
-		if (!skb->len) {
-			skb_dst_set(skb, NULL);
-			__skb_unlink(skb, &sk->sk_receive_queue);
-			kfree_skb(skb);
-
-			if (!copied && !timeo) {
-				copied = -EAGAIN;
-				break;
-			}
-
-			if (copied < target) {
-				release_sock(sk);
-				lock_sock(sk);
-				continue;
-			}
-			break;
-		}
-		offset = hws->copied_seq;
-		avail = skb->len - offset;
-		if (len < avail)
-			avail = len;
-
-		if (unlikely(tp->urg_data)) {
-			u32 urg_offset = tp->urg_seq - tp->copied_seq;
-
-			if (urg_offset < avail) {
-				if (urg_offset) {
-					avail = urg_offset;
-				} else if (!sock_flag(sk, SOCK_URGINLINE)) {
-					/* First byte is urgent, skip */
-					tp->copied_seq++;
-					offset++;
-					avail--;
-					if (!avail)
-						goto skip_copy;
-				}
-			}
-		}
-		/* Set record type if not already done. For a non-data record,
-		 * do not proceed if record type could not be copied.
-		 */
-		if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_TLS_HDR) {
-			struct tls_hdr *thdr = (struct tls_hdr *)skb->data;
-			int cerr = 0;
-
-			cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE,
-					sizeof(thdr->type), &thdr->type);
-
-			if (cerr && thdr->type != TLS_RECORD_TYPE_DATA)
-				return -EIO;
-			/*  don't send tls header, skip copy */
-			goto skip_copy;
-		}
-
-		if (skb_copy_datagram_msg(skb, offset, msg, avail)) {
-			if (!copied) {
-				copied = -EFAULT;
-				break;
-			}
-		}
-
-		copied += avail;
-		len -= avail;
-		hws->copied_seq += avail;
-skip_copy:
-		if (tp->urg_data && after(tp->copied_seq, tp->urg_seq))
-			tp->urg_data = 0;
-
-		if ((avail + offset) >= skb->len) {
-			if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_TLS_HDR) {
-				tp->copied_seq += skb->len;
-				hws->rcvpld = skb->hdr_len;
-			} else {
-				atomic_inc(&adap->chcr_stats.tls_pdu_rx);
-				tp->copied_seq += hws->rcvpld;
-			}
-			chtls_free_skb(sk, skb);
-			buffers_freed++;
-			hws->copied_seq = 0;
-			if (copied >= target &&
-			    !skb_peek(&sk->sk_receive_queue))
-				break;
-		}
-	} while (len > 0);
-
-	if (buffers_freed)
-		chtls_cleanup_rbuf(sk, copied);
-	release_sock(sk);
-	return copied;
-}
-
-/*
- * Peek at data in a socket's receive buffer.
- */
-static int peekmsg(struct sock *sk, struct msghdr *msg,
-		   size_t len, int nonblock, int flags)
-{
-	struct tcp_sock *tp = tcp_sk(sk);
-	u32 peek_seq, offset;
-	struct sk_buff *skb;
-	int copied = 0;
-	size_t avail;          /* amount of available data in current skb */
-	long timeo;
-
-	lock_sock(sk);
-	timeo = sock_rcvtimeo(sk, nonblock);
-	peek_seq = tp->copied_seq;
-
-	do {
-		if (unlikely(tp->urg_data && tp->urg_seq == peek_seq)) {
-			if (copied)
-				break;
-			if (signal_pending(current)) {
-				copied = timeo ? sock_intr_errno(timeo) :
-				-EAGAIN;
-				break;
-			}
-		}
-
-		skb_queue_walk(&sk->sk_receive_queue, skb) {
-			offset = peek_seq - ULP_SKB_CB(skb)->seq;
-			if (offset < skb->len)
-				goto found_ok_skb;
-		}
-
-		/* empty receive queue */
-		if (copied)
-			break;
-		if (sock_flag(sk, SOCK_DONE))
-			break;
-		if (sk->sk_err) {
-			copied = sock_error(sk);
-			break;
-		}
-		if (sk->sk_shutdown & RCV_SHUTDOWN)
-			break;
-		if (sk->sk_state == TCP_CLOSE) {
-			copied = -ENOTCONN;
-			break;
-		}
-		if (!timeo) {
-			copied = -EAGAIN;
-			break;
-		}
-		if (signal_pending(current)) {
-			copied = sock_intr_errno(timeo);
-			break;
-		}
-
-		if (READ_ONCE(sk->sk_backlog.tail)) {
-			/* Do not sleep, just process backlog. */
-			release_sock(sk);
-			lock_sock(sk);
-		} else {
-			sk_wait_data(sk, &timeo, NULL);
-		}
-
-		if (unlikely(peek_seq != tp->copied_seq)) {
-			if (net_ratelimit())
-				pr_info("TCP(%s:%d), race in MSG_PEEK.\n",
-					current->comm, current->pid);
-			peek_seq = tp->copied_seq;
-		}
-		continue;
-
-found_ok_skb:
-		avail = skb->len - offset;
-		if (len < avail)
-			avail = len;
-		/*
-		 * Do we have urgent data here?  We need to skip over the
-		 * urgent byte.
-		 */
-		if (unlikely(tp->urg_data)) {
-			u32 urg_offset = tp->urg_seq - peek_seq;
-
-			if (urg_offset < avail) {
-				/*
-				 * The amount of data we are preparing to copy
-				 * contains urgent data.
-				 */
-				if (!urg_offset) { /* First byte is urgent */
-					if (!sock_flag(sk, SOCK_URGINLINE)) {
-						peek_seq++;
-						offset++;
-						avail--;
-					}
-					if (!avail)
-						continue;
-				} else {
-					/* stop short of the urgent data */
-					avail = urg_offset;
-				}
-			}
-		}
-
-		/*
-		 * If MSG_TRUNC is specified the data is discarded.
-		 */
-		if (likely(!(flags & MSG_TRUNC)))
-			if (skb_copy_datagram_msg(skb, offset, msg, len)) {
-				if (!copied) {
-					copied = -EFAULT;
-					break;
-				}
-			}
-		peek_seq += avail;
-		copied += avail;
-		len -= avail;
-	} while (len > 0);
-
-	release_sock(sk);
-	return copied;
-}
-
-int chtls_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
-		  int nonblock, int flags, int *addr_len)
-{
-	struct tcp_sock *tp = tcp_sk(sk);
-	struct chtls_sock *csk;
-	unsigned long avail;    /* amount of available data in current skb */
-	int buffers_freed;
-	int copied = 0;
-	long timeo;
-	int target;             /* Read at least this many bytes */
-
-	buffers_freed = 0;
-
-	if (unlikely(flags & MSG_OOB))
-		return tcp_prot.recvmsg(sk, msg, len, nonblock, flags,
-					addr_len);
-
-	if (unlikely(flags & MSG_PEEK))
-		return peekmsg(sk, msg, len, nonblock, flags);
-
-	if (sk_can_busy_loop(sk) &&
-	    skb_queue_empty_lockless(&sk->sk_receive_queue) &&
-	    sk->sk_state == TCP_ESTABLISHED)
-		sk_busy_loop(sk, nonblock);
-
-	lock_sock(sk);
-	csk = rcu_dereference_sk_user_data(sk);
-
-	if (is_tls_rx(csk))
-		return chtls_pt_recvmsg(sk, msg, len, nonblock,
-					flags, addr_len);
-
-	timeo = sock_rcvtimeo(sk, nonblock);
-	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
-
-	if (unlikely(csk_flag(sk, CSK_UPDATE_RCV_WND)))
-		chtls_cleanup_rbuf(sk, copied);
-
-	do {
-		struct sk_buff *skb;
-		u32 offset;
-
-		if (unlikely(tp->urg_data && tp->urg_seq == tp->copied_seq)) {
-			if (copied)
-				break;
-			if (signal_pending(current)) {
-				copied = timeo ? sock_intr_errno(timeo) :
-					-EAGAIN;
-				break;
-			}
-		}
-
-		skb = skb_peek(&sk->sk_receive_queue);
-		if (skb)
-			goto found_ok_skb;
-
-		if (csk->wr_credits &&
-		    skb_queue_len(&csk->txq) &&
-		    chtls_push_frames(csk, csk->wr_credits ==
-				      csk->wr_max_credits))
-			sk->sk_write_space(sk);
-
-		if (copied >= target && !READ_ONCE(sk->sk_backlog.tail))
-			break;
-
-		if (copied) {
-			if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
-			    (sk->sk_shutdown & RCV_SHUTDOWN) ||
-			    signal_pending(current))
-				break;
-		} else {
-			if (sock_flag(sk, SOCK_DONE))
-				break;
-			if (sk->sk_err) {
-				copied = sock_error(sk);
-				break;
-			}
-			if (sk->sk_shutdown & RCV_SHUTDOWN)
-				break;
-			if (sk->sk_state == TCP_CLOSE) {
-				copied = -ENOTCONN;
-				break;
-			}
-			if (!timeo) {
-				copied = -EAGAIN;
-				break;
-			}
-			if (signal_pending(current)) {
-				copied = sock_intr_errno(timeo);
-				break;
-			}
-		}
-
-		if (READ_ONCE(sk->sk_backlog.tail)) {
-			release_sock(sk);
-			lock_sock(sk);
-			chtls_cleanup_rbuf(sk, copied);
-			continue;
-		}
-
-		if (copied >= target)
-			break;
-		chtls_cleanup_rbuf(sk, copied);
-		sk_wait_data(sk, &timeo, NULL);
-		continue;
-
-found_ok_skb:
-		if (!skb->len) {
-			chtls_kfree_skb(sk, skb);
-			if (!copied && !timeo) {
-				copied = -EAGAIN;
-				break;
-			}
-
-			if (copied < target)
-				continue;
-
-			break;
-		}
-
-		offset = tp->copied_seq - ULP_SKB_CB(skb)->seq;
-		avail = skb->len - offset;
-		if (len < avail)
-			avail = len;
-
-		if (unlikely(tp->urg_data)) {
-			u32 urg_offset = tp->urg_seq - tp->copied_seq;
-
-			if (urg_offset < avail) {
-				if (urg_offset) {
-					avail = urg_offset;
-				} else if (!sock_flag(sk, SOCK_URGINLINE)) {
-					tp->copied_seq++;
-					offset++;
-					avail--;
-					if (!avail)
-						goto skip_copy;
-				}
-			}
-		}
-
-		if (likely(!(flags & MSG_TRUNC))) {
-			if (skb_copy_datagram_msg(skb, offset,
-						  msg, avail)) {
-				if (!copied) {
-					copied = -EFAULT;
-					break;
-				}
-			}
-		}
-
-		tp->copied_seq += avail;
-		copied += avail;
-		len -= avail;
-
-skip_copy:
-		if (tp->urg_data && after(tp->copied_seq, tp->urg_seq))
-			tp->urg_data = 0;
-
-		if (avail + offset >= skb->len) {
-			chtls_free_skb(sk, skb);
-			buffers_freed++;
-
-			if  (copied >= target &&
-			     !skb_peek(&sk->sk_receive_queue))
-				break;
-		}
-	} while (len > 0);
-
-	if (buffers_freed)
-		chtls_cleanup_rbuf(sk, copied);
-
-	release_sock(sk);
-	return copied;
-}
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c b/drivers/crypto/chelsio/chtls/chtls_main.c
deleted file mode 100644
index 66d247efd561..000000000000
--- a/drivers/crypto/chelsio/chtls/chtls_main.c
+++ /dev/null
@@ -1,641 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018 Chelsio Communications, Inc.
- *
- * Written by: Atul Gupta (atul.gupta@chelsio.com)
- */
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/skbuff.h>
-#include <linux/socket.h>
-#include <linux/hash.h>
-#include <linux/in.h>
-#include <linux/net.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <net/ipv6.h>
-#include <net/transp_v6.h>
-#include <net/tcp.h>
-#include <net/tls.h>
-
-#include "chtls.h"
-#include "chtls_cm.h"
-
-#define DRV_NAME "chtls"
-
-/*
- * chtls device management
- * maintains a list of the chtls devices
- */
-static LIST_HEAD(cdev_list);
-static DEFINE_MUTEX(cdev_mutex);
-
-static DEFINE_MUTEX(notify_mutex);
-static RAW_NOTIFIER_HEAD(listen_notify_list);
-static struct proto chtls_cpl_prot, chtls_cpl_protv6;
-struct request_sock_ops chtls_rsk_ops, chtls_rsk_opsv6;
-static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT;
-
-static void register_listen_notifier(struct notifier_block *nb)
-{
-	mutex_lock(&notify_mutex);
-	raw_notifier_chain_register(&listen_notify_list, nb);
-	mutex_unlock(&notify_mutex);
-}
-
-static void unregister_listen_notifier(struct notifier_block *nb)
-{
-	mutex_lock(&notify_mutex);
-	raw_notifier_chain_unregister(&listen_notify_list, nb);
-	mutex_unlock(&notify_mutex);
-}
-
-static int listen_notify_handler(struct notifier_block *this,
-				 unsigned long event, void *data)
-{
-	struct chtls_listen *clisten;
-	int ret = NOTIFY_DONE;
-
-	clisten = (struct chtls_listen *)data;
-
-	switch (event) {
-	case CHTLS_LISTEN_START:
-		ret = chtls_listen_start(clisten->cdev, clisten->sk);
-		kfree(clisten);
-		break;
-	case CHTLS_LISTEN_STOP:
-		chtls_listen_stop(clisten->cdev, clisten->sk);
-		kfree(clisten);
-		break;
-	}
-	return ret;
-}
-
-static struct notifier_block listen_notifier = {
-	.notifier_call = listen_notify_handler
-};
-
-static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb)
-{
-	if (likely(skb_transport_header(skb) != skb_network_header(skb)))
-		return tcp_v4_do_rcv(sk, skb);
-	BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
-	return 0;
-}
-
-static int chtls_start_listen(struct chtls_dev *cdev, struct sock *sk)
-{
-	struct chtls_listen *clisten;
-
-	if (sk->sk_protocol != IPPROTO_TCP)
-		return -EPROTONOSUPPORT;
-
-	if (sk->sk_family == PF_INET &&
-	    LOOPBACK(inet_sk(sk)->inet_rcv_saddr))
-		return -EADDRNOTAVAIL;
-
-	sk->sk_backlog_rcv = listen_backlog_rcv;
-	clisten = kmalloc(sizeof(*clisten), GFP_KERNEL);
-	if (!clisten)
-		return -ENOMEM;
-	clisten->cdev = cdev;
-	clisten->sk = sk;
-	mutex_lock(&notify_mutex);
-	raw_notifier_call_chain(&listen_notify_list,
-				      CHTLS_LISTEN_START, clisten);
-	mutex_unlock(&notify_mutex);
-	return 0;
-}
-
-static void chtls_stop_listen(struct chtls_dev *cdev, struct sock *sk)
-{
-	struct chtls_listen *clisten;
-
-	if (sk->sk_protocol != IPPROTO_TCP)
-		return;
-
-	clisten = kmalloc(sizeof(*clisten), GFP_KERNEL);
-	if (!clisten)
-		return;
-	clisten->cdev = cdev;
-	clisten->sk = sk;
-	mutex_lock(&notify_mutex);
-	raw_notifier_call_chain(&listen_notify_list,
-				CHTLS_LISTEN_STOP, clisten);
-	mutex_unlock(&notify_mutex);
-}
-
-static int chtls_inline_feature(struct tls_toe_device *dev)
-{
-	struct net_device *netdev;
-	struct chtls_dev *cdev;
-	int i;
-
-	cdev = to_chtls_dev(dev);
-
-	for (i = 0; i < cdev->lldi->nports; i++) {
-		netdev = cdev->ports[i];
-		if (netdev->features & NETIF_F_HW_TLS_RECORD)
-			return 1;
-	}
-	return 0;
-}
-
-static int chtls_create_hash(struct tls_toe_device *dev, struct sock *sk)
-{
-	struct chtls_dev *cdev = to_chtls_dev(dev);
-
-	if (sk->sk_state == TCP_LISTEN)
-		return chtls_start_listen(cdev, sk);
-	return 0;
-}
-
-static void chtls_destroy_hash(struct tls_toe_device *dev, struct sock *sk)
-{
-	struct chtls_dev *cdev = to_chtls_dev(dev);
-
-	if (sk->sk_state == TCP_LISTEN)
-		chtls_stop_listen(cdev, sk);
-}
-
-static void chtls_free_uld(struct chtls_dev *cdev)
-{
-	int i;
-
-	tls_toe_unregister_device(&cdev->tlsdev);
-	kvfree(cdev->kmap.addr);
-	idr_destroy(&cdev->hwtid_idr);
-	for (i = 0; i < (1 << RSPQ_HASH_BITS); i++)
-		kfree_skb(cdev->rspq_skb_cache[i]);
-	kfree(cdev->lldi);
-	kfree_skb(cdev->askb);
-	kfree(cdev);
-}
-
-static inline void chtls_dev_release(struct kref *kref)
-{
-	struct tls_toe_device *dev;
-	struct chtls_dev *cdev;
-	struct adapter *adap;
-
-	dev = container_of(kref, struct tls_toe_device, kref);
-	cdev = to_chtls_dev(dev);
-
-	/* Reset tls rx/tx stats */
-	adap = pci_get_drvdata(cdev->pdev);
-	atomic_set(&adap->chcr_stats.tls_pdu_tx, 0);
-	atomic_set(&adap->chcr_stats.tls_pdu_rx, 0);
-
-	chtls_free_uld(cdev);
-}
-
-static void chtls_register_dev(struct chtls_dev *cdev)
-{
-	struct tls_toe_device *tlsdev = &cdev->tlsdev;
-
-	strlcpy(tlsdev->name, "chtls", TLS_TOE_DEVICE_NAME_MAX);
-	strlcat(tlsdev->name, cdev->lldi->ports[0]->name,
-		TLS_TOE_DEVICE_NAME_MAX);
-	tlsdev->feature = chtls_inline_feature;
-	tlsdev->hash = chtls_create_hash;
-	tlsdev->unhash = chtls_destroy_hash;
-	tlsdev->release = chtls_dev_release;
-	kref_init(&tlsdev->kref);
-	tls_toe_register_device(tlsdev);
-	cdev->cdev_state = CHTLS_CDEV_STATE_UP;
-}
-
-static void process_deferq(struct work_struct *task_param)
-{
-	struct chtls_dev *cdev = container_of(task_param,
-				struct chtls_dev, deferq_task);
-	struct sk_buff *skb;
-
-	spin_lock_bh(&cdev->deferq.lock);
-	while ((skb = __skb_dequeue(&cdev->deferq)) != NULL) {
-		spin_unlock_bh(&cdev->deferq.lock);
-		DEFERRED_SKB_CB(skb)->handler(cdev, skb);
-		spin_lock_bh(&cdev->deferq.lock);
-	}
-	spin_unlock_bh(&cdev->deferq.lock);
-}
-
-static int chtls_get_skb(struct chtls_dev *cdev)
-{
-	cdev->askb = alloc_skb(sizeof(struct tcphdr), GFP_KERNEL);
-	if (!cdev->askb)
-		return -ENOMEM;
-
-	skb_put(cdev->askb, sizeof(struct tcphdr));
-	skb_reset_transport_header(cdev->askb);
-	memset(cdev->askb->data, 0, cdev->askb->len);
-	return 0;
-}
-
-static void *chtls_uld_add(const struct cxgb4_lld_info *info)
-{
-	struct cxgb4_lld_info *lldi;
-	struct chtls_dev *cdev;
-	int i, j;
-
-	cdev = kzalloc(sizeof(*cdev), GFP_KERNEL);
-	if (!cdev)
-		goto out;
-
-	lldi = kzalloc(sizeof(*lldi), GFP_KERNEL);
-	if (!lldi)
-		goto out_lldi;
-
-	if (chtls_get_skb(cdev))
-		goto out_skb;
-
-	*lldi = *info;
-	cdev->lldi = lldi;
-	cdev->pdev = lldi->pdev;
-	cdev->tids = lldi->tids;
-	cdev->ports = lldi->ports;
-	cdev->mtus = lldi->mtus;
-	cdev->tids = lldi->tids;
-	cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0]))
-			<< FW_VIID_PFN_S;
-
-	for (i = 0; i < (1 << RSPQ_HASH_BITS); i++) {
-		unsigned int size = 64 - sizeof(struct rsp_ctrl) - 8;
-
-		cdev->rspq_skb_cache[i] = __alloc_skb(size,
-						      gfp_any(), 0,
-						      lldi->nodeid);
-		if (unlikely(!cdev->rspq_skb_cache[i]))
-			goto out_rspq_skb;
-	}
-
-	idr_init(&cdev->hwtid_idr);
-	INIT_WORK(&cdev->deferq_task, process_deferq);
-	spin_lock_init(&cdev->listen_lock);
-	spin_lock_init(&cdev->idr_lock);
-	cdev->send_page_order = min_t(uint, get_order(32768),
-				      send_page_order);
-	cdev->max_host_sndbuf = 48 * 1024;
-
-	if (lldi->vr->key.size)
-		if (chtls_init_kmap(cdev, lldi))
-			goto out_rspq_skb;
-
-	mutex_lock(&cdev_mutex);
-	list_add_tail(&cdev->list, &cdev_list);
-	mutex_unlock(&cdev_mutex);
-
-	return cdev;
-out_rspq_skb:
-	for (j = 0; j < i; j++)
-		kfree_skb(cdev->rspq_skb_cache[j]);
-	kfree_skb(cdev->askb);
-out_skb:
-	kfree(lldi);
-out_lldi:
-	kfree(cdev);
-out:
-	return NULL;
-}
-
-static void chtls_free_all_uld(void)
-{
-	struct chtls_dev *cdev, *tmp;
-
-	mutex_lock(&cdev_mutex);
-	list_for_each_entry_safe(cdev, tmp, &cdev_list, list) {
-		if (cdev->cdev_state == CHTLS_CDEV_STATE_UP) {
-			list_del(&cdev->list);
-			kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release);
-		}
-	}
-	mutex_unlock(&cdev_mutex);
-}
-
-static int chtls_uld_state_change(void *handle, enum cxgb4_state new_state)
-{
-	struct chtls_dev *cdev = handle;
-
-	switch (new_state) {
-	case CXGB4_STATE_UP:
-		chtls_register_dev(cdev);
-		break;
-	case CXGB4_STATE_DOWN:
-		break;
-	case CXGB4_STATE_START_RECOVERY:
-		break;
-	case CXGB4_STATE_DETACH:
-		mutex_lock(&cdev_mutex);
-		list_del(&cdev->list);
-		mutex_unlock(&cdev_mutex);
-		kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release);
-		break;
-	default:
-		break;
-	}
-	return 0;
-}
-
-static struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl,
-					  const __be64 *rsp,
-					  u32 pktshift)
-{
-	struct sk_buff *skb;
-
-	/* Allocate space for cpl_pass_accpet_req which will be synthesized by
-	 * driver. Once driver synthesizes cpl_pass_accpet_req the skb will go
-	 * through the regular cpl_pass_accept_req processing in TOM.
-	 */
-	skb = alloc_skb(gl->tot_len + sizeof(struct cpl_pass_accept_req)
-			- pktshift, GFP_ATOMIC);
-	if (unlikely(!skb))
-		return NULL;
-	__skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req)
-		   - pktshift);
-	/* For now we will copy  cpl_rx_pkt in the skb */
-	skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_rx_pkt));
-	skb_copy_to_linear_data_offset(skb, sizeof(struct cpl_pass_accept_req)
-				       , gl->va + pktshift,
-				       gl->tot_len - pktshift);
-
-	return skb;
-}
-
-static int chtls_recv_packet(struct chtls_dev *cdev,
-			     const struct pkt_gl *gl, const __be64 *rsp)
-{
-	unsigned int opcode = *(u8 *)rsp;
-	struct sk_buff *skb;
-	int ret;
-
-	skb = copy_gl_to_skb_pkt(gl, rsp, cdev->lldi->sge_pktshift);
-	if (!skb)
-		return -ENOMEM;
-
-	ret = chtls_handlers[opcode](cdev, skb);
-	if (ret & CPL_RET_BUF_DONE)
-		kfree_skb(skb);
-
-	return 0;
-}
-
-static int chtls_recv_rsp(struct chtls_dev *cdev, const __be64 *rsp)
-{
-	unsigned long rspq_bin;
-	unsigned int opcode;
-	struct sk_buff *skb;
-	unsigned int len;
-	int ret;
-
-	len = 64 - sizeof(struct rsp_ctrl) - 8;
-	opcode = *(u8 *)rsp;
-
-	rspq_bin = hash_ptr((void *)rsp, RSPQ_HASH_BITS);
-	skb = cdev->rspq_skb_cache[rspq_bin];
-	if (skb && !skb_is_nonlinear(skb) &&
-	    !skb_shared(skb) && !skb_cloned(skb)) {
-		refcount_inc(&skb->users);
-		if (refcount_read(&skb->users) == 2) {
-			__skb_trim(skb, 0);
-			if (skb_tailroom(skb) >= len)
-				goto copy_out;
-		}
-		refcount_dec(&skb->users);
-	}
-	skb = alloc_skb(len, GFP_ATOMIC);
-	if (unlikely(!skb))
-		return -ENOMEM;
-
-copy_out:
-	__skb_put(skb, len);
-	skb_copy_to_linear_data(skb, rsp, len);
-	skb_reset_network_header(skb);
-	skb_reset_transport_header(skb);
-	ret = chtls_handlers[opcode](cdev, skb);
-
-	if (ret & CPL_RET_BUF_DONE)
-		kfree_skb(skb);
-	return 0;
-}
-
-static void chtls_recv(struct chtls_dev *cdev,
-		       struct sk_buff **skbs, const __be64 *rsp)
-{
-	struct sk_buff *skb = *skbs;
-	unsigned int opcode;
-	int ret;
-
-	opcode = *(u8 *)rsp;
-
-	__skb_push(skb, sizeof(struct rss_header));
-	skb_copy_to_linear_data(skb, rsp, sizeof(struct rss_header));
-
-	ret = chtls_handlers[opcode](cdev, skb);
-	if (ret & CPL_RET_BUF_DONE)
-		kfree_skb(skb);
-}
-
-static int chtls_uld_rx_handler(void *handle, const __be64 *rsp,
-				const struct pkt_gl *gl)
-{
-	struct chtls_dev *cdev = handle;
-	unsigned int opcode;
-	struct sk_buff *skb;
-
-	opcode = *(u8 *)rsp;
-
-	if (unlikely(opcode == CPL_RX_PKT)) {
-		if (chtls_recv_packet(cdev, gl, rsp) < 0)
-			goto nomem;
-		return 0;
-	}
-
-	if (!gl)
-		return chtls_recv_rsp(cdev, rsp);
-
-#define RX_PULL_LEN 128
-	skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
-	if (unlikely(!skb))
-		goto nomem;
-	chtls_recv(cdev, &skb, rsp);
-	return 0;
-
-nomem:
-	return -ENOMEM;
-}
-
-static int do_chtls_getsockopt(struct sock *sk, char __user *optval,
-			       int __user *optlen)
-{
-	struct tls_crypto_info crypto_info = { 0 };
-
-	crypto_info.version = TLS_1_2_VERSION;
-	if (copy_to_user(optval, &crypto_info, sizeof(struct tls_crypto_info)))
-		return -EFAULT;
-	return 0;
-}
-
-static int chtls_getsockopt(struct sock *sk, int level, int optname,
-			    char __user *optval, int __user *optlen)
-{
-	struct tls_context *ctx = tls_get_ctx(sk);
-
-	if (level != SOL_TLS)
-		return ctx->sk_proto->getsockopt(sk, level,
-						 optname, optval, optlen);
-
-	return do_chtls_getsockopt(sk, optval, optlen);
-}
-
-static int do_chtls_setsockopt(struct sock *sk, int optname,
-			       sockptr_t optval, unsigned int optlen)
-{
-	struct tls_crypto_info *crypto_info, tmp_crypto_info;
-	struct chtls_sock *csk;
-	int keylen;
-	int cipher_type;
-	int rc = 0;
-
-	csk = rcu_dereference_sk_user_data(sk);
-
-	if (sockptr_is_null(optval) || optlen < sizeof(*crypto_info)) {
-		rc = -EINVAL;
-		goto out;
-	}
-
-	rc = copy_from_sockptr(&tmp_crypto_info, optval, sizeof(*crypto_info));
-	if (rc) {
-		rc = -EFAULT;
-		goto out;
-	}
-
-	/* check version */
-	if (tmp_crypto_info.version != TLS_1_2_VERSION) {
-		rc = -ENOTSUPP;
-		goto out;
-	}
-
-	crypto_info = (struct tls_crypto_info *)&csk->tlshws.crypto_info;
-
-	/* GCM mode of AES supports 128 and 256 bit encryption, so
-	 * copy keys from user based on GCM cipher type.
-	 */
-	switch (tmp_crypto_info.cipher_type) {
-	case TLS_CIPHER_AES_GCM_128: {
-		/* Obtain version and type from previous copy */
-		crypto_info[0] = tmp_crypto_info;
-		/* Now copy the following data */
-		rc = copy_from_sockptr_offset((char *)crypto_info +
-				sizeof(*crypto_info),
-				optval, sizeof(*crypto_info),
-				sizeof(struct tls12_crypto_info_aes_gcm_128)
-				- sizeof(*crypto_info));
-
-		if (rc) {
-			rc = -EFAULT;
-			goto out;
-		}
-
-		keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
-		cipher_type = TLS_CIPHER_AES_GCM_128;
-		break;
-	}
-	case TLS_CIPHER_AES_GCM_256: {
-		crypto_info[0] = tmp_crypto_info;
-		rc = copy_from_sockptr_offset((char *)crypto_info +
-				sizeof(*crypto_info),
-				optval, sizeof(*crypto_info),
-				sizeof(struct tls12_crypto_info_aes_gcm_256)
-				- sizeof(*crypto_info));
-
-		if (rc) {
-			rc = -EFAULT;
-			goto out;
-		}
-
-		keylen = TLS_CIPHER_AES_GCM_256_KEY_SIZE;
-		cipher_type = TLS_CIPHER_AES_GCM_256;
-		break;
-	}
-	default:
-		rc = -EINVAL;
-		goto out;
-	}
-	rc = chtls_setkey(csk, keylen, optname, cipher_type);
-out:
-	return rc;
-}
-
-static int chtls_setsockopt(struct sock *sk, int level, int optname,
-			    sockptr_t optval, unsigned int optlen)
-{
-	struct tls_context *ctx = tls_get_ctx(sk);
-
-	if (level != SOL_TLS)
-		return ctx->sk_proto->setsockopt(sk, level,
-						 optname, optval, optlen);
-
-	return do_chtls_setsockopt(sk, optname, optval, optlen);
-}
-
-static struct cxgb4_uld_info chtls_uld_info = {
-	.name = DRV_NAME,
-	.nrxq = MAX_ULD_QSETS,
-	.ntxq = MAX_ULD_QSETS,
-	.rxq_size = 1024,
-	.add = chtls_uld_add,
-	.state_change = chtls_uld_state_change,
-	.rx_handler = chtls_uld_rx_handler,
-};
-
-void chtls_install_cpl_ops(struct sock *sk)
-{
-	if (sk->sk_family == AF_INET)
-		sk->sk_prot = &chtls_cpl_prot;
-	else
-		sk->sk_prot = &chtls_cpl_protv6;
-}
-
-static void __init chtls_init_ulp_ops(void)
-{
-	chtls_cpl_prot			= tcp_prot;
-	chtls_init_rsk_ops(&chtls_cpl_prot, &chtls_rsk_ops,
-			   &tcp_prot, PF_INET);
-	chtls_cpl_prot.close		= chtls_close;
-	chtls_cpl_prot.disconnect	= chtls_disconnect;
-	chtls_cpl_prot.destroy		= chtls_destroy_sock;
-	chtls_cpl_prot.shutdown		= chtls_shutdown;
-	chtls_cpl_prot.sendmsg		= chtls_sendmsg;
-	chtls_cpl_prot.sendpage		= chtls_sendpage;
-	chtls_cpl_prot.recvmsg		= chtls_recvmsg;
-	chtls_cpl_prot.setsockopt	= chtls_setsockopt;
-	chtls_cpl_prot.getsockopt	= chtls_getsockopt;
-#if IS_ENABLED(CONFIG_IPV6)
-	chtls_cpl_protv6		= chtls_cpl_prot;
-	chtls_init_rsk_ops(&chtls_cpl_protv6, &chtls_rsk_opsv6,
-			   &tcpv6_prot, PF_INET6);
-#endif
-}
-
-static int __init chtls_register(void)
-{
-	chtls_init_ulp_ops();
-	register_listen_notifier(&listen_notifier);
-	cxgb4_register_uld(CXGB4_ULD_TLS, &chtls_uld_info);
-	return 0;
-}
-
-static void __exit chtls_unregister(void)
-{
-	unregister_listen_notifier(&listen_notifier);
-	chtls_free_all_uld();
-	cxgb4_unregister_uld(CXGB4_ULD_TLS);
-}
-
-module_init(chtls_register);
-module_exit(chtls_unregister);
-
-MODULE_DESCRIPTION("Chelsio TLS Inline driver");
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Chelsio Communications");
-MODULE_VERSION(DRV_VERSION);