net: reorganize struct sock for better data locality
authorEric Dumazet <edumazet@google.com>
Sat, 3 Dec 2016 19:14:56 +0000 (11:14 -0800)
committerDavid S. Miller <davem@davemloft.net>
Mon, 5 Dec 2016 18:32:24 +0000 (13:32 -0500)
commit9115e8cd2a0c6eaaa900c462721f12e1d45f326c
treeb6fc9852f970033942f70a2265e1e4a3a076a549
parent12a59abc22d6664f7d3944f625ceefee92de8820
net: reorganize struct sock for better data locality

Group fields used in TX path, and keep some cache lines mostly read
to permit sharing among cpus.

Gained two 4 bytes holes on 64bit arches.

Added a place holder for tcp tsq_flags, next to sk_wmem_alloc
to speed up tcp_wfree() in the following patch.

I have not added ____cacheline_aligned_in_smp, this might be done later.
I prefer doing this once inet and tcp/udp sockets reorg is also done.

Tested with both TCP and UDP.

UDP receiver performance under flood increased by ~20 % :
Accessing sk_filter/sk_wq/sk_napi_id no longer stalls because sk_drops
was moved away from a critical cache line, now mostly read and shared.

/* --- cacheline 4 boundary (256 bytes) --- */
unsigned int               sk_napi_id;           /* 0x100   0x4 */
int                        sk_rcvbuf;            /* 0x104   0x4 */
struct sk_filter *         sk_filter;            /* 0x108   0x8 */
union {
struct socket_wq * sk_wq;                /*         0x8 */
struct socket_wq * sk_wq_raw;            /*         0x8 */
};                                               /* 0x110   0x8 */
struct xfrm_policy *       sk_policy[2];         /* 0x118  0x10 */
struct dst_entry *         sk_rx_dst;            /* 0x128   0x8 */
struct dst_entry *         sk_dst_cache;         /* 0x130   0x8 */
atomic_t                   sk_omem_alloc;        /* 0x138   0x4 */
int                        sk_sndbuf;            /* 0x13c   0x4 */
/* --- cacheline 5 boundary (320 bytes) --- */
int                        sk_wmem_queued;       /* 0x140   0x4 */
atomic_t                   sk_wmem_alloc;        /* 0x144   0x4 */
long unsigned int          sk_tsq_flags;         /* 0x148   0x8 */
struct sk_buff *           sk_send_head;         /* 0x150   0x8 */
struct sk_buff_head        sk_write_queue;       /* 0x158  0x18 */
__s32                      sk_peek_off;          /* 0x170   0x4 */
int                        sk_write_pending;     /* 0x174   0x4 */
long int                   sk_sndtimeo;          /* 0x178   0x8 */

Signed-off-by: Eric Dumazet <edumazet@google.com>
Tested-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
include/net/sock.h