#include <linux/pfkeyv2.h>
#include <linux/ipsec.h>
#include <linux/module.h>
+#include <linux/cache.h>
#include <asm/uaccess.h>
+#include "xfrm_hash.h"
+
struct sock *xfrm_nl;
EXPORT_SYMBOL(xfrm_nl);
/* Each xfrm_state may be linked to two tables:
1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl)
- 2. Hash table by daddr to find what SAs exist for given
+ 2. Hash table by (daddr,family,reqid) to find what SAs exist for given
destination/tunnel endpoint. (output)
*/
static DEFINE_SPINLOCK(xfrm_state_lock);
-#define XFRM_DST_HSIZE 1024
-
/* Hash table to find appropriate SA towards given target (endpoint
* of tunnel or destination of transport mode) allowed by selector.
*
* Main use is finding SA after policy selected tunnel or transport mode.
* Also, it can be used by ah/esp icmp error handler to find offending SA.
*/
-static struct hlist_head xfrm_state_bydst[XFRM_DST_HSIZE];
-static struct hlist_head xfrm_state_bysrc[XFRM_DST_HSIZE];
-static struct hlist_head xfrm_state_byspi[XFRM_DST_HSIZE];
-
-static __inline__
-unsigned __xfrm4_dst_hash(xfrm_address_t *addr)
+static struct hlist_head *xfrm_state_bydst __read_mostly;
+static struct hlist_head *xfrm_state_bysrc __read_mostly;
+static struct hlist_head *xfrm_state_byspi __read_mostly;
+static unsigned int xfrm_state_hmask __read_mostly;
+static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024;
+static unsigned int xfrm_state_num;
+static unsigned int xfrm_state_genid;
+
+static inline unsigned int xfrm_dst_hash(xfrm_address_t *daddr,
+ xfrm_address_t *saddr,
+ u32 reqid,
+ unsigned short family)
{
- unsigned h;
- h = ntohl(addr->a4);
- h = (h ^ (h>>16)) % XFRM_DST_HSIZE;
- return h;
+ return __xfrm_dst_hash(daddr, saddr, reqid, family, xfrm_state_hmask);
}
-static __inline__
-unsigned __xfrm6_dst_hash(xfrm_address_t *addr)
+static inline unsigned int xfrm_src_hash(xfrm_address_t *addr,
+ unsigned short family)
{
- unsigned h;
- h = ntohl(addr->a6[2]^addr->a6[3]);
- h = (h ^ (h>>16)) % XFRM_DST_HSIZE;
- return h;
+ return __xfrm_src_hash(addr, family, xfrm_state_hmask);
}
-static __inline__
-unsigned __xfrm4_src_hash(xfrm_address_t *addr)
+static inline unsigned int
+xfrm_spi_hash(xfrm_address_t *daddr, u32 spi, u8 proto, unsigned short family)
{
- return __xfrm4_dst_hash(addr);
+ return __xfrm_spi_hash(daddr, spi, proto, family, xfrm_state_hmask);
}
-static __inline__
-unsigned __xfrm6_src_hash(xfrm_address_t *addr)
+static void xfrm_hash_transfer(struct hlist_head *list,
+ struct hlist_head *ndsttable,
+ struct hlist_head *nsrctable,
+ struct hlist_head *nspitable,
+ unsigned int nhashmask)
{
- return __xfrm6_dst_hash(addr);
-}
+ struct hlist_node *entry, *tmp;
+ struct xfrm_state *x;
-static __inline__
-unsigned xfrm_src_hash(xfrm_address_t *addr, unsigned short family)
-{
- switch (family) {
- case AF_INET:
- return __xfrm4_src_hash(addr);
- case AF_INET6:
- return __xfrm6_src_hash(addr);
- }
- return 0;
-}
+ hlist_for_each_entry_safe(x, entry, tmp, list, bydst) {
+ unsigned int h;
-static __inline__
-unsigned xfrm_dst_hash(xfrm_address_t *addr, unsigned short family)
-{
- switch (family) {
- case AF_INET:
- return __xfrm4_dst_hash(addr);
- case AF_INET6:
- return __xfrm6_dst_hash(addr);
+ h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr,
+ x->props.reqid, x->props.family,
+ nhashmask);
+ hlist_add_head(&x->bydst, ndsttable+h);
+
+ h = __xfrm_src_hash(&x->props.saddr, x->props.family,
+ nhashmask);
+ hlist_add_head(&x->bysrc, nsrctable+h);
+
+ h = __xfrm_spi_hash(&x->id.daddr, x->id.spi, x->id.proto,
+ x->props.family, nhashmask);
+ hlist_add_head(&x->byspi, nspitable+h);
}
- return 0;
}
-static __inline__
-unsigned __xfrm4_spi_hash(xfrm_address_t *addr, u32 spi, u8 proto)
+static unsigned long xfrm_hash_new_size(void)
{
- unsigned h;
- h = ntohl(addr->a4^spi^proto);
- h = (h ^ (h>>10) ^ (h>>20)) % XFRM_DST_HSIZE;
- return h;
+ return ((xfrm_state_hmask + 1) << 1) *
+ sizeof(struct hlist_head);
}
-static __inline__
-unsigned __xfrm6_spi_hash(xfrm_address_t *addr, u32 spi, u8 proto)
-{
- unsigned h;
- h = ntohl(addr->a6[2]^addr->a6[3]^spi^proto);
- h = (h ^ (h>>10) ^ (h>>20)) % XFRM_DST_HSIZE;
- return h;
-}
+static DEFINE_MUTEX(hash_resize_mutex);
-static __inline__
-unsigned xfrm_spi_hash(xfrm_address_t *addr, u32 spi, u8 proto, unsigned short family)
+static void xfrm_hash_resize(void *__unused)
{
- switch (family) {
- case AF_INET:
- return __xfrm4_spi_hash(addr, spi, proto);
- case AF_INET6:
- return __xfrm6_spi_hash(addr, spi, proto);
+ struct hlist_head *ndst, *nsrc, *nspi, *odst, *osrc, *ospi;
+ unsigned long nsize, osize;
+ unsigned int nhashmask, ohashmask;
+ int i;
+
+ mutex_lock(&hash_resize_mutex);
+
+ nsize = xfrm_hash_new_size();
+ ndst = xfrm_hash_alloc(nsize);
+ if (!ndst)
+ goto out_unlock;
+ nsrc = xfrm_hash_alloc(nsize);
+ if (!nsrc) {
+ xfrm_hash_free(ndst, nsize);
+ goto out_unlock;
+ }
+ nspi = xfrm_hash_alloc(nsize);
+ if (!nspi) {
+ xfrm_hash_free(ndst, nsize);
+ xfrm_hash_free(nsrc, nsize);
+ goto out_unlock;
}
- return 0; /*XXX*/
+
+ spin_lock_bh(&xfrm_state_lock);
+
+ nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;
+ for (i = xfrm_state_hmask; i >= 0; i--)
+ xfrm_hash_transfer(xfrm_state_bydst+i, ndst, nsrc, nspi,
+ nhashmask);
+
+ odst = xfrm_state_bydst;
+ osrc = xfrm_state_bysrc;
+ ospi = xfrm_state_byspi;
+ ohashmask = xfrm_state_hmask;
+
+ xfrm_state_bydst = ndst;
+ xfrm_state_bysrc = nsrc;
+ xfrm_state_byspi = nspi;
+ xfrm_state_hmask = nhashmask;
+
+ spin_unlock_bh(&xfrm_state_lock);
+
+ osize = (ohashmask + 1) * sizeof(struct hlist_head);
+ xfrm_hash_free(odst, osize);
+ xfrm_hash_free(osrc, osize);
+ xfrm_hash_free(ospi, osize);
+
+out_unlock:
+ mutex_unlock(&hash_resize_mutex);
}
+static DECLARE_WORK(xfrm_hash_work, xfrm_hash_resize, NULL);
+
DECLARE_WAIT_QUEUE_HEAD(km_waitq);
EXPORT_SYMBOL(km_waitq);
static HLIST_HEAD(xfrm_state_gc_list);
static DEFINE_SPINLOCK(xfrm_state_gc_lock);
-static int xfrm_state_gc_flush_bundles;
-
int __xfrm_state_delete(struct xfrm_state *x);
static struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned short family);
static void xfrm_state_gc_destroy(struct xfrm_state *x)
{
- if (del_timer(&x->timer))
- BUG();
- if (del_timer(&x->rtimer))
- BUG();
+ del_timer_sync(&x->timer);
+ del_timer_sync(&x->rtimer);
kfree(x->aalg);
kfree(x->ealg);
kfree(x->calg);
struct hlist_node *entry, *tmp;
struct hlist_head gc_list;
- if (xfrm_state_gc_flush_bundles) {
- xfrm_state_gc_flush_bundles = 0;
- xfrm_flush_bundles();
- }
-
spin_lock_bh(&xfrm_state_gc_lock);
gc_list.first = xfrm_state_gc_list.first;
INIT_HLIST_HEAD(&xfrm_state_gc_list);
if (warn)
km_state_expired(x, 0, 0);
resched:
- if (next != LONG_MAX &&
- !mod_timer(&x->timer, jiffies + make_jiffies(next)))
- xfrm_state_hold(x);
+ if (next != LONG_MAX)
+ mod_timer(&x->timer, jiffies + make_jiffies(next));
+
goto out;
expired:
out:
spin_unlock(&x->lock);
- xfrm_state_put(x);
}
static void xfrm_replay_timer_handler(unsigned long data);
x->km.state = XFRM_STATE_DEAD;
spin_lock(&xfrm_state_lock);
hlist_del(&x->bydst);
- __xfrm_state_put(x);
hlist_del(&x->bysrc);
- __xfrm_state_put(x);
- if (x->id.spi) {
+ if (x->id.spi)
hlist_del(&x->byspi);
- __xfrm_state_put(x);
- }
+ xfrm_state_num--;
spin_unlock(&xfrm_state_lock);
- if (del_timer(&x->timer))
- __xfrm_state_put(x);
- if (del_timer(&x->rtimer))
- __xfrm_state_put(x);
-
- /* The number two in this test is the reference
- * mentioned in the comment below plus the reference
- * our caller holds. A larger value means that
- * there are DSTs attached to this xfrm_state.
- */
- if (atomic_read(&x->refcnt) > 2) {
- xfrm_state_gc_flush_bundles = 1;
- schedule_work(&xfrm_state_gc_work);
- }
/* All xfrm_state objects are created by xfrm_state_alloc.
* The xfrm_state_alloc call gives a reference, and that
int i;
spin_lock_bh(&xfrm_state_lock);
- for (i = 0; i < XFRM_DST_HSIZE; i++) {
+ for (i = 0; i <= xfrm_state_hmask; i++) {
struct hlist_node *entry;
struct xfrm_state *x;
restart:
return 0;
}
-static struct xfrm_state *__xfrm_state_lookup(xfrm_address_t *daddr, u32 spi, u8 proto, unsigned short family)
+static struct xfrm_state *__xfrm_state_lookup(xfrm_address_t *daddr, __be32 spi, u8 proto, unsigned short family)
{
unsigned int h = xfrm_spi_hash(daddr, spi, proto, family);
struct xfrm_state *x;
struct xfrm_policy *pol, int *err,
unsigned short family)
{
- unsigned int h = xfrm_dst_hash(daddr, family);
+ unsigned int h = xfrm_dst_hash(daddr, saddr, tmpl->reqid, family);
struct hlist_node *entry;
struct xfrm_state *x, *x0;
int acquire_in_progress = 0;
if (km_query(x, tmpl, pol) == 0) {
x->km.state = XFRM_STATE_ACQ;
hlist_add_head(&x->bydst, xfrm_state_bydst+h);
- xfrm_state_hold(x);
h = xfrm_src_hash(saddr, family);
hlist_add_head(&x->bysrc, xfrm_state_bysrc+h);
- xfrm_state_hold(x);
if (x->id.spi) {
h = xfrm_spi_hash(&x->id.daddr, x->id.spi, x->id.proto, family);
hlist_add_head(&x->byspi, xfrm_state_byspi+h);
- xfrm_state_hold(x);
}
x->lft.hard_add_expires_seconds = XFRM_ACQ_EXPIRES;
- xfrm_state_hold(x);
x->timer.expires = jiffies + XFRM_ACQ_EXPIRES*HZ;
add_timer(&x->timer);
} else {
static void __xfrm_state_insert(struct xfrm_state *x)
{
- unsigned h = xfrm_dst_hash(&x->id.daddr, x->props.family);
+ unsigned int h;
+ x->genid = ++xfrm_state_genid;
+
+ h = xfrm_dst_hash(&x->id.daddr, &x->props.saddr,
+ x->props.reqid, x->props.family);
hlist_add_head(&x->bydst, xfrm_state_bydst+h);
- xfrm_state_hold(x);
h = xfrm_src_hash(&x->props.saddr, x->props.family);
-
hlist_add_head(&x->bysrc, xfrm_state_bysrc+h);
- xfrm_state_hold(x);
if (xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY)) {
h = xfrm_spi_hash(&x->id.daddr, x->id.spi, x->id.proto,
x->props.family);
hlist_add_head(&x->byspi, xfrm_state_byspi+h);
- xfrm_state_hold(x);
}
- if (!mod_timer(&x->timer, jiffies + HZ))
- xfrm_state_hold(x);
-
- if (x->replay_maxage &&
- !mod_timer(&x->rtimer, jiffies + x->replay_maxage))
- xfrm_state_hold(x);
+ mod_timer(&x->timer, jiffies + HZ);
+ if (x->replay_maxage)
+ mod_timer(&x->rtimer, jiffies + x->replay_maxage);
wake_up(&km_waitq);
+
+ xfrm_state_num++;
+
+ if (x->bydst.next != NULL &&
+ (xfrm_state_hmask + 1) < xfrm_state_hashmax &&
+ xfrm_state_num > xfrm_state_hmask)
+ schedule_work(&xfrm_hash_work);
+}
+
+/* xfrm_state_lock is held */
+static void __xfrm_state_bump_genids(struct xfrm_state *xnew)
+{
+ unsigned short family = xnew->props.family;
+ u32 reqid = xnew->props.reqid;
+ struct xfrm_state *x;
+ struct hlist_node *entry;
+ unsigned int h;
+
+ h = xfrm_dst_hash(&xnew->id.daddr, &xnew->props.saddr, reqid, family);
+ hlist_for_each_entry(x, entry, xfrm_state_bydst+h, bydst) {
+ if (x->props.family == family &&
+ x->props.reqid == reqid &&
+ !xfrm_addr_cmp(&x->id.daddr, &xnew->id.daddr, family) &&
+ !xfrm_addr_cmp(&x->props.saddr, &xnew->props.saddr, family))
+ x->genid = xfrm_state_genid;
+ }
}
void xfrm_state_insert(struct xfrm_state *x)
{
spin_lock_bh(&xfrm_state_lock);
+ __xfrm_state_bump_genids(x);
__xfrm_state_insert(x);
spin_unlock_bh(&xfrm_state_lock);
-
- xfrm_flush_all_bundles();
}
EXPORT_SYMBOL(xfrm_state_insert);
/* xfrm_state_lock is held */
static struct xfrm_state *__find_acq_core(unsigned short family, u8 mode, u32 reqid, u8 proto, xfrm_address_t *daddr, xfrm_address_t *saddr, int create)
{
- unsigned int h = xfrm_dst_hash(daddr, family);
+ unsigned int h = xfrm_dst_hash(daddr, saddr, reqid, family);
struct hlist_node *entry;
struct xfrm_state *x;
xfrm_state_hold(x);
x->timer.expires = jiffies + XFRM_ACQ_EXPIRES*HZ;
add_timer(&x->timer);
- xfrm_state_hold(x);
hlist_add_head(&x->bydst, xfrm_state_bydst+h);
h = xfrm_src_hash(saddr, family);
- xfrm_state_hold(x);
hlist_add_head(&x->bysrc, xfrm_state_bysrc+h);
wake_up(&km_waitq);
}
x->id.proto,
&x->id.daddr, &x->props.saddr, 0);
+ __xfrm_state_bump_genids(x);
__xfrm_state_insert(x);
err = 0;
out:
spin_unlock_bh(&xfrm_state_lock);
- if (!err)
- xfrm_flush_all_bundles();
-
if (x1) {
xfrm_state_delete(x1);
xfrm_state_put(x1);
memcpy(&x1->lft, &x->lft, sizeof(x1->lft));
x1->km.dying = 0;
- if (!mod_timer(&x1->timer, jiffies + HZ))
- xfrm_state_hold(x1);
+ mod_timer(&x1->timer, jiffies + HZ);
if (x1->curlft.use_time)
xfrm_state_check_expire(x1);
if (x->curlft.bytes >= x->lft.hard_byte_limit ||
x->curlft.packets >= x->lft.hard_packet_limit) {
x->km.state = XFRM_STATE_EXPIRED;
- if (!mod_timer(&x->timer, jiffies))
- xfrm_state_hold(x);
+ mod_timer(&x->timer, jiffies);
return -EINVAL;
}
EXPORT_SYMBOL(xfrm_state_check);
struct xfrm_state *
-xfrm_state_lookup(xfrm_address_t *daddr, u32 spi, u8 proto,
+xfrm_state_lookup(xfrm_address_t *daddr, __be32 spi, u8 proto,
unsigned short family)
{
struct xfrm_state *x;
{
int i;
- for (i = 0; i < XFRM_DST_HSIZE; i++) {
+ for (i = 0; i <= xfrm_state_hmask; i++) {
struct hlist_node *entry;
struct xfrm_state *x;
EXPORT_SYMBOL(xfrm_get_acqseq);
void
-xfrm_alloc_spi(struct xfrm_state *x, u32 minspi, u32 maxspi)
+xfrm_alloc_spi(struct xfrm_state *x, __be32 minspi, __be32 maxspi)
{
- u32 h;
+ unsigned int h;
struct xfrm_state *x0;
if (x->id.spi)
x->id.spi = minspi;
} else {
u32 spi = 0;
- minspi = ntohl(minspi);
- maxspi = ntohl(maxspi);
- for (h=0; h<maxspi-minspi+1; h++) {
- spi = minspi + net_random()%(maxspi-minspi+1);
+ u32 low = ntohl(minspi);
+ u32 high = ntohl(maxspi);
+ for (h=0; h<high-low+1; h++) {
+ spi = low + net_random()%(high-low+1);
x0 = xfrm_state_lookup(&x->id.daddr, htonl(spi), x->id.proto, x->props.family);
if (x0 == NULL) {
x->id.spi = htonl(spi);
spin_lock_bh(&xfrm_state_lock);
h = xfrm_spi_hash(&x->id.daddr, x->id.spi, x->id.proto, x->props.family);
hlist_add_head(&x->byspi, xfrm_state_byspi+h);
- xfrm_state_hold(x);
spin_unlock_bh(&xfrm_state_lock);
wake_up(&km_waitq);
}
int err = 0;
spin_lock_bh(&xfrm_state_lock);
- for (i = 0; i < XFRM_DST_HSIZE; i++) {
+ for (i = 0; i <= xfrm_state_hmask; i++) {
hlist_for_each_entry(x, entry, xfrm_state_bydst+i, bydst) {
if (xfrm_id_proto_match(x->id.proto, proto))
count++;
goto out;
}
- for (i = 0; i < XFRM_DST_HSIZE; i++) {
+ for (i = 0; i <= xfrm_state_hmask; i++) {
hlist_for_each_entry(x, entry, xfrm_state_bydst+i, bydst) {
if (!xfrm_id_proto_match(x->id.proto, proto))
continue;
km_state_notify(x, &c);
if (x->replay_maxage &&
- !mod_timer(&x->rtimer, jiffies + x->replay_maxage)) {
- xfrm_state_hold(x);
+ !mod_timer(&x->rtimer, jiffies + x->replay_maxage))
x->xflags &= ~XFRM_TIME_DEFER;
- }
}
EXPORT_SYMBOL(xfrm_replay_notify);
}
spin_unlock(&x->lock);
- xfrm_state_put(x);
}
int xfrm_replay_check(struct xfrm_state *x, u32 seq)
void __init xfrm_state_init(void)
{
- int i;
+ unsigned int sz;
+
+ sz = sizeof(struct hlist_head) * 8;
+
+ xfrm_state_bydst = xfrm_hash_alloc(sz);
+ xfrm_state_bysrc = xfrm_hash_alloc(sz);
+ xfrm_state_byspi = xfrm_hash_alloc(sz);
+ if (!xfrm_state_bydst || !xfrm_state_bysrc || !xfrm_state_byspi)
+ panic("XFRM: Cannot allocate bydst/bysrc/byspi hashes.");
+ xfrm_state_hmask = ((sz / sizeof(struct hlist_head)) - 1);
- for (i=0; i<XFRM_DST_HSIZE; i++) {
- INIT_HLIST_HEAD(&xfrm_state_bydst[i]);
- INIT_HLIST_HEAD(&xfrm_state_bysrc[i]);
- INIT_HLIST_HEAD(&xfrm_state_byspi[i]);
- }
INIT_WORK(&xfrm_state_gc_work, xfrm_state_gc_task, NULL);
}
projects / ~andy / linux / blobdiff