xen-project/xen
1
0 Fork 1
mirror of https://github.com/xen-project/xen.git synced 2026-10-28 18:21:39 +01:00
Code Issues Activity
Files
8e2c94de2619693197d9fbdbec5220e18b77baec
xen/tools/memshr/bidir-hash.c
T
Ian Campbell 443701ef0c Replace FSF street address with canonical URL 
As recommended in http://www.gnu.org/licenses/gpl-howto.en.html.

This is the result of:
$ git grep -El Mass\|Temple\|Franklin | xargs ./fsf.pl

Where fsf.pl is:
    #!/usr/bin/perl -w -pi.bak -0777
    my $repl = 'If not, see <http://www.gnu.org/licenses/>.';
    my $br = qr/(?:\s*\n\s*(?:[\*\#]|\/\/|\.\\" )?\s*|\s+)/;

    my $inwt = qr/[Ii]f${br}not,${br}write${br}(?:to${br})?the${br}Free${br}Software${br}Foundation,(?:${br}Inc\.,)?/;

    my $mass = qr/675${br}Mass${br}Ave,?${br}Cambridge,?${br}MA${br}02139,?${br}USA,?\.?/;
    my $franklin = qr/51${br}Franklin${br}St(?:reet)?(?:,${br}| - )Fifth${br}Floor,?${br}Boston,?${br}MA,?${br}02110-1301,?${br}USA,?\.?/;
    my $temple = qr/59${br}Temple${br}Place(?:,${br}| - )Suite${br}330,?${br}Boston,?${br}MA,?${br}021110?-1307,?${br}USA,?\.?/;

    s|$inwt$br$mass|$repl|m;
    s|$inwt$br$franklin|$repl|m;
    s|$inwt$br$temple|$repl|m;

The only remaining mentions of these addresses are in COPYING files which I
haven't touched.

Some of the changed files are imports from elsewhere, however
filtering them out is tricky, I think it is tolerable to have these
files be modified here and then perhaps reverted on the next sync,
since it's only 1-2 lines and obvious what is going on.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
2015-07-29 11:14:07 +01:00

1356 lines
44 KiB
C
Raw Blame History

/******************************************************************************
*
* Copyright (c) 2009 Citrix Systems, Inc. (Grzegorz Milos)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; If not, see <http://www.gnu.org/licenses/>.
*/
#include <assert.h>
#include <errno.h>
#include <math.h>
#include <pthread.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include "bidir-hash.h"
static const uint32_t hash_sizes[] = {53, 97, 193, 389, 769, 1543, 3079, 6151,
12289, 24593, 49157, 98317, 196613, 393241, 786433, 1572869, 3145739,
6291469, 12582917, 25165843, 50331653, 100663319, 201326611, 402653189,
805306457, 1610612741};
static const uint16_t hash_sizes_len =
sizeof(hash_sizes)/sizeof(hash_sizes[0]);
static const float hash_max_load_fact = 0.65;
static const float hash_min_load_fact = 0.10;
/* How many buckets will be covered by a single rw lock */
#define BUCKETS_PER_LOCK 64
#define nr_locks(_nr_buckets) (1 + (_nr_buckets) / BUCKETS_PER_LOCK)
#define HASH_LOCK \
pthread_rwlock_t hash_lock
#define BUCKET_LOCK \
pthread_rwlock_t bucket_lock
struct hash_entry
{
__k_t key;
__v_t value;
/* This structure will belong to two buckets, one in each hash table */
struct hash_entry *key_next;
struct hash_entry *value_next;
};
struct bucket
{
struct hash_entry *hash_entry;
};
struct bucket_lock
{
BUCKET_LOCK;
};
struct __hash
{
int lock_alive;
HASH_LOCK; /* protects:
* *_tab, tab_size, size_idx, *_load
* (all writes with wrlock)
*/
uint32_t nr_ent; /* # entries held in hashtables */
struct bucket *key_tab; /* forward mapping hashtable */
struct bucket *value_tab; /* backward mapping hashtable */
struct bucket_lock *key_lock_tab; /* key table bucket locks */
struct bucket_lock *value_lock_tab; /* value table bucket locks */
uint32_t tab_size; /* # buckets is hashtables */
uint16_t size_idx; /* table size index */
uint32_t max_load; /* # entries before rehash */
uint32_t min_load; /* # entries before rehash */
};
struct __hash *__hash_init (struct __hash *h, uint32_t min_size);
int __key_lookup (struct __hash *h, __k_t k, __v_t *vp);
int __value_lookup(struct __hash *h, __v_t v, __k_t *kp);
int __insert (struct __hash *h, __k_t k, __v_t v);
int __key_remove (struct __hash *h, __k_t k, __v_t *vp);
int __value_remove(struct __hash *h, __v_t v, __k_t *kp);
int __hash_destroy(struct __hash *h,
void (*entry_consumer)(__k_t k, __v_t v, void *p),
void *d);
int __hash_iterator(struct __hash *h,
int (*entry_consumer)(__k_t k, __v_t v, void *p),
void *d);
static void hash_resize(struct __hash *h);
#if defined(__arm__)
static inline void atomic_inc(uint32_t *v)
{
unsigned long tmp;
int result;
__asm__ __volatile__("@ atomic_inc\n"
"1: ldrex %0, [%3]\n"
" add %0, %0, #1\n"
" strex %1, %0, [%3]\n"
" teq %1, #0\n"
" bne 1b"
: "=&r" (result), "=&r" (tmp), "+Qo" (*v)
: "r" (v)
: "cc");
}
static inline void atomic_dec(uint32_t *v)
{
unsigned long tmp;
int result;
__asm__ __volatile__("@ atomic_dec\n"
"1: ldrex %0, [%3]\n"
" sub %0, %0, #1\n"
" strex %1, %0, [%3]\n"
" teq %1, #0\n"
" bne 1b"
: "=&r" (result), "=&r" (tmp), "+Qo" (*v)
: "r" (v)
: "cc");
}
#elif defined(__aarch64__)
static inline void atomic_inc(uint32_t *v)
{
unsigned long tmp;
int result;
asm volatile("// atomic_inc\n"
"1: ldxr %w0, [%3]\n"
" add %w0, %w0, #1\n"
" stxr %w1, %w0, [%3]\n"
" cbnz %w1, 1b"
: "=&r" (result), "=&r" (tmp), "+o" (v)
: "r" (v)
: "cc");
}
static inline void atomic_dec(uint32_t *v)
{
unsigned long tmp;
int result;
asm volatile("// atomic_dec\n"
"1: ldxr %w0, [%3]\n"
" sub %w0, %w0, #1\n"
" stxr %w1, %w0, [%3]\n"
" cbnz %w1, 1b"
: "=&r" (result), "=&r" (tmp), "+o" (v)
: "r" (v)
: "cc");
}
#else /* __x86__ */
static inline void atomic_inc(uint32_t *v)
{
asm volatile (
"lock ; incl %0"
: "=m" (*(volatile uint32_t *)v)
: "m" (*(volatile uint32_t *)v) );
}
static inline void atomic_dec(uint32_t *v)
{
asm volatile (
"lock ; decl %0"
: "=m" (*(volatile uint32_t *)v)
: "m" (*(volatile uint32_t *)v) );
}
#endif
#ifdef BIDIR_USE_STDMALLOC
static void* alloc_entry(struct __hash *h, int size)
{
return malloc(size);
}
static void alloc_buckets(struct __hash *h,
int nr_buckets,
struct bucket **bucket_tab,
struct bucket_lock **bucket_locks_tab)
{
*bucket_tab = (struct bucket*)
malloc(nr_buckets * sizeof(struct bucket));
*bucket_locks_tab = (struct bucket_lock*)
malloc(nr_locks(nr_buckets) * sizeof(struct bucket_lock));
}
static void free_entry(struct __hash *h, void *p)
{
free(p);
}
static void free_buckets(struct __hash *h,
struct bucket *buckets,
struct bucket_lock *bucket_locks)
{
free(buckets);
free(bucket_locks);
}
static int max_entries(struct __hash *h)
{
/* There are no explicit restrictions to how many entries we can store */
return -1;
}
#else
/*****************************************************************************/
/** Memory allocator for shared memory region **/
/*****************************************************************************/
#define SHM_TABLE_SLOTS 4
struct shm_hdr
{
int hash_allocated;
pthread_mutex_t mutex;
int free_tab_slots[SHM_TABLE_SLOTS];
unsigned long freelist_offset;
unsigned long entries_offset;
unsigned long nr_entries;
unsigned long tabs_offset;
unsigned long max_tab_size;
unsigned long max_lock_tab_size;
struct __hash hash;
};
static unsigned long get_shm_baddr(void *hdr)
{
return ((unsigned long)hdr - offsetof(struct shm_hdr, hash));
}
/** Locations of various structures/memory areas **/
static struct shm_hdr* get_shm_hdr(struct __hash *h)
{
return (struct shm_hdr *)
((unsigned long)h - offsetof(struct shm_hdr, hash));
}
static uint32_t* get_shm_freelist(struct shm_hdr *hdr)
{
unsigned long shm_baddr = (unsigned long)hdr;
return ((uint32_t *)(shm_baddr + hdr->freelist_offset));
}
static struct hash_entry* get_shm_entries(struct shm_hdr *hdr)
{
unsigned long shm_baddr = (unsigned long)hdr;
return ((struct hash_entry *)(shm_baddr + hdr->entries_offset));
}
static struct bucket* get_shm_tab(struct shm_hdr *hdr, int i)
{
unsigned long shm_baddr = (unsigned long)hdr;
return ((struct bucket *)
((shm_baddr + hdr->tabs_offset) +
i * (hdr->max_tab_size + hdr->max_lock_tab_size)));
}
static struct bucket_lock* get_shm_lock_tab(struct shm_hdr *hdr, int i)
{
unsigned long shm_baddr = (unsigned long)hdr;
return ((struct bucket_lock *)
((shm_baddr + hdr->tabs_offset) +
i * (hdr->max_tab_size + hdr->max_lock_tab_size) +
hdr->max_tab_size));
}
static int get_shm_slot(struct shm_hdr *hdr, void *p)
{
unsigned long shm_baddr = (unsigned long)hdr;
return ((unsigned long)p - (shm_baddr + hdr->tabs_offset)) /
(hdr->max_tab_size + hdr->max_lock_tab_size);
}
/* Shared memory allocator locks */
static int shm_mutex_init(struct shm_hdr *h)
{
int ret;
pthread_mutexattr_t _attr;
ret = pthread_mutexattr_init(&_attr);
if(ret == 0)
ret = pthread_mutexattr_setpshared(&_attr, PTHREAD_PROCESS_SHARED);
if(ret == 0)
ret = pthread_mutex_init(&h->mutex, &_attr);
if(ret == 0)
ret = pthread_mutexattr_destroy(&_attr);
return ret;
};
static int shm_mutex_lock(struct shm_hdr *h)
{
return pthread_mutex_lock(&h->mutex);
}
static int shm_mutex_unlock(struct shm_hdr *h)
{
return pthread_mutex_unlock(&h->mutex);
}
/* Shared memory allocator freelist */
static void shm_add_to_freelist(struct shm_hdr *hdr, uint32_t sl)
{
uint32_t *freelist = get_shm_freelist(hdr);
shm_mutex_lock(hdr);
freelist[sl+1] = freelist[0];
freelist[0] = sl;
shm_mutex_unlock(hdr);
}
static uint32_t shm_get_from_freelist(struct shm_hdr *hdr)
{
uint32_t *freelist = get_shm_freelist(hdr);
uint32_t slot;
shm_mutex_lock(hdr);
slot = freelist[0];
freelist[0] = freelist[slot+1];
shm_mutex_unlock(hdr);
return (slot == 0 ? -1 : slot);
}
#define SHM_ALLOC_MAIN(_n)
static unsigned long shm_init_offsets(
struct shm_hdr *hdr, int nr_entries)
{
hdr->freelist_offset = sizeof(struct shm_hdr);
/* Freelist needs one extra slot in the array for the freelist head */
hdr->entries_offset =
hdr->freelist_offset + (nr_entries + 1) * sizeof(uint32_t);
hdr->nr_entries = nr_entries;
hdr->tabs_offset = hdr->entries_offset +
nr_entries * sizeof(struct hash_entry);
/* We want to allocate table 1.5 larger than the number of entries
we want to hold in it */
hdr->max_tab_size =
(nr_entries * 3 / 2) * sizeof(struct bucket);
hdr->max_lock_tab_size =
nr_locks(hdr->max_tab_size) * sizeof(struct bucket_lock);
return hdr->tabs_offset + (hdr->max_tab_size + hdr->max_lock_tab_size) * 4;
}
struct __hash* __shm_hash_init(unsigned long shm_baddr, unsigned long shm_size)
{
uint32_t i;
struct shm_hdr *hdr;
/* Some sanity checks */
hdr = (struct shm_hdr *)shm_baddr;
memset(hdr, 0, sizeof(struct shm_hdr));
/* Find the maximum number of entries we can store in the given shm_size */
for(i=1; shm_init_offsets(hdr, i) < shm_size; i++){};
shm_init_offsets(hdr, (i-1));
memset(get_shm_freelist(hdr), 0,
(hdr->nr_entries + 1) * sizeof(uint32_t));
if(shm_mutex_init(hdr) != 0)
return NULL;
for(i=0; i<hdr->nr_entries; i++)
shm_add_to_freelist(hdr, i);
for(i=0; i<SHM_TABLE_SLOTS; i++)
hdr->free_tab_slots[i] = 1;
shm_mutex_lock(hdr);
assert(!hdr->hash_allocated);
hdr->hash_allocated = 1;
shm_mutex_unlock(hdr);
return __hash_init(&hdr->hash, 1000);
}
struct __hash* __shm_hash_get(unsigned long shm_baddr)
{
struct shm_hdr *hdr = (struct shm_hdr *)shm_baddr;
return (hdr->hash_allocated ? &hdr->hash : NULL);
}
static void* alloc_entry(struct __hash *h, int size)
{
struct shm_hdr *hdr = get_shm_hdr(h);
uint32_t slot = shm_get_from_freelist(hdr);
assert(size == sizeof(struct hash_entry));
if(slot == -1)
return NULL;
return (get_shm_entries(hdr) + slot);
}
static void alloc_buckets(struct __hash *h,
int nr_buckets,
struct bucket **buckets_tab,
struct bucket_lock **bucket_locks_tab)
{
struct shm_hdr *hdr = get_shm_hdr(h);
int free_slot;
*buckets_tab = NULL;
*bucket_locks_tab = NULL;
if(((nr_buckets * sizeof(struct bucket)) > hdr->max_tab_size) ||
((nr_locks(nr_buckets) * sizeof(struct bucket_lock)) >
hdr->max_lock_tab_size))
return;
shm_mutex_lock(hdr);
for(free_slot=0; free_slot<SHM_TABLE_SLOTS; free_slot++)
if(hdr->free_tab_slots[free_slot])
break;
if(free_slot == SHM_TABLE_SLOTS)
{
shm_mutex_unlock(hdr);
return;
}
hdr->free_tab_slots[free_slot] = 0;
shm_mutex_unlock(hdr);
*buckets_tab = get_shm_tab(hdr, free_slot);
*bucket_locks_tab = get_shm_lock_tab(hdr, free_slot);
}
static void free_entry(struct __hash *h, void *p)
{
struct shm_hdr *hdr = get_shm_hdr(h);
uint32_t slot;
slot = ((uint32_t)((struct hash_entry *)p -
get_shm_entries(hdr)));
shm_add_to_freelist(hdr, slot);
}
static void free_buckets(struct __hash *h,
struct bucket *buckets,
struct bucket_lock *bucket_locks)
{
struct shm_hdr *hdr = get_shm_hdr(h);
int slot;
if(!buckets || !bucket_locks)
{
assert(!buckets && !bucket_locks);
return;
}
slot = get_shm_slot(hdr, buckets);
assert(slot < SHM_TABLE_SLOTS);
assert((char *)bucket_locks == (char *)buckets + hdr->max_tab_size);
shm_mutex_lock(hdr);
assert(hdr->free_tab_slots[slot] == 0);
hdr->free_tab_slots[slot] = 1;
shm_mutex_unlock(hdr);
}
static int max_entries(struct __hash *h)
{
struct shm_hdr *hdr = get_shm_hdr(h);
return hdr->nr_entries;
}
#endif /* !BIDIR_USE_STDMALLOC */
/* The structures may be stored in shared memory region, with base address */
/* stored in shm_base_addr. All the pointers in the above structures need */
/* to be relative to this base address (otherwise they would not make */
/* sense to other processes). Bellow accessor functions are used to */
/* convert between canonical (base address relative) and local addresses. */
/* C2L stands for CANONICAL_TO_LOCAL, and vice versa */
#define C2L(_h, _p) ((typeof(_p))((unsigned long)(_p) + \
get_shm_baddr(_h)))
#define L2C(_h, _p) ((typeof(_p))((unsigned long)(_p) - \
get_shm_baddr(_h)))
#define HASH_LOCK_INIT(_h) ({ \
int _ret; \
pthread_rwlockattr_t _attr; \
\
h->lock_alive = 1; \
_ret = pthread_rwlockattr_init(&_attr); \
if(_ret == 0) \
_ret = pthread_rwlockattr_setpshared(&_attr, \
PTHREAD_PROCESS_SHARED); \
if(_ret == 0) \
_ret = pthread_rwlock_init(&(_h)->hash_lock, &_attr); \
if(_ret == 0) \
_ret = pthread_rwlockattr_destroy(&_attr); \
\
_ret; \
})
#define HASH_LOCK_RDLOCK(_h) ({ \
int _ret; \
\
if(!_h->lock_alive) _ret = ENOLCK; \
else \
{ \
struct timespec _ts; \
/* 10s timeout, long but ~matches disk spin-up times */ \
_ts.tv_sec = time(NULL) + 10; \
_ts.tv_nsec = 0; \
_ret = pthread_rwlock_timedrdlock(&(_h)->hash_lock, &_ts); \
if(_ret == ETIMEDOUT) _h->lock_alive = 0; \
} \
_ret; \
})
#define HASH_LOCK_RDUNLOCK(_h) \
pthread_rwlock_unlock(&(_h)->hash_lock)
#define HASH_LOCK_WRLOCK(_h) ({ \
int _ret; \
\
if(!_h->lock_alive) _ret = ENOLCK; \
else \
{ \
struct timespec _ts; \
_ts.tv_sec = time(NULL) + 10; \
_ts.tv_nsec = 0UL; \
_ret = pthread_rwlock_timedwrlock(&(_h)->hash_lock, &_ts); \
if(_ret == ETIMEDOUT) _h->lock_alive = 0; \
} \
_ret; \
})
#define HASH_LOCK_TRYWRLOCK(_h) ({ \
int _ret = (_h->lock_alive ? \
pthread_rwlock_trywrlock(&(_h)->hash_lock) : \
ENOLCK); \
_ret; \
})
#define HASH_LOCK_WRUNLOCK(_h) \
pthread_rwlock_unlock(&(_h)->hash_lock)
#define BUCKET_LOCK_INIT(_h, _b) ({ \
int _ret; \
pthread_rwlockattr_t _attr; \
\
_ret = pthread_rwlockattr_init(&_attr); \
if(_ret == 0) \
_ret = pthread_rwlockattr_setpshared(&_attr, \
PTHREAD_PROCESS_SHARED); \
if(_ret == 0) \
_ret = pthread_rwlock_init(&(_b)->bucket_lock, &_attr); \
if(_ret == 0) \
_ret = pthread_rwlockattr_destroy(&_attr); \
\
_ret; \
})
#define BUCKET_LOCK_RDLOCK(_h, _lock_tab, _idx) ({ \
int _ret; \
struct timespec _ts; \
struct bucket_lock *_lock = &(_lock_tab)[(_idx) / BUCKETS_PER_LOCK]; \
\
_ts.tv_sec = time(NULL) + 10; \
_ts.tv_nsec = 0; \
_ret = pthread_rwlock_timedrdlock(&(_lock)->bucket_lock, &_ts); \
if(_ret == ETIMEDOUT) (_h)->lock_alive = 0; \
_ret; \
})
#define BUCKET_LOCK_RDUNLOCK(_h, _lock_tab, _idx) ({ \
struct bucket_lock *_lock = &(_lock_tab)[(_idx) / BUCKETS_PER_LOCK]; \
pthread_rwlock_unlock(&(_lock)->bucket_lock); \
})
#define BUCKET_LOCK_WRLOCK(_h, _lock_tab, _idx) ({ \
int _ret; \
struct timespec _ts; \
struct bucket_lock *_lock = &(_lock_tab)[(_idx) / BUCKETS_PER_LOCK]; \
\
_ts.tv_sec = time(NULL) + 10; \
_ts.tv_nsec = 0; \
_ret = pthread_rwlock_timedwrlock(&(_lock)->bucket_lock, &_ts); \
if(_ret == ETIMEDOUT) (_h)->lock_alive = 0; \
_ret; \
})
#define BUCKET_LOCK_WRUNLOCK(_h, _lock_tab, _idx) ({ \
struct bucket_lock *_lock = &(_lock_tab)[(_idx) / BUCKETS_PER_LOCK]; \
pthread_rwlock_unlock(&(_lock)->bucket_lock); \
})
#define TWO_BUCKETS_LOCK_WRLOCK(_h, _blt1, _idx1, _blt2, _idx2) ({ \
int _ret; \
pthread_rwlock_t *_l1, *_l2; \
struct timespec _ts; \
struct bucket_lock *_bl1 = &(_blt1)[(_idx1) / BUCKETS_PER_LOCK]; \
struct bucket_lock *_bl2 = &(_blt2)[(_idx2) / BUCKETS_PER_LOCK]; \
\
assert((_bl1) != (_bl2)); \
if((_bl1) < (_bl2)) \
{ \
_l1 = &(_bl1)->bucket_lock; \
_l2 = &(_bl2)->bucket_lock; \
} \
else \
{ \
_l1 = &(_bl2)->bucket_lock; \
_l2 = &(_bl1)->bucket_lock; \
} \
_ts.tv_sec = time(NULL) + 10; \
_ts.tv_nsec = 0; \
_ret = pthread_rwlock_timedwrlock(_l1, &_ts); \
_ts.tv_sec = time(NULL) + 10; \
_ts.tv_nsec = 0; \
if(_ret == 0) \
_ret = pthread_rwlock_timedwrlock(_l2, &_ts); \
if(_ret == ETIMEDOUT) (_h)->lock_alive = 0; \
\
_ret; \
})
#define TWO_BUCKETS_LOCK_WRUNLOCK(_h, _blt1, _idx1, _blt2, _idx2) ({ \
int _ret; \
struct bucket_lock *_bl1 = &(_blt1)[(_idx1) / BUCKETS_PER_LOCK]; \
struct bucket_lock *_bl2 = &(_blt2)[(_idx2) / BUCKETS_PER_LOCK]; \
\
_ret = pthread_rwlock_unlock(&(_bl1)->bucket_lock); \
if(_ret == 0) \
_ret = pthread_rwlock_unlock(&(_bl2)->bucket_lock); \
\
_ret; \
})
static uint32_t hash_to_idx(struct __hash *h, uint32_t hash)
{
return (hash % h->tab_size);
}
static void alloc_tab(struct __hash *h,
int size,
struct bucket **buckets_tab,
struct bucket_lock **bucket_locks_tab)
{
int i;
alloc_buckets(h, size, buckets_tab, bucket_locks_tab);
if(!(*buckets_tab) || !(*bucket_locks_tab))
goto error_out;
memset(*buckets_tab, 0, size * sizeof(struct bucket));
memset(*bucket_locks_tab, 0, nr_locks(size) * sizeof(struct bucket_lock));
for(i=0; i<nr_locks(size); i++)
if(BUCKET_LOCK_INIT(h, *bucket_locks_tab + i) != 0)
goto error_out;
return;
error_out:
free_buckets(h, *buckets_tab, *bucket_locks_tab);
*buckets_tab = NULL;
*bucket_locks_tab = NULL;
return;
}
struct __hash *__hash_init(struct __hash *h, uint32_t min_size)
{
uint32_t size;
uint16_t size_idx;
struct bucket *buckets;
struct bucket_lock *bucket_locks;
/* Sanity check on args */
if (min_size > hash_sizes[hash_sizes_len-1]) return NULL;
/* Find least size greater than init_size */
for(size_idx = 0; size_idx < hash_sizes_len; size_idx++)
if(hash_sizes[size_idx] >= min_size)
break;
size = hash_sizes[size_idx];
if(!h) return NULL;
alloc_tab(h, size, &buckets, &bucket_locks);
if(!buckets || !bucket_locks) goto alloc_fail;
h->key_tab = L2C(h, buckets);
h->key_lock_tab = L2C(h, bucket_locks);
alloc_tab(h, size, &buckets, &bucket_locks);
if(!buckets || !bucket_locks) goto alloc_fail;
h->value_tab = L2C(h, buckets);
h->value_lock_tab = L2C(h, bucket_locks);
/* Init all h variables */
if(HASH_LOCK_INIT(h) != 0) goto alloc_fail;
h->nr_ent = 0;
h->tab_size = size;
h->size_idx = size_idx;
h->max_load = (uint32_t)ceilf(hash_max_load_fact * size);
h->min_load = (uint32_t)ceilf(hash_min_load_fact * size);
return h;
alloc_fail:
if(h->key_tab || h->key_lock_tab)
free_buckets(h, C2L(h, h->key_tab), C2L(h, h->key_lock_tab));
return NULL;
}
#undef __prim
#undef __prim_t
#undef __prim_tab
#undef __prim_lock_tab
#undef __prim_hash
#undef __prim_cmp
#undef __prim_next
#undef __sec
#undef __sec_t
#define __prim key
#define __prim_t __k_t
#define __prim_tab key_tab
#define __prim_lock_tab key_lock_tab
#define __prim_hash __key_hash
#define __prim_cmp __key_cmp
#define __prim_next key_next
#define __sec value
#define __sec_t __v_t
int __key_lookup(struct __hash *h, __prim_t k, __sec_t *vp)
{
struct hash_entry *entry;
struct bucket *b;
struct bucket_lock *blt;
uint32_t idx;
if(HASH_LOCK_RDLOCK(h) != 0) return -ENOLCK;
idx = hash_to_idx(h, __prim_hash(k));
b = C2L(h, &h->__prim_tab[idx]);
blt = C2L(h, h->__prim_lock_tab);
if(BUCKET_LOCK_RDLOCK(h, blt, idx) != 0) return -ENOLCK;
entry = b->hash_entry;
while(entry != NULL)
{
entry = C2L(h, entry);
if(__prim_cmp(k, entry->__prim))
{
/* Unlock here */
*vp = entry->__sec;
BUCKET_LOCK_RDUNLOCK(h, blt, idx);
HASH_LOCK_RDUNLOCK(h);
return 1;
}
entry = entry->__prim_next;
}
BUCKET_LOCK_RDUNLOCK(h, blt, idx);
HASH_LOCK_RDUNLOCK(h);
return 0;
}
/* value lookup is an almost exact copy of key lookup */
#undef __prim
#undef __prim_t
#undef __prim_tab
#undef __prim_lock_tab
#undef __prim_hash
#undef __prim_cmp
#undef __prim_next
#undef __sec
#undef __sec_t
#define __prim value
#define __prim_t __v_t
#define __prim_tab value_tab
#define __prim_lock_tab value_lock_tab
#define __prim_hash __value_hash
#define __prim_cmp __value_cmp
#define __prim_next value_next
#define __sec key
#define __sec_t __k_t
int __value_lookup(struct __hash *h, __prim_t k, __sec_t *vp)
{
struct hash_entry *entry;
struct bucket *b;
struct bucket_lock *blt;
uint32_t idx;
if(HASH_LOCK_RDLOCK(h) != 0) return -ENOLCK;
idx = hash_to_idx(h, __prim_hash(k));
b = C2L(h, &h->__prim_tab[idx]);
blt = C2L(h, h->__prim_lock_tab);
if(BUCKET_LOCK_RDLOCK(h, blt, idx) != 0) return -ENOLCK;
entry = b->hash_entry;
while(entry != NULL)
{
entry = C2L(h, entry);
if(__prim_cmp(k, entry->__prim))
{
/* Unlock here */
*vp = entry->__sec;
BUCKET_LOCK_RDUNLOCK(h, blt, idx);
HASH_LOCK_RDUNLOCK(h);
return 1;
}
entry = entry->__prim_next;
}
BUCKET_LOCK_RDUNLOCK(h, blt, idx);
HASH_LOCK_RDUNLOCK(h);
return 0;
}
int __insert(struct __hash *h, __k_t k, __v_t v)
{
uint32_t k_idx, v_idx;
struct hash_entry *entry;
struct bucket *bk, *bv;
struct bucket_lock *bltk, *bltv;
/* Allocate new entry before any locks (in case it fails) */
entry = (struct hash_entry*)
alloc_entry(h, sizeof(struct hash_entry));
if(!entry) return 0;
if(HASH_LOCK_RDLOCK(h) != 0) return -ENOLCK;
/* Read from nr_ent is atomic(TODO check), no need for fancy accessors */
if(h->nr_ent+1 > h->max_load)
{
/* Resize needs the write lock, drop read lock temporarily */
HASH_LOCK_RDUNLOCK(h);
hash_resize(h);
if(HASH_LOCK_RDLOCK(h) != 0) return -ENOLCK;
}
/* Init the entry */
entry->key = k;
entry->value = v;
/* Work out the indicies */
k_idx = hash_to_idx(h, __key_hash(k));
v_idx = hash_to_idx(h, __value_hash(v));
/* Insert */
bk = C2L(h, &h->key_tab[k_idx]);
bv = C2L(h, &h->value_tab[v_idx]);
bltk = C2L(h, h->key_lock_tab);
bltv = C2L(h, h->value_lock_tab);
if(TWO_BUCKETS_LOCK_WRLOCK(h, bltk, k_idx, bltv, v_idx) != 0)
return -ENOLCK;
entry->key_next = bk->hash_entry;
bk->hash_entry = L2C(h, entry);
entry->value_next = bv->hash_entry;
bv->hash_entry = L2C(h, entry);
TWO_BUCKETS_LOCK_WRUNLOCK(h, bltk, k_idx, bltv, v_idx);
/* Book keeping */
atomic_inc(&h->nr_ent);
HASH_LOCK_RDUNLOCK(h);
return 1;
}
#undef __prim
#undef __prim_t
#undef __prim_tab
#undef __prim_lock_tab
#undef __prim_hash
#undef __prim_cmp
#undef __prim_next
#undef __sec
#undef __sec_t
#undef __sec_tab
#undef __sec_lock_tab
#undef __sec_hash
#undef __sec_next
#define __prim key
#define __prim_t __k_t
#define __prim_tab key_tab
#define __prim_lock_tab key_lock_tab
#define __prim_hash __key_hash
#define __prim_cmp __key_cmp
#define __prim_next key_next
#define __sec value
#define __sec_t __v_t
#define __sec_tab value_tab
#define __sec_lock_tab value_lock_tab
#define __sec_hash __value_hash
#define __sec_next value_next
int __key_remove(struct __hash *h, __prim_t k, __sec_t *vp)
{
struct hash_entry *e, *es, **pek, **pev;
struct bucket *bk, *bv;
struct bucket_lock *bltk, *bltv;
uint32_t old_kidx, kidx, vidx, min_load, nr_ent;
__prim_t ks;
__sec_t vs;
if(HASH_LOCK_RDLOCK(h) != 0) return -ENOLCK;
again:
old_kidx = kidx = hash_to_idx(h, __prim_hash(k));
bk = C2L(h, &h->__prim_tab[kidx]);
bltk = C2L(h, h->__prim_lock_tab);
if(BUCKET_LOCK_RDLOCK(h, bltk, kidx) != 0) return -ENOLCK;
pek = &(bk->hash_entry);
e = *pek;
while(e != NULL)
{
e = C2L(h, e);
if(__prim_cmp(k, e->__prim))
{
goto found;
}
pek = &(e->__prim_next);
e = *pek;
}
BUCKET_LOCK_RDUNLOCK(h, bltk, kidx);
HASH_LOCK_RDUNLOCK(h);
return 0;
found:
/*
* Make local copy of key and value.
*/
es = e;
ks = e->__prim;
vs = e->__sec;
kidx = hash_to_idx(h, __prim_hash(ks));
/* Being paranoid: check if kidx has not changed, so that we unlock the
* right bucket */
assert(old_kidx == kidx);
vidx = hash_to_idx(h, __sec_hash(vs));
bk = C2L(h, &h->__prim_tab[kidx]);
bv = C2L(h, &h->__sec_tab[vidx]);
bltk = C2L(h, h->__prim_lock_tab);
bltv = C2L(h, h->__sec_lock_tab);
BUCKET_LOCK_RDUNLOCK(h, bltk, kidx);
if(TWO_BUCKETS_LOCK_WRLOCK(h, bltk, kidx, bltv, vidx) != 0) return -ENOLCK;
pek = &(bk->hash_entry);
pev = &(bv->hash_entry);
/* Find the entry in both tables */
e = *pek;
while(e != NULL)
{
e = C2L(h, e);
if(e == es)
{
/* Being paranoid: make sure that the key and value are
* still the same. This is still not 100%, because, in
* principle, the entry could have got deleted, when we
* didn't hold the locks for a little while, and exactly
* the same entry reinserted. If the __k_t & __v_t are
* simple types than it probably doesn't matter, but if
* either is a pointer type, the actual structure might
* now be different. The chances that happens are very
* slim, but still, if that's a problem, the user needs to
* pay attention to the structure re-allocation */
if((memcmp(&(e->__prim), &ks, sizeof(__prim_t))) ||
(memcmp(&(e->__sec), &vs, sizeof(__sec_t))))
break;
goto found_again;
}
pek = &(e->__prim_next);
e = *pek;
}
TWO_BUCKETS_LOCK_WRUNLOCK(h, bltk, kidx, bltv, vidx);
/* Entry got removed in the meantime, try again */
goto again;
found_again:
/* We are now comitted to the removal */
e = *pev;
while(e != NULL)
{
e = C2L(h, e);
if(e == es)
{
/* Both pek and pev are pointing to the right place, remove */
*pek = e->__prim_next;
*pev = e->__sec_next;
atomic_dec(&h->nr_ent);
nr_ent = h->nr_ent;
/* read min_load still under the hash lock! */
min_load = h->min_load;
TWO_BUCKETS_LOCK_WRUNLOCK(h, bltk, kidx, bltv, vidx);
HASH_LOCK_RDUNLOCK(h);
if(nr_ent < min_load)
hash_resize(h);
if(vp != NULL)
*vp = e->__sec;
free_entry(h, e);
return 1;
}
pev = &(e->__sec_next);
e = *pev;
}
/* We should never get here!, no need to unlock anything */
return -ENOLCK;
}
#undef __prim
#undef __prim_t
#undef __prim_tab
#undef __prim_lock_tab
#undef __prim_hash
#undef __prim_cmp
#undef __prim_next
#undef __sec
#undef __sec_t
#undef __sec_tab
#undef __sec_lock_tab
#undef __sec_hash
#undef __sec_next
#define __prim value
#define __prim_t __v_t
#define __prim_tab value_tab
#define __prim_lock_tab value_lock_tab
#define __prim_hash __value_hash
#define __prim_cmp __value_cmp
#define __prim_next value_next
#define __sec key
#define __sec_t __k_t
#define __sec_tab key_tab
#define __sec_lock_tab key_lock_tab
#define __sec_hash __key_hash
#define __sec_next key_next
int __value_remove(struct __hash *h, __prim_t k, __sec_t *vp)
{
struct hash_entry *e, *es, **pek, **pev;
struct bucket *bk, *bv;
struct bucket_lock *bltk, *bltv;
uint32_t old_kidx, kidx, vidx, min_load, nr_ent;
__prim_t ks;
__sec_t vs;
if(HASH_LOCK_RDLOCK(h) != 0) return -ENOLCK;
again:
old_kidx = kidx = hash_to_idx(h, __prim_hash(k));
bk = C2L(h, &h->__prim_tab[kidx]);
bltk = C2L(h, h->__prim_lock_tab);
if(BUCKET_LOCK_RDLOCK(h, bltk, kidx) != 0) return -ENOLCK;
pek = &(bk->hash_entry);
e = *pek;
while(e != NULL)
{
e = C2L(h, e);
if(__prim_cmp(k, e->__prim))
{
goto found;
}
pek = &(e->__prim_next);
e = *pek;
}
BUCKET_LOCK_RDUNLOCK(h, bltk, kidx);
HASH_LOCK_RDUNLOCK(h);
return 0;
found:
/*
* Make local copy of key and value.
*/
es = e;
ks = e->__prim;
vs = e->__sec;
kidx = hash_to_idx(h, __prim_hash(ks));
/* Being paranoid: check if kidx has not changed, so that we unlock the
* right bucket */
assert(old_kidx == kidx);
vidx = hash_to_idx(h, __sec_hash(vs));
bk = C2L(h, &h->__prim_tab[kidx]);
bv = C2L(h, &h->__sec_tab[vidx]);
bltk = C2L(h, h->__prim_lock_tab);
bltv = C2L(h, h->__sec_lock_tab);
BUCKET_LOCK_RDUNLOCK(h, bltk, kidx);
if(TWO_BUCKETS_LOCK_WRLOCK(h, bltk, kidx, bltv, vidx) != 0) return -ENOLCK;
pek = &(bk->hash_entry);
pev = &(bv->hash_entry);
/* Find the entry in both tables */
e = *pek;
while(e != NULL)
{
e = C2L(h, e);
if(e == es)
{
/* Being paranoid: make sure that the key and value are
* still the same. This is still not 100%, because, in
* principle, the entry could have got deleted, when we
* didn't hold the locks for a little while, and exactly
* the same entry reinserted. If the __k_t & __v_t are
* simple types than it probably doesn't matter, but if
* either is a pointer type, the actual structure might
* now be different. The chances that happens are very
* slim, but still, if that's a problem, the user needs to
* pay attention to the structure re-allocation */
if((memcmp(&(e->__prim), &ks, sizeof(__prim_t))) ||
(memcmp(&(e->__sec), &vs, sizeof(__sec_t))))
break;
goto found_again;
}
pek = &(e->__prim_next);
e = *pek;
}
TWO_BUCKETS_LOCK_WRUNLOCK(h, bltk, kidx, bltv, vidx);
/* Entry got removed in the meantime, try again */
goto again;
found_again:
/* We are now comitted to the removal */
e = *pev;
while(e != NULL)
{
e = C2L(h, e);
if(e == es)
{
/* Both pek and pev are pointing to the right place, remove */
*pek = e->__prim_next;
*pev = e->__sec_next;
atomic_dec(&h->nr_ent);
nr_ent = h->nr_ent;
/* read min_load still under the hash lock! */
min_load = h->min_load;
TWO_BUCKETS_LOCK_WRUNLOCK(h, bltk, kidx, bltv, vidx);
HASH_LOCK_RDUNLOCK(h);
if(nr_ent < min_load)
hash_resize(h);
if(vp != NULL)
*vp = e->__sec;
free_entry(h, e);
return 1;
}
pev = &(e->__sec_next);
e = *pev;
}
/* We should never get here!, no need to unlock anything */
return -ENOLCK;
}
int __hash_destroy(struct __hash *h,
void (*entry_consumer)(__k_t k, __v_t v, void *p),
void *d)
{
struct hash_entry *e, *n;
struct bucket *b;
int i;
if(HASH_LOCK_WRLOCK(h) != 0) return -ENOLCK;
/* No need to lock individual buckets, with hash write lock */
for(i=0; i < h->tab_size; i++)
{
b = C2L(h, &h->key_tab[i]);
e = b->hash_entry;
while(e != NULL)
{
e = C2L(h, e);
n = e->key_next;
if(entry_consumer)
entry_consumer(e->key, e->value, d);
free_entry(h, e);
e = n;
}
}
free_buckets(h, C2L(h, h->key_tab), C2L(h, h->key_lock_tab));
free_buckets(h, C2L(h, h->value_tab), C2L(h, h->value_lock_tab));
HASH_LOCK_WRUNLOCK(h);
h->lock_alive = 0;
return 0;
}
static void hash_resize(struct __hash *h)
{
int new_size_idx, i, lock_ret;
uint32_t size, old_size, kidx, vidx;
struct bucket *t1, *t2, *b;
struct bucket_lock *l1, *l2;
struct hash_entry *e, *n;
/* We may fail to allocate the lock, if the resize is triggered while
we are iterating (under read lock) */
lock_ret = HASH_LOCK_TRYWRLOCK(h);
if(lock_ret != 0) return;
new_size_idx = h->size_idx;
/* Work out the new size */
if(h->nr_ent >= h->max_load)
new_size_idx = h->size_idx+1;
if(h->nr_ent < h->min_load)
new_size_idx = h->size_idx-1;
if((new_size_idx == h->size_idx) ||
(new_size_idx >= hash_sizes_len) ||
(new_size_idx < 0))
{
HASH_LOCK_WRUNLOCK(h);
return;
}
size = hash_sizes[new_size_idx];
/* Allocate the new sizes */
t1 = t2 = NULL;
l1 = l2 = NULL;
alloc_tab(h, size, &t1, &l1);
if(!t1 || !l1) goto alloc_fail;
alloc_tab(h, size, &t2, &l2);
if(!t2 || !l2) goto alloc_fail;
old_size = h->tab_size;
h->tab_size = size;
h->size_idx = new_size_idx;
h->max_load = (uint32_t)ceilf(hash_max_load_fact * size);
h->min_load = (uint32_t)ceilf(hash_min_load_fact * size);
/* Move the entries */
for(i=0; i < old_size; i++)
{
b = C2L(h, &h->key_tab[i]);
e = b->hash_entry;
while(e != NULL)
{
e = C2L(h, e);
n = e->key_next;
kidx =hash_to_idx(h, __key_hash(e->key));
vidx =hash_to_idx(h, __value_hash(e->value));
/* Move to the correct bucket */
e->key_next = t1[kidx].hash_entry;
t1[kidx].hash_entry = L2C(h, e);
e->value_next = t2[vidx].hash_entry;
t2[vidx].hash_entry = L2C(h, e);
e = n;
}
}
free_buckets(h, C2L(h, h->key_tab), C2L(h, h->key_lock_tab));
free_buckets(h, C2L(h, h->value_tab), C2L(h, h->value_lock_tab));
h->key_tab = L2C(h, t1);
h->key_lock_tab = L2C(h, l1);
h->value_tab = L2C(h, t2);
h->value_lock_tab = L2C(h, l2);
HASH_LOCK_WRUNLOCK(h);
return;
alloc_fail:
/* If we failed to resize, adjust max/min load. This will stop us from
* retrying resize too frequently */
if(new_size_idx > h->size_idx)
h->max_load = (h->max_load + 2 * h->tab_size) / 2 + 1;
else
if (new_size_idx < h->size_idx)
h->min_load = h->min_load / 2;
HASH_LOCK_WRUNLOCK(h);
if(t1 || l1) free_buckets(h, t1, l1);
if(t2 || l2) free_buckets(h, t2, l2);
return;
}
int __hash_iterator(struct __hash *h,
int (*entry_consumer)(__k_t k, __v_t v, void *p),
void *d)
{
struct hash_entry *e, *n;
struct bucket *b;
struct bucket_lock *blt;
int i, brk_early;
if(HASH_LOCK_RDLOCK(h) != 0) return -ENOLCK;
for(i=0; i < h->tab_size; i++)
{
b = C2L(h, &h->key_tab[i]);
blt = C2L(h, h->key_lock_tab);
if(BUCKET_LOCK_RDLOCK(h, blt, i) != 0) return -ENOLCK;
e = b->hash_entry;
while(e != NULL)
{
e = C2L(h, e);
n = e->key_next;
brk_early = entry_consumer(e->key, e->value, d);
if(brk_early)
{
BUCKET_LOCK_RDUNLOCK(h, blt, i);
goto out;
}
e = n;
}
BUCKET_LOCK_RDUNLOCK(h, blt, i);
}
out:
HASH_LOCK_RDUNLOCK(h);
return 0;
}
void __hash_sizes(struct __hash *h,
uint32_t *nr_ent,
uint32_t *max_nr_ent,
uint32_t *tab_size,
uint32_t *max_load,
uint32_t *min_load)
{
if(nr_ent != NULL) *nr_ent = h->nr_ent;
if(max_nr_ent != NULL) *max_nr_ent = max_entries(h);
if(tab_size != NULL) *tab_size = h->tab_size;
if(max_load != NULL) *max_load = h->max_load;
if(min_load != NULL) *min_load = h->min_load;
}
Reference in new issue View Git Blame Copy Permalink