lib/rhashtable.c

   1 /*
   2  * Resizable, Scalable, Concurrent Hash Table
   3  *
   4  * Copyright (c) 2014-2015 Thomas Graf <tgraf@suug.ch>
   5  * Copyright (c) 2008-2014 Patrick McHardy <kaber@trash.net>
   6  *
   7  * Based on the following paper:
   8  * https://www.usenix.org/legacy/event/atc11/tech/final_files/Triplett.pdf
   9  *
  10  * Code partially derived from nft_hash
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  */
  16
  17 #include <linux/kernel.h>
  18 #include <linux/init.h>
  19 #include <linux/log2.h>
  20 #include <linux/sched.h>
  21 #include <linux/slab.h>
  22 #include <linux/vmalloc.h>
  23 #include <linux/mm.h>
  24 #include <linux/jhash.h>
  25 #include <linux/random.h>
  26 #include <linux/rhashtable.h>
  27 #include <linux/err.h>
  28
  29 #define HASH_DEFAULT_SIZE       64UL
  30 #define HASH_MIN_SIZE           4UL
  31 #define BUCKET_LOCKS_PER_CPU   128UL
  32
  33 /* Base bits plus 1 bit for nulls marker */
  34 #define HASH_RESERVED_SPACE     (RHT_BASE_BITS + 1)
  35
  36 enum {
  37         RHT_LOCK_NORMAL,
  38         RHT_LOCK_NESTED,
  39 };
  40
  41 /* The bucket lock is selected based on the hash and protects mutations
  42  * on a group of hash buckets.
  43  *
  44  * A maximum of tbl->size/2 bucket locks is allocated. This ensures that
  45  * a single lock always covers both buckets which may both contains
  46  * entries which link to the same bucket of the old table during resizing.
  47  * This allows to simplify the locking as locking the bucket in both
  48  * tables during resize always guarantee protection.
  49  *
  50  * IMPORTANT: When holding the bucket lock of both the old and new table
  51  * during expansions and shrinking, the old bucket lock must always be
  52  * acquired first.
  53  */
  54 static spinlock_t *bucket_lock(const struct bucket_table *tbl, u32 hash)
  55 {
  56         return &tbl->locks[hash & tbl->locks_mask];
  57 }
  58
  59 static void *rht_obj(const struct rhashtable *ht, const struct rhash_head *he)
  60 {
  61         return (void *) he - ht->p.head_offset;
  62 }
  63
  64 static u32 rht_bucket_index(const struct bucket_table *tbl, u32 hash)
  65 {
  66         return hash & (tbl->size - 1);
  67 }
  68
  69 static u32 obj_raw_hashfn(struct rhashtable *ht, const void *ptr)
  70 {
  71         struct bucket_table *tbl = rht_dereference_rcu(ht->tbl, ht);
  72         u32 hash;
  73
  74         if (unlikely(!ht->p.key_len))
  75                 hash = ht->p.obj_hashfn(ptr, tbl->hash_rnd);
  76         else
  77                 hash = ht->p.hashfn(ptr + ht->p.key_offset, ht->p.key_len,
  78                                     tbl->hash_rnd);
  79
  80         return hash >> HASH_RESERVED_SPACE;
  81 }
  82
  83 static u32 key_hashfn(struct rhashtable *ht, const void *key, u32 len)
  84 {
  85         struct bucket_table *tbl = rht_dereference_rcu(ht->tbl, ht);
  86
  87         return ht->p.hashfn(key, len, tbl->hash_rnd) >> HASH_RESERVED_SPACE;
  88 }
  89
  90 static u32 head_hashfn(struct rhashtable *ht,
  91                        const struct bucket_table *tbl,
  92                        const struct rhash_head *he)
  93 {
  94         return rht_bucket_index(tbl, obj_raw_hashfn(ht, rht_obj(ht, he)));
  95 }
  96
  97 #ifdef CONFIG_PROVE_LOCKING
  98 static void debug_dump_buckets(struct rhashtable *ht,
  99                                const struct bucket_table *tbl)
 100 {
 101         struct rhash_head *he;
 102         unsigned int i, hash;
 103
 104         for (i = 0; i < tbl->size; i++) {
 105                 pr_warn(" [Bucket %d] ", i);
 106                 rht_for_each_rcu(he, tbl, i) {
 107                         hash = head_hashfn(ht, tbl, he);
 108                         pr_cont("[hash = %#x, lock = %p] ",
 109                                 hash, bucket_lock(tbl, hash));
 110                 }
 111                 pr_cont("\n");
 112         }
 113
 114 }
 115
 116 static void debug_dump_table(struct rhashtable *ht,
 117                              const struct bucket_table *tbl,
 118                              unsigned int hash)
 119 {
 120         struct bucket_table *old_tbl, *future_tbl;
 121
 122         pr_emerg("BUG: lock for hash %#x in table %p not held\n",
 123                  hash, tbl);
 124
 125         rcu_read_lock();
 126         future_tbl = rht_dereference_rcu(ht->future_tbl, ht);
 127         old_tbl = rht_dereference_rcu(ht->tbl, ht);
 128         if (future_tbl != old_tbl) {
 129                 pr_warn("Future table %p (size: %zd)\n",
 130                         future_tbl, future_tbl->size);
 131                 debug_dump_buckets(ht, future_tbl);
 132         }
 133
 134         pr_warn("Table %p (size: %zd)\n", old_tbl, old_tbl->size);
 135         debug_dump_buckets(ht, old_tbl);
 136
 137         rcu_read_unlock();
 138 }
 139
 140 #define ASSERT_RHT_MUTEX(HT) BUG_ON(!lockdep_rht_mutex_is_held(HT))
 141 #define ASSERT_BUCKET_LOCK(HT, TBL, HASH)                               \
 142         do {                                                            \
 143                 if (unlikely(!lockdep_rht_bucket_is_held(TBL, HASH))) { \
 144                         debug_dump_table(HT, TBL, HASH);                \
 145                         BUG();                                          \
 146                 }                                                       \
 147         } while (0)
 148
 149 int lockdep_rht_mutex_is_held(struct rhashtable *ht)
 150 {
 151         return (debug_locks) ? lockdep_is_held(&ht->mutex) : 1;
 152 }
 153 EXPORT_SYMBOL_GPL(lockdep_rht_mutex_is_held);
 154
 155 int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash)
 156 {
 157         spinlock_t *lock = bucket_lock(tbl, hash);
 158
 159         return (debug_locks) ? lockdep_is_held(lock) : 1;
 160 }
 161 EXPORT_SYMBOL_GPL(lockdep_rht_bucket_is_held);
 162 #else
 163 #define ASSERT_RHT_MUTEX(HT)
 164 #define ASSERT_BUCKET_LOCK(HT, TBL, HASH)
 165 #endif
 166
 167
 168 static struct rhash_head __rcu **bucket_tail(struct bucket_table *tbl, u32 n)
 169 {
 170         struct rhash_head __rcu **pprev;
 171
 172         for (pprev = &tbl->buckets[n];
 173              !rht_is_a_nulls(rht_dereference_bucket(*pprev, tbl, n));
 174              pprev = &rht_dereference_bucket(*pprev, tbl, n)->next)
 175                 ;
 176
 177         return pprev;
 178 }
 179
 180 static int alloc_bucket_locks(struct rhashtable *ht, struct bucket_table *tbl)
 181 {
 182         unsigned int i, size;
 183 #if defined(CONFIG_PROVE_LOCKING)
 184         unsigned int nr_pcpus = 2;
 185 #else
 186         unsigned int nr_pcpus = num_possible_cpus();
 187 #endif
 188
 189         nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
 190         size = roundup_pow_of_two(nr_pcpus * ht->p.locks_mul);
 191
 192         /* Never allocate more than 0.5 locks per bucket */
 193         size = min_t(unsigned int, size, tbl->size >> 1);
 194
 195         if (sizeof(spinlock_t) != 0) {
 196 #ifdef CONFIG_NUMA
 197                 if (size * sizeof(spinlock_t) > PAGE_SIZE)
 198                         tbl->locks = vmalloc(size * sizeof(spinlock_t));
 199                 else
 200 #endif
 201                 tbl->locks = kmalloc_array(size, sizeof(spinlock_t),
 202                                            GFP_KERNEL);
 203                 if (!tbl->locks)
 204                         return -ENOMEM;
 205                 for (i = 0; i < size; i++)
 206                         spin_lock_init(&tbl->locks[i]);
 207         }
 208         tbl->locks_mask = size - 1;
 209
 210         return 0;
 211 }
 212
 213 static void bucket_table_free(const struct bucket_table *tbl)
 214 {
 215         if (tbl)
 216                 kvfree(tbl->locks);
 217
 218         kvfree(tbl);
 219 }
 220
 221 static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
 222                                                size_t nbuckets)
 223 {
 224         struct bucket_table *tbl = NULL;
 225         size_t size;
 226         int i;
 227
 228         size = sizeof(*tbl) + nbuckets * sizeof(tbl->buckets[0]);
 229         if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER))
 230                 tbl = kzalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
 231         if (tbl == NULL)
 232                 tbl = vzalloc(size);
 233         if (tbl == NULL)
 234                 return NULL;
 235
 236         tbl->size = nbuckets;
 237
 238         if (alloc_bucket_locks(ht, tbl) < 0) {
 239                 bucket_table_free(tbl);
 240                 return NULL;
 241         }
 242
 243         for (i = 0; i < nbuckets; i++)
 244                 INIT_RHT_NULLS_HEAD(tbl->buckets[i], ht, i);
 245
 246         return tbl;
 247 }
 248
 249 /**
 250  * rht_grow_above_75 - returns true if nelems > 0.75 * table-size
 251  * @ht:         hash table
 252  * @new_size:   new table size
 253  */
 254 static bool rht_grow_above_75(const struct rhashtable *ht, size_t new_size)
 255 {
 256         /* Expand table when exceeding 75% load */
 257         return atomic_read(&ht->nelems) > (new_size / 4 * 3) &&
 258                (!ht->p.max_shift || atomic_read(&ht->shift) < ht->p.max_shift);
 259 }
 260
 261 /**
 262  * rht_shrink_below_30 - returns true if nelems < 0.3 * table-size
 263  * @ht:         hash table
 264  * @new_size:   new table size
 265  */
 266 static bool rht_shrink_below_30(const struct rhashtable *ht, size_t new_size)
 267 {
 268         /* Shrink table beneath 30% load */
 269         return atomic_read(&ht->nelems) < (new_size * 3 / 10) &&
 270                (atomic_read(&ht->shift) > ht->p.min_shift);
 271 }
 272
 273 static void lock_buckets(struct bucket_table *new_tbl,
 274                          struct bucket_table *old_tbl, unsigned int hash)
 275         __acquires(old_bucket_lock)
 276 {
 277         spin_lock_bh(bucket_lock(old_tbl, hash));
 278         if (new_tbl != old_tbl)
 279                 spin_lock_bh_nested(bucket_lock(new_tbl, hash),
 280                                     RHT_LOCK_NESTED);
 281 }
 282
 283 static void unlock_buckets(struct bucket_table *new_tbl,
 284                            struct bucket_table *old_tbl, unsigned int hash)
 285         __releases(old_bucket_lock)
 286 {
 287         if (new_tbl != old_tbl)
 288                 spin_unlock_bh(bucket_lock(new_tbl, hash));
 289         spin_unlock_bh(bucket_lock(old_tbl, hash));
 290 }
 291
 292 /**
 293  * Unlink entries on bucket which hash to different bucket.
 294  *
 295  * Returns true if no more work needs to be performed on the bucket.
 296  */
 297 static bool hashtable_chain_unzip(struct rhashtable *ht,
 298                                   const struct bucket_table *new_tbl,
 299                                   struct bucket_table *old_tbl,
 300                                   size_t old_hash)
 301 {
 302         struct rhash_head *he, *p, *next;
 303         unsigned int new_hash, new_hash2;
 304
 305         ASSERT_BUCKET_LOCK(ht, old_tbl, old_hash);
 306
 307         /* Old bucket empty, no work needed. */
 308         p = rht_dereference_bucket(old_tbl->buckets[old_hash], old_tbl,
 309                                    old_hash);
 310         if (rht_is_a_nulls(p))
 311                 return false;
 312
 313         new_hash = head_hashfn(ht, new_tbl, p);
 314         ASSERT_BUCKET_LOCK(ht, new_tbl, new_hash);
 315
 316         /* Advance the old bucket pointer one or more times until it
 317          * reaches a node that doesn't hash to the same bucket as the
 318          * previous node p. Call the previous node p;
 319          */
 320         rht_for_each_continue(he, p->next, old_tbl, old_hash) {
 321                 new_hash2 = head_hashfn(ht, new_tbl, he);
 322                 ASSERT_BUCKET_LOCK(ht, new_tbl, new_hash2);
 323
 324                 if (new_hash != new_hash2)
 325                         break;
 326                 p = he;
 327         }
 328         rcu_assign_pointer(old_tbl->buckets[old_hash], p->next);
 329
 330         /* Find the subsequent node which does hash to the same
 331          * bucket as node P, or NULL if no such node exists.
 332          */
 333         INIT_RHT_NULLS_HEAD(next, ht, old_hash);
 334         if (!rht_is_a_nulls(he)) {
 335                 rht_for_each_continue(he, he->next, old_tbl, old_hash) {
 336                         if (head_hashfn(ht, new_tbl, he) == new_hash) {
 337                                 next = he;
 338                                 break;
 339                         }
 340                 }
 341         }
 342
 343         /* Set p's next pointer to that subsequent node pointer,
 344          * bypassing the nodes which do not hash to p's bucket
 345          */
 346         rcu_assign_pointer(p->next, next);
 347
 348         p = rht_dereference_bucket(old_tbl->buckets[old_hash], old_tbl,
 349                                    old_hash);
 350
 351         return !rht_is_a_nulls(p);
 352 }
 353
 354 static void link_old_to_new(struct rhashtable *ht, struct bucket_table *new_tbl,
 355                             unsigned int new_hash, struct rhash_head *entry)
 356 {
 357         ASSERT_BUCKET_LOCK(ht, new_tbl, new_hash);
 358
 359         rcu_assign_pointer(*bucket_tail(new_tbl, new_hash), entry);
 360 }
 361
 362 /**
 363  * rhashtable_expand - Expand hash table while allowing concurrent lookups
 364  * @ht:         the hash table to expand
 365  *
 366  * A secondary bucket array is allocated and the hash entries are migrated
 367  * while keeping them on both lists until the end of the RCU grace period.
 368  *
 369  * This function may only be called in a context where it is safe to call
 370  * synchronize_rcu(), e.g. not within a rcu_read_lock() section.
 371  *
 372  * The caller must ensure that no concurrent resizing occurs by holding
 373  * ht->mutex.
 374  *
 375  * It is valid to have concurrent insertions and deletions protected by per
 376  * bucket locks or concurrent RCU protected lookups and traversals.
 377  */
 378 int rhashtable_expand(struct rhashtable *ht)
 379 {
 380         struct bucket_table *new_tbl, *old_tbl = rht_dereference(ht->tbl, ht);
 381         struct rhash_head *he;
 382         unsigned int new_hash, old_hash;
 383         bool complete = false;
 384
 385         ASSERT_RHT_MUTEX(ht);
 386
 387         new_tbl = bucket_table_alloc(ht, old_tbl->size * 2);
 388         if (new_tbl == NULL)
 389                 return -ENOMEM;
 390
 391         new_tbl->hash_rnd = old_tbl->hash_rnd;
 392
 393         atomic_inc(&ht->shift);
 394
 395         /* Make insertions go into the new, empty table right away. Deletions
 396          * and lookups will be attempted in both tables until we synchronize.
 397          * The synchronize_rcu() guarantees for the new table to be picked up
 398          * so no new additions go into the old table while we relink.
 399          */
 400         rcu_assign_pointer(ht->future_tbl, new_tbl);
 401         synchronize_rcu();
 402
 403         /* For each new bucket, search the corresponding old bucket for the
 404          * first entry that hashes to the new bucket, and link the end of
 405          * newly formed bucket chain (containing entries added to future
 406          * table) to that entry. Since all the entries which will end up in
 407          * the new bucket appear in the same old bucket, this constructs an
 408          * entirely valid new hash table, but with multiple buckets
 409          * "zipped" together into a single imprecise chain.
 410          */
 411         for (new_hash = 0; new_hash < new_tbl->size; new_hash++) {
 412                 old_hash = rht_bucket_index(old_tbl, new_hash);
 413                 lock_buckets(new_tbl, old_tbl, new_hash);
 414                 rht_for_each(he, old_tbl, old_hash) {
 415                         if (head_hashfn(ht, new_tbl, he) == new_hash) {
 416                                 link_old_to_new(ht, new_tbl, new_hash, he);
 417                                 break;
 418                         }
 419                 }
 420                 unlock_buckets(new_tbl, old_tbl, new_hash);
 421                 cond_resched();
 422         }
 423
 424         /* Unzip interleaved hash chains */
 425         while (!complete && !ht->being_destroyed) {
 426                 /* Wait for readers. All new readers will see the new
 427                  * table, and thus no references to the old table will
 428                  * remain.
 429                  */
 430                 synchronize_rcu();
 431
 432                 /* For each bucket in the old table (each of which
 433                  * contains items from multiple buckets of the new
 434                  * table): ...
 435                  */
 436                 complete = true;
 437                 for (old_hash = 0; old_hash < old_tbl->size; old_hash++) {
 438                         lock_buckets(new_tbl, old_tbl, old_hash);
 439
 440                         if (hashtable_chain_unzip(ht, new_tbl, old_tbl,
 441                                                   old_hash))
 442                                 complete = false;
 443
 444                         unlock_buckets(new_tbl, old_tbl, old_hash);
 445                         cond_resched();
 446                 }
 447         }
 448
 449         rcu_assign_pointer(ht->tbl, new_tbl);
 450         synchronize_rcu();
 451
 452         bucket_table_free(old_tbl);
 453         return 0;
 454 }
 455 EXPORT_SYMBOL_GPL(rhashtable_expand);
 456
 457 /**
 458  * rhashtable_shrink - Shrink hash table while allowing concurrent lookups
 459  * @ht:         the hash table to shrink
 460  *
 461  * This function may only be called in a context where it is safe to call
 462  * synchronize_rcu(), e.g. not within a rcu_read_lock() section.
 463  *
 464  * The caller must ensure that no concurrent resizing occurs by holding
 465  * ht->mutex.
 466  *
 467  * The caller must ensure that no concurrent table mutations take place.
 468  * It is however valid to have concurrent lookups if they are RCU protected.
 469  *
 470  * It is valid to have concurrent insertions and deletions protected by per
 471  * bucket locks or concurrent RCU protected lookups and traversals.
 472  */
 473 int rhashtable_shrink(struct rhashtable *ht)
 474 {
 475         struct bucket_table *new_tbl, *tbl = rht_dereference(ht->tbl, ht);
 476         unsigned int new_hash;
 477
 478         ASSERT_RHT_MUTEX(ht);
 479
 480         new_tbl = bucket_table_alloc(ht, tbl->size / 2);
 481         if (new_tbl == NULL)
 482                 return -ENOMEM;
 483
 484         new_tbl->hash_rnd = tbl->hash_rnd;
 485
 486         rcu_assign_pointer(ht->future_tbl, new_tbl);
 487         synchronize_rcu();
 488
 489         /* Link the first entry in the old bucket to the end of the
 490          * bucket in the new table. As entries are concurrently being
 491          * added to the new table, lock down the new bucket. As we
 492          * always divide the size in half when shrinking, each bucket
 493          * in the new table maps to exactly two buckets in the old
 494          * table.
 495          */
 496         for (new_hash = 0; new_hash < new_tbl->size; new_hash++) {
 497                 lock_buckets(new_tbl, tbl, new_hash);
 498
 499                 rcu_assign_pointer(*bucket_tail(new_tbl, new_hash),
 500                                    tbl->buckets[new_hash]);
 501                 ASSERT_BUCKET_LOCK(ht, tbl, new_hash + new_tbl->size);
 502                 rcu_assign_pointer(*bucket_tail(new_tbl, new_hash),
 503                                    tbl->buckets[new_hash + new_tbl->size]);
 504
 505                 unlock_buckets(new_tbl, tbl, new_hash);
 506                 cond_resched();
 507         }
 508
 509         /* Publish the new, valid hash table */
 510         rcu_assign_pointer(ht->tbl, new_tbl);
 511         atomic_dec(&ht->shift);
 512
 513         /* Wait for readers. No new readers will have references to the
 514          * old hash table.
 515          */
 516         synchronize_rcu();
 517
 518         bucket_table_free(tbl);
 519
 520         return 0;
 521 }
 522 EXPORT_SYMBOL_GPL(rhashtable_shrink);
 523
 524 static void rht_deferred_worker(struct work_struct *work)
 525 {
 526         struct rhashtable *ht;
 527         struct bucket_table *tbl;
 528         struct rhashtable_walker *walker;
 529
 530         ht = container_of(work, struct rhashtable, run_work);
 531         mutex_lock(&ht->mutex);
 532         if (ht->being_destroyed)
 533                 goto unlock;
 534
 535         tbl = rht_dereference(ht->tbl, ht);
 536
 537         list_for_each_entry(walker, &ht->walkers, list)
 538                 walker->resize = true;
 539
 540         if (rht_grow_above_75(ht, tbl->size))
 541                 rhashtable_expand(ht);
 542         else if (rht_shrink_below_30(ht, tbl->size))
 543                 rhashtable_shrink(ht);
 544 unlock:
 545         mutex_unlock(&ht->mutex);
 546 }
 547
 548 static void __rhashtable_insert(struct rhashtable *ht, struct rhash_head *obj,
 549                                 struct bucket_table *tbl,
 550                                 const struct bucket_table *old_tbl, u32 hash)
 551 {
 552         bool no_resize_running = tbl == old_tbl;
 553         struct rhash_head *head;
 554
 555         hash = rht_bucket_index(tbl, hash);
 556         head = rht_dereference_bucket(tbl->buckets[hash], tbl, hash);
 557
 558         ASSERT_BUCKET_LOCK(ht, tbl, hash);
 559
 560         if (rht_is_a_nulls(head))
 561                 INIT_RHT_NULLS_HEAD(obj->next, ht, hash);
 562         else
 563                 RCU_INIT_POINTER(obj->next, head);
 564
 565         rcu_assign_pointer(tbl->buckets[hash], obj);
 566
 567         atomic_inc(&ht->nelems);
 568         if (no_resize_running && rht_grow_above_75(ht, tbl->size))
 569                 schedule_work(&ht->run_work);
 570 }
 571
 572 /**
 573  * rhashtable_insert - insert object into hash table
 574  * @ht:         hash table
 575  * @obj:        pointer to hash head inside object
 576  *
 577  * Will take a per bucket spinlock to protect against mutual mutations
 578  * on the same bucket. Multiple insertions may occur in parallel unless
 579  * they map to the same bucket lock.
 580  *
 581  * It is safe to call this function from atomic context.
 582  *
 583  * Will trigger an automatic deferred table resizing if the size grows
 584  * beyond the watermark indicated by grow_decision() which can be passed
 585  * to rhashtable_init().
 586  */
 587 void rhashtable_insert(struct rhashtable *ht, struct rhash_head *obj)
 588 {
 589         struct bucket_table *tbl, *old_tbl;
 590         unsigned hash;
 591
 592         rcu_read_lock();
 593
 594         tbl = rht_dereference_rcu(ht->future_tbl, ht);
 595         old_tbl = rht_dereference_rcu(ht->tbl, ht);
 596         hash = obj_raw_hashfn(ht, rht_obj(ht, obj));
 597
 598         lock_buckets(tbl, old_tbl, hash);
 599         __rhashtable_insert(ht, obj, tbl, old_tbl, hash);
 600         unlock_buckets(tbl, old_tbl, hash);
 601
 602         rcu_read_unlock();
 603 }
 604 EXPORT_SYMBOL_GPL(rhashtable_insert);
 605
 606 /**
 607  * rhashtable_remove - remove object from hash table
 608  * @ht:         hash table
 609  * @obj:        pointer to hash head inside object
 610  *
 611  * Since the hash chain is single linked, the removal operation needs to
 612  * walk the bucket chain upon removal. The removal operation is thus
 613  * considerable slow if the hash table is not correctly sized.
 614  *
 615  * Will automatically shrink the table via rhashtable_expand() if the
 616  * shrink_decision function specified at rhashtable_init() returns true.
 617  *
 618  * The caller must ensure that no concurrent table mutations occur. It is
 619  * however valid to have concurrent lookups if they are RCU protected.
 620  */
 621 bool rhashtable_remove(struct rhashtable *ht, struct rhash_head *obj)
 622 {
 623         struct bucket_table *tbl, *new_tbl, *old_tbl;
 624         struct rhash_head __rcu **pprev;
 625         struct rhash_head *he, *he2;
 626         unsigned int hash, new_hash;
 627         bool ret = false;
 628
 629         rcu_read_lock();
 630         old_tbl = rht_dereference_rcu(ht->tbl, ht);
 631         tbl = new_tbl = rht_dereference_rcu(ht->future_tbl, ht);
 632         new_hash = obj_raw_hashfn(ht, rht_obj(ht, obj));
 633
 634         lock_buckets(new_tbl, old_tbl, new_hash);
 635 restart:
 636         hash = rht_bucket_index(tbl, new_hash);
 637         pprev = &tbl->buckets[hash];
 638         rht_for_each(he, tbl, hash) {
 639                 if (he != obj) {
 640                         pprev = &he->next;
 641                         continue;
 642                 }
 643
 644                 ASSERT_BUCKET_LOCK(ht, tbl, hash);
 645
 646                 if (old_tbl->size > new_tbl->size && tbl == old_tbl &&
 647                     !rht_is_a_nulls(obj->next) &&
 648                     head_hashfn(ht, tbl, obj->next) != hash) {
 649                         rcu_assign_pointer(*pprev, (struct rhash_head *) rht_marker(ht, hash));
 650                 } else if (unlikely(old_tbl->size < new_tbl->size && tbl == new_tbl)) {
 651                         rht_for_each_continue(he2, obj->next, tbl, hash) {
 652                                 if (head_hashfn(ht, tbl, he2) == hash) {
 653                                         rcu_assign_pointer(*pprev, he2);
 654                                         goto found;
 655                                 }
 656                         }
 657
 658                         rcu_assign_pointer(*pprev, (struct rhash_head *) rht_marker(ht, hash));
 659                 } else {
 660                         rcu_assign_pointer(*pprev, obj->next);
 661                 }
 662
 663 found:
 664                 ret = true;
 665                 break;
 666         }
 667
 668         /* The entry may be linked in either 'tbl', 'future_tbl', or both.
 669          * 'future_tbl' only exists for a short period of time during
 670          * resizing. Thus traversing both is fine and the added cost is
 671          * very rare.
 672          */
 673         if (tbl != old_tbl) {
 674                 tbl = old_tbl;
 675                 goto restart;
 676         }
 677
 678         unlock_buckets(new_tbl, old_tbl, new_hash);
 679
 680         if (ret) {
 681                 bool no_resize_running = new_tbl == old_tbl;
 682
 683                 atomic_dec(&ht->nelems);
 684                 if (no_resize_running && rht_shrink_below_30(ht, new_tbl->size))
 685                         schedule_work(&ht->run_work);
 686         }
 687
 688         rcu_read_unlock();
 689
 690         return ret;
 691 }
 692 EXPORT_SYMBOL_GPL(rhashtable_remove);
 693
 694 struct rhashtable_compare_arg {
 695         struct rhashtable *ht;
 696         const void *key;
 697 };
 698
 699 static bool rhashtable_compare(void *ptr, void *arg)
 700 {
 701         struct rhashtable_compare_arg *x = arg;
 702         struct rhashtable *ht = x->ht;
 703
 704         return !memcmp(ptr + ht->p.key_offset, x->key, ht->p.key_len);
 705 }
 706
 707 /**
 708  * rhashtable_lookup - lookup key in hash table
 709  * @ht:         hash table
 710  * @key:        pointer to key
 711  *
 712  * Computes the hash value for the key and traverses the bucket chain looking
 713  * for a entry with an identical key. The first matching entry is returned.
 714  *
 715  * This lookup function may only be used for fixed key hash table (key_len
 716  * parameter set). It will BUG() if used inappropriately.
 717  *
 718  * Lookups may occur in parallel with hashtable mutations and resizing.
 719  */
 720 void *rhashtable_lookup(struct rhashtable *ht, const void *key)
 721 {
 722         struct rhashtable_compare_arg arg = {
 723                 .ht = ht,
 724                 .key = key,
 725         };
 726
 727         BUG_ON(!ht->p.key_len);
 728
 729         return rhashtable_lookup_compare(ht, key, &rhashtable_compare, &arg);
 730 }
 731 EXPORT_SYMBOL_GPL(rhashtable_lookup);
 732
 733 /**
 734  * rhashtable_lookup_compare - search hash table with compare function
 735  * @ht:         hash table
 736  * @key:        the pointer to the key
 737  * @compare:    compare function, must return true on match
 738  * @arg:        argument passed on to compare function
 739  *
 740  * Traverses the bucket chain behind the provided hash value and calls the
 741  * specified compare function for each entry.
 742  *
 743  * Lookups may occur in parallel with hashtable mutations and resizing.
 744  *
 745  * Returns the first entry on which the compare function returned true.
 746  */
 747 void *rhashtable_lookup_compare(struct rhashtable *ht, const void *key,
 748                                 bool (*compare)(void *, void *), void *arg)
 749 {
 750         const struct bucket_table *tbl, *old_tbl;
 751         struct rhash_head *he;
 752         u32 hash;
 753
 754         rcu_read_lock();
 755
 756         old_tbl = rht_dereference_rcu(ht->tbl, ht);
 757         tbl = rht_dereference_rcu(ht->future_tbl, ht);
 758         hash = key_hashfn(ht, key, ht->p.key_len);
 759 restart:
 760         rht_for_each_rcu(he, tbl, rht_bucket_index(tbl, hash)) {
 761                 if (!compare(rht_obj(ht, he), arg))
 762                         continue;
 763                 rcu_read_unlock();
 764                 return rht_obj(ht, he);
 765         }
 766
 767         if (unlikely(tbl != old_tbl)) {
 768                 tbl = old_tbl;
 769                 goto restart;
 770         }
 771         rcu_read_unlock();
 772
 773         return NULL;
 774 }
 775 EXPORT_SYMBOL_GPL(rhashtable_lookup_compare);
 776
 777 /**
 778  * rhashtable_lookup_insert - lookup and insert object into hash table
 779  * @ht:         hash table
 780  * @obj:        pointer to hash head inside object
 781  *
 782  * Locks down the bucket chain in both the old and new table if a resize
 783  * is in progress to ensure that writers can't remove from the old table
 784  * and can't insert to the new table during the atomic operation of search
 785  * and insertion. Searches for duplicates in both the old and new table if
 786  * a resize is in progress.
 787  *
 788  * This lookup function may only be used for fixed key hash table (key_len
 789  * parameter set). It will BUG() if used inappropriately.
 790  *
 791  * It is safe to call this function from atomic context.
 792  *
 793  * Will trigger an automatic deferred table resizing if the size grows
 794  * beyond the watermark indicated by grow_decision() which can be passed
 795  * to rhashtable_init().
 796  */
 797 bool rhashtable_lookup_insert(struct rhashtable *ht, struct rhash_head *obj)
 798 {
 799         struct rhashtable_compare_arg arg = {
 800                 .ht = ht,
 801                 .key = rht_obj(ht, obj) + ht->p.key_offset,
 802         };
 803
 804         BUG_ON(!ht->p.key_len);
 805
 806         return rhashtable_lookup_compare_insert(ht, obj, &rhashtable_compare,
 807                                                 &arg);
 808 }
 809 EXPORT_SYMBOL_GPL(rhashtable_lookup_insert);
 810
 811 /**
 812  * rhashtable_lookup_compare_insert - search and insert object to hash table
 813  *                                    with compare function
 814  * @ht:         hash table
 815  * @obj:        pointer to hash head inside object
 816  * @compare:    compare function, must return true on match
 817  * @arg:        argument passed on to compare function
 818  *
 819  * Locks down the bucket chain in both the old and new table if a resize
 820  * is in progress to ensure that writers can't remove from the old table
 821  * and can't insert to the new table during the atomic operation of search
 822  * and insertion. Searches for duplicates in both the old and new table if
 823  * a resize is in progress.
 824  *
 825  * Lookups may occur in parallel with hashtable mutations and resizing.
 826  *
 827  * Will trigger an automatic deferred table resizing if the size grows
 828  * beyond the watermark indicated by grow_decision() which can be passed
 829  * to rhashtable_init().
 830  */
 831 bool rhashtable_lookup_compare_insert(struct rhashtable *ht,
 832                                       struct rhash_head *obj,
 833                                       bool (*compare)(void *, void *),
 834                                       void *arg)
 835 {
 836         struct bucket_table *new_tbl, *old_tbl;
 837         u32 new_hash;
 838         bool success = true;
 839
 840         BUG_ON(!ht->p.key_len);
 841
 842         rcu_read_lock();
 843         old_tbl = rht_dereference_rcu(ht->tbl, ht);
 844         new_tbl = rht_dereference_rcu(ht->future_tbl, ht);
 845         new_hash = obj_raw_hashfn(ht, rht_obj(ht, obj));
 846
 847         lock_buckets(new_tbl, old_tbl, new_hash);
 848
 849         if (rhashtable_lookup_compare(ht, rht_obj(ht, obj) + ht->p.key_offset,
 850                                       compare, arg)) {
 851                 success = false;
 852                 goto exit;
 853         }
 854
 855         __rhashtable_insert(ht, obj, new_tbl, old_tbl, new_hash);
 856
 857 exit:
 858         unlock_buckets(new_tbl, old_tbl, new_hash);
 859         rcu_read_unlock();
 860
 861         return success;
 862 }
 863 EXPORT_SYMBOL_GPL(rhashtable_lookup_compare_insert);
 864
 865 /**
 866  * rhashtable_walk_init - Initialise an iterator
 867  * @ht:         Table to walk over
 868  * @iter:       Hash table Iterator
 869  *
 870  * This function prepares a hash table walk.
 871  *
 872  * Note that if you restart a walk after rhashtable_walk_stop you
 873  * may see the same object twice.  Also, you may miss objects if
 874  * there are removals in between rhashtable_walk_stop and the next
 875  * call to rhashtable_walk_start.
 876  *
 877  * For a completely stable walk you should construct your own data
 878  * structure outside the hash table.
 879  *
 880  * This function may sleep so you must not call it from interrupt
 881  * context or with spin locks held.
 882  *
 883  * You must call rhashtable_walk_exit if this function returns
 884  * successfully.
 885  */
 886 int rhashtable_walk_init(struct rhashtable *ht, struct rhashtable_iter *iter)
 887 {
 888         iter->ht = ht;
 889         iter->p = NULL;
 890         iter->slot = 0;
 891         iter->skip = 0;
 892
 893         iter->walker = kmalloc(sizeof(*iter->walker), GFP_KERNEL);
 894         if (!iter->walker)
 895                 return -ENOMEM;
 896
 897         INIT_LIST_HEAD(&iter->walker->list);
 898         iter->walker->resize = false;
 899
 900         mutex_lock(&ht->mutex);
 901         list_add(&iter->walker->list, &ht->walkers);
 902         mutex_unlock(&ht->mutex);
 903
 904         return 0;
 905 }
 906 EXPORT_SYMBOL_GPL(rhashtable_walk_init);
 907
 908 /**
 909  * rhashtable_walk_exit - Free an iterator
 910  * @iter:       Hash table Iterator
 911  *
 912  * This function frees resources allocated by rhashtable_walk_init.
 913  */
 914 void rhashtable_walk_exit(struct rhashtable_iter *iter)
 915 {
 916         mutex_lock(&iter->ht->mutex);
 917         list_del(&iter->walker->list);
 918         mutex_unlock(&iter->ht->mutex);
 919         kfree(iter->walker);
 920 }
 921 EXPORT_SYMBOL_GPL(rhashtable_walk_exit);
 922
 923 /**
 924  * rhashtable_walk_start - Start a hash table walk
 925  * @iter:       Hash table iterator
 926  *
 927  * Start a hash table walk.  Note that we take the RCU lock in all
 928  * cases including when we return an error.  So you must always call
 929  * rhashtable_walk_stop to clean up.
 930  *
 931  * Returns zero if successful.
 932  *
 933  * Returns -EAGAIN if resize event occured.  Note that the iterator
 934  * will rewind back to the beginning and you may use it immediately
 935  * by calling rhashtable_walk_next.
 936  */
 937 int rhashtable_walk_start(struct rhashtable_iter *iter)
 938 {
 939         rcu_read_lock();
 940
 941         if (iter->walker->resize) {
 942                 iter->slot = 0;
 943                 iter->skip = 0;
 944                 iter->walker->resize = false;
 945                 return -EAGAIN;
 946         }
 947
 948         return 0;
 949 }
 950 EXPORT_SYMBOL_GPL(rhashtable_walk_start);
 951
 952 /**
 953  * rhashtable_walk_next - Return the next object and advance the iterator
 954  * @iter:       Hash table iterator
 955  *
 956  * Note that you must call rhashtable_walk_stop when you are finished
 957  * with the walk.
 958  *
 959  * Returns the next object or NULL when the end of the table is reached.
 960  *
 961  * Returns -EAGAIN if resize event occured.  Note that the iterator
 962  * will rewind back to the beginning and you may continue to use it.
 963  */
 964 void *rhashtable_walk_next(struct rhashtable_iter *iter)
 965 {
 966         const struct bucket_table *tbl;
 967         struct rhashtable *ht = iter->ht;
 968         struct rhash_head *p = iter->p;
 969         void *obj = NULL;
 970
 971         tbl = rht_dereference_rcu(ht->tbl, ht);
 972
 973         if (p) {
 974                 p = rht_dereference_bucket_rcu(p->next, tbl, iter->slot);
 975                 goto next;
 976         }
 977
 978         for (; iter->slot < tbl->size; iter->slot++) {
 979                 int skip = iter->skip;
 980
 981                 rht_for_each_rcu(p, tbl, iter->slot) {
 982                         if (!skip)
 983                                 break;
 984                         skip--;
 985                 }
 986
 987 next:
 988                 if (!rht_is_a_nulls(p)) {
 989                         iter->skip++;
 990                         iter->p = p;
 991                         obj = rht_obj(ht, p);
 992                         goto out;
 993                 }
 994
 995                 iter->skip = 0;
 996         }
 997
 998         iter->p = NULL;
 999
1000 out:
1001         if (iter->walker->resize) {
1002                 iter->p = NULL;
1003                 iter->slot = 0;
1004                 iter->skip = 0;
1005                 iter->walker->resize = false;
1006                 return ERR_PTR(-EAGAIN);
1007         }
1008
1009         return obj;
1010 }
1011 EXPORT_SYMBOL_GPL(rhashtable_walk_next);
1012
1013 /**
1014  * rhashtable_walk_stop - Finish a hash table walk
1015  * @iter:       Hash table iterator
1016  *
1017  * Finish a hash table walk.
1018  */
1019 void rhashtable_walk_stop(struct rhashtable_iter *iter)
1020 {
1021         rcu_read_unlock();
1022         iter->p = NULL;
1023 }
1024 EXPORT_SYMBOL_GPL(rhashtable_walk_stop);
1025
1026 static size_t rounded_hashtable_size(struct rhashtable_params *params)
1027 {
1028         return max(roundup_pow_of_two(params->nelem_hint * 4 / 3),
1029                    1UL << params->min_shift);
1030 }
1031
1032 /**
1033  * rhashtable_init - initialize a new hash table
1034  * @ht:         hash table to be initialized
1035  * @params:     configuration parameters
1036  *
1037  * Initializes a new hash table based on the provided configuration
1038  * parameters. A table can be configured either with a variable or
1039  * fixed length key:
1040  *
1041  * Configuration Example 1: Fixed length keys
1042  * struct test_obj {
1043  *      int                     key;
1044  *      void *                  my_member;
1045  *      struct rhash_head       node;
1046  * };
1047  *
1048  * struct rhashtable_params params = {
1049  *      .head_offset = offsetof(struct test_obj, node),
1050  *      .key_offset = offsetof(struct test_obj, key),
1051  *      .key_len = sizeof(int),
1052  *      .hashfn = jhash,
1053  *      .nulls_base = (1U << RHT_BASE_SHIFT),
1054  * };
1055  *
1056  * Configuration Example 2: Variable length keys
1057  * struct test_obj {
1058  *      [...]
1059  *      struct rhash_head       node;
1060  * };
1061  *
1062  * u32 my_hash_fn(const void *data, u32 seed)
1063  * {
1064  *      struct test_obj *obj = data;
1065  *
1066  *      return [... hash ...];
1067  * }
1068  *
1069  * struct rhashtable_params params = {
1070  *      .head_offset = offsetof(struct test_obj, node),
1071  *      .hashfn = jhash,
1072  *      .obj_hashfn = my_hash_fn,
1073  * };
1074  */
1075 int rhashtable_init(struct rhashtable *ht, struct rhashtable_params *params)
1076 {
1077         struct bucket_table *tbl;
1078         size_t size;
1079
1080         size = HASH_DEFAULT_SIZE;
1081
1082         if ((params->key_len && !params->hashfn) ||
1083             (!params->key_len && !params->obj_hashfn))
1084                 return -EINVAL;
1085
1086         if (params->nulls_base && params->nulls_base < (1U << RHT_BASE_SHIFT))
1087                 return -EINVAL;
1088
1089         params->min_shift = max_t(size_t, params->min_shift,
1090                                   ilog2(HASH_MIN_SIZE));
1091
1092         if (params->nelem_hint)
1093                 size = rounded_hashtable_size(params);
1094
1095         memset(ht, 0, sizeof(*ht));
1096         mutex_init(&ht->mutex);
1097         memcpy(&ht->p, params, sizeof(*params));
1098         INIT_LIST_HEAD(&ht->walkers);
1099
1100         if (params->locks_mul)
1101                 ht->p.locks_mul = roundup_pow_of_two(params->locks_mul);
1102         else
1103                 ht->p.locks_mul = BUCKET_LOCKS_PER_CPU;
1104
1105         tbl = bucket_table_alloc(ht, size);
1106         if (tbl == NULL)
1107                 return -ENOMEM;
1108
1109         get_random_bytes(&tbl->hash_rnd, sizeof(tbl->hash_rnd));
1110
1111         atomic_set(&ht->nelems, 0);
1112         atomic_set(&ht->shift, ilog2(tbl->size));
1113         RCU_INIT_POINTER(ht->tbl, tbl);
1114         RCU_INIT_POINTER(ht->future_tbl, tbl);
1115
1116         INIT_WORK(&ht->run_work, rht_deferred_worker);
1117
1118         return 0;
1119 }
1120 EXPORT_SYMBOL_GPL(rhashtable_init);
1121
1122 /**
1123  * rhashtable_destroy - destroy hash table
1124  * @ht:         the hash table to destroy
1125  *
1126  * Frees the bucket array. This function is not rcu safe, therefore the caller
1127  * has to make sure that no resizing may happen by unpublishing the hashtable
1128  * and waiting for the quiescent cycle before releasing the bucket array.
1129  */
1130 void rhashtable_destroy(struct rhashtable *ht)
1131 {
1132         ht->being_destroyed = true;
1133
1134         cancel_work_sync(&ht->run_work);
1135
1136         mutex_lock(&ht->mutex);
1137         bucket_table_free(rht_dereference(ht->tbl, ht));
1138         mutex_unlock(&ht->mutex);
1139 }
1140 EXPORT_SYMBOL_GPL(rhashtable_destroy);