mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-28 20:02:17 +00:00
![Marco Elver](/assets/img/avatar_default.png)
Numerous production kernel configs (see [1, 2]) are choosing to enable CONFIG_DEBUG_LIST, which is also being recommended by KSPP for hardened configs [3]. The motivation behind this is that the option can be used as a security hardening feature (e.g. CVE-2019-2215 and CVE-2019-2025 are mitigated by the option [4]). The feature has never been designed with performance in mind, yet common list manipulation is happening across hot paths all over the kernel. Introduce CONFIG_LIST_HARDENED, which performs list pointer checking inline, and only upon list corruption calls the reporting slow path. To generate optimal machine code with CONFIG_LIST_HARDENED: 1. Elide checking for pointer values which upon dereference would result in an immediate access fault (i.e. minimal hardening checks). The trade-off is lower-quality error reports. 2. Use the __preserve_most function attribute (available with Clang, but not yet with GCC) to minimize the code footprint for calling the reporting slow path. As a result, function size of callers is reduced by avoiding saving registers before calling the rarely called reporting slow path. Note that all TUs in lib/Makefile already disable function tracing, including list_debug.c, and __preserve_most's implied notrace has no effect in this case. 3. Because the inline checks are a subset of the full set of checks in __list_*_valid_or_report(), always return false if the inline checks failed. This avoids redundant compare and conditional branch right after return from the slow path. As a side-effect of the checks being inline, if the compiler can prove some condition to always be true, it can completely elide some checks. Since DEBUG_LIST is functionally a superset of LIST_HARDENED, the Kconfig variables are changed to reflect that: DEBUG_LIST selects LIST_HARDENED, whereas LIST_HARDENED itself has no dependency on DEBUG_LIST. Running netperf with CONFIG_LIST_HARDENED (using a Clang compiler with "preserve_most") shows throughput improvements, in my case of ~7% on average (up to 20-30% on some test cases). Link: https://r.android.com/1266735 [1] Link: https://gitlab.archlinux.org/archlinux/packaging/packages/linux/-/blob/main/config [2] Link: https://kernsec.org/wiki/index.php/Kernel_Self_Protection_Project/Recommended_Settings [3] Link: https://googleprojectzero.blogspot.com/2019/11/bad-binder-android-in-wild-exploit.html [4] Signed-off-by: Marco Elver <elver@google.com> Link: https://lore.kernel.org/r/20230811151847.1594958-3-elver@google.com Signed-off-by: Kees Cook <keescook@chromium.org>
1170 lines
34 KiB
C
1170 lines
34 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _LINUX_LIST_H
|
|
#define _LINUX_LIST_H
|
|
|
|
#include <linux/container_of.h>
|
|
#include <linux/types.h>
|
|
#include <linux/stddef.h>
|
|
#include <linux/poison.h>
|
|
#include <linux/const.h>
|
|
|
|
#include <asm/barrier.h>
|
|
|
|
/*
|
|
* Circular doubly linked list implementation.
|
|
*
|
|
* Some of the internal functions ("__xxx") are useful when
|
|
* manipulating whole lists rather than single entries, as
|
|
* sometimes we already know the next/prev entries and we can
|
|
* generate better code by using them directly rather than
|
|
* using the generic single-entry routines.
|
|
*/
|
|
|
|
#define LIST_HEAD_INIT(name) { &(name), &(name) }
|
|
|
|
#define LIST_HEAD(name) \
|
|
struct list_head name = LIST_HEAD_INIT(name)
|
|
|
|
/**
|
|
* INIT_LIST_HEAD - Initialize a list_head structure
|
|
* @list: list_head structure to be initialized.
|
|
*
|
|
* Initializes the list_head to point to itself. If it is a list header,
|
|
* the result is an empty list.
|
|
*/
|
|
static inline void INIT_LIST_HEAD(struct list_head *list)
|
|
{
|
|
WRITE_ONCE(list->next, list);
|
|
WRITE_ONCE(list->prev, list);
|
|
}
|
|
|
|
#ifdef CONFIG_LIST_HARDENED
|
|
|
|
#ifdef CONFIG_DEBUG_LIST
|
|
# define __list_valid_slowpath
|
|
#else
|
|
# define __list_valid_slowpath __cold __preserve_most
|
|
#endif
|
|
|
|
/*
|
|
* Performs the full set of list corruption checks before __list_add().
|
|
* On list corruption reports a warning, and returns false.
|
|
*/
|
|
extern bool __list_valid_slowpath __list_add_valid_or_report(struct list_head *new,
|
|
struct list_head *prev,
|
|
struct list_head *next);
|
|
|
|
/*
|
|
* Performs list corruption checks before __list_add(). Returns false if a
|
|
* corruption is detected, true otherwise.
|
|
*
|
|
* With CONFIG_LIST_HARDENED only, performs minimal list integrity checking
|
|
* inline to catch non-faulting corruptions, and only if a corruption is
|
|
* detected calls the reporting function __list_add_valid_or_report().
|
|
*/
|
|
static __always_inline bool __list_add_valid(struct list_head *new,
|
|
struct list_head *prev,
|
|
struct list_head *next)
|
|
{
|
|
bool ret = true;
|
|
|
|
if (!IS_ENABLED(CONFIG_DEBUG_LIST)) {
|
|
/*
|
|
* With the hardening version, elide checking if next and prev
|
|
* are NULL, since the immediate dereference of them below would
|
|
* result in a fault if NULL.
|
|
*
|
|
* With the reduced set of checks, we can afford to inline the
|
|
* checks, which also gives the compiler a chance to elide some
|
|
* of them completely if they can be proven at compile-time. If
|
|
* one of the pre-conditions does not hold, the slow-path will
|
|
* show a report which pre-condition failed.
|
|
*/
|
|
if (likely(next->prev == prev && prev->next == next && new != prev && new != next))
|
|
return true;
|
|
ret = false;
|
|
}
|
|
|
|
ret &= __list_add_valid_or_report(new, prev, next);
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Performs the full set of list corruption checks before __list_del_entry().
|
|
* On list corruption reports a warning, and returns false.
|
|
*/
|
|
extern bool __list_valid_slowpath __list_del_entry_valid_or_report(struct list_head *entry);
|
|
|
|
/*
|
|
* Performs list corruption checks before __list_del_entry(). Returns false if a
|
|
* corruption is detected, true otherwise.
|
|
*
|
|
* With CONFIG_LIST_HARDENED only, performs minimal list integrity checking
|
|
* inline to catch non-faulting corruptions, and only if a corruption is
|
|
* detected calls the reporting function __list_del_entry_valid_or_report().
|
|
*/
|
|
static __always_inline bool __list_del_entry_valid(struct list_head *entry)
|
|
{
|
|
bool ret = true;
|
|
|
|
if (!IS_ENABLED(CONFIG_DEBUG_LIST)) {
|
|
struct list_head *prev = entry->prev;
|
|
struct list_head *next = entry->next;
|
|
|
|
/*
|
|
* With the hardening version, elide checking if next and prev
|
|
* are NULL, LIST_POISON1 or LIST_POISON2, since the immediate
|
|
* dereference of them below would result in a fault.
|
|
*/
|
|
if (likely(prev->next == entry && next->prev == entry))
|
|
return true;
|
|
ret = false;
|
|
}
|
|
|
|
ret &= __list_del_entry_valid_or_report(entry);
|
|
return ret;
|
|
}
|
|
#else
|
|
static inline bool __list_add_valid(struct list_head *new,
|
|
struct list_head *prev,
|
|
struct list_head *next)
|
|
{
|
|
return true;
|
|
}
|
|
static inline bool __list_del_entry_valid(struct list_head *entry)
|
|
{
|
|
return true;
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* Insert a new entry between two known consecutive entries.
|
|
*
|
|
* This is only for internal list manipulation where we know
|
|
* the prev/next entries already!
|
|
*/
|
|
static inline void __list_add(struct list_head *new,
|
|
struct list_head *prev,
|
|
struct list_head *next)
|
|
{
|
|
if (!__list_add_valid(new, prev, next))
|
|
return;
|
|
|
|
next->prev = new;
|
|
new->next = next;
|
|
new->prev = prev;
|
|
WRITE_ONCE(prev->next, new);
|
|
}
|
|
|
|
/**
|
|
* list_add - add a new entry
|
|
* @new: new entry to be added
|
|
* @head: list head to add it after
|
|
*
|
|
* Insert a new entry after the specified head.
|
|
* This is good for implementing stacks.
|
|
*/
|
|
static inline void list_add(struct list_head *new, struct list_head *head)
|
|
{
|
|
__list_add(new, head, head->next);
|
|
}
|
|
|
|
|
|
/**
|
|
* list_add_tail - add a new entry
|
|
* @new: new entry to be added
|
|
* @head: list head to add it before
|
|
*
|
|
* Insert a new entry before the specified head.
|
|
* This is useful for implementing queues.
|
|
*/
|
|
static inline void list_add_tail(struct list_head *new, struct list_head *head)
|
|
{
|
|
__list_add(new, head->prev, head);
|
|
}
|
|
|
|
/*
|
|
* Delete a list entry by making the prev/next entries
|
|
* point to each other.
|
|
*
|
|
* This is only for internal list manipulation where we know
|
|
* the prev/next entries already!
|
|
*/
|
|
static inline void __list_del(struct list_head * prev, struct list_head * next)
|
|
{
|
|
next->prev = prev;
|
|
WRITE_ONCE(prev->next, next);
|
|
}
|
|
|
|
/*
|
|
* Delete a list entry and clear the 'prev' pointer.
|
|
*
|
|
* This is a special-purpose list clearing method used in the networking code
|
|
* for lists allocated as per-cpu, where we don't want to incur the extra
|
|
* WRITE_ONCE() overhead of a regular list_del_init(). The code that uses this
|
|
* needs to check the node 'prev' pointer instead of calling list_empty().
|
|
*/
|
|
static inline void __list_del_clearprev(struct list_head *entry)
|
|
{
|
|
__list_del(entry->prev, entry->next);
|
|
entry->prev = NULL;
|
|
}
|
|
|
|
static inline void __list_del_entry(struct list_head *entry)
|
|
{
|
|
if (!__list_del_entry_valid(entry))
|
|
return;
|
|
|
|
__list_del(entry->prev, entry->next);
|
|
}
|
|
|
|
/**
|
|
* list_del - deletes entry from list.
|
|
* @entry: the element to delete from the list.
|
|
* Note: list_empty() on entry does not return true after this, the entry is
|
|
* in an undefined state.
|
|
*/
|
|
static inline void list_del(struct list_head *entry)
|
|
{
|
|
__list_del_entry(entry);
|
|
entry->next = LIST_POISON1;
|
|
entry->prev = LIST_POISON2;
|
|
}
|
|
|
|
/**
|
|
* list_replace - replace old entry by new one
|
|
* @old : the element to be replaced
|
|
* @new : the new element to insert
|
|
*
|
|
* If @old was empty, it will be overwritten.
|
|
*/
|
|
static inline void list_replace(struct list_head *old,
|
|
struct list_head *new)
|
|
{
|
|
new->next = old->next;
|
|
new->next->prev = new;
|
|
new->prev = old->prev;
|
|
new->prev->next = new;
|
|
}
|
|
|
|
/**
|
|
* list_replace_init - replace old entry by new one and initialize the old one
|
|
* @old : the element to be replaced
|
|
* @new : the new element to insert
|
|
*
|
|
* If @old was empty, it will be overwritten.
|
|
*/
|
|
static inline void list_replace_init(struct list_head *old,
|
|
struct list_head *new)
|
|
{
|
|
list_replace(old, new);
|
|
INIT_LIST_HEAD(old);
|
|
}
|
|
|
|
/**
|
|
* list_swap - replace entry1 with entry2 and re-add entry1 at entry2's position
|
|
* @entry1: the location to place entry2
|
|
* @entry2: the location to place entry1
|
|
*/
|
|
static inline void list_swap(struct list_head *entry1,
|
|
struct list_head *entry2)
|
|
{
|
|
struct list_head *pos = entry2->prev;
|
|
|
|
list_del(entry2);
|
|
list_replace(entry1, entry2);
|
|
if (pos == entry1)
|
|
pos = entry2;
|
|
list_add(entry1, pos);
|
|
}
|
|
|
|
/**
|
|
* list_del_init - deletes entry from list and reinitialize it.
|
|
* @entry: the element to delete from the list.
|
|
*/
|
|
static inline void list_del_init(struct list_head *entry)
|
|
{
|
|
__list_del_entry(entry);
|
|
INIT_LIST_HEAD(entry);
|
|
}
|
|
|
|
/**
|
|
* list_move - delete from one list and add as another's head
|
|
* @list: the entry to move
|
|
* @head: the head that will precede our entry
|
|
*/
|
|
static inline void list_move(struct list_head *list, struct list_head *head)
|
|
{
|
|
__list_del_entry(list);
|
|
list_add(list, head);
|
|
}
|
|
|
|
/**
|
|
* list_move_tail - delete from one list and add as another's tail
|
|
* @list: the entry to move
|
|
* @head: the head that will follow our entry
|
|
*/
|
|
static inline void list_move_tail(struct list_head *list,
|
|
struct list_head *head)
|
|
{
|
|
__list_del_entry(list);
|
|
list_add_tail(list, head);
|
|
}
|
|
|
|
/**
|
|
* list_bulk_move_tail - move a subsection of a list to its tail
|
|
* @head: the head that will follow our entry
|
|
* @first: first entry to move
|
|
* @last: last entry to move, can be the same as first
|
|
*
|
|
* Move all entries between @first and including @last before @head.
|
|
* All three entries must belong to the same linked list.
|
|
*/
|
|
static inline void list_bulk_move_tail(struct list_head *head,
|
|
struct list_head *first,
|
|
struct list_head *last)
|
|
{
|
|
first->prev->next = last->next;
|
|
last->next->prev = first->prev;
|
|
|
|
head->prev->next = first;
|
|
first->prev = head->prev;
|
|
|
|
last->next = head;
|
|
head->prev = last;
|
|
}
|
|
|
|
/**
|
|
* list_is_first -- tests whether @list is the first entry in list @head
|
|
* @list: the entry to test
|
|
* @head: the head of the list
|
|
*/
|
|
static inline int list_is_first(const struct list_head *list, const struct list_head *head)
|
|
{
|
|
return list->prev == head;
|
|
}
|
|
|
|
/**
|
|
* list_is_last - tests whether @list is the last entry in list @head
|
|
* @list: the entry to test
|
|
* @head: the head of the list
|
|
*/
|
|
static inline int list_is_last(const struct list_head *list, const struct list_head *head)
|
|
{
|
|
return list->next == head;
|
|
}
|
|
|
|
/**
|
|
* list_is_head - tests whether @list is the list @head
|
|
* @list: the entry to test
|
|
* @head: the head of the list
|
|
*/
|
|
static inline int list_is_head(const struct list_head *list, const struct list_head *head)
|
|
{
|
|
return list == head;
|
|
}
|
|
|
|
/**
|
|
* list_empty - tests whether a list is empty
|
|
* @head: the list to test.
|
|
*/
|
|
static inline int list_empty(const struct list_head *head)
|
|
{
|
|
return READ_ONCE(head->next) == head;
|
|
}
|
|
|
|
/**
|
|
* list_del_init_careful - deletes entry from list and reinitialize it.
|
|
* @entry: the element to delete from the list.
|
|
*
|
|
* This is the same as list_del_init(), except designed to be used
|
|
* together with list_empty_careful() in a way to guarantee ordering
|
|
* of other memory operations.
|
|
*
|
|
* Any memory operations done before a list_del_init_careful() are
|
|
* guaranteed to be visible after a list_empty_careful() test.
|
|
*/
|
|
static inline void list_del_init_careful(struct list_head *entry)
|
|
{
|
|
__list_del_entry(entry);
|
|
WRITE_ONCE(entry->prev, entry);
|
|
smp_store_release(&entry->next, entry);
|
|
}
|
|
|
|
/**
|
|
* list_empty_careful - tests whether a list is empty and not being modified
|
|
* @head: the list to test
|
|
*
|
|
* Description:
|
|
* tests whether a list is empty _and_ checks that no other CPU might be
|
|
* in the process of modifying either member (next or prev)
|
|
*
|
|
* NOTE: using list_empty_careful() without synchronization
|
|
* can only be safe if the only activity that can happen
|
|
* to the list entry is list_del_init(). Eg. it cannot be used
|
|
* if another CPU could re-list_add() it.
|
|
*/
|
|
static inline int list_empty_careful(const struct list_head *head)
|
|
{
|
|
struct list_head *next = smp_load_acquire(&head->next);
|
|
return list_is_head(next, head) && (next == READ_ONCE(head->prev));
|
|
}
|
|
|
|
/**
|
|
* list_rotate_left - rotate the list to the left
|
|
* @head: the head of the list
|
|
*/
|
|
static inline void list_rotate_left(struct list_head *head)
|
|
{
|
|
struct list_head *first;
|
|
|
|
if (!list_empty(head)) {
|
|
first = head->next;
|
|
list_move_tail(first, head);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* list_rotate_to_front() - Rotate list to specific item.
|
|
* @list: The desired new front of the list.
|
|
* @head: The head of the list.
|
|
*
|
|
* Rotates list so that @list becomes the new front of the list.
|
|
*/
|
|
static inline void list_rotate_to_front(struct list_head *list,
|
|
struct list_head *head)
|
|
{
|
|
/*
|
|
* Deletes the list head from the list denoted by @head and
|
|
* places it as the tail of @list, this effectively rotates the
|
|
* list so that @list is at the front.
|
|
*/
|
|
list_move_tail(head, list);
|
|
}
|
|
|
|
/**
|
|
* list_is_singular - tests whether a list has just one entry.
|
|
* @head: the list to test.
|
|
*/
|
|
static inline int list_is_singular(const struct list_head *head)
|
|
{
|
|
return !list_empty(head) && (head->next == head->prev);
|
|
}
|
|
|
|
static inline void __list_cut_position(struct list_head *list,
|
|
struct list_head *head, struct list_head *entry)
|
|
{
|
|
struct list_head *new_first = entry->next;
|
|
list->next = head->next;
|
|
list->next->prev = list;
|
|
list->prev = entry;
|
|
entry->next = list;
|
|
head->next = new_first;
|
|
new_first->prev = head;
|
|
}
|
|
|
|
/**
|
|
* list_cut_position - cut a list into two
|
|
* @list: a new list to add all removed entries
|
|
* @head: a list with entries
|
|
* @entry: an entry within head, could be the head itself
|
|
* and if so we won't cut the list
|
|
*
|
|
* This helper moves the initial part of @head, up to and
|
|
* including @entry, from @head to @list. You should
|
|
* pass on @entry an element you know is on @head. @list
|
|
* should be an empty list or a list you do not care about
|
|
* losing its data.
|
|
*
|
|
*/
|
|
static inline void list_cut_position(struct list_head *list,
|
|
struct list_head *head, struct list_head *entry)
|
|
{
|
|
if (list_empty(head))
|
|
return;
|
|
if (list_is_singular(head) && !list_is_head(entry, head) && (entry != head->next))
|
|
return;
|
|
if (list_is_head(entry, head))
|
|
INIT_LIST_HEAD(list);
|
|
else
|
|
__list_cut_position(list, head, entry);
|
|
}
|
|
|
|
/**
|
|
* list_cut_before - cut a list into two, before given entry
|
|
* @list: a new list to add all removed entries
|
|
* @head: a list with entries
|
|
* @entry: an entry within head, could be the head itself
|
|
*
|
|
* This helper moves the initial part of @head, up to but
|
|
* excluding @entry, from @head to @list. You should pass
|
|
* in @entry an element you know is on @head. @list should
|
|
* be an empty list or a list you do not care about losing
|
|
* its data.
|
|
* If @entry == @head, all entries on @head are moved to
|
|
* @list.
|
|
*/
|
|
static inline void list_cut_before(struct list_head *list,
|
|
struct list_head *head,
|
|
struct list_head *entry)
|
|
{
|
|
if (head->next == entry) {
|
|
INIT_LIST_HEAD(list);
|
|
return;
|
|
}
|
|
list->next = head->next;
|
|
list->next->prev = list;
|
|
list->prev = entry->prev;
|
|
list->prev->next = list;
|
|
head->next = entry;
|
|
entry->prev = head;
|
|
}
|
|
|
|
static inline void __list_splice(const struct list_head *list,
|
|
struct list_head *prev,
|
|
struct list_head *next)
|
|
{
|
|
struct list_head *first = list->next;
|
|
struct list_head *last = list->prev;
|
|
|
|
first->prev = prev;
|
|
prev->next = first;
|
|
|
|
last->next = next;
|
|
next->prev = last;
|
|
}
|
|
|
|
/**
|
|
* list_splice - join two lists, this is designed for stacks
|
|
* @list: the new list to add.
|
|
* @head: the place to add it in the first list.
|
|
*/
|
|
static inline void list_splice(const struct list_head *list,
|
|
struct list_head *head)
|
|
{
|
|
if (!list_empty(list))
|
|
__list_splice(list, head, head->next);
|
|
}
|
|
|
|
/**
|
|
* list_splice_tail - join two lists, each list being a queue
|
|
* @list: the new list to add.
|
|
* @head: the place to add it in the first list.
|
|
*/
|
|
static inline void list_splice_tail(struct list_head *list,
|
|
struct list_head *head)
|
|
{
|
|
if (!list_empty(list))
|
|
__list_splice(list, head->prev, head);
|
|
}
|
|
|
|
/**
|
|
* list_splice_init - join two lists and reinitialise the emptied list.
|
|
* @list: the new list to add.
|
|
* @head: the place to add it in the first list.
|
|
*
|
|
* The list at @list is reinitialised
|
|
*/
|
|
static inline void list_splice_init(struct list_head *list,
|
|
struct list_head *head)
|
|
{
|
|
if (!list_empty(list)) {
|
|
__list_splice(list, head, head->next);
|
|
INIT_LIST_HEAD(list);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* list_splice_tail_init - join two lists and reinitialise the emptied list
|
|
* @list: the new list to add.
|
|
* @head: the place to add it in the first list.
|
|
*
|
|
* Each of the lists is a queue.
|
|
* The list at @list is reinitialised
|
|
*/
|
|
static inline void list_splice_tail_init(struct list_head *list,
|
|
struct list_head *head)
|
|
{
|
|
if (!list_empty(list)) {
|
|
__list_splice(list, head->prev, head);
|
|
INIT_LIST_HEAD(list);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* list_entry - get the struct for this entry
|
|
* @ptr: the &struct list_head pointer.
|
|
* @type: the type of the struct this is embedded in.
|
|
* @member: the name of the list_head within the struct.
|
|
*/
|
|
#define list_entry(ptr, type, member) \
|
|
container_of(ptr, type, member)
|
|
|
|
/**
|
|
* list_first_entry - get the first element from a list
|
|
* @ptr: the list head to take the element from.
|
|
* @type: the type of the struct this is embedded in.
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Note, that list is expected to be not empty.
|
|
*/
|
|
#define list_first_entry(ptr, type, member) \
|
|
list_entry((ptr)->next, type, member)
|
|
|
|
/**
|
|
* list_last_entry - get the last element from a list
|
|
* @ptr: the list head to take the element from.
|
|
* @type: the type of the struct this is embedded in.
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Note, that list is expected to be not empty.
|
|
*/
|
|
#define list_last_entry(ptr, type, member) \
|
|
list_entry((ptr)->prev, type, member)
|
|
|
|
/**
|
|
* list_first_entry_or_null - get the first element from a list
|
|
* @ptr: the list head to take the element from.
|
|
* @type: the type of the struct this is embedded in.
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Note that if the list is empty, it returns NULL.
|
|
*/
|
|
#define list_first_entry_or_null(ptr, type, member) ({ \
|
|
struct list_head *head__ = (ptr); \
|
|
struct list_head *pos__ = READ_ONCE(head__->next); \
|
|
pos__ != head__ ? list_entry(pos__, type, member) : NULL; \
|
|
})
|
|
|
|
/**
|
|
* list_next_entry - get the next element in list
|
|
* @pos: the type * to cursor
|
|
* @member: the name of the list_head within the struct.
|
|
*/
|
|
#define list_next_entry(pos, member) \
|
|
list_entry((pos)->member.next, typeof(*(pos)), member)
|
|
|
|
/**
|
|
* list_next_entry_circular - get the next element in list
|
|
* @pos: the type * to cursor.
|
|
* @head: the list head to take the element from.
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Wraparound if pos is the last element (return the first element).
|
|
* Note, that list is expected to be not empty.
|
|
*/
|
|
#define list_next_entry_circular(pos, head, member) \
|
|
(list_is_last(&(pos)->member, head) ? \
|
|
list_first_entry(head, typeof(*(pos)), member) : list_next_entry(pos, member))
|
|
|
|
/**
|
|
* list_prev_entry - get the prev element in list
|
|
* @pos: the type * to cursor
|
|
* @member: the name of the list_head within the struct.
|
|
*/
|
|
#define list_prev_entry(pos, member) \
|
|
list_entry((pos)->member.prev, typeof(*(pos)), member)
|
|
|
|
/**
|
|
* list_prev_entry_circular - get the prev element in list
|
|
* @pos: the type * to cursor.
|
|
* @head: the list head to take the element from.
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Wraparound if pos is the first element (return the last element).
|
|
* Note, that list is expected to be not empty.
|
|
*/
|
|
#define list_prev_entry_circular(pos, head, member) \
|
|
(list_is_first(&(pos)->member, head) ? \
|
|
list_last_entry(head, typeof(*(pos)), member) : list_prev_entry(pos, member))
|
|
|
|
/**
|
|
* list_for_each - iterate over a list
|
|
* @pos: the &struct list_head to use as a loop cursor.
|
|
* @head: the head for your list.
|
|
*/
|
|
#define list_for_each(pos, head) \
|
|
for (pos = (head)->next; !list_is_head(pos, (head)); pos = pos->next)
|
|
|
|
/**
|
|
* list_for_each_rcu - Iterate over a list in an RCU-safe fashion
|
|
* @pos: the &struct list_head to use as a loop cursor.
|
|
* @head: the head for your list.
|
|
*/
|
|
#define list_for_each_rcu(pos, head) \
|
|
for (pos = rcu_dereference((head)->next); \
|
|
!list_is_head(pos, (head)); \
|
|
pos = rcu_dereference(pos->next))
|
|
|
|
/**
|
|
* list_for_each_continue - continue iteration over a list
|
|
* @pos: the &struct list_head to use as a loop cursor.
|
|
* @head: the head for your list.
|
|
*
|
|
* Continue to iterate over a list, continuing after the current position.
|
|
*/
|
|
#define list_for_each_continue(pos, head) \
|
|
for (pos = pos->next; !list_is_head(pos, (head)); pos = pos->next)
|
|
|
|
/**
|
|
* list_for_each_prev - iterate over a list backwards
|
|
* @pos: the &struct list_head to use as a loop cursor.
|
|
* @head: the head for your list.
|
|
*/
|
|
#define list_for_each_prev(pos, head) \
|
|
for (pos = (head)->prev; !list_is_head(pos, (head)); pos = pos->prev)
|
|
|
|
/**
|
|
* list_for_each_safe - iterate over a list safe against removal of list entry
|
|
* @pos: the &struct list_head to use as a loop cursor.
|
|
* @n: another &struct list_head to use as temporary storage
|
|
* @head: the head for your list.
|
|
*/
|
|
#define list_for_each_safe(pos, n, head) \
|
|
for (pos = (head)->next, n = pos->next; \
|
|
!list_is_head(pos, (head)); \
|
|
pos = n, n = pos->next)
|
|
|
|
/**
|
|
* list_for_each_prev_safe - iterate over a list backwards safe against removal of list entry
|
|
* @pos: the &struct list_head to use as a loop cursor.
|
|
* @n: another &struct list_head to use as temporary storage
|
|
* @head: the head for your list.
|
|
*/
|
|
#define list_for_each_prev_safe(pos, n, head) \
|
|
for (pos = (head)->prev, n = pos->prev; \
|
|
!list_is_head(pos, (head)); \
|
|
pos = n, n = pos->prev)
|
|
|
|
/**
|
|
* list_count_nodes - count nodes in the list
|
|
* @head: the head for your list.
|
|
*/
|
|
static inline size_t list_count_nodes(struct list_head *head)
|
|
{
|
|
struct list_head *pos;
|
|
size_t count = 0;
|
|
|
|
list_for_each(pos, head)
|
|
count++;
|
|
|
|
return count;
|
|
}
|
|
|
|
/**
|
|
* list_entry_is_head - test if the entry points to the head of the list
|
|
* @pos: the type * to cursor
|
|
* @head: the head for your list.
|
|
* @member: the name of the list_head within the struct.
|
|
*/
|
|
#define list_entry_is_head(pos, head, member) \
|
|
(&pos->member == (head))
|
|
|
|
/**
|
|
* list_for_each_entry - iterate over list of given type
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @head: the head for your list.
|
|
* @member: the name of the list_head within the struct.
|
|
*/
|
|
#define list_for_each_entry(pos, head, member) \
|
|
for (pos = list_first_entry(head, typeof(*pos), member); \
|
|
!list_entry_is_head(pos, head, member); \
|
|
pos = list_next_entry(pos, member))
|
|
|
|
/**
|
|
* list_for_each_entry_reverse - iterate backwards over list of given type.
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @head: the head for your list.
|
|
* @member: the name of the list_head within the struct.
|
|
*/
|
|
#define list_for_each_entry_reverse(pos, head, member) \
|
|
for (pos = list_last_entry(head, typeof(*pos), member); \
|
|
!list_entry_is_head(pos, head, member); \
|
|
pos = list_prev_entry(pos, member))
|
|
|
|
/**
|
|
* list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue()
|
|
* @pos: the type * to use as a start point
|
|
* @head: the head of the list
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Prepares a pos entry for use as a start point in list_for_each_entry_continue().
|
|
*/
|
|
#define list_prepare_entry(pos, head, member) \
|
|
((pos) ? : list_entry(head, typeof(*pos), member))
|
|
|
|
/**
|
|
* list_for_each_entry_continue - continue iteration over list of given type
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @head: the head for your list.
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Continue to iterate over list of given type, continuing after
|
|
* the current position.
|
|
*/
|
|
#define list_for_each_entry_continue(pos, head, member) \
|
|
for (pos = list_next_entry(pos, member); \
|
|
!list_entry_is_head(pos, head, member); \
|
|
pos = list_next_entry(pos, member))
|
|
|
|
/**
|
|
* list_for_each_entry_continue_reverse - iterate backwards from the given point
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @head: the head for your list.
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Start to iterate over list of given type backwards, continuing after
|
|
* the current position.
|
|
*/
|
|
#define list_for_each_entry_continue_reverse(pos, head, member) \
|
|
for (pos = list_prev_entry(pos, member); \
|
|
!list_entry_is_head(pos, head, member); \
|
|
pos = list_prev_entry(pos, member))
|
|
|
|
/**
|
|
* list_for_each_entry_from - iterate over list of given type from the current point
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @head: the head for your list.
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Iterate over list of given type, continuing from current position.
|
|
*/
|
|
#define list_for_each_entry_from(pos, head, member) \
|
|
for (; !list_entry_is_head(pos, head, member); \
|
|
pos = list_next_entry(pos, member))
|
|
|
|
/**
|
|
* list_for_each_entry_from_reverse - iterate backwards over list of given type
|
|
* from the current point
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @head: the head for your list.
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Iterate backwards over list of given type, continuing from current position.
|
|
*/
|
|
#define list_for_each_entry_from_reverse(pos, head, member) \
|
|
for (; !list_entry_is_head(pos, head, member); \
|
|
pos = list_prev_entry(pos, member))
|
|
|
|
/**
|
|
* list_for_each_entry_safe - iterate over list of given type safe against removal of list entry
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @n: another type * to use as temporary storage
|
|
* @head: the head for your list.
|
|
* @member: the name of the list_head within the struct.
|
|
*/
|
|
#define list_for_each_entry_safe(pos, n, head, member) \
|
|
for (pos = list_first_entry(head, typeof(*pos), member), \
|
|
n = list_next_entry(pos, member); \
|
|
!list_entry_is_head(pos, head, member); \
|
|
pos = n, n = list_next_entry(n, member))
|
|
|
|
/**
|
|
* list_for_each_entry_safe_continue - continue list iteration safe against removal
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @n: another type * to use as temporary storage
|
|
* @head: the head for your list.
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Iterate over list of given type, continuing after current point,
|
|
* safe against removal of list entry.
|
|
*/
|
|
#define list_for_each_entry_safe_continue(pos, n, head, member) \
|
|
for (pos = list_next_entry(pos, member), \
|
|
n = list_next_entry(pos, member); \
|
|
!list_entry_is_head(pos, head, member); \
|
|
pos = n, n = list_next_entry(n, member))
|
|
|
|
/**
|
|
* list_for_each_entry_safe_from - iterate over list from current point safe against removal
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @n: another type * to use as temporary storage
|
|
* @head: the head for your list.
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Iterate over list of given type from current point, safe against
|
|
* removal of list entry.
|
|
*/
|
|
#define list_for_each_entry_safe_from(pos, n, head, member) \
|
|
for (n = list_next_entry(pos, member); \
|
|
!list_entry_is_head(pos, head, member); \
|
|
pos = n, n = list_next_entry(n, member))
|
|
|
|
/**
|
|
* list_for_each_entry_safe_reverse - iterate backwards over list safe against removal
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @n: another type * to use as temporary storage
|
|
* @head: the head for your list.
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* Iterate backwards over list of given type, safe against removal
|
|
* of list entry.
|
|
*/
|
|
#define list_for_each_entry_safe_reverse(pos, n, head, member) \
|
|
for (pos = list_last_entry(head, typeof(*pos), member), \
|
|
n = list_prev_entry(pos, member); \
|
|
!list_entry_is_head(pos, head, member); \
|
|
pos = n, n = list_prev_entry(n, member))
|
|
|
|
/**
|
|
* list_safe_reset_next - reset a stale list_for_each_entry_safe loop
|
|
* @pos: the loop cursor used in the list_for_each_entry_safe loop
|
|
* @n: temporary storage used in list_for_each_entry_safe
|
|
* @member: the name of the list_head within the struct.
|
|
*
|
|
* list_safe_reset_next is not safe to use in general if the list may be
|
|
* modified concurrently (eg. the lock is dropped in the loop body). An
|
|
* exception to this is if the cursor element (pos) is pinned in the list,
|
|
* and list_safe_reset_next is called after re-taking the lock and before
|
|
* completing the current iteration of the loop body.
|
|
*/
|
|
#define list_safe_reset_next(pos, n, member) \
|
|
n = list_next_entry(pos, member)
|
|
|
|
/*
|
|
* Double linked lists with a single pointer list head.
|
|
* Mostly useful for hash tables where the two pointer list head is
|
|
* too wasteful.
|
|
* You lose the ability to access the tail in O(1).
|
|
*/
|
|
|
|
#define HLIST_HEAD_INIT { .first = NULL }
|
|
#define HLIST_HEAD(name) struct hlist_head name = { .first = NULL }
|
|
#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL)
|
|
static inline void INIT_HLIST_NODE(struct hlist_node *h)
|
|
{
|
|
h->next = NULL;
|
|
h->pprev = NULL;
|
|
}
|
|
|
|
/**
|
|
* hlist_unhashed - Has node been removed from list and reinitialized?
|
|
* @h: Node to be checked
|
|
*
|
|
* Not that not all removal functions will leave a node in unhashed
|
|
* state. For example, hlist_nulls_del_init_rcu() does leave the
|
|
* node in unhashed state, but hlist_nulls_del() does not.
|
|
*/
|
|
static inline int hlist_unhashed(const struct hlist_node *h)
|
|
{
|
|
return !h->pprev;
|
|
}
|
|
|
|
/**
|
|
* hlist_unhashed_lockless - Version of hlist_unhashed for lockless use
|
|
* @h: Node to be checked
|
|
*
|
|
* This variant of hlist_unhashed() must be used in lockless contexts
|
|
* to avoid potential load-tearing. The READ_ONCE() is paired with the
|
|
* various WRITE_ONCE() in hlist helpers that are defined below.
|
|
*/
|
|
static inline int hlist_unhashed_lockless(const struct hlist_node *h)
|
|
{
|
|
return !READ_ONCE(h->pprev);
|
|
}
|
|
|
|
/**
|
|
* hlist_empty - Is the specified hlist_head structure an empty hlist?
|
|
* @h: Structure to check.
|
|
*/
|
|
static inline int hlist_empty(const struct hlist_head *h)
|
|
{
|
|
return !READ_ONCE(h->first);
|
|
}
|
|
|
|
static inline void __hlist_del(struct hlist_node *n)
|
|
{
|
|
struct hlist_node *next = n->next;
|
|
struct hlist_node **pprev = n->pprev;
|
|
|
|
WRITE_ONCE(*pprev, next);
|
|
if (next)
|
|
WRITE_ONCE(next->pprev, pprev);
|
|
}
|
|
|
|
/**
|
|
* hlist_del - Delete the specified hlist_node from its list
|
|
* @n: Node to delete.
|
|
*
|
|
* Note that this function leaves the node in hashed state. Use
|
|
* hlist_del_init() or similar instead to unhash @n.
|
|
*/
|
|
static inline void hlist_del(struct hlist_node *n)
|
|
{
|
|
__hlist_del(n);
|
|
n->next = LIST_POISON1;
|
|
n->pprev = LIST_POISON2;
|
|
}
|
|
|
|
/**
|
|
* hlist_del_init - Delete the specified hlist_node from its list and initialize
|
|
* @n: Node to delete.
|
|
*
|
|
* Note that this function leaves the node in unhashed state.
|
|
*/
|
|
static inline void hlist_del_init(struct hlist_node *n)
|
|
{
|
|
if (!hlist_unhashed(n)) {
|
|
__hlist_del(n);
|
|
INIT_HLIST_NODE(n);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* hlist_add_head - add a new entry at the beginning of the hlist
|
|
* @n: new entry to be added
|
|
* @h: hlist head to add it after
|
|
*
|
|
* Insert a new entry after the specified head.
|
|
* This is good for implementing stacks.
|
|
*/
|
|
static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
|
|
{
|
|
struct hlist_node *first = h->first;
|
|
WRITE_ONCE(n->next, first);
|
|
if (first)
|
|
WRITE_ONCE(first->pprev, &n->next);
|
|
WRITE_ONCE(h->first, n);
|
|
WRITE_ONCE(n->pprev, &h->first);
|
|
}
|
|
|
|
/**
|
|
* hlist_add_before - add a new entry before the one specified
|
|
* @n: new entry to be added
|
|
* @next: hlist node to add it before, which must be non-NULL
|
|
*/
|
|
static inline void hlist_add_before(struct hlist_node *n,
|
|
struct hlist_node *next)
|
|
{
|
|
WRITE_ONCE(n->pprev, next->pprev);
|
|
WRITE_ONCE(n->next, next);
|
|
WRITE_ONCE(next->pprev, &n->next);
|
|
WRITE_ONCE(*(n->pprev), n);
|
|
}
|
|
|
|
/**
|
|
* hlist_add_behind - add a new entry after the one specified
|
|
* @n: new entry to be added
|
|
* @prev: hlist node to add it after, which must be non-NULL
|
|
*/
|
|
static inline void hlist_add_behind(struct hlist_node *n,
|
|
struct hlist_node *prev)
|
|
{
|
|
WRITE_ONCE(n->next, prev->next);
|
|
WRITE_ONCE(prev->next, n);
|
|
WRITE_ONCE(n->pprev, &prev->next);
|
|
|
|
if (n->next)
|
|
WRITE_ONCE(n->next->pprev, &n->next);
|
|
}
|
|
|
|
/**
|
|
* hlist_add_fake - create a fake hlist consisting of a single headless node
|
|
* @n: Node to make a fake list out of
|
|
*
|
|
* This makes @n appear to be its own predecessor on a headless hlist.
|
|
* The point of this is to allow things like hlist_del() to work correctly
|
|
* in cases where there is no list.
|
|
*/
|
|
static inline void hlist_add_fake(struct hlist_node *n)
|
|
{
|
|
n->pprev = &n->next;
|
|
}
|
|
|
|
/**
|
|
* hlist_fake: Is this node a fake hlist?
|
|
* @h: Node to check for being a self-referential fake hlist.
|
|
*/
|
|
static inline bool hlist_fake(struct hlist_node *h)
|
|
{
|
|
return h->pprev == &h->next;
|
|
}
|
|
|
|
/**
|
|
* hlist_is_singular_node - is node the only element of the specified hlist?
|
|
* @n: Node to check for singularity.
|
|
* @h: Header for potentially singular list.
|
|
*
|
|
* Check whether the node is the only node of the head without
|
|
* accessing head, thus avoiding unnecessary cache misses.
|
|
*/
|
|
static inline bool
|
|
hlist_is_singular_node(struct hlist_node *n, struct hlist_head *h)
|
|
{
|
|
return !n->next && n->pprev == &h->first;
|
|
}
|
|
|
|
/**
|
|
* hlist_move_list - Move an hlist
|
|
* @old: hlist_head for old list.
|
|
* @new: hlist_head for new list.
|
|
*
|
|
* Move a list from one list head to another. Fixup the pprev
|
|
* reference of the first entry if it exists.
|
|
*/
|
|
static inline void hlist_move_list(struct hlist_head *old,
|
|
struct hlist_head *new)
|
|
{
|
|
new->first = old->first;
|
|
if (new->first)
|
|
new->first->pprev = &new->first;
|
|
old->first = NULL;
|
|
}
|
|
|
|
#define hlist_entry(ptr, type, member) container_of(ptr,type,member)
|
|
|
|
#define hlist_for_each(pos, head) \
|
|
for (pos = (head)->first; pos ; pos = pos->next)
|
|
|
|
#define hlist_for_each_safe(pos, n, head) \
|
|
for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \
|
|
pos = n)
|
|
|
|
#define hlist_entry_safe(ptr, type, member) \
|
|
({ typeof(ptr) ____ptr = (ptr); \
|
|
____ptr ? hlist_entry(____ptr, type, member) : NULL; \
|
|
})
|
|
|
|
/**
|
|
* hlist_for_each_entry - iterate over list of given type
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @head: the head for your list.
|
|
* @member: the name of the hlist_node within the struct.
|
|
*/
|
|
#define hlist_for_each_entry(pos, head, member) \
|
|
for (pos = hlist_entry_safe((head)->first, typeof(*(pos)), member);\
|
|
pos; \
|
|
pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))
|
|
|
|
/**
|
|
* hlist_for_each_entry_continue - iterate over a hlist continuing after current point
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @member: the name of the hlist_node within the struct.
|
|
*/
|
|
#define hlist_for_each_entry_continue(pos, member) \
|
|
for (pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member);\
|
|
pos; \
|
|
pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))
|
|
|
|
/**
|
|
* hlist_for_each_entry_from - iterate over a hlist continuing from current point
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @member: the name of the hlist_node within the struct.
|
|
*/
|
|
#define hlist_for_each_entry_from(pos, member) \
|
|
for (; pos; \
|
|
pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))
|
|
|
|
/**
|
|
* hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry
|
|
* @pos: the type * to use as a loop cursor.
|
|
* @n: a &struct hlist_node to use as temporary storage
|
|
* @head: the head for your list.
|
|
* @member: the name of the hlist_node within the struct.
|
|
*/
|
|
#define hlist_for_each_entry_safe(pos, n, head, member) \
|
|
for (pos = hlist_entry_safe((head)->first, typeof(*pos), member);\
|
|
pos && ({ n = pos->member.next; 1; }); \
|
|
pos = hlist_entry_safe(n, typeof(*pos), member))
|
|
|
|
#endif
|