mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-23 09:19:51 +00:00
drm/i915: Coordinate i915_active with its own mutex
Forgo the struct_mutex serialisation for i915_active, and interpose its own mutex handling for active/retire. This is a multi-layered sleight-of-hand. First, we had to ensure that no active/retire callbacks accidentally inverted the mutex ordering rules, nor assumed that they were themselves serialised by struct_mutex. More challenging though, is the rule over updating elements of the active rbtree. Instead of the whole i915_active now being serialised by struct_mutex, allocations/rotations of the tree are serialised by the i915_active.mutex and individual nodes are serialised by the caller using the i915_timeline.mutex (we need to use nested spinlocks to interact with the dma_fence callback lists). The pain point here is that instead of a single mutex around execbuf, we now have to take a mutex for active tracker (one for each vma, context, etc) and a couple of spinlocks for each fence update. The improvement in fine grained locking allowing for multiple concurrent clients (eventually!) should be worth it in typical loads. v2: Add some comments that barely elucidate anything :( Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com> Link: https://patchwork.freedesktop.org/patch/msgid/20191004134015.13204-6-chris@chris-wilson.co.uk
This commit is contained in:
parent
274cbf20fd
commit
b1e3177bd1
24 changed files with 298 additions and 572 deletions
|
@ -257,7 +257,7 @@ intel_frontbuffer_get(struct drm_i915_gem_object *obj)
|
|||
front->obj = obj;
|
||||
kref_init(&front->ref);
|
||||
atomic_set(&front->bits, 0);
|
||||
i915_active_init(i915, &front->write,
|
||||
i915_active_init(&front->write,
|
||||
frontbuffer_active,
|
||||
i915_active_may_sleep(frontbuffer_retire));
|
||||
|
||||
|
|
|
@ -1360,8 +1360,7 @@ void intel_overlay_setup(struct drm_i915_private *dev_priv)
|
|||
overlay->contrast = 75;
|
||||
overlay->saturation = 146;
|
||||
|
||||
i915_active_init(dev_priv,
|
||||
&overlay->last_flip,
|
||||
i915_active_init(&overlay->last_flip,
|
||||
NULL, intel_overlay_last_flip_retire);
|
||||
|
||||
ret = get_registers(overlay, OVERLAY_NEEDS_PHYSICAL(dev_priv));
|
||||
|
|
|
@ -868,20 +868,18 @@ static int context_barrier_task(struct i915_gem_context *ctx,
|
|||
void (*task)(void *data),
|
||||
void *data)
|
||||
{
|
||||
struct drm_i915_private *i915 = ctx->i915;
|
||||
struct context_barrier_task *cb;
|
||||
struct i915_gem_engines_iter it;
|
||||
struct intel_context *ce;
|
||||
int err = 0;
|
||||
|
||||
lockdep_assert_held(&i915->drm.struct_mutex);
|
||||
GEM_BUG_ON(!task);
|
||||
|
||||
cb = kmalloc(sizeof(*cb), GFP_KERNEL);
|
||||
if (!cb)
|
||||
return -ENOMEM;
|
||||
|
||||
i915_active_init(i915, &cb->base, NULL, cb_retire);
|
||||
i915_active_init(&cb->base, NULL, cb_retire);
|
||||
err = i915_active_acquire(&cb->base);
|
||||
if (err) {
|
||||
kfree(cb);
|
||||
|
|
|
@ -8,6 +8,7 @@
|
|||
#define __I915_GEM_OBJECT_TYPES_H__
|
||||
|
||||
#include <drm/drm_gem.h>
|
||||
#include <uapi/drm/i915_drm.h>
|
||||
|
||||
#include "i915_active.h"
|
||||
#include "i915_selftest.h"
|
||||
|
|
|
@ -16,14 +16,11 @@ static void call_idle_barriers(struct intel_engine_cs *engine)
|
|||
struct llist_node *node, *next;
|
||||
|
||||
llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) {
|
||||
struct i915_active_request *active =
|
||||
struct dma_fence_cb *cb =
|
||||
container_of((struct list_head *)node,
|
||||
typeof(*active), link);
|
||||
typeof(*cb), node);
|
||||
|
||||
INIT_LIST_HEAD(&active->link);
|
||||
RCU_INIT_POINTER(active->request, NULL);
|
||||
|
||||
active->retire(active, NULL);
|
||||
cb->func(NULL, cb);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -240,7 +240,7 @@ intel_context_init(struct intel_context *ce,
|
|||
|
||||
mutex_init(&ce->pin_mutex);
|
||||
|
||||
i915_active_init(ctx->i915, &ce->active,
|
||||
i915_active_init(&ce->active,
|
||||
__intel_context_active, __intel_context_retire);
|
||||
}
|
||||
|
||||
|
@ -307,7 +307,7 @@ int intel_context_prepare_remote_request(struct intel_context *ce,
|
|||
return err;
|
||||
|
||||
/* Queue this switch after current activity by this context. */
|
||||
err = i915_active_request_set(&tl->last_request, rq);
|
||||
err = i915_active_fence_set(&tl->last_request, rq);
|
||||
mutex_unlock(&tl->mutex);
|
||||
if (err)
|
||||
return err;
|
||||
|
|
|
@ -95,7 +95,7 @@ node_create(struct intel_engine_pool *pool, size_t sz)
|
|||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
node->pool = pool;
|
||||
i915_active_init(engine->i915, &node->active, pool_active, pool_retire);
|
||||
i915_active_init(&node->active, pool_active, pool_retire);
|
||||
|
||||
obj = i915_gem_object_create_internal(engine->i915, sz);
|
||||
if (IS_ERR(obj)) {
|
||||
|
|
|
@ -844,10 +844,10 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt)
|
|||
*/
|
||||
spin_lock_irqsave(&timelines->lock, flags);
|
||||
list_for_each_entry(tl, &timelines->active_list, link) {
|
||||
struct i915_request *rq;
|
||||
struct dma_fence *fence;
|
||||
|
||||
rq = i915_active_request_get_unlocked(&tl->last_request);
|
||||
if (!rq)
|
||||
fence = i915_active_fence_get(&tl->last_request);
|
||||
if (!fence)
|
||||
continue;
|
||||
|
||||
spin_unlock_irqrestore(&timelines->lock, flags);
|
||||
|
@ -859,8 +859,8 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt)
|
|||
* (I915_FENCE_TIMEOUT) so this wait should not be unbounded
|
||||
* in the worst case.
|
||||
*/
|
||||
dma_fence_default_wait(&rq->fence, false, MAX_SCHEDULE_TIMEOUT);
|
||||
i915_request_put(rq);
|
||||
dma_fence_default_wait(fence, false, MAX_SCHEDULE_TIMEOUT);
|
||||
dma_fence_put(fence);
|
||||
|
||||
/* Restart iteration after droping lock */
|
||||
spin_lock_irqsave(&timelines->lock, flags);
|
||||
|
|
|
@ -178,8 +178,7 @@ cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline)
|
|||
cl->hwsp = hwsp;
|
||||
cl->vaddr = page_pack_bits(vaddr, cacheline);
|
||||
|
||||
i915_active_init(hwsp->gt->i915, &cl->active,
|
||||
__cacheline_active, __cacheline_retire);
|
||||
i915_active_init(&cl->active, __cacheline_active, __cacheline_retire);
|
||||
|
||||
return cl;
|
||||
}
|
||||
|
@ -255,7 +254,7 @@ int intel_timeline_init(struct intel_timeline *timeline,
|
|||
|
||||
mutex_init(&timeline->mutex);
|
||||
|
||||
INIT_ACTIVE_REQUEST(&timeline->last_request, &timeline->mutex);
|
||||
INIT_ACTIVE_FENCE(&timeline->last_request, &timeline->mutex);
|
||||
INIT_LIST_HEAD(&timeline->requests);
|
||||
|
||||
i915_syncmap_init(&timeline->sync);
|
||||
|
@ -443,7 +442,7 @@ __intel_timeline_get_seqno(struct intel_timeline *tl,
|
|||
* free it after the current request is retired, which ensures that
|
||||
* all writes into the cacheline from previous requests are complete.
|
||||
*/
|
||||
err = i915_active_ref(&tl->hwsp_cacheline->active, tl, rq);
|
||||
err = i915_active_ref(&tl->hwsp_cacheline->active, tl, &rq->fence);
|
||||
if (err)
|
||||
goto err_cacheline;
|
||||
|
||||
|
|
|
@ -58,12 +58,13 @@ struct intel_timeline {
|
|||
*/
|
||||
struct list_head requests;
|
||||
|
||||
/* Contains an RCU guarded pointer to the last request. No reference is
|
||||
/*
|
||||
* Contains an RCU guarded pointer to the last request. No reference is
|
||||
* held to the request, users must carefully acquire a reference to
|
||||
* the request using i915_active_request_get_request_rcu(), or hold the
|
||||
* struct_mutex.
|
||||
* the request using i915_active_fence_get(), or manage the RCU
|
||||
* protection themselves (cf the i915_active_fence API).
|
||||
*/
|
||||
struct i915_active_request last_request;
|
||||
struct i915_active_fence last_request;
|
||||
|
||||
/**
|
||||
* We track the most recent seqno that we wait on in every context so
|
||||
|
|
|
@ -47,24 +47,20 @@ static int context_sync(struct intel_context *ce)
|
|||
|
||||
mutex_lock(&tl->mutex);
|
||||
do {
|
||||
struct i915_request *rq;
|
||||
struct dma_fence *fence;
|
||||
long timeout;
|
||||
|
||||
rcu_read_lock();
|
||||
rq = rcu_dereference(tl->last_request.request);
|
||||
if (rq)
|
||||
rq = i915_request_get_rcu(rq);
|
||||
rcu_read_unlock();
|
||||
if (!rq)
|
||||
fence = i915_active_fence_get(&tl->last_request);
|
||||
if (!fence)
|
||||
break;
|
||||
|
||||
timeout = i915_request_wait(rq, 0, HZ / 10);
|
||||
timeout = dma_fence_wait_timeout(fence, false, HZ / 10);
|
||||
if (timeout < 0)
|
||||
err = timeout;
|
||||
else
|
||||
i915_request_retire_upto(rq);
|
||||
i915_request_retire_upto(to_request(fence));
|
||||
|
||||
i915_request_put(rq);
|
||||
dma_fence_put(fence);
|
||||
} while (!err);
|
||||
mutex_unlock(&tl->mutex);
|
||||
|
||||
|
|
|
@ -1172,9 +1172,13 @@ static struct i915_request *dummy_request(struct intel_engine_cs *engine)
|
|||
if (!rq)
|
||||
return NULL;
|
||||
|
||||
INIT_LIST_HEAD(&rq->active_list);
|
||||
rq->engine = engine;
|
||||
|
||||
spin_lock_init(&rq->lock);
|
||||
INIT_LIST_HEAD(&rq->fence.cb_list);
|
||||
rq->fence.lock = &rq->lock;
|
||||
rq->fence.ops = &i915_fence_ops;
|
||||
|
||||
i915_sched_node_init(&rq->sched);
|
||||
|
||||
/* mark this request as permanently incomplete */
|
||||
|
@ -1267,8 +1271,8 @@ static int live_suppress_wait_preempt(void *arg)
|
|||
}
|
||||
|
||||
/* Disable NEWCLIENT promotion */
|
||||
__i915_active_request_set(&i915_request_timeline(rq[i])->last_request,
|
||||
dummy);
|
||||
__i915_active_fence_set(&i915_request_timeline(rq[i])->last_request,
|
||||
&dummy->fence);
|
||||
i915_request_add(rq[i]);
|
||||
}
|
||||
|
||||
|
|
|
@ -15,7 +15,7 @@ void mock_timeline_init(struct intel_timeline *timeline, u64 context)
|
|||
|
||||
mutex_init(&timeline->mutex);
|
||||
|
||||
INIT_ACTIVE_REQUEST(&timeline->last_request, &timeline->mutex);
|
||||
INIT_ACTIVE_FENCE(&timeline->last_request, &timeline->mutex);
|
||||
INIT_LIST_HEAD(&timeline->requests);
|
||||
|
||||
i915_syncmap_init(&timeline->sync);
|
||||
|
|
|
@ -385,11 +385,8 @@ intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload)
|
|||
{
|
||||
struct intel_vgpu *vgpu = workload->vgpu;
|
||||
struct intel_vgpu_submission *s = &vgpu->submission;
|
||||
struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
|
||||
struct i915_request *rq;
|
||||
|
||||
lockdep_assert_held(&dev_priv->drm.struct_mutex);
|
||||
|
||||
if (workload->req)
|
||||
return 0;
|
||||
|
||||
|
|
|
@ -12,8 +12,6 @@
|
|||
#include "i915_active.h"
|
||||
#include "i915_globals.h"
|
||||
|
||||
#define BKL(ref) (&(ref)->i915->drm.struct_mutex)
|
||||
|
||||
/*
|
||||
* Active refs memory management
|
||||
*
|
||||
|
@ -27,35 +25,35 @@ static struct i915_global_active {
|
|||
} global;
|
||||
|
||||
struct active_node {
|
||||
struct i915_active_request base;
|
||||
struct i915_active_fence base;
|
||||
struct i915_active *ref;
|
||||
struct rb_node node;
|
||||
u64 timeline;
|
||||
};
|
||||
|
||||
static inline struct active_node *
|
||||
node_from_active(struct i915_active_request *active)
|
||||
node_from_active(struct i915_active_fence *active)
|
||||
{
|
||||
return container_of(active, struct active_node, base);
|
||||
}
|
||||
|
||||
#define take_preallocated_barriers(x) llist_del_all(&(x)->preallocated_barriers)
|
||||
|
||||
static inline bool is_barrier(const struct i915_active_request *active)
|
||||
static inline bool is_barrier(const struct i915_active_fence *active)
|
||||
{
|
||||
return IS_ERR(rcu_access_pointer(active->request));
|
||||
return IS_ERR(rcu_access_pointer(active->fence));
|
||||
}
|
||||
|
||||
static inline struct llist_node *barrier_to_ll(struct active_node *node)
|
||||
{
|
||||
GEM_BUG_ON(!is_barrier(&node->base));
|
||||
return (struct llist_node *)&node->base.link;
|
||||
return (struct llist_node *)&node->base.cb.node;
|
||||
}
|
||||
|
||||
static inline struct intel_engine_cs *
|
||||
__barrier_to_engine(struct active_node *node)
|
||||
{
|
||||
return (struct intel_engine_cs *)READ_ONCE(node->base.link.prev);
|
||||
return (struct intel_engine_cs *)READ_ONCE(node->base.cb.node.prev);
|
||||
}
|
||||
|
||||
static inline struct intel_engine_cs *
|
||||
|
@ -68,7 +66,7 @@ barrier_to_engine(struct active_node *node)
|
|||
static inline struct active_node *barrier_from_ll(struct llist_node *x)
|
||||
{
|
||||
return container_of((struct list_head *)x,
|
||||
struct active_node, base.link);
|
||||
struct active_node, base.cb.node);
|
||||
}
|
||||
|
||||
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) && IS_ENABLED(CONFIG_DEBUG_OBJECTS)
|
||||
|
@ -147,15 +145,18 @@ __active_retire(struct i915_active *ref)
|
|||
if (!retire)
|
||||
return;
|
||||
|
||||
GEM_BUG_ON(rcu_access_pointer(ref->excl));
|
||||
GEM_BUG_ON(rcu_access_pointer(ref->excl.fence));
|
||||
rbtree_postorder_for_each_entry_safe(it, n, &root, node) {
|
||||
GEM_BUG_ON(i915_active_request_isset(&it->base));
|
||||
GEM_BUG_ON(i915_active_fence_isset(&it->base));
|
||||
kmem_cache_free(global.slab_cache, it);
|
||||
}
|
||||
|
||||
/* After the final retire, the entire struct may be freed */
|
||||
if (ref->retire)
|
||||
ref->retire(ref);
|
||||
|
||||
/* ... except if you wait on it, you must manage your own references! */
|
||||
wake_up_var(ref);
|
||||
}
|
||||
|
||||
static void
|
||||
|
@ -189,12 +190,20 @@ active_retire(struct i915_active *ref)
|
|||
}
|
||||
|
||||
static void
|
||||
node_retire(struct i915_active_request *base, struct i915_request *rq)
|
||||
node_retire(struct dma_fence *fence, struct dma_fence_cb *cb)
|
||||
{
|
||||
active_retire(node_from_active(base)->ref);
|
||||
i915_active_fence_cb(fence, cb);
|
||||
active_retire(container_of(cb, struct active_node, base.cb)->ref);
|
||||
}
|
||||
|
||||
static struct i915_active_request *
|
||||
static void
|
||||
excl_retire(struct dma_fence *fence, struct dma_fence_cb *cb)
|
||||
{
|
||||
i915_active_fence_cb(fence, cb);
|
||||
active_retire(container_of(cb, struct i915_active, excl.cb));
|
||||
}
|
||||
|
||||
static struct i915_active_fence *
|
||||
active_instance(struct i915_active *ref, struct intel_timeline *tl)
|
||||
{
|
||||
struct active_node *node, *prealloc;
|
||||
|
@ -238,7 +247,7 @@ active_instance(struct i915_active *ref, struct intel_timeline *tl)
|
|||
}
|
||||
|
||||
node = prealloc;
|
||||
i915_active_request_init(&node->base, &tl->mutex, NULL, node_retire);
|
||||
__i915_active_fence_init(&node->base, &tl->mutex, NULL, node_retire);
|
||||
node->ref = ref;
|
||||
node->timeline = idx;
|
||||
|
||||
|
@ -253,8 +262,7 @@ active_instance(struct i915_active *ref, struct intel_timeline *tl)
|
|||
return &node->base;
|
||||
}
|
||||
|
||||
void __i915_active_init(struct drm_i915_private *i915,
|
||||
struct i915_active *ref,
|
||||
void __i915_active_init(struct i915_active *ref,
|
||||
int (*active)(struct i915_active *ref),
|
||||
void (*retire)(struct i915_active *ref),
|
||||
struct lock_class_key *key)
|
||||
|
@ -263,19 +271,18 @@ void __i915_active_init(struct drm_i915_private *i915,
|
|||
|
||||
debug_active_init(ref);
|
||||
|
||||
ref->i915 = i915;
|
||||
ref->flags = 0;
|
||||
ref->active = active;
|
||||
ref->retire = ptr_unpack_bits(retire, &bits, 2);
|
||||
if (bits & I915_ACTIVE_MAY_SLEEP)
|
||||
ref->flags |= I915_ACTIVE_RETIRE_SLEEPS;
|
||||
|
||||
ref->excl = NULL;
|
||||
ref->tree = RB_ROOT;
|
||||
ref->cache = NULL;
|
||||
init_llist_head(&ref->preallocated_barriers);
|
||||
atomic_set(&ref->count, 0);
|
||||
__mutex_init(&ref->mutex, "i915_active", key);
|
||||
__i915_active_fence_init(&ref->excl, &ref->mutex, NULL, excl_retire);
|
||||
INIT_WORK(&ref->work, active_work);
|
||||
}
|
||||
|
||||
|
@ -329,9 +336,9 @@ __active_del_barrier(struct i915_active *ref, struct active_node *node)
|
|||
|
||||
int i915_active_ref(struct i915_active *ref,
|
||||
struct intel_timeline *tl,
|
||||
struct i915_request *rq)
|
||||
struct dma_fence *fence)
|
||||
{
|
||||
struct i915_active_request *active;
|
||||
struct i915_active_fence *active;
|
||||
int err;
|
||||
|
||||
lockdep_assert_held(&tl->mutex);
|
||||
|
@ -354,66 +361,44 @@ int i915_active_ref(struct i915_active *ref,
|
|||
* request that we want to emit on the kernel_context.
|
||||
*/
|
||||
__active_del_barrier(ref, node_from_active(active));
|
||||
RCU_INIT_POINTER(active->request, NULL);
|
||||
INIT_LIST_HEAD(&active->link);
|
||||
} else {
|
||||
if (!i915_active_request_isset(active))
|
||||
atomic_inc(&ref->count);
|
||||
RCU_INIT_POINTER(active->fence, NULL);
|
||||
atomic_dec(&ref->count);
|
||||
}
|
||||
GEM_BUG_ON(!atomic_read(&ref->count));
|
||||
__i915_active_request_set(active, rq);
|
||||
if (!__i915_active_fence_set(active, fence))
|
||||
atomic_inc(&ref->count);
|
||||
|
||||
out:
|
||||
i915_active_release(ref);
|
||||
return err;
|
||||
}
|
||||
|
||||
static void excl_cb(struct dma_fence *f, struct dma_fence_cb *cb)
|
||||
{
|
||||
struct i915_active *ref = container_of(cb, typeof(*ref), excl_cb);
|
||||
|
||||
RCU_INIT_POINTER(ref->excl, NULL);
|
||||
dma_fence_put(f);
|
||||
|
||||
active_retire(ref);
|
||||
}
|
||||
|
||||
void i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f)
|
||||
{
|
||||
/* We expect the caller to manage the exclusive timeline ordering */
|
||||
GEM_BUG_ON(i915_active_is_idle(ref));
|
||||
|
||||
dma_fence_get(f);
|
||||
/*
|
||||
* As we don't know which mutex the caller is using, we told a small
|
||||
* lie to the debug code that it is using the i915_active.mutex;
|
||||
* and now we must stick to that lie.
|
||||
*/
|
||||
mutex_acquire(&ref->mutex.dep_map, 0, 0, _THIS_IP_);
|
||||
if (!__i915_active_fence_set(&ref->excl, f))
|
||||
atomic_inc(&ref->count);
|
||||
mutex_release(&ref->mutex.dep_map, 0, _THIS_IP_);
|
||||
}
|
||||
|
||||
rcu_read_lock();
|
||||
if (rcu_access_pointer(ref->excl)) {
|
||||
struct dma_fence *old;
|
||||
|
||||
old = dma_fence_get_rcu_safe(&ref->excl);
|
||||
if (old) {
|
||||
if (dma_fence_remove_callback(old, &ref->excl_cb))
|
||||
atomic_dec(&ref->count);
|
||||
dma_fence_put(old);
|
||||
}
|
||||
}
|
||||
rcu_read_unlock();
|
||||
|
||||
atomic_inc(&ref->count);
|
||||
rcu_assign_pointer(ref->excl, f);
|
||||
|
||||
if (dma_fence_add_callback(f, &ref->excl_cb, excl_cb)) {
|
||||
RCU_INIT_POINTER(ref->excl, NULL);
|
||||
atomic_dec(&ref->count);
|
||||
dma_fence_put(f);
|
||||
}
|
||||
bool i915_active_acquire_if_busy(struct i915_active *ref)
|
||||
{
|
||||
debug_active_assert(ref);
|
||||
return atomic_add_unless(&ref->count, 1, 0);
|
||||
}
|
||||
|
||||
int i915_active_acquire(struct i915_active *ref)
|
||||
{
|
||||
int err;
|
||||
|
||||
debug_active_assert(ref);
|
||||
if (atomic_add_unless(&ref->count, 1, 0))
|
||||
if (i915_active_acquire_if_busy(ref))
|
||||
return 0;
|
||||
|
||||
err = mutex_lock_interruptible(&ref->mutex);
|
||||
|
@ -438,121 +423,57 @@ void i915_active_release(struct i915_active *ref)
|
|||
active_retire(ref);
|
||||
}
|
||||
|
||||
static void __active_ungrab(struct i915_active *ref)
|
||||
static void enable_signaling(struct i915_active_fence *active)
|
||||
{
|
||||
clear_and_wake_up_bit(I915_ACTIVE_GRAB_BIT, &ref->flags);
|
||||
}
|
||||
struct dma_fence *fence;
|
||||
|
||||
bool i915_active_trygrab(struct i915_active *ref)
|
||||
{
|
||||
debug_active_assert(ref);
|
||||
fence = i915_active_fence_get(active);
|
||||
if (!fence)
|
||||
return;
|
||||
|
||||
if (test_and_set_bit(I915_ACTIVE_GRAB_BIT, &ref->flags))
|
||||
return false;
|
||||
|
||||
if (!atomic_add_unless(&ref->count, 1, 0)) {
|
||||
__active_ungrab(ref);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void i915_active_ungrab(struct i915_active *ref)
|
||||
{
|
||||
GEM_BUG_ON(!test_bit(I915_ACTIVE_GRAB_BIT, &ref->flags));
|
||||
|
||||
active_retire(ref);
|
||||
__active_ungrab(ref);
|
||||
}
|
||||
|
||||
static int excl_wait(struct i915_active *ref)
|
||||
{
|
||||
struct dma_fence *old;
|
||||
int err = 0;
|
||||
|
||||
if (!rcu_access_pointer(ref->excl))
|
||||
return 0;
|
||||
|
||||
rcu_read_lock();
|
||||
old = dma_fence_get_rcu_safe(&ref->excl);
|
||||
rcu_read_unlock();
|
||||
if (old) {
|
||||
err = dma_fence_wait(old, true);
|
||||
dma_fence_put(old);
|
||||
}
|
||||
|
||||
return err;
|
||||
dma_fence_enable_sw_signaling(fence);
|
||||
dma_fence_put(fence);
|
||||
}
|
||||
|
||||
int i915_active_wait(struct i915_active *ref)
|
||||
{
|
||||
struct active_node *it, *n;
|
||||
int err;
|
||||
int err = 0;
|
||||
|
||||
might_sleep();
|
||||
might_lock(&ref->mutex);
|
||||
|
||||
if (i915_active_is_idle(ref))
|
||||
if (!i915_active_acquire_if_busy(ref))
|
||||
return 0;
|
||||
|
||||
err = mutex_lock_interruptible(&ref->mutex);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
if (!atomic_add_unless(&ref->count, 1, 0)) {
|
||||
mutex_unlock(&ref->mutex);
|
||||
return 0;
|
||||
}
|
||||
|
||||
err = excl_wait(ref);
|
||||
if (err)
|
||||
goto out;
|
||||
|
||||
/* Flush lazy signals */
|
||||
enable_signaling(&ref->excl);
|
||||
rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
|
||||
if (is_barrier(&it->base)) { /* unconnected idle-barrier */
|
||||
err = -EBUSY;
|
||||
break;
|
||||
}
|
||||
if (is_barrier(&it->base)) /* unconnected idle barrier */
|
||||
continue;
|
||||
|
||||
err = i915_active_request_retire(&it->base, BKL(ref));
|
||||
if (err)
|
||||
break;
|
||||
enable_signaling(&it->base);
|
||||
}
|
||||
/* Any fence added after the wait begins will not be auto-signaled */
|
||||
|
||||
out:
|
||||
__active_retire(ref);
|
||||
i915_active_release(ref);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
if (wait_on_bit(&ref->flags, I915_ACTIVE_GRAB_BIT, TASK_KILLABLE))
|
||||
if (wait_var_event_interruptible(ref, i915_active_is_idle(ref)))
|
||||
return -EINTR;
|
||||
|
||||
flush_work(&ref->work);
|
||||
if (!i915_active_is_idle(ref))
|
||||
return -EBUSY;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int i915_request_await_active_request(struct i915_request *rq,
|
||||
struct i915_active_request *active)
|
||||
{
|
||||
struct i915_request *barrier =
|
||||
i915_active_request_raw(active, &rq->i915->drm.struct_mutex);
|
||||
|
||||
return barrier ? i915_request_await_dma_fence(rq, &barrier->fence) : 0;
|
||||
}
|
||||
|
||||
int i915_request_await_active(struct i915_request *rq, struct i915_active *ref)
|
||||
{
|
||||
int err = 0;
|
||||
|
||||
if (rcu_access_pointer(ref->excl)) {
|
||||
if (rcu_access_pointer(ref->excl.fence)) {
|
||||
struct dma_fence *fence;
|
||||
|
||||
rcu_read_lock();
|
||||
fence = dma_fence_get_rcu_safe(&ref->excl);
|
||||
fence = dma_fence_get_rcu_safe(&ref->excl.fence);
|
||||
rcu_read_unlock();
|
||||
if (fence) {
|
||||
err = i915_request_await_dma_fence(rq, fence);
|
||||
|
@ -578,7 +499,7 @@ void i915_active_fini(struct i915_active *ref)
|
|||
|
||||
static inline bool is_idle_barrier(struct active_node *node, u64 idx)
|
||||
{
|
||||
return node->timeline == idx && !i915_active_request_isset(&node->base);
|
||||
return node->timeline == idx && !i915_active_fence_isset(&node->base);
|
||||
}
|
||||
|
||||
static struct active_node *reuse_idle_barrier(struct i915_active *ref, u64 idx)
|
||||
|
@ -698,13 +619,13 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref,
|
|||
node->base.lock =
|
||||
&engine->kernel_context->timeline->mutex;
|
||||
#endif
|
||||
RCU_INIT_POINTER(node->base.request, NULL);
|
||||
node->base.retire = node_retire;
|
||||
RCU_INIT_POINTER(node->base.fence, NULL);
|
||||
node->base.cb.func = node_retire;
|
||||
node->timeline = idx;
|
||||
node->ref = ref;
|
||||
}
|
||||
|
||||
if (!i915_active_request_isset(&node->base)) {
|
||||
if (!i915_active_fence_isset(&node->base)) {
|
||||
/*
|
||||
* Mark this as being *our* unconnected proto-node.
|
||||
*
|
||||
|
@ -714,8 +635,8 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref,
|
|||
* and then we can use the rb_node and list pointers
|
||||
* for our tracking of the pending barrier.
|
||||
*/
|
||||
RCU_INIT_POINTER(node->base.request, ERR_PTR(-EAGAIN));
|
||||
node->base.link.prev = (void *)engine;
|
||||
RCU_INIT_POINTER(node->base.fence, ERR_PTR(-EAGAIN));
|
||||
node->base.cb.node.prev = (void *)engine;
|
||||
atomic_inc(&ref->count);
|
||||
}
|
||||
|
||||
|
@ -782,44 +703,113 @@ void i915_request_add_active_barriers(struct i915_request *rq)
|
|||
{
|
||||
struct intel_engine_cs *engine = rq->engine;
|
||||
struct llist_node *node, *next;
|
||||
unsigned long flags;
|
||||
|
||||
GEM_BUG_ON(intel_engine_is_virtual(engine));
|
||||
GEM_BUG_ON(i915_request_timeline(rq) != engine->kernel_context->timeline);
|
||||
|
||||
node = llist_del_all(&engine->barrier_tasks);
|
||||
if (!node)
|
||||
return;
|
||||
/*
|
||||
* Attach the list of proto-fences to the in-flight request such
|
||||
* that the parent i915_active will be released when this request
|
||||
* is retired.
|
||||
*/
|
||||
llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) {
|
||||
RCU_INIT_POINTER(barrier_from_ll(node)->base.request, rq);
|
||||
spin_lock_irqsave(&rq->lock, flags);
|
||||
llist_for_each_safe(node, next, node) {
|
||||
RCU_INIT_POINTER(barrier_from_ll(node)->base.fence, &rq->fence);
|
||||
smp_wmb(); /* serialise with reuse_idle_barrier */
|
||||
list_add_tail((struct list_head *)node, &rq->active_list);
|
||||
list_add_tail((struct list_head *)node, &rq->fence.cb_list);
|
||||
}
|
||||
spin_unlock_irqrestore(&rq->lock, flags);
|
||||
}
|
||||
|
||||
int i915_active_request_set(struct i915_active_request *active,
|
||||
struct i915_request *rq)
|
||||
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
|
||||
#define active_is_held(active) lockdep_is_held((active)->lock)
|
||||
#else
|
||||
#define active_is_held(active) true
|
||||
#endif
|
||||
|
||||
/*
|
||||
* __i915_active_fence_set: Update the last active fence along its timeline
|
||||
* @active: the active tracker
|
||||
* @fence: the new fence (under construction)
|
||||
*
|
||||
* Records the new @fence as the last active fence along its timeline in
|
||||
* this active tracker, moving the tracking callbacks from the previous
|
||||
* fence onto this one. Returns the previous fence (if not already completed),
|
||||
* which the caller must ensure is executed before the new fence. To ensure
|
||||
* that the order of fences within the timeline of the i915_active_fence is
|
||||
* maintained, it must be locked by the caller.
|
||||
*/
|
||||
struct dma_fence *
|
||||
__i915_active_fence_set(struct i915_active_fence *active,
|
||||
struct dma_fence *fence)
|
||||
{
|
||||
int err;
|
||||
struct dma_fence *prev;
|
||||
unsigned long flags;
|
||||
|
||||
/* NB: must be serialised by an outer timeline mutex (active->lock) */
|
||||
spin_lock_irqsave(fence->lock, flags);
|
||||
GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags));
|
||||
|
||||
prev = rcu_dereference_protected(active->fence, active_is_held(active));
|
||||
if (prev) {
|
||||
GEM_BUG_ON(prev == fence);
|
||||
spin_lock_nested(prev->lock, SINGLE_DEPTH_NESTING);
|
||||
__list_del_entry(&active->cb.node);
|
||||
spin_unlock(prev->lock); /* serialise with prev->cb_list */
|
||||
|
||||
/*
|
||||
* active->fence is reset by the callback from inside
|
||||
* interrupt context. We need to serialise our list
|
||||
* manipulation with the fence->lock to prevent the prev
|
||||
* being lost inside an interrupt (it can't be replaced as
|
||||
* no other caller is allowed to enter __i915_active_fence_set
|
||||
* as we hold the timeline lock). After serialising with
|
||||
* the callback, we need to double check which ran first,
|
||||
* our list_del() [decoupling prev from the callback] or
|
||||
* the callback...
|
||||
*/
|
||||
prev = rcu_access_pointer(active->fence);
|
||||
}
|
||||
|
||||
rcu_assign_pointer(active->fence, fence);
|
||||
list_add_tail(&active->cb.node, &fence->cb_list);
|
||||
|
||||
spin_unlock_irqrestore(fence->lock, flags);
|
||||
|
||||
return prev;
|
||||
}
|
||||
|
||||
int i915_active_fence_set(struct i915_active_fence *active,
|
||||
struct i915_request *rq)
|
||||
{
|
||||
struct dma_fence *fence;
|
||||
int err = 0;
|
||||
|
||||
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
|
||||
lockdep_assert_held(active->lock);
|
||||
#endif
|
||||
|
||||
/* Must maintain ordering wrt previous active requests */
|
||||
err = i915_request_await_active_request(rq, active);
|
||||
if (err)
|
||||
return err;
|
||||
/* Must maintain timeline ordering wrt previous active requests */
|
||||
rcu_read_lock();
|
||||
fence = __i915_active_fence_set(active, &rq->fence);
|
||||
if (fence) /* but the previous fence may not belong to that timeline! */
|
||||
fence = dma_fence_get_rcu(fence);
|
||||
rcu_read_unlock();
|
||||
if (fence) {
|
||||
err = i915_request_await_dma_fence(rq, fence);
|
||||
dma_fence_put(fence);
|
||||
}
|
||||
|
||||
__i915_active_request_set(active, rq);
|
||||
return 0;
|
||||
return err;
|
||||
}
|
||||
|
||||
void i915_active_retire_noop(struct i915_active_request *active,
|
||||
struct i915_request *request)
|
||||
void i915_active_noop(struct dma_fence *fence, struct dma_fence_cb *cb)
|
||||
{
|
||||
/* Space left intentionally blank */
|
||||
i915_active_fence_cb(fence, cb);
|
||||
}
|
||||
|
||||
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
|
||||
|
|
|
@ -12,6 +12,10 @@
|
|||
#include "i915_active_types.h"
|
||||
#include "i915_request.h"
|
||||
|
||||
struct i915_request;
|
||||
struct intel_engine_cs;
|
||||
struct intel_timeline;
|
||||
|
||||
/*
|
||||
* We treat requests as fences. This is not be to confused with our
|
||||
* "fence registers" but pipeline synchronisation objects ala GL_ARB_sync.
|
||||
|
@ -28,308 +32,108 @@
|
|||
* write access so that we can perform concurrent read operations between
|
||||
* the CPU and GPU engines, as well as waiting for all rendering to
|
||||
* complete, or waiting for the last GPU user of a "fence register". The
|
||||
* object then embeds a #i915_active_request to track the most recent (in
|
||||
* object then embeds a #i915_active_fence to track the most recent (in
|
||||
* retirement order) request relevant for the desired mode of access.
|
||||
* The #i915_active_request is updated with i915_active_request_set() to
|
||||
* The #i915_active_fence is updated with i915_active_fence_set() to
|
||||
* track the most recent fence request, typically this is done as part of
|
||||
* i915_vma_move_to_active().
|
||||
*
|
||||
* When the #i915_active_request completes (is retired), it will
|
||||
* When the #i915_active_fence completes (is retired), it will
|
||||
* signal its completion to the owner through a callback as well as mark
|
||||
* itself as idle (i915_active_request.request == NULL). The owner
|
||||
* itself as idle (i915_active_fence.request == NULL). The owner
|
||||
* can then perform any action, such as delayed freeing of an active
|
||||
* resource including itself.
|
||||
*/
|
||||
|
||||
void i915_active_retire_noop(struct i915_active_request *active,
|
||||
struct i915_request *request);
|
||||
void i915_active_noop(struct dma_fence *fence, struct dma_fence_cb *cb);
|
||||
|
||||
/**
|
||||
* i915_active_request_init - prepares the activity tracker for use
|
||||
* __i915_active_fence_init - prepares the activity tracker for use
|
||||
* @active - the active tracker
|
||||
* @rq - initial request to track, can be NULL
|
||||
* @fence - initial fence to track, can be NULL
|
||||
* @func - a callback when then the tracker is retired (becomes idle),
|
||||
* can be NULL
|
||||
*
|
||||
* i915_active_request_init() prepares the embedded @active struct for use as
|
||||
* an activity tracker, that is for tracking the last known active request
|
||||
* associated with it. When the last request becomes idle, when it is retired
|
||||
* i915_active_fence_init() prepares the embedded @active struct for use as
|
||||
* an activity tracker, that is for tracking the last known active fence
|
||||
* associated with it. When the last fence becomes idle, when it is retired
|
||||
* after completion, the optional callback @func is invoked.
|
||||
*/
|
||||
static inline void
|
||||
i915_active_request_init(struct i915_active_request *active,
|
||||
__i915_active_fence_init(struct i915_active_fence *active,
|
||||
struct mutex *lock,
|
||||
struct i915_request *rq,
|
||||
i915_active_retire_fn retire)
|
||||
void *fence,
|
||||
dma_fence_func_t fn)
|
||||
{
|
||||
RCU_INIT_POINTER(active->request, rq);
|
||||
INIT_LIST_HEAD(&active->link);
|
||||
active->retire = retire ?: i915_active_retire_noop;
|
||||
RCU_INIT_POINTER(active->fence, fence);
|
||||
active->cb.func = fn ?: i915_active_noop;
|
||||
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
|
||||
active->lock = lock;
|
||||
#endif
|
||||
}
|
||||
|
||||
#define INIT_ACTIVE_REQUEST(name, lock) \
|
||||
i915_active_request_init((name), (lock), NULL, NULL)
|
||||
#define INIT_ACTIVE_FENCE(A, LOCK) \
|
||||
__i915_active_fence_init((A), (LOCK), NULL, NULL)
|
||||
|
||||
struct dma_fence *
|
||||
__i915_active_fence_set(struct i915_active_fence *active,
|
||||
struct dma_fence *fence);
|
||||
|
||||
/**
|
||||
* i915_active_request_set - updates the tracker to watch the current request
|
||||
* i915_active_fence_set - updates the tracker to watch the current fence
|
||||
* @active - the active tracker
|
||||
* @request - the request to watch
|
||||
* @rq - the request to watch
|
||||
*
|
||||
* __i915_active_request_set() watches the given @request for completion. Whilst
|
||||
* that @request is busy, the @active reports busy. When that @request is
|
||||
* retired, the @active tracker is updated to report idle.
|
||||
* i915_active_fence_set() watches the given @rq for completion. While
|
||||
* that @rq is busy, the @active reports busy. When that @rq is signaled
|
||||
* (or else retired) the @active tracker is updated to report idle.
|
||||
*/
|
||||
static inline void
|
||||
__i915_active_request_set(struct i915_active_request *active,
|
||||
struct i915_request *request)
|
||||
{
|
||||
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
|
||||
lockdep_assert_held(active->lock);
|
||||
#endif
|
||||
list_move(&active->link, &request->active_list);
|
||||
rcu_assign_pointer(active->request, request);
|
||||
}
|
||||
|
||||
int __must_check
|
||||
i915_active_request_set(struct i915_active_request *active,
|
||||
struct i915_request *rq);
|
||||
|
||||
i915_active_fence_set(struct i915_active_fence *active,
|
||||
struct i915_request *rq);
|
||||
/**
|
||||
* i915_active_request_raw - return the active request
|
||||
* i915_active_fence_get - return a reference to the active fence
|
||||
* @active - the active tracker
|
||||
*
|
||||
* i915_active_request_raw() returns the current request being tracked, or NULL.
|
||||
* It does not obtain a reference on the request for the caller, so the caller
|
||||
* must hold struct_mutex.
|
||||
*/
|
||||
static inline struct i915_request *
|
||||
i915_active_request_raw(const struct i915_active_request *active,
|
||||
struct mutex *mutex)
|
||||
{
|
||||
return rcu_dereference_protected(active->request,
|
||||
lockdep_is_held(mutex));
|
||||
}
|
||||
|
||||
/**
|
||||
* i915_active_request_peek - report the active request being monitored
|
||||
* @active - the active tracker
|
||||
*
|
||||
* i915_active_request_peek() returns the current request being tracked if
|
||||
* still active, or NULL. It does not obtain a reference on the request
|
||||
* for the caller, so the caller must hold struct_mutex.
|
||||
*/
|
||||
static inline struct i915_request *
|
||||
i915_active_request_peek(const struct i915_active_request *active,
|
||||
struct mutex *mutex)
|
||||
{
|
||||
struct i915_request *request;
|
||||
|
||||
request = i915_active_request_raw(active, mutex);
|
||||
if (!request || i915_request_completed(request))
|
||||
return NULL;
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
/**
|
||||
* i915_active_request_get - return a reference to the active request
|
||||
* @active - the active tracker
|
||||
*
|
||||
* i915_active_request_get() returns a reference to the active request, or NULL
|
||||
* if the active tracker is idle. The caller must hold struct_mutex.
|
||||
*/
|
||||
static inline struct i915_request *
|
||||
i915_active_request_get(const struct i915_active_request *active,
|
||||
struct mutex *mutex)
|
||||
{
|
||||
return i915_request_get(i915_active_request_peek(active, mutex));
|
||||
}
|
||||
|
||||
/**
|
||||
* __i915_active_request_get_rcu - return a reference to the active request
|
||||
* @active - the active tracker
|
||||
*
|
||||
* __i915_active_request_get() returns a reference to the active request,
|
||||
* or NULL if the active tracker is idle. The caller must hold the RCU read
|
||||
* lock, but the returned pointer is safe to use outside of RCU.
|
||||
*/
|
||||
static inline struct i915_request *
|
||||
__i915_active_request_get_rcu(const struct i915_active_request *active)
|
||||
{
|
||||
/*
|
||||
* Performing a lockless retrieval of the active request is super
|
||||
* tricky. SLAB_TYPESAFE_BY_RCU merely guarantees that the backing
|
||||
* slab of request objects will not be freed whilst we hold the
|
||||
* RCU read lock. It does not guarantee that the request itself
|
||||
* will not be freed and then *reused*. Viz,
|
||||
*
|
||||
* Thread A Thread B
|
||||
*
|
||||
* rq = active.request
|
||||
* retire(rq) -> free(rq);
|
||||
* (rq is now first on the slab freelist)
|
||||
* active.request = NULL
|
||||
*
|
||||
* rq = new submission on a new object
|
||||
* ref(rq)
|
||||
*
|
||||
* To prevent the request from being reused whilst the caller
|
||||
* uses it, we take a reference like normal. Whilst acquiring
|
||||
* the reference we check that it is not in a destroyed state
|
||||
* (refcnt == 0). That prevents the request being reallocated
|
||||
* whilst the caller holds on to it. To check that the request
|
||||
* was not reallocated as we acquired the reference we have to
|
||||
* check that our request remains the active request across
|
||||
* the lookup, in the same manner as a seqlock. The visibility
|
||||
* of the pointer versus the reference counting is controlled
|
||||
* by using RCU barriers (rcu_dereference and rcu_assign_pointer).
|
||||
*
|
||||
* In the middle of all that, we inspect whether the request is
|
||||
* complete. Retiring is lazy so the request may be completed long
|
||||
* before the active tracker is updated. Querying whether the
|
||||
* request is complete is far cheaper (as it involves no locked
|
||||
* instructions setting cachelines to exclusive) than acquiring
|
||||
* the reference, so we do it first. The RCU read lock ensures the
|
||||
* pointer dereference is valid, but does not ensure that the
|
||||
* seqno nor HWS is the right one! However, if the request was
|
||||
* reallocated, that means the active tracker's request was complete.
|
||||
* If the new request is also complete, then both are and we can
|
||||
* just report the active tracker is idle. If the new request is
|
||||
* incomplete, then we acquire a reference on it and check that
|
||||
* it remained the active request.
|
||||
*
|
||||
* It is then imperative that we do not zero the request on
|
||||
* reallocation, so that we can chase the dangling pointers!
|
||||
* See i915_request_alloc().
|
||||
*/
|
||||
do {
|
||||
struct i915_request *request;
|
||||
|
||||
request = rcu_dereference(active->request);
|
||||
if (!request || i915_request_completed(request))
|
||||
return NULL;
|
||||
|
||||
/*
|
||||
* An especially silly compiler could decide to recompute the
|
||||
* result of i915_request_completed, more specifically
|
||||
* re-emit the load for request->fence.seqno. A race would catch
|
||||
* a later seqno value, which could flip the result from true to
|
||||
* false. Which means part of the instructions below might not
|
||||
* be executed, while later on instructions are executed. Due to
|
||||
* barriers within the refcounting the inconsistency can't reach
|
||||
* past the call to i915_request_get_rcu, but not executing
|
||||
* that while still executing i915_request_put() creates
|
||||
* havoc enough. Prevent this with a compiler barrier.
|
||||
*/
|
||||
barrier();
|
||||
|
||||
request = i915_request_get_rcu(request);
|
||||
|
||||
/*
|
||||
* What stops the following rcu_access_pointer() from occurring
|
||||
* before the above i915_request_get_rcu()? If we were
|
||||
* to read the value before pausing to get the reference to
|
||||
* the request, we may not notice a change in the active
|
||||
* tracker.
|
||||
*
|
||||
* The rcu_access_pointer() is a mere compiler barrier, which
|
||||
* means both the CPU and compiler are free to perform the
|
||||
* memory read without constraint. The compiler only has to
|
||||
* ensure that any operations after the rcu_access_pointer()
|
||||
* occur afterwards in program order. This means the read may
|
||||
* be performed earlier by an out-of-order CPU, or adventurous
|
||||
* compiler.
|
||||
*
|
||||
* The atomic operation at the heart of
|
||||
* i915_request_get_rcu(), see dma_fence_get_rcu(), is
|
||||
* atomic_inc_not_zero() which is only a full memory barrier
|
||||
* when successful. That is, if i915_request_get_rcu()
|
||||
* returns the request (and so with the reference counted
|
||||
* incremented) then the following read for rcu_access_pointer()
|
||||
* must occur after the atomic operation and so confirm
|
||||
* that this request is the one currently being tracked.
|
||||
*
|
||||
* The corresponding write barrier is part of
|
||||
* rcu_assign_pointer().
|
||||
*/
|
||||
if (!request || request == rcu_access_pointer(active->request))
|
||||
return rcu_pointer_handoff(request);
|
||||
|
||||
i915_request_put(request);
|
||||
} while (1);
|
||||
}
|
||||
|
||||
/**
|
||||
* i915_active_request_get_unlocked - return a reference to the active request
|
||||
* @active - the active tracker
|
||||
*
|
||||
* i915_active_request_get_unlocked() returns a reference to the active request,
|
||||
* i915_active_fence_get() returns a reference to the active fence,
|
||||
* or NULL if the active tracker is idle. The reference is obtained under RCU,
|
||||
* so no locking is required by the caller.
|
||||
*
|
||||
* The reference should be freed with i915_request_put().
|
||||
* The reference should be freed with dma_fence_put().
|
||||
*/
|
||||
static inline struct i915_request *
|
||||
i915_active_request_get_unlocked(const struct i915_active_request *active)
|
||||
static inline struct dma_fence *
|
||||
i915_active_fence_get(struct i915_active_fence *active)
|
||||
{
|
||||
struct i915_request *request;
|
||||
struct dma_fence *fence;
|
||||
|
||||
rcu_read_lock();
|
||||
request = __i915_active_request_get_rcu(active);
|
||||
fence = dma_fence_get_rcu_safe(&active->fence);
|
||||
rcu_read_unlock();
|
||||
|
||||
return request;
|
||||
return fence;
|
||||
}
|
||||
|
||||
/**
|
||||
* i915_active_request_isset - report whether the active tracker is assigned
|
||||
* i915_active_fence_isset - report whether the active tracker is assigned
|
||||
* @active - the active tracker
|
||||
*
|
||||
* i915_active_request_isset() returns true if the active tracker is currently
|
||||
* assigned to a request. Due to the lazy retiring, that request may be idle
|
||||
* i915_active_fence_isset() returns true if the active tracker is currently
|
||||
* assigned to a fence. Due to the lazy retiring, that fence may be idle
|
||||
* and this may report stale information.
|
||||
*/
|
||||
static inline bool
|
||||
i915_active_request_isset(const struct i915_active_request *active)
|
||||
i915_active_fence_isset(const struct i915_active_fence *active)
|
||||
{
|
||||
return rcu_access_pointer(active->request);
|
||||
return rcu_access_pointer(active->fence);
|
||||
}
|
||||
|
||||
/**
|
||||
* i915_active_request_retire - waits until the request is retired
|
||||
* @active - the active request on which to wait
|
||||
*
|
||||
* i915_active_request_retire() waits until the request is completed,
|
||||
* and then ensures that at least the retirement handler for this
|
||||
* @active tracker is called before returning. If the @active
|
||||
* tracker is idle, the function returns immediately.
|
||||
*/
|
||||
static inline int __must_check
|
||||
i915_active_request_retire(struct i915_active_request *active,
|
||||
struct mutex *mutex)
|
||||
static inline void
|
||||
i915_active_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
|
||||
{
|
||||
struct i915_request *request;
|
||||
long ret;
|
||||
struct i915_active_fence *active =
|
||||
container_of(cb, typeof(*active), cb);
|
||||
|
||||
request = i915_active_request_raw(active, mutex);
|
||||
if (!request)
|
||||
return 0;
|
||||
|
||||
ret = i915_request_wait(request,
|
||||
I915_WAIT_INTERRUPTIBLE,
|
||||
MAX_SCHEDULE_TIMEOUT);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
list_del_init(&active->link);
|
||||
RCU_INIT_POINTER(active->request, NULL);
|
||||
|
||||
active->retire(active, request);
|
||||
|
||||
return 0;
|
||||
RCU_INIT_POINTER(active->fence, NULL);
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -358,47 +162,40 @@ i915_active_request_retire(struct i915_active_request *active,
|
|||
* synchronisation.
|
||||
*/
|
||||
|
||||
void __i915_active_init(struct drm_i915_private *i915,
|
||||
struct i915_active *ref,
|
||||
void __i915_active_init(struct i915_active *ref,
|
||||
int (*active)(struct i915_active *ref),
|
||||
void (*retire)(struct i915_active *ref),
|
||||
struct lock_class_key *key);
|
||||
#define i915_active_init(i915, ref, active, retire) do { \
|
||||
#define i915_active_init(ref, active, retire) do { \
|
||||
static struct lock_class_key __key; \
|
||||
\
|
||||
__i915_active_init(i915, ref, active, retire, &__key); \
|
||||
__i915_active_init(ref, active, retire, &__key); \
|
||||
} while (0)
|
||||
|
||||
int i915_active_ref(struct i915_active *ref,
|
||||
struct intel_timeline *tl,
|
||||
struct i915_request *rq);
|
||||
struct dma_fence *fence);
|
||||
|
||||
static inline int
|
||||
i915_active_add_request(struct i915_active *ref, struct i915_request *rq)
|
||||
{
|
||||
return i915_active_ref(ref, i915_request_timeline(rq), rq);
|
||||
return i915_active_ref(ref, i915_request_timeline(rq), &rq->fence);
|
||||
}
|
||||
|
||||
void i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f);
|
||||
|
||||
static inline bool i915_active_has_exclusive(struct i915_active *ref)
|
||||
{
|
||||
return rcu_access_pointer(ref->excl);
|
||||
return rcu_access_pointer(ref->excl.fence);
|
||||
}
|
||||
|
||||
int i915_active_wait(struct i915_active *ref);
|
||||
|
||||
int i915_request_await_active(struct i915_request *rq,
|
||||
struct i915_active *ref);
|
||||
int i915_request_await_active_request(struct i915_request *rq,
|
||||
struct i915_active_request *active);
|
||||
int i915_request_await_active(struct i915_request *rq, struct i915_active *ref);
|
||||
|
||||
int i915_active_acquire(struct i915_active *ref);
|
||||
bool i915_active_acquire_if_busy(struct i915_active *ref);
|
||||
void i915_active_release(struct i915_active *ref);
|
||||
void __i915_active_release_nested(struct i915_active *ref, int subclass);
|
||||
|
||||
bool i915_active_trygrab(struct i915_active *ref);
|
||||
void i915_active_ungrab(struct i915_active *ref);
|
||||
|
||||
static inline bool
|
||||
i915_active_is_idle(const struct i915_active *ref)
|
||||
|
|
|
@ -17,17 +17,9 @@
|
|||
|
||||
#include "i915_utils.h"
|
||||
|
||||
struct drm_i915_private;
|
||||
struct i915_active_request;
|
||||
struct i915_request;
|
||||
|
||||
typedef void (*i915_active_retire_fn)(struct i915_active_request *,
|
||||
struct i915_request *);
|
||||
|
||||
struct i915_active_request {
|
||||
struct i915_request __rcu *request;
|
||||
struct list_head link;
|
||||
i915_active_retire_fn retire;
|
||||
struct i915_active_fence {
|
||||
struct dma_fence __rcu *fence;
|
||||
struct dma_fence_cb cb;
|
||||
#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
|
||||
/*
|
||||
* Incorporeal!
|
||||
|
@ -53,20 +45,17 @@ struct active_node;
|
|||
#define i915_active_may_sleep(fn) ptr_pack_bits(&(fn), I915_ACTIVE_MAY_SLEEP, 2)
|
||||
|
||||
struct i915_active {
|
||||
struct drm_i915_private *i915;
|
||||
atomic_t count;
|
||||
struct mutex mutex;
|
||||
|
||||
struct active_node *cache;
|
||||
struct rb_root tree;
|
||||
struct mutex mutex;
|
||||
atomic_t count;
|
||||
|
||||
/* Preallocated "exclusive" node */
|
||||
struct dma_fence __rcu *excl;
|
||||
struct dma_fence_cb excl_cb;
|
||||
struct i915_active_fence excl;
|
||||
|
||||
unsigned long flags;
|
||||
#define I915_ACTIVE_RETIRE_SLEEPS BIT(0)
|
||||
#define I915_ACTIVE_GRAB_BIT 1
|
||||
|
||||
int (*active)(struct i915_active *ref);
|
||||
void (*retire)(struct i915_active *ref);
|
||||
|
|
|
@ -892,28 +892,38 @@ wait_for_timelines(struct intel_gt *gt, unsigned int wait, long timeout)
|
|||
|
||||
spin_lock_irqsave(&timelines->lock, flags);
|
||||
list_for_each_entry(tl, &timelines->active_list, link) {
|
||||
struct i915_request *rq;
|
||||
struct dma_fence *fence;
|
||||
|
||||
rq = i915_active_request_get_unlocked(&tl->last_request);
|
||||
if (!rq)
|
||||
fence = i915_active_fence_get(&tl->last_request);
|
||||
if (!fence)
|
||||
continue;
|
||||
|
||||
spin_unlock_irqrestore(&timelines->lock, flags);
|
||||
|
||||
/*
|
||||
* "Race-to-idle".
|
||||
*
|
||||
* Switching to the kernel context is often used a synchronous
|
||||
* step prior to idling, e.g. in suspend for flushing all
|
||||
* current operations to memory before sleeping. These we
|
||||
* want to complete as quickly as possible to avoid prolonged
|
||||
* stalls, so allow the gpu to boost to maximum clocks.
|
||||
*/
|
||||
if (wait & I915_WAIT_FOR_IDLE_BOOST)
|
||||
gen6_rps_boost(rq);
|
||||
if (!dma_fence_is_i915(fence)) {
|
||||
timeout = dma_fence_wait_timeout(fence,
|
||||
flags & I915_WAIT_INTERRUPTIBLE,
|
||||
timeout);
|
||||
} else {
|
||||
struct i915_request *rq = to_request(fence);
|
||||
|
||||
timeout = i915_request_wait(rq, wait, timeout);
|
||||
i915_request_put(rq);
|
||||
/*
|
||||
* "Race-to-idle".
|
||||
*
|
||||
* Switching to the kernel context is often used as
|
||||
* a synchronous step prior to idling, e.g. in suspend
|
||||
* for flushing all current operations to memory before
|
||||
* sleeping. These we want to complete as quickly as
|
||||
* possible to avoid prolonged stalls, so allow the gpu
|
||||
* to boost to maximum clocks.
|
||||
*/
|
||||
if (flags & I915_WAIT_FOR_IDLE_BOOST)
|
||||
gen6_rps_boost(rq);
|
||||
|
||||
timeout = i915_request_wait(rq, flags, timeout);
|
||||
}
|
||||
|
||||
dma_fence_put(fence);
|
||||
if (timeout < 0)
|
||||
return timeout;
|
||||
|
||||
|
|
|
@ -1861,7 +1861,6 @@ static const struct i915_vma_ops pd_vma_ops = {
|
|||
|
||||
static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size)
|
||||
{
|
||||
struct drm_i915_private *i915 = ppgtt->base.vm.i915;
|
||||
struct i915_ggtt *ggtt = ppgtt->base.vm.gt->ggtt;
|
||||
struct i915_vma *vma;
|
||||
|
||||
|
@ -1872,7 +1871,7 @@ static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size)
|
|||
if (!vma)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
i915_active_init(i915, &vma->active, NULL, NULL);
|
||||
i915_active_init(&vma->active, NULL, NULL);
|
||||
|
||||
mutex_init(&vma->pages_mutex);
|
||||
vma->vm = i915_vm_get(&ggtt->vm);
|
||||
|
|
|
@ -1299,7 +1299,7 @@ capture_vma(struct capture_vma *next,
|
|||
if (!c)
|
||||
return next;
|
||||
|
||||
if (!i915_active_trygrab(&vma->active)) {
|
||||
if (!i915_active_acquire_if_busy(&vma->active)) {
|
||||
kfree(c);
|
||||
return next;
|
||||
}
|
||||
|
@ -1439,7 +1439,7 @@ gem_record_rings(struct i915_gpu_state *error, struct compress *compress)
|
|||
*this->slot =
|
||||
i915_error_object_create(i915, vma, compress);
|
||||
|
||||
i915_active_ungrab(&vma->active);
|
||||
i915_active_release(&vma->active);
|
||||
i915_vma_put(vma);
|
||||
|
||||
capture = this->next;
|
||||
|
|
|
@ -218,8 +218,6 @@ static void remove_from_engine(struct i915_request *rq)
|
|||
|
||||
static bool i915_request_retire(struct i915_request *rq)
|
||||
{
|
||||
struct i915_active_request *active, *next;
|
||||
|
||||
if (!i915_request_completed(rq))
|
||||
return false;
|
||||
|
||||
|
@ -244,35 +242,6 @@ static bool i915_request_retire(struct i915_request *rq)
|
|||
&i915_request_timeline(rq)->requests));
|
||||
rq->ring->head = rq->postfix;
|
||||
|
||||
/*
|
||||
* Walk through the active list, calling retire on each. This allows
|
||||
* objects to track their GPU activity and mark themselves as idle
|
||||
* when their *last* active request is completed (updating state
|
||||
* tracking lists for eviction, active references for GEM, etc).
|
||||
*
|
||||
* As the ->retire() may free the node, we decouple it first and
|
||||
* pass along the auxiliary information (to avoid dereferencing
|
||||
* the node after the callback).
|
||||
*/
|
||||
list_for_each_entry_safe(active, next, &rq->active_list, link) {
|
||||
/*
|
||||
* In microbenchmarks or focusing upon time inside the kernel,
|
||||
* we may spend an inordinate amount of time simply handling
|
||||
* the retirement of requests and processing their callbacks.
|
||||
* Of which, this loop itself is particularly hot due to the
|
||||
* cache misses when jumping around the list of
|
||||
* i915_active_request. So we try to keep this loop as
|
||||
* streamlined as possible and also prefetch the next
|
||||
* i915_active_request to try and hide the likely cache miss.
|
||||
*/
|
||||
prefetchw(next);
|
||||
|
||||
INIT_LIST_HEAD(&active->link);
|
||||
RCU_INIT_POINTER(active->request, NULL);
|
||||
|
||||
active->retire(active, rq);
|
||||
}
|
||||
|
||||
local_irq_disable();
|
||||
|
||||
/*
|
||||
|
@ -704,7 +673,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp)
|
|||
rq->flags = 0;
|
||||
rq->execution_mask = ALL_ENGINES;
|
||||
|
||||
INIT_LIST_HEAD(&rq->active_list);
|
||||
INIT_LIST_HEAD(&rq->execute_cb);
|
||||
|
||||
/*
|
||||
|
@ -743,7 +711,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp)
|
|||
ce->ring->emit = rq->head;
|
||||
|
||||
/* Make sure we didn't add ourselves to external state before freeing */
|
||||
GEM_BUG_ON(!list_empty(&rq->active_list));
|
||||
GEM_BUG_ON(!list_empty(&rq->sched.signalers_list));
|
||||
GEM_BUG_ON(!list_empty(&rq->sched.waiters_list));
|
||||
|
||||
|
@ -1174,8 +1141,8 @@ __i915_request_add_to_timeline(struct i915_request *rq)
|
|||
* precludes optimising to use semaphores serialisation of a single
|
||||
* timeline across engines.
|
||||
*/
|
||||
prev = rcu_dereference_protected(timeline->last_request.request,
|
||||
lockdep_is_held(&timeline->mutex));
|
||||
prev = to_request(__i915_active_fence_set(&timeline->last_request,
|
||||
&rq->fence));
|
||||
if (prev && !i915_request_completed(prev)) {
|
||||
if (is_power_of_2(prev->engine->mask | rq->engine->mask))
|
||||
i915_sw_fence_await_sw_fence(&rq->submit,
|
||||
|
@ -1200,7 +1167,6 @@ __i915_request_add_to_timeline(struct i915_request *rq)
|
|||
* us, the timeline will hold its seqno which is later than ours.
|
||||
*/
|
||||
GEM_BUG_ON(timeline->seqno != rq->fence.seqno);
|
||||
__i915_active_request_set(&timeline->last_request, rq);
|
||||
|
||||
return prev;
|
||||
}
|
||||
|
|
|
@ -211,7 +211,6 @@ struct i915_request {
|
|||
* on the active_list (of their final request).
|
||||
*/
|
||||
struct i915_capture_list *capture_list;
|
||||
struct list_head active_list;
|
||||
|
||||
/** Time at which this request was emitted, in jiffies. */
|
||||
unsigned long emitted_jiffies;
|
||||
|
|
|
@ -120,8 +120,7 @@ vma_create(struct drm_i915_gem_object *obj,
|
|||
vma->size = obj->base.size;
|
||||
vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
|
||||
|
||||
i915_active_init(vm->i915, &vma->active,
|
||||
__i915_vma_active, __i915_vma_retire);
|
||||
i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire);
|
||||
|
||||
/* Declare ourselves safe for use inside shrinkers */
|
||||
if (IS_ENABLED(CONFIG_LOCKDEP)) {
|
||||
|
@ -1148,6 +1147,7 @@ int __i915_vma_unbind(struct i915_vma *vma)
|
|||
if (ret)
|
||||
return ret;
|
||||
|
||||
GEM_BUG_ON(i915_vma_is_active(vma));
|
||||
if (i915_vma_is_pinned(vma)) {
|
||||
vma_print_allocator(vma, "is pinned");
|
||||
return -EBUSY;
|
||||
|
|
|
@ -68,7 +68,7 @@ static struct live_active *__live_alloc(struct drm_i915_private *i915)
|
|||
return NULL;
|
||||
|
||||
kref_init(&active->ref);
|
||||
i915_active_init(i915, &active->base, __live_active, __live_retire);
|
||||
i915_active_init(&active->base, __live_active, __live_retire);
|
||||
|
||||
return active;
|
||||
}
|
||||
|
@ -146,19 +146,13 @@ static int live_active_wait(void *arg)
|
|||
{
|
||||
struct drm_i915_private *i915 = arg;
|
||||
struct live_active *active;
|
||||
intel_wakeref_t wakeref;
|
||||
int err = 0;
|
||||
|
||||
/* Check that we get a callback when requests retire upon waiting */
|
||||
|
||||
mutex_lock(&i915->drm.struct_mutex);
|
||||
wakeref = intel_runtime_pm_get(&i915->runtime_pm);
|
||||
|
||||
active = __live_active_setup(i915);
|
||||
if (IS_ERR(active)) {
|
||||
err = PTR_ERR(active);
|
||||
goto err;
|
||||
}
|
||||
if (IS_ERR(active))
|
||||
return PTR_ERR(active);
|
||||
|
||||
i915_active_wait(&active->base);
|
||||
if (!READ_ONCE(active->retired)) {
|
||||
|
@ -168,11 +162,9 @@ static int live_active_wait(void *arg)
|
|||
|
||||
__live_put(active);
|
||||
|
||||
mutex_lock(&i915->drm.struct_mutex);
|
||||
if (igt_flush_test(i915, I915_WAIT_LOCKED))
|
||||
err = -EIO;
|
||||
|
||||
err:
|
||||
intel_runtime_pm_put(&i915->runtime_pm, wakeref);
|
||||
mutex_unlock(&i915->drm.struct_mutex);
|
||||
|
||||
return err;
|
||||
|
@ -182,23 +174,19 @@ static int live_active_retire(void *arg)
|
|||
{
|
||||
struct drm_i915_private *i915 = arg;
|
||||
struct live_active *active;
|
||||
intel_wakeref_t wakeref;
|
||||
int err = 0;
|
||||
|
||||
/* Check that we get a callback when requests are indirectly retired */
|
||||
|
||||
mutex_lock(&i915->drm.struct_mutex);
|
||||
wakeref = intel_runtime_pm_get(&i915->runtime_pm);
|
||||
|
||||
active = __live_active_setup(i915);
|
||||
if (IS_ERR(active)) {
|
||||
err = PTR_ERR(active);
|
||||
goto err;
|
||||
}
|
||||
if (IS_ERR(active))
|
||||
return PTR_ERR(active);
|
||||
|
||||
/* waits for & retires all requests */
|
||||
mutex_lock(&i915->drm.struct_mutex);
|
||||
if (igt_flush_test(i915, I915_WAIT_LOCKED))
|
||||
err = -EIO;
|
||||
mutex_unlock(&i915->drm.struct_mutex);
|
||||
|
||||
if (!READ_ONCE(active->retired)) {
|
||||
pr_err("i915_active not retired after flushing!\n");
|
||||
|
@ -207,10 +195,6 @@ static int live_active_retire(void *arg)
|
|||
|
||||
__live_put(active);
|
||||
|
||||
err:
|
||||
intel_runtime_pm_put(&i915->runtime_pm, wakeref);
|
||||
mutex_unlock(&i915->drm.struct_mutex);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue