2015-04-20 20:55:21 +00:00
|
|
|
/*
|
|
|
|
* Copyright 2009 Jerome Glisse.
|
|
|
|
* All Rights Reserved.
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
|
|
* copy of this software and associated documentation files (the
|
|
|
|
* "Software"), to deal in the Software without restriction, including
|
|
|
|
* without limitation the rights to use, copy, modify, merge, publish,
|
|
|
|
* distribute, sub license, and/or sell copies of the Software, and to
|
|
|
|
* permit persons to whom the Software is furnished to do so, subject to
|
|
|
|
* the following conditions:
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
|
|
|
|
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
|
|
|
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
|
|
* USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice (including the
|
|
|
|
* next paragraph) shall be included in all copies or substantial portions
|
|
|
|
* of the Software.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
/*
|
|
|
|
* Authors:
|
|
|
|
* Jerome Glisse <glisse@freedesktop.org>
|
|
|
|
* Dave Airlie
|
|
|
|
*/
|
|
|
|
#include <linux/seq_file.h>
|
|
|
|
#include <linux/atomic.h>
|
|
|
|
#include <linux/wait.h>
|
|
|
|
#include <linux/kref.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/firmware.h>
|
|
|
|
#include <drm/drmP.h>
|
|
|
|
#include "amdgpu.h"
|
|
|
|
#include "amdgpu_trace.h"
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Fences
|
|
|
|
* Fences mark an event in the GPUs pipeline and are used
|
|
|
|
* for GPU/CPU synchronization. When the fence is written,
|
|
|
|
* it is expected that all buffers associated with that fence
|
|
|
|
* are no longer in use by the associated ring on the GPU and
|
|
|
|
* that the the relevant GPU caches have been flushed.
|
|
|
|
*/
|
|
|
|
|
2016-03-11 14:12:53 +00:00
|
|
|
struct amdgpu_fence {
|
2016-10-25 12:00:45 +00:00
|
|
|
struct dma_fence base;
|
2016-03-11 14:12:53 +00:00
|
|
|
|
|
|
|
/* RB, DMA, etc. */
|
|
|
|
struct amdgpu_ring *ring;
|
|
|
|
};
|
|
|
|
|
2015-11-05 03:28:28 +00:00
|
|
|
static struct kmem_cache *amdgpu_fence_slab;
|
|
|
|
|
2016-05-12 05:27:28 +00:00
|
|
|
int amdgpu_fence_slab_init(void)
|
|
|
|
{
|
|
|
|
amdgpu_fence_slab = kmem_cache_create(
|
|
|
|
"amdgpu_fence", sizeof(struct amdgpu_fence), 0,
|
|
|
|
SLAB_HWCACHE_ALIGN, NULL);
|
|
|
|
if (!amdgpu_fence_slab)
|
|
|
|
return -ENOMEM;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void amdgpu_fence_slab_fini(void)
|
|
|
|
{
|
2016-10-23 18:31:43 +00:00
|
|
|
rcu_barrier();
|
2016-05-12 05:27:28 +00:00
|
|
|
kmem_cache_destroy(amdgpu_fence_slab);
|
|
|
|
}
|
2016-03-11 14:12:53 +00:00
|
|
|
/*
|
|
|
|
* Cast helper
|
|
|
|
*/
|
2016-10-25 12:00:45 +00:00
|
|
|
static const struct dma_fence_ops amdgpu_fence_ops;
|
|
|
|
static inline struct amdgpu_fence *to_amdgpu_fence(struct dma_fence *f)
|
2016-03-11 14:12:53 +00:00
|
|
|
{
|
|
|
|
struct amdgpu_fence *__f = container_of(f, struct amdgpu_fence, base);
|
|
|
|
|
|
|
|
if (__f->base.ops == &amdgpu_fence_ops)
|
|
|
|
return __f;
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2015-04-20 20:55:21 +00:00
|
|
|
/**
|
|
|
|
* amdgpu_fence_write - write a fence value
|
|
|
|
*
|
|
|
|
* @ring: ring the fence is associated with
|
|
|
|
* @seq: sequence number to write
|
|
|
|
*
|
|
|
|
* Writes a fence value to memory (all asics).
|
|
|
|
*/
|
|
|
|
static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
|
|
|
|
{
|
|
|
|
struct amdgpu_fence_driver *drv = &ring->fence_drv;
|
|
|
|
|
|
|
|
if (drv->cpu_addr)
|
|
|
|
*drv->cpu_addr = cpu_to_le32(seq);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* amdgpu_fence_read - read a fence value
|
|
|
|
*
|
|
|
|
* @ring: ring the fence is associated with
|
|
|
|
*
|
|
|
|
* Reads a fence value from memory (all asics).
|
|
|
|
* Returns the value of the fence read from memory.
|
|
|
|
*/
|
|
|
|
static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
|
|
|
|
{
|
|
|
|
struct amdgpu_fence_driver *drv = &ring->fence_drv;
|
|
|
|
u32 seq = 0;
|
|
|
|
|
|
|
|
if (drv->cpu_addr)
|
|
|
|
seq = le32_to_cpu(*drv->cpu_addr);
|
|
|
|
else
|
2016-03-14 14:46:06 +00:00
|
|
|
seq = atomic_read(&drv->last_seq);
|
2015-04-20 20:55:21 +00:00
|
|
|
|
|
|
|
return seq;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* amdgpu_fence_emit - emit a fence on the requested ring
|
|
|
|
*
|
|
|
|
* @ring: ring the fence is associated with
|
2016-02-16 16:39:39 +00:00
|
|
|
* @f: resulting fence object
|
2015-04-20 20:55:21 +00:00
|
|
|
*
|
|
|
|
* Emits a fence command on the requested ring (all asics).
|
|
|
|
* Returns 0 on success, -ENOMEM on failure.
|
|
|
|
*/
|
2016-10-25 12:00:45 +00:00
|
|
|
int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f)
|
2015-04-20 20:55:21 +00:00
|
|
|
{
|
|
|
|
struct amdgpu_device *adev = ring->adev;
|
2016-02-16 16:39:39 +00:00
|
|
|
struct amdgpu_fence *fence;
|
2016-10-25 12:00:45 +00:00
|
|
|
struct dma_fence *old, **ptr;
|
2016-03-14 14:46:06 +00:00
|
|
|
uint32_t seq;
|
2015-04-20 20:55:21 +00:00
|
|
|
|
2016-02-16 16:39:39 +00:00
|
|
|
fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
|
|
|
|
if (fence == NULL)
|
2015-04-20 20:55:21 +00:00
|
|
|
return -ENOMEM;
|
2016-02-16 16:39:39 +00:00
|
|
|
|
2016-03-14 14:46:06 +00:00
|
|
|
seq = ++ring->fence_drv.sync_seq;
|
2016-02-16 16:39:39 +00:00
|
|
|
fence->ring = ring;
|
2016-10-25 12:00:45 +00:00
|
|
|
dma_fence_init(&fence->base, &amdgpu_fence_ops,
|
|
|
|
&ring->fence_drv.lock,
|
|
|
|
adev->fence_context + ring->idx,
|
|
|
|
seq);
|
2015-06-01 06:35:03 +00:00
|
|
|
amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
|
2016-03-14 14:46:06 +00:00
|
|
|
seq, AMDGPU_FENCE_FLAG_INT);
|
2016-03-13 18:19:48 +00:00
|
|
|
|
2016-03-14 14:46:06 +00:00
|
|
|
ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
|
2016-03-13 18:19:48 +00:00
|
|
|
/* This function can't be called concurrently anyway, otherwise
|
|
|
|
* emitting the fence would mess up the hardware ring buffer.
|
|
|
|
*/
|
2016-03-31 03:07:14 +00:00
|
|
|
old = rcu_dereference_protected(*ptr, 1);
|
2016-10-25 12:00:45 +00:00
|
|
|
if (old && !dma_fence_is_signaled(old)) {
|
2016-03-31 03:07:14 +00:00
|
|
|
DRM_INFO("rcu slot is busy\n");
|
2016-10-25 12:00:45 +00:00
|
|
|
dma_fence_wait(old, false);
|
2016-03-31 03:07:14 +00:00
|
|
|
}
|
2016-03-13 18:19:48 +00:00
|
|
|
|
2016-10-25 12:00:45 +00:00
|
|
|
rcu_assign_pointer(*ptr, dma_fence_get(&fence->base));
|
2016-03-13 18:19:48 +00:00
|
|
|
|
2016-02-16 16:39:39 +00:00
|
|
|
*f = &fence->base;
|
2016-03-13 18:19:48 +00:00
|
|
|
|
2015-04-20 20:55:21 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-10-13 07:38:35 +00:00
|
|
|
/**
|
|
|
|
* amdgpu_fence_emit_polling - emit a fence on the requeste ring
|
|
|
|
*
|
|
|
|
* @ring: ring the fence is associated with
|
|
|
|
* @s: resulting sequence number
|
|
|
|
*
|
|
|
|
* Emits a fence command on the requested ring (all asics).
|
|
|
|
* Used For polling fence.
|
|
|
|
* Returns 0 on success, -ENOMEM on failure.
|
|
|
|
*/
|
|
|
|
int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s)
|
|
|
|
{
|
|
|
|
uint32_t seq;
|
|
|
|
|
|
|
|
if (!s)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
seq = ++ring->fence_drv.sync_seq;
|
|
|
|
amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
|
2017-12-04 12:46:17 +00:00
|
|
|
seq, 0);
|
2017-10-13 07:38:35 +00:00
|
|
|
|
|
|
|
*s = seq;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-11-03 12:27:39 +00:00
|
|
|
/**
|
|
|
|
* amdgpu_fence_schedule_fallback - schedule fallback check
|
|
|
|
*
|
|
|
|
* @ring: pointer to struct amdgpu_ring
|
|
|
|
*
|
|
|
|
* Start a timer as fallback to our interrupts.
|
|
|
|
*/
|
|
|
|
static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)
|
|
|
|
{
|
|
|
|
mod_timer(&ring->fence_drv.fallback_timer,
|
|
|
|
jiffies + AMDGPU_FENCE_JIFFIES_TIMEOUT);
|
|
|
|
}
|
|
|
|
|
2015-04-20 20:55:21 +00:00
|
|
|
/**
|
2016-03-11 16:57:56 +00:00
|
|
|
* amdgpu_fence_process - check for fence activity
|
2015-04-20 20:55:21 +00:00
|
|
|
*
|
|
|
|
* @ring: pointer to struct amdgpu_ring
|
|
|
|
*
|
|
|
|
* Checks the current fence value and calculates the last
|
2016-03-11 16:57:56 +00:00
|
|
|
* signalled fence value. Wakes the fence queue if the
|
|
|
|
* sequence number has increased.
|
2015-04-20 20:55:21 +00:00
|
|
|
*/
|
2016-03-11 16:57:56 +00:00
|
|
|
void amdgpu_fence_process(struct amdgpu_ring *ring)
|
2015-04-20 20:55:21 +00:00
|
|
|
{
|
2016-03-14 13:29:46 +00:00
|
|
|
struct amdgpu_fence_driver *drv = &ring->fence_drv;
|
2016-03-14 14:46:06 +00:00
|
|
|
uint32_t seq, last_seq;
|
2016-03-14 13:29:46 +00:00
|
|
|
int r;
|
2015-04-20 20:55:21 +00:00
|
|
|
|
|
|
|
do {
|
2016-03-14 14:46:06 +00:00
|
|
|
last_seq = atomic_read(&ring->fence_drv.last_seq);
|
2015-04-20 20:55:21 +00:00
|
|
|
seq = amdgpu_fence_read(ring);
|
|
|
|
|
2016-03-14 14:46:06 +00:00
|
|
|
} while (atomic_cmpxchg(&drv->last_seq, last_seq, seq) != last_seq);
|
2015-04-20 20:55:21 +00:00
|
|
|
|
2016-03-14 14:46:06 +00:00
|
|
|
if (seq != ring->fence_drv.sync_seq)
|
2015-11-03 12:27:39 +00:00
|
|
|
amdgpu_fence_schedule_fallback(ring);
|
2015-04-20 20:55:21 +00:00
|
|
|
|
2016-07-12 11:57:03 +00:00
|
|
|
if (unlikely(seq == last_seq))
|
|
|
|
return;
|
|
|
|
|
2016-06-24 19:11:51 +00:00
|
|
|
last_seq &= drv->num_fences_mask;
|
|
|
|
seq &= drv->num_fences_mask;
|
|
|
|
|
2016-07-12 11:57:03 +00:00
|
|
|
do {
|
2016-10-25 12:00:45 +00:00
|
|
|
struct dma_fence *fence, **ptr;
|
2016-03-14 13:29:46 +00:00
|
|
|
|
2016-06-24 19:11:51 +00:00
|
|
|
++last_seq;
|
|
|
|
last_seq &= drv->num_fences_mask;
|
|
|
|
ptr = &drv->fences[last_seq];
|
2016-03-14 13:29:46 +00:00
|
|
|
|
|
|
|
/* There is always exactly one thread signaling this fence slot */
|
|
|
|
fence = rcu_dereference_protected(*ptr, 1);
|
2016-04-30 19:00:24 +00:00
|
|
|
RCU_INIT_POINTER(*ptr, NULL);
|
2016-03-14 13:29:46 +00:00
|
|
|
|
2016-06-24 19:11:51 +00:00
|
|
|
if (!fence)
|
|
|
|
continue;
|
2016-03-14 13:29:46 +00:00
|
|
|
|
2016-10-25 12:00:45 +00:00
|
|
|
r = dma_fence_signal(fence);
|
2016-03-14 13:29:46 +00:00
|
|
|
if (!r)
|
2016-10-25 12:00:45 +00:00
|
|
|
DMA_FENCE_TRACE(fence, "signaled from irq context\n");
|
2016-03-14 13:29:46 +00:00
|
|
|
else
|
|
|
|
BUG();
|
|
|
|
|
2016-10-25 12:00:45 +00:00
|
|
|
dma_fence_put(fence);
|
2016-07-12 11:57:03 +00:00
|
|
|
} while (last_seq != seq);
|
2015-04-20 20:55:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2015-11-03 12:27:39 +00:00
|
|
|
* amdgpu_fence_fallback - fallback for hardware interrupts
|
2015-04-20 20:55:21 +00:00
|
|
|
*
|
2015-11-03 12:27:39 +00:00
|
|
|
* @work: delayed work item
|
2015-04-20 20:55:21 +00:00
|
|
|
*
|
2015-11-03 12:27:39 +00:00
|
|
|
* Checks for fence activity.
|
2015-04-20 20:55:21 +00:00
|
|
|
*/
|
treewide: setup_timer() -> timer_setup() (2 field)
This converts all remaining setup_timer() calls that use a nested field
to reach a struct timer_list. Coccinelle does not have an easy way to
match multiple fields, so a new script is needed to change the matches of
"&_E->_timer" into "&_E->_field1._timer" in all the rules.
spatch --very-quiet --all-includes --include-headers \
-I ./arch/x86/include -I ./arch/x86/include/generated \
-I ./include -I ./arch/x86/include/uapi \
-I ./arch/x86/include/generated/uapi -I ./include/uapi \
-I ./include/generated/uapi --include ./include/linux/kconfig.h \
--dir . \
--cocci-file ~/src/data/timer_setup-2fields.cocci
@fix_address_of depends@
expression e;
@@
setup_timer(
-&(e)
+&e
, ...)
// Update any raw setup_timer() usages that have a NULL callback, but
// would otherwise match change_timer_function_usage, since the latter
// will update all function assignments done in the face of a NULL
// function initialization in setup_timer().
@change_timer_function_usage_NULL@
expression _E;
identifier _field1;
identifier _timer;
type _cast_data;
@@
(
-setup_timer(&_E->_field1._timer, NULL, _E);
+timer_setup(&_E->_field1._timer, NULL, 0);
|
-setup_timer(&_E->_field1._timer, NULL, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, NULL, 0);
|
-setup_timer(&_E._field1._timer, NULL, &_E);
+timer_setup(&_E._field1._timer, NULL, 0);
|
-setup_timer(&_E._field1._timer, NULL, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, NULL, 0);
)
@change_timer_function_usage@
expression _E;
identifier _field1;
identifier _timer;
struct timer_list _stl;
identifier _callback;
type _cast_func, _cast_data;
@@
(
-setup_timer(&_E->_field1._timer, _callback, _E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, &_callback, _E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, _callback, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, &_callback, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, (_cast_func)_callback, _E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, (_cast_func)&_callback, _E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, (_cast_func)_callback, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, (_cast_func)&_callback, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, (_cast_data)_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, &_callback, (_cast_data)_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, &_callback, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, (_cast_func)_callback, (_cast_data)_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, (_cast_func)_callback, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, (_cast_func)&_callback, (_cast_data)_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, (_cast_func)&_callback, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
_E->_field1._timer@_stl.function = _callback;
|
_E->_field1._timer@_stl.function = &_callback;
|
_E->_field1._timer@_stl.function = (_cast_func)_callback;
|
_E->_field1._timer@_stl.function = (_cast_func)&_callback;
|
_E._field1._timer@_stl.function = _callback;
|
_E._field1._timer@_stl.function = &_callback;
|
_E._field1._timer@_stl.function = (_cast_func)_callback;
|
_E._field1._timer@_stl.function = (_cast_func)&_callback;
)
// callback(unsigned long arg)
@change_callback_handle_cast
depends on change_timer_function_usage@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
type _origtype;
identifier _origarg;
type _handletype;
identifier _handle;
@@
void _callback(
-_origtype _origarg
+struct timer_list *t
)
{
(
... when != _origarg
_handletype *_handle =
-(_handletype *)_origarg;
+from_timer(_handle, t, _field1._timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle =
-(void *)_origarg;
+from_timer(_handle, t, _field1._timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle;
... when != _handle
_handle =
-(_handletype *)_origarg;
+from_timer(_handle, t, _field1._timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle;
... when != _handle
_handle =
-(void *)_origarg;
+from_timer(_handle, t, _field1._timer);
... when != _origarg
)
}
// callback(unsigned long arg) without existing variable
@change_callback_handle_cast_no_arg
depends on change_timer_function_usage &&
!change_callback_handle_cast@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
type _origtype;
identifier _origarg;
type _handletype;
@@
void _callback(
-_origtype _origarg
+struct timer_list *t
)
{
+ _handletype *_origarg = from_timer(_origarg, t, _field1._timer);
+
... when != _origarg
- (_handletype *)_origarg
+ _origarg
... when != _origarg
}
// Avoid already converted callbacks.
@match_callback_converted
depends on change_timer_function_usage &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg@
identifier change_timer_function_usage._callback;
identifier t;
@@
void _callback(struct timer_list *t)
{ ... }
// callback(struct something *handle)
@change_callback_handle_arg
depends on change_timer_function_usage &&
!match_callback_converted &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
type _handletype;
identifier _handle;
@@
void _callback(
-_handletype *_handle
+struct timer_list *t
)
{
+ _handletype *_handle = from_timer(_handle, t, _field1._timer);
...
}
// If change_callback_handle_arg ran on an empty function, remove
// the added handler.
@unchange_callback_handle_arg
depends on change_timer_function_usage &&
change_callback_handle_arg@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
type _handletype;
identifier _handle;
identifier t;
@@
void _callback(struct timer_list *t)
{
- _handletype *_handle = from_timer(_handle, t, _field1._timer);
}
// We only want to refactor the setup_timer() data argument if we've found
// the matching callback. This undoes changes in change_timer_function_usage.
@unchange_timer_function_usage
depends on change_timer_function_usage &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg &&
!change_callback_handle_arg@
expression change_timer_function_usage._E;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type change_timer_function_usage._cast_data;
@@
(
-timer_setup(&_E->_field1._timer, _callback, 0);
+setup_timer(&_E->_field1._timer, _callback, (_cast_data)_E);
|
-timer_setup(&_E._field1._timer, _callback, 0);
+setup_timer(&_E._field1._timer, _callback, (_cast_data)&_E);
)
// If we fixed a callback from a .function assignment, fix the
// assignment cast now.
@change_timer_function_assignment
depends on change_timer_function_usage &&
(change_callback_handle_cast ||
change_callback_handle_cast_no_arg ||
change_callback_handle_arg)@
expression change_timer_function_usage._E;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type _cast_func;
typedef TIMER_FUNC_TYPE;
@@
(
_E->_field1._timer.function =
-_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_field1._timer.function =
-&_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_field1._timer.function =
-(_cast_func)_callback;
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_field1._timer.function =
-(_cast_func)&_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._field1._timer.function =
-_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._field1._timer.function =
-&_callback;
+(TIMER_FUNC_TYPE)_callback
;
|
_E._field1._timer.function =
-(_cast_func)_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._field1._timer.function =
-(_cast_func)&_callback
+(TIMER_FUNC_TYPE)_callback
;
)
// Sometimes timer functions are called directly. Replace matched args.
@change_timer_function_calls
depends on change_timer_function_usage &&
(change_callback_handle_cast ||
change_callback_handle_cast_no_arg ||
change_callback_handle_arg)@
expression _E;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type _cast_data;
@@
_callback(
(
-(_cast_data)_E
+&_E->_field1._timer
|
-(_cast_data)&_E
+&_E._field1._timer
|
-_E
+&_E->_field1._timer
)
)
// If a timer has been configured without a data argument, it can be
// converted without regard to the callback argument, since it is unused.
@match_timer_function_unused_data@
expression _E;
identifier _field1;
identifier _timer;
identifier _callback;
@@
(
-setup_timer(&_E->_field1._timer, _callback, 0);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, _callback, 0L);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, _callback, 0UL);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, 0);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, 0L);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, 0UL);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_field1._timer, _callback, 0);
+timer_setup(&_field1._timer, _callback, 0);
|
-setup_timer(&_field1._timer, _callback, 0L);
+timer_setup(&_field1._timer, _callback, 0);
|
-setup_timer(&_field1._timer, _callback, 0UL);
+timer_setup(&_field1._timer, _callback, 0);
|
-setup_timer(_field1._timer, _callback, 0);
+timer_setup(_field1._timer, _callback, 0);
|
-setup_timer(_field1._timer, _callback, 0L);
+timer_setup(_field1._timer, _callback, 0);
|
-setup_timer(_field1._timer, _callback, 0UL);
+timer_setup(_field1._timer, _callback, 0);
)
@change_callback_unused_data
depends on match_timer_function_unused_data@
identifier match_timer_function_unused_data._callback;
type _origtype;
identifier _origarg;
@@
void _callback(
-_origtype _origarg
+struct timer_list *unused
)
{
... when != _origarg
}
Signed-off-by: Kees Cook <keescook@chromium.org>
2017-10-18 03:21:24 +00:00
|
|
|
static void amdgpu_fence_fallback(struct timer_list *t)
|
2015-04-20 20:55:21 +00:00
|
|
|
{
|
treewide: setup_timer() -> timer_setup() (2 field)
This converts all remaining setup_timer() calls that use a nested field
to reach a struct timer_list. Coccinelle does not have an easy way to
match multiple fields, so a new script is needed to change the matches of
"&_E->_timer" into "&_E->_field1._timer" in all the rules.
spatch --very-quiet --all-includes --include-headers \
-I ./arch/x86/include -I ./arch/x86/include/generated \
-I ./include -I ./arch/x86/include/uapi \
-I ./arch/x86/include/generated/uapi -I ./include/uapi \
-I ./include/generated/uapi --include ./include/linux/kconfig.h \
--dir . \
--cocci-file ~/src/data/timer_setup-2fields.cocci
@fix_address_of depends@
expression e;
@@
setup_timer(
-&(e)
+&e
, ...)
// Update any raw setup_timer() usages that have a NULL callback, but
// would otherwise match change_timer_function_usage, since the latter
// will update all function assignments done in the face of a NULL
// function initialization in setup_timer().
@change_timer_function_usage_NULL@
expression _E;
identifier _field1;
identifier _timer;
type _cast_data;
@@
(
-setup_timer(&_E->_field1._timer, NULL, _E);
+timer_setup(&_E->_field1._timer, NULL, 0);
|
-setup_timer(&_E->_field1._timer, NULL, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, NULL, 0);
|
-setup_timer(&_E._field1._timer, NULL, &_E);
+timer_setup(&_E._field1._timer, NULL, 0);
|
-setup_timer(&_E._field1._timer, NULL, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, NULL, 0);
)
@change_timer_function_usage@
expression _E;
identifier _field1;
identifier _timer;
struct timer_list _stl;
identifier _callback;
type _cast_func, _cast_data;
@@
(
-setup_timer(&_E->_field1._timer, _callback, _E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, &_callback, _E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, _callback, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, &_callback, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, (_cast_func)_callback, _E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, (_cast_func)&_callback, _E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, (_cast_func)_callback, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, (_cast_func)&_callback, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, (_cast_data)_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, &_callback, (_cast_data)_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, &_callback, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, (_cast_func)_callback, (_cast_data)_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, (_cast_func)_callback, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, (_cast_func)&_callback, (_cast_data)_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, (_cast_func)&_callback, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
_E->_field1._timer@_stl.function = _callback;
|
_E->_field1._timer@_stl.function = &_callback;
|
_E->_field1._timer@_stl.function = (_cast_func)_callback;
|
_E->_field1._timer@_stl.function = (_cast_func)&_callback;
|
_E._field1._timer@_stl.function = _callback;
|
_E._field1._timer@_stl.function = &_callback;
|
_E._field1._timer@_stl.function = (_cast_func)_callback;
|
_E._field1._timer@_stl.function = (_cast_func)&_callback;
)
// callback(unsigned long arg)
@change_callback_handle_cast
depends on change_timer_function_usage@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
type _origtype;
identifier _origarg;
type _handletype;
identifier _handle;
@@
void _callback(
-_origtype _origarg
+struct timer_list *t
)
{
(
... when != _origarg
_handletype *_handle =
-(_handletype *)_origarg;
+from_timer(_handle, t, _field1._timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle =
-(void *)_origarg;
+from_timer(_handle, t, _field1._timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle;
... when != _handle
_handle =
-(_handletype *)_origarg;
+from_timer(_handle, t, _field1._timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle;
... when != _handle
_handle =
-(void *)_origarg;
+from_timer(_handle, t, _field1._timer);
... when != _origarg
)
}
// callback(unsigned long arg) without existing variable
@change_callback_handle_cast_no_arg
depends on change_timer_function_usage &&
!change_callback_handle_cast@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
type _origtype;
identifier _origarg;
type _handletype;
@@
void _callback(
-_origtype _origarg
+struct timer_list *t
)
{
+ _handletype *_origarg = from_timer(_origarg, t, _field1._timer);
+
... when != _origarg
- (_handletype *)_origarg
+ _origarg
... when != _origarg
}
// Avoid already converted callbacks.
@match_callback_converted
depends on change_timer_function_usage &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg@
identifier change_timer_function_usage._callback;
identifier t;
@@
void _callback(struct timer_list *t)
{ ... }
// callback(struct something *handle)
@change_callback_handle_arg
depends on change_timer_function_usage &&
!match_callback_converted &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
type _handletype;
identifier _handle;
@@
void _callback(
-_handletype *_handle
+struct timer_list *t
)
{
+ _handletype *_handle = from_timer(_handle, t, _field1._timer);
...
}
// If change_callback_handle_arg ran on an empty function, remove
// the added handler.
@unchange_callback_handle_arg
depends on change_timer_function_usage &&
change_callback_handle_arg@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
type _handletype;
identifier _handle;
identifier t;
@@
void _callback(struct timer_list *t)
{
- _handletype *_handle = from_timer(_handle, t, _field1._timer);
}
// We only want to refactor the setup_timer() data argument if we've found
// the matching callback. This undoes changes in change_timer_function_usage.
@unchange_timer_function_usage
depends on change_timer_function_usage &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg &&
!change_callback_handle_arg@
expression change_timer_function_usage._E;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type change_timer_function_usage._cast_data;
@@
(
-timer_setup(&_E->_field1._timer, _callback, 0);
+setup_timer(&_E->_field1._timer, _callback, (_cast_data)_E);
|
-timer_setup(&_E._field1._timer, _callback, 0);
+setup_timer(&_E._field1._timer, _callback, (_cast_data)&_E);
)
// If we fixed a callback from a .function assignment, fix the
// assignment cast now.
@change_timer_function_assignment
depends on change_timer_function_usage &&
(change_callback_handle_cast ||
change_callback_handle_cast_no_arg ||
change_callback_handle_arg)@
expression change_timer_function_usage._E;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type _cast_func;
typedef TIMER_FUNC_TYPE;
@@
(
_E->_field1._timer.function =
-_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_field1._timer.function =
-&_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_field1._timer.function =
-(_cast_func)_callback;
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_field1._timer.function =
-(_cast_func)&_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._field1._timer.function =
-_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._field1._timer.function =
-&_callback;
+(TIMER_FUNC_TYPE)_callback
;
|
_E._field1._timer.function =
-(_cast_func)_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._field1._timer.function =
-(_cast_func)&_callback
+(TIMER_FUNC_TYPE)_callback
;
)
// Sometimes timer functions are called directly. Replace matched args.
@change_timer_function_calls
depends on change_timer_function_usage &&
(change_callback_handle_cast ||
change_callback_handle_cast_no_arg ||
change_callback_handle_arg)@
expression _E;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type _cast_data;
@@
_callback(
(
-(_cast_data)_E
+&_E->_field1._timer
|
-(_cast_data)&_E
+&_E._field1._timer
|
-_E
+&_E->_field1._timer
)
)
// If a timer has been configured without a data argument, it can be
// converted without regard to the callback argument, since it is unused.
@match_timer_function_unused_data@
expression _E;
identifier _field1;
identifier _timer;
identifier _callback;
@@
(
-setup_timer(&_E->_field1._timer, _callback, 0);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, _callback, 0L);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, _callback, 0UL);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, 0);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, 0L);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, 0UL);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_field1._timer, _callback, 0);
+timer_setup(&_field1._timer, _callback, 0);
|
-setup_timer(&_field1._timer, _callback, 0L);
+timer_setup(&_field1._timer, _callback, 0);
|
-setup_timer(&_field1._timer, _callback, 0UL);
+timer_setup(&_field1._timer, _callback, 0);
|
-setup_timer(_field1._timer, _callback, 0);
+timer_setup(_field1._timer, _callback, 0);
|
-setup_timer(_field1._timer, _callback, 0L);
+timer_setup(_field1._timer, _callback, 0);
|
-setup_timer(_field1._timer, _callback, 0UL);
+timer_setup(_field1._timer, _callback, 0);
)
@change_callback_unused_data
depends on match_timer_function_unused_data@
identifier match_timer_function_unused_data._callback;
type _origtype;
identifier _origarg;
@@
void _callback(
-_origtype _origarg
+struct timer_list *unused
)
{
... when != _origarg
}
Signed-off-by: Kees Cook <keescook@chromium.org>
2017-10-18 03:21:24 +00:00
|
|
|
struct amdgpu_ring *ring = from_timer(ring, t,
|
|
|
|
fence_drv.fallback_timer);
|
2015-11-03 12:27:39 +00:00
|
|
|
|
|
|
|
amdgpu_fence_process(ring);
|
2015-04-20 20:55:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* amdgpu_fence_wait_empty - wait for all fences to signal
|
|
|
|
*
|
|
|
|
* @adev: amdgpu device pointer
|
|
|
|
* @ring: ring index the fence is associated with
|
|
|
|
*
|
|
|
|
* Wait for all fences on the requested ring to signal (all asics).
|
|
|
|
* Returns 0 if the fences have passed, error for all other cases.
|
|
|
|
*/
|
|
|
|
int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
|
|
|
|
{
|
locking/atomics: COCCINELLE/treewide: Convert trivial ACCESS_ONCE() patterns to READ_ONCE()/WRITE_ONCE()
Please do not apply this to mainline directly, instead please re-run the
coccinelle script shown below and apply its output.
For several reasons, it is desirable to use {READ,WRITE}_ONCE() in
preference to ACCESS_ONCE(), and new code is expected to use one of the
former. So far, there's been no reason to change most existing uses of
ACCESS_ONCE(), as these aren't harmful, and changing them results in
churn.
However, for some features, the read/write distinction is critical to
correct operation. To distinguish these cases, separate read/write
accessors must be used. This patch migrates (most) remaining
ACCESS_ONCE() instances to {READ,WRITE}_ONCE(), using the following
coccinelle script:
----
// Convert trivial ACCESS_ONCE() uses to equivalent READ_ONCE() and
// WRITE_ONCE()
// $ make coccicheck COCCI=/home/mark/once.cocci SPFLAGS="--include-headers" MODE=patch
virtual patch
@ depends on patch @
expression E1, E2;
@@
- ACCESS_ONCE(E1) = E2
+ WRITE_ONCE(E1, E2)
@ depends on patch @
expression E;
@@
- ACCESS_ONCE(E)
+ READ_ONCE(E)
----
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: davem@davemloft.net
Cc: linux-arch@vger.kernel.org
Cc: mpe@ellerman.id.au
Cc: shuah@kernel.org
Cc: snitzer@redhat.com
Cc: thor.thayer@linux.intel.com
Cc: tj@kernel.org
Cc: viro@zeniv.linux.org.uk
Cc: will.deacon@arm.com
Link: http://lkml.kernel.org/r/1508792849-3115-19-git-send-email-paulmck@linux.vnet.ibm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-10-23 21:07:29 +00:00
|
|
|
uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);
|
2016-10-25 12:00:45 +00:00
|
|
|
struct dma_fence *fence, **ptr;
|
2016-03-13 18:37:01 +00:00
|
|
|
int r;
|
2015-08-07 14:15:36 +00:00
|
|
|
|
2015-07-30 10:28:12 +00:00
|
|
|
if (!seq)
|
2015-04-20 20:55:21 +00:00
|
|
|
return 0;
|
|
|
|
|
2016-03-13 18:37:01 +00:00
|
|
|
ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
|
|
|
|
rcu_read_lock();
|
|
|
|
fence = rcu_dereference(*ptr);
|
2016-10-25 12:00:45 +00:00
|
|
|
if (!fence || !dma_fence_get_rcu(fence)) {
|
2016-03-13 18:37:01 +00:00
|
|
|
rcu_read_unlock();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
2016-10-25 12:00:45 +00:00
|
|
|
r = dma_fence_wait(fence, false);
|
|
|
|
dma_fence_put(fence);
|
2016-03-13 18:37:01 +00:00
|
|
|
return r;
|
2015-04-20 20:55:21 +00:00
|
|
|
}
|
|
|
|
|
2017-10-13 07:38:35 +00:00
|
|
|
/**
|
|
|
|
* amdgpu_fence_wait_polling - busy wait for givn sequence number
|
|
|
|
*
|
|
|
|
* @ring: ring index the fence is associated with
|
|
|
|
* @wait_seq: sequence number to wait
|
|
|
|
* @timeout: the timeout for waiting in usecs
|
|
|
|
*
|
|
|
|
* Wait for all fences on the requested ring to signal (all asics).
|
|
|
|
* Returns left time if no timeout, 0 or minus if timeout.
|
|
|
|
*/
|
|
|
|
signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,
|
|
|
|
uint32_t wait_seq,
|
|
|
|
signed long timeout)
|
|
|
|
{
|
|
|
|
uint32_t seq;
|
|
|
|
|
|
|
|
do {
|
|
|
|
seq = amdgpu_fence_read(ring);
|
|
|
|
udelay(5);
|
|
|
|
timeout -= 5;
|
|
|
|
} while ((int32_t)(wait_seq - seq) > 0 && timeout > 0);
|
|
|
|
|
|
|
|
return timeout > 0 ? timeout : 0;
|
|
|
|
}
|
2015-04-20 20:55:21 +00:00
|
|
|
/**
|
|
|
|
* amdgpu_fence_count_emitted - get the count of emitted fences
|
|
|
|
*
|
|
|
|
* @ring: ring the fence is associated with
|
|
|
|
*
|
|
|
|
* Get the number of fences emitted on the requested ring (all asics).
|
|
|
|
* Returns the number of emitted fences on the ring. Used by the
|
|
|
|
* dynpm code to ring track activity.
|
|
|
|
*/
|
|
|
|
unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring)
|
|
|
|
{
|
|
|
|
uint64_t emitted;
|
|
|
|
|
|
|
|
/* We are not protected by ring lock when reading the last sequence
|
|
|
|
* but it's ok to report slightly wrong fence count here.
|
|
|
|
*/
|
|
|
|
amdgpu_fence_process(ring);
|
2016-03-14 14:46:06 +00:00
|
|
|
emitted = 0x100000000ull;
|
|
|
|
emitted -= atomic_read(&ring->fence_drv.last_seq);
|
locking/atomics: COCCINELLE/treewide: Convert trivial ACCESS_ONCE() patterns to READ_ONCE()/WRITE_ONCE()
Please do not apply this to mainline directly, instead please re-run the
coccinelle script shown below and apply its output.
For several reasons, it is desirable to use {READ,WRITE}_ONCE() in
preference to ACCESS_ONCE(), and new code is expected to use one of the
former. So far, there's been no reason to change most existing uses of
ACCESS_ONCE(), as these aren't harmful, and changing them results in
churn.
However, for some features, the read/write distinction is critical to
correct operation. To distinguish these cases, separate read/write
accessors must be used. This patch migrates (most) remaining
ACCESS_ONCE() instances to {READ,WRITE}_ONCE(), using the following
coccinelle script:
----
// Convert trivial ACCESS_ONCE() uses to equivalent READ_ONCE() and
// WRITE_ONCE()
// $ make coccicheck COCCI=/home/mark/once.cocci SPFLAGS="--include-headers" MODE=patch
virtual patch
@ depends on patch @
expression E1, E2;
@@
- ACCESS_ONCE(E1) = E2
+ WRITE_ONCE(E1, E2)
@ depends on patch @
expression E;
@@
- ACCESS_ONCE(E)
+ READ_ONCE(E)
----
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: davem@davemloft.net
Cc: linux-arch@vger.kernel.org
Cc: mpe@ellerman.id.au
Cc: shuah@kernel.org
Cc: snitzer@redhat.com
Cc: thor.thayer@linux.intel.com
Cc: tj@kernel.org
Cc: viro@zeniv.linux.org.uk
Cc: will.deacon@arm.com
Link: http://lkml.kernel.org/r/1508792849-3115-19-git-send-email-paulmck@linux.vnet.ibm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-10-23 21:07:29 +00:00
|
|
|
emitted += READ_ONCE(ring->fence_drv.sync_seq);
|
2016-03-14 14:46:06 +00:00
|
|
|
return lower_32_bits(emitted);
|
2015-04-20 20:55:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* amdgpu_fence_driver_start_ring - make the fence driver
|
|
|
|
* ready for use on the requested ring.
|
|
|
|
*
|
|
|
|
* @ring: ring to start the fence driver on
|
|
|
|
* @irq_src: interrupt source to use for this ring
|
|
|
|
* @irq_type: interrupt type to use for this ring
|
|
|
|
*
|
|
|
|
* Make the fence driver ready for processing (all asics).
|
|
|
|
* Not all asics have all rings, so each asic will only
|
|
|
|
* start the fence driver on the rings it has.
|
|
|
|
* Returns 0 for success, errors for failure.
|
|
|
|
*/
|
|
|
|
int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
|
|
|
|
struct amdgpu_irq_src *irq_src,
|
|
|
|
unsigned irq_type)
|
|
|
|
{
|
|
|
|
struct amdgpu_device *adev = ring->adev;
|
|
|
|
uint64_t index;
|
|
|
|
|
|
|
|
if (ring != &adev->uvd.ring) {
|
|
|
|
ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs];
|
|
|
|
ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4);
|
|
|
|
} else {
|
|
|
|
/* put fence directly behind firmware */
|
|
|
|
index = ALIGN(adev->uvd.fw->size, 8);
|
|
|
|
ring->fence_drv.cpu_addr = adev->uvd.cpu_addr + index;
|
|
|
|
ring->fence_drv.gpu_addr = adev->uvd.gpu_addr + index;
|
|
|
|
}
|
2016-03-14 14:46:06 +00:00
|
|
|
amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));
|
2015-06-01 06:14:32 +00:00
|
|
|
amdgpu_irq_get(adev, irq_src, irq_type);
|
|
|
|
|
2015-04-20 20:55:21 +00:00
|
|
|
ring->fence_drv.irq_src = irq_src;
|
|
|
|
ring->fence_drv.irq_type = irq_type;
|
2015-06-01 06:14:32 +00:00
|
|
|
ring->fence_drv.initialized = true;
|
|
|
|
|
2017-10-26 01:30:38 +00:00
|
|
|
dev_dbg(adev->dev, "fence driver on ring %d use gpu addr 0x%016llx, "
|
|
|
|
"cpu addr 0x%p\n", ring->idx,
|
|
|
|
ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr);
|
2015-04-20 20:55:21 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* amdgpu_fence_driver_init_ring - init the fence driver
|
|
|
|
* for the requested ring.
|
|
|
|
*
|
|
|
|
* @ring: ring to init the fence driver on
|
2016-03-15 13:52:26 +00:00
|
|
|
* @num_hw_submission: number of entries on the hardware queue
|
2015-04-20 20:55:21 +00:00
|
|
|
*
|
|
|
|
* Init the fence driver for the requested ring (all asics).
|
|
|
|
* Helper function for amdgpu_fence_driver_init().
|
|
|
|
*/
|
2016-03-15 13:52:26 +00:00
|
|
|
int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
|
|
|
|
unsigned num_hw_submission)
|
2015-04-20 20:55:21 +00:00
|
|
|
{
|
2016-01-18 14:16:53 +00:00
|
|
|
int r;
|
2015-04-20 20:55:21 +00:00
|
|
|
|
2016-03-15 13:52:26 +00:00
|
|
|
/* Check that num_hw_submission is a power of two */
|
|
|
|
if ((num_hw_submission & (num_hw_submission - 1)) != 0)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2015-04-20 20:55:21 +00:00
|
|
|
ring->fence_drv.cpu_addr = NULL;
|
|
|
|
ring->fence_drv.gpu_addr = 0;
|
2016-01-18 14:16:53 +00:00
|
|
|
ring->fence_drv.sync_seq = 0;
|
2016-03-14 14:46:06 +00:00
|
|
|
atomic_set(&ring->fence_drv.last_seq, 0);
|
2015-04-20 20:55:21 +00:00
|
|
|
ring->fence_drv.initialized = false;
|
|
|
|
|
treewide: setup_timer() -> timer_setup() (2 field)
This converts all remaining setup_timer() calls that use a nested field
to reach a struct timer_list. Coccinelle does not have an easy way to
match multiple fields, so a new script is needed to change the matches of
"&_E->_timer" into "&_E->_field1._timer" in all the rules.
spatch --very-quiet --all-includes --include-headers \
-I ./arch/x86/include -I ./arch/x86/include/generated \
-I ./include -I ./arch/x86/include/uapi \
-I ./arch/x86/include/generated/uapi -I ./include/uapi \
-I ./include/generated/uapi --include ./include/linux/kconfig.h \
--dir . \
--cocci-file ~/src/data/timer_setup-2fields.cocci
@fix_address_of depends@
expression e;
@@
setup_timer(
-&(e)
+&e
, ...)
// Update any raw setup_timer() usages that have a NULL callback, but
// would otherwise match change_timer_function_usage, since the latter
// will update all function assignments done in the face of a NULL
// function initialization in setup_timer().
@change_timer_function_usage_NULL@
expression _E;
identifier _field1;
identifier _timer;
type _cast_data;
@@
(
-setup_timer(&_E->_field1._timer, NULL, _E);
+timer_setup(&_E->_field1._timer, NULL, 0);
|
-setup_timer(&_E->_field1._timer, NULL, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, NULL, 0);
|
-setup_timer(&_E._field1._timer, NULL, &_E);
+timer_setup(&_E._field1._timer, NULL, 0);
|
-setup_timer(&_E._field1._timer, NULL, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, NULL, 0);
)
@change_timer_function_usage@
expression _E;
identifier _field1;
identifier _timer;
struct timer_list _stl;
identifier _callback;
type _cast_func, _cast_data;
@@
(
-setup_timer(&_E->_field1._timer, _callback, _E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, &_callback, _E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, _callback, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, &_callback, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, (_cast_func)_callback, _E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, (_cast_func)&_callback, _E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, (_cast_func)_callback, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, (_cast_func)&_callback, (_cast_data)_E);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, (_cast_data)_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, &_callback, (_cast_data)_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, &_callback, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, (_cast_func)_callback, (_cast_data)_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, (_cast_func)_callback, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, (_cast_func)&_callback, (_cast_data)_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, (_cast_func)&_callback, (_cast_data)&_E);
+timer_setup(&_E._field1._timer, _callback, 0);
|
_E->_field1._timer@_stl.function = _callback;
|
_E->_field1._timer@_stl.function = &_callback;
|
_E->_field1._timer@_stl.function = (_cast_func)_callback;
|
_E->_field1._timer@_stl.function = (_cast_func)&_callback;
|
_E._field1._timer@_stl.function = _callback;
|
_E._field1._timer@_stl.function = &_callback;
|
_E._field1._timer@_stl.function = (_cast_func)_callback;
|
_E._field1._timer@_stl.function = (_cast_func)&_callback;
)
// callback(unsigned long arg)
@change_callback_handle_cast
depends on change_timer_function_usage@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
type _origtype;
identifier _origarg;
type _handletype;
identifier _handle;
@@
void _callback(
-_origtype _origarg
+struct timer_list *t
)
{
(
... when != _origarg
_handletype *_handle =
-(_handletype *)_origarg;
+from_timer(_handle, t, _field1._timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle =
-(void *)_origarg;
+from_timer(_handle, t, _field1._timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle;
... when != _handle
_handle =
-(_handletype *)_origarg;
+from_timer(_handle, t, _field1._timer);
... when != _origarg
|
... when != _origarg
_handletype *_handle;
... when != _handle
_handle =
-(void *)_origarg;
+from_timer(_handle, t, _field1._timer);
... when != _origarg
)
}
// callback(unsigned long arg) without existing variable
@change_callback_handle_cast_no_arg
depends on change_timer_function_usage &&
!change_callback_handle_cast@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
type _origtype;
identifier _origarg;
type _handletype;
@@
void _callback(
-_origtype _origarg
+struct timer_list *t
)
{
+ _handletype *_origarg = from_timer(_origarg, t, _field1._timer);
+
... when != _origarg
- (_handletype *)_origarg
+ _origarg
... when != _origarg
}
// Avoid already converted callbacks.
@match_callback_converted
depends on change_timer_function_usage &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg@
identifier change_timer_function_usage._callback;
identifier t;
@@
void _callback(struct timer_list *t)
{ ... }
// callback(struct something *handle)
@change_callback_handle_arg
depends on change_timer_function_usage &&
!match_callback_converted &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
type _handletype;
identifier _handle;
@@
void _callback(
-_handletype *_handle
+struct timer_list *t
)
{
+ _handletype *_handle = from_timer(_handle, t, _field1._timer);
...
}
// If change_callback_handle_arg ran on an empty function, remove
// the added handler.
@unchange_callback_handle_arg
depends on change_timer_function_usage &&
change_callback_handle_arg@
identifier change_timer_function_usage._callback;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
type _handletype;
identifier _handle;
identifier t;
@@
void _callback(struct timer_list *t)
{
- _handletype *_handle = from_timer(_handle, t, _field1._timer);
}
// We only want to refactor the setup_timer() data argument if we've found
// the matching callback. This undoes changes in change_timer_function_usage.
@unchange_timer_function_usage
depends on change_timer_function_usage &&
!change_callback_handle_cast &&
!change_callback_handle_cast_no_arg &&
!change_callback_handle_arg@
expression change_timer_function_usage._E;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type change_timer_function_usage._cast_data;
@@
(
-timer_setup(&_E->_field1._timer, _callback, 0);
+setup_timer(&_E->_field1._timer, _callback, (_cast_data)_E);
|
-timer_setup(&_E._field1._timer, _callback, 0);
+setup_timer(&_E._field1._timer, _callback, (_cast_data)&_E);
)
// If we fixed a callback from a .function assignment, fix the
// assignment cast now.
@change_timer_function_assignment
depends on change_timer_function_usage &&
(change_callback_handle_cast ||
change_callback_handle_cast_no_arg ||
change_callback_handle_arg)@
expression change_timer_function_usage._E;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type _cast_func;
typedef TIMER_FUNC_TYPE;
@@
(
_E->_field1._timer.function =
-_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_field1._timer.function =
-&_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_field1._timer.function =
-(_cast_func)_callback;
+(TIMER_FUNC_TYPE)_callback
;
|
_E->_field1._timer.function =
-(_cast_func)&_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._field1._timer.function =
-_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._field1._timer.function =
-&_callback;
+(TIMER_FUNC_TYPE)_callback
;
|
_E._field1._timer.function =
-(_cast_func)_callback
+(TIMER_FUNC_TYPE)_callback
;
|
_E._field1._timer.function =
-(_cast_func)&_callback
+(TIMER_FUNC_TYPE)_callback
;
)
// Sometimes timer functions are called directly. Replace matched args.
@change_timer_function_calls
depends on change_timer_function_usage &&
(change_callback_handle_cast ||
change_callback_handle_cast_no_arg ||
change_callback_handle_arg)@
expression _E;
identifier change_timer_function_usage._field1;
identifier change_timer_function_usage._timer;
identifier change_timer_function_usage._callback;
type _cast_data;
@@
_callback(
(
-(_cast_data)_E
+&_E->_field1._timer
|
-(_cast_data)&_E
+&_E._field1._timer
|
-_E
+&_E->_field1._timer
)
)
// If a timer has been configured without a data argument, it can be
// converted without regard to the callback argument, since it is unused.
@match_timer_function_unused_data@
expression _E;
identifier _field1;
identifier _timer;
identifier _callback;
@@
(
-setup_timer(&_E->_field1._timer, _callback, 0);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, _callback, 0L);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E->_field1._timer, _callback, 0UL);
+timer_setup(&_E->_field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, 0);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, 0L);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_E._field1._timer, _callback, 0UL);
+timer_setup(&_E._field1._timer, _callback, 0);
|
-setup_timer(&_field1._timer, _callback, 0);
+timer_setup(&_field1._timer, _callback, 0);
|
-setup_timer(&_field1._timer, _callback, 0L);
+timer_setup(&_field1._timer, _callback, 0);
|
-setup_timer(&_field1._timer, _callback, 0UL);
+timer_setup(&_field1._timer, _callback, 0);
|
-setup_timer(_field1._timer, _callback, 0);
+timer_setup(_field1._timer, _callback, 0);
|
-setup_timer(_field1._timer, _callback, 0L);
+timer_setup(_field1._timer, _callback, 0);
|
-setup_timer(_field1._timer, _callback, 0UL);
+timer_setup(_field1._timer, _callback, 0);
)
@change_callback_unused_data
depends on match_timer_function_unused_data@
identifier match_timer_function_unused_data._callback;
type _origtype;
identifier _origarg;
@@
void _callback(
-_origtype _origarg
+struct timer_list *unused
)
{
... when != _origarg
}
Signed-off-by: Kees Cook <keescook@chromium.org>
2017-10-18 03:21:24 +00:00
|
|
|
timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);
|
2015-08-17 02:55:02 +00:00
|
|
|
|
2016-04-14 02:27:28 +00:00
|
|
|
ring->fence_drv.num_fences_mask = num_hw_submission * 2 - 1;
|
2016-03-14 13:29:46 +00:00
|
|
|
spin_lock_init(&ring->fence_drv.lock);
|
2016-04-14 02:27:28 +00:00
|
|
|
ring->fence_drv.fences = kcalloc(num_hw_submission * 2, sizeof(void *),
|
2016-03-13 18:19:48 +00:00
|
|
|
GFP_KERNEL);
|
|
|
|
if (!ring->fence_drv.fences)
|
|
|
|
return -ENOMEM;
|
2015-09-07 16:43:02 +00:00
|
|
|
|
2016-11-02 09:43:44 +00:00
|
|
|
/* No need to setup the GPU scheduler for KIQ ring */
|
|
|
|
if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ) {
|
2017-12-06 16:49:39 +00:00
|
|
|
r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
|
2017-10-17 05:40:54 +00:00
|
|
|
num_hw_submission, amdgpu_job_hang_limit,
|
2017-12-13 19:36:53 +00:00
|
|
|
msecs_to_jiffies(amdgpu_lockup_timeout), ring->name);
|
2016-11-02 09:43:44 +00:00
|
|
|
if (r) {
|
|
|
|
DRM_ERROR("Failed to create scheduler on ring %s.\n",
|
|
|
|
ring->name);
|
|
|
|
return r;
|
|
|
|
}
|
2015-08-17 02:55:02 +00:00
|
|
|
}
|
2015-09-08 18:22:31 +00:00
|
|
|
|
|
|
|
return 0;
|
2015-04-20 20:55:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* amdgpu_fence_driver_init - init the fence driver
|
|
|
|
* for all possible rings.
|
|
|
|
*
|
|
|
|
* @adev: amdgpu device pointer
|
|
|
|
*
|
|
|
|
* Init the fence driver for all possible rings (all asics).
|
|
|
|
* Not all asics have all rings, so each asic will only
|
|
|
|
* start the fence driver on the rings it has using
|
|
|
|
* amdgpu_fence_driver_start_ring().
|
|
|
|
* Returns 0 for success.
|
|
|
|
*/
|
|
|
|
int amdgpu_fence_driver_init(struct amdgpu_device *adev)
|
|
|
|
{
|
|
|
|
if (amdgpu_debugfs_fence_init(adev))
|
|
|
|
dev_err(adev->dev, "fence debugfs file creation failed\n");
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* amdgpu_fence_driver_fini - tear down the fence driver
|
|
|
|
* for all possible rings.
|
|
|
|
*
|
|
|
|
* @adev: amdgpu device pointer
|
|
|
|
*
|
|
|
|
* Tear down the fence driver for all possible rings (all asics).
|
|
|
|
*/
|
|
|
|
void amdgpu_fence_driver_fini(struct amdgpu_device *adev)
|
|
|
|
{
|
2016-03-13 18:19:48 +00:00
|
|
|
unsigned i, j;
|
|
|
|
int r;
|
2015-04-20 20:55:21 +00:00
|
|
|
|
|
|
|
for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
|
|
|
|
struct amdgpu_ring *ring = adev->rings[i];
|
2015-11-03 12:27:39 +00:00
|
|
|
|
2015-04-20 20:55:21 +00:00
|
|
|
if (!ring || !ring->fence_drv.initialized)
|
|
|
|
continue;
|
|
|
|
r = amdgpu_fence_wait_empty(ring);
|
|
|
|
if (r) {
|
|
|
|
/* no need to trigger GPU reset as we are unloading */
|
2017-10-16 06:38:10 +00:00
|
|
|
amdgpu_fence_driver_force_completion(ring);
|
2015-04-20 20:55:21 +00:00
|
|
|
}
|
2015-06-01 06:14:32 +00:00
|
|
|
amdgpu_irq_put(adev, ring->fence_drv.irq_src,
|
|
|
|
ring->fence_drv.irq_type);
|
2017-12-06 16:49:39 +00:00
|
|
|
drm_sched_fini(&ring->sched);
|
2015-11-03 12:27:39 +00:00
|
|
|
del_timer_sync(&ring->fence_drv.fallback_timer);
|
2016-03-13 18:19:48 +00:00
|
|
|
for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)
|
2016-10-25 12:00:45 +00:00
|
|
|
dma_fence_put(ring->fence_drv.fences[j]);
|
2016-03-13 18:19:48 +00:00
|
|
|
kfree(ring->fence_drv.fences);
|
2016-09-25 20:34:46 +00:00
|
|
|
ring->fence_drv.fences = NULL;
|
2015-04-20 20:55:21 +00:00
|
|
|
ring->fence_drv.initialized = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-08-05 16:41:48 +00:00
|
|
|
/**
|
|
|
|
* amdgpu_fence_driver_suspend - suspend the fence driver
|
|
|
|
* for all possible rings.
|
|
|
|
*
|
|
|
|
* @adev: amdgpu device pointer
|
|
|
|
*
|
|
|
|
* Suspend the fence driver for all possible rings (all asics).
|
|
|
|
*/
|
|
|
|
void amdgpu_fence_driver_suspend(struct amdgpu_device *adev)
|
|
|
|
{
|
|
|
|
int i, r;
|
|
|
|
|
|
|
|
for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
|
|
|
|
struct amdgpu_ring *ring = adev->rings[i];
|
|
|
|
if (!ring || !ring->fence_drv.initialized)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* wait for gpu to finish processing current batch */
|
|
|
|
r = amdgpu_fence_wait_empty(ring);
|
|
|
|
if (r) {
|
|
|
|
/* delay GPU reset to resume */
|
2017-10-16 06:38:10 +00:00
|
|
|
amdgpu_fence_driver_force_completion(ring);
|
2015-08-05 16:41:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* disable the interrupt */
|
|
|
|
amdgpu_irq_put(adev, ring->fence_drv.irq_src,
|
|
|
|
ring->fence_drv.irq_type);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* amdgpu_fence_driver_resume - resume the fence driver
|
|
|
|
* for all possible rings.
|
|
|
|
*
|
|
|
|
* @adev: amdgpu device pointer
|
|
|
|
*
|
|
|
|
* Resume the fence driver for all possible rings (all asics).
|
|
|
|
* Not all asics have all rings, so each asic will only
|
|
|
|
* start the fence driver on the rings it has using
|
|
|
|
* amdgpu_fence_driver_start_ring().
|
|
|
|
* Returns 0 for success.
|
|
|
|
*/
|
|
|
|
void amdgpu_fence_driver_resume(struct amdgpu_device *adev)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
|
|
|
|
struct amdgpu_ring *ring = adev->rings[i];
|
|
|
|
if (!ring || !ring->fence_drv.initialized)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* enable the interrupt */
|
|
|
|
amdgpu_irq_get(adev, ring->fence_drv.irq_src,
|
|
|
|
ring->fence_drv.irq_type);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-04-20 20:55:21 +00:00
|
|
|
/**
|
2017-10-16 06:38:10 +00:00
|
|
|
* amdgpu_fence_driver_force_completion - force signal latest fence of ring
|
2015-04-20 20:55:21 +00:00
|
|
|
*
|
2017-10-16 06:38:10 +00:00
|
|
|
* @ring: fence of the ring to signal
|
2015-04-20 20:55:21 +00:00
|
|
|
*
|
|
|
|
*/
|
2017-10-16 06:38:10 +00:00
|
|
|
void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)
|
2015-04-20 20:55:21 +00:00
|
|
|
{
|
2017-10-16 06:38:10 +00:00
|
|
|
amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
|
|
|
|
amdgpu_fence_process(ring);
|
drm/amdgpu/SRIOV:implement guilty job TDR for(V2)
1,TDR will kickout guilty job if it hang exceed the threshold
of the given one from kernel paramter "job_hang_limit", that
way a bad command stream will not infinitly cause GPU hang.
by default this threshold is 1 so a job will be kicked out
after it hang.
2,if a job timeout TDR routine will not reset all sched/ring,
instead if will only reset on the givn one which is indicated
by @job of amdgpu_sriov_gpu_reset, that way we don't need to
reset and recover each sched/ring if we already know which job
cause GPU hang.
3,unblock sriov_gpu_reset for AI family.
V2:
1:put kickout guilty job after sched parked.
2:since parking scheduler prior to kickout already occupies a
while, we can do last check on the in question job before
doing hw_reset.
TODO:
1:when a job is considered as guilty, we should mark some flag
in its fence status flag, and let UMD side aware that this
fence signaling is not due to job complete but job hang.
2:if gpu reset cause all video memory lost, we need introduce
a new policy to implement TDR, like drop all jobs not yet
signaled, and all IOCTL on this device will return ERROR
DEVICE_LOST.
this will be implemented later.
Signed-off-by: Monk Liu <Monk.Liu@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
2017-05-11 05:36:44 +00:00
|
|
|
}
|
|
|
|
|
2015-11-03 11:21:57 +00:00
|
|
|
/*
|
|
|
|
* Common fence implementation
|
|
|
|
*/
|
|
|
|
|
2016-10-25 12:00:45 +00:00
|
|
|
static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
|
2015-11-03 11:21:57 +00:00
|
|
|
{
|
|
|
|
return "amdgpu";
|
|
|
|
}
|
|
|
|
|
2016-10-25 12:00:45 +00:00
|
|
|
static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
|
2015-11-03 11:21:57 +00:00
|
|
|
{
|
|
|
|
struct amdgpu_fence *fence = to_amdgpu_fence(f);
|
|
|
|
return (const char *)fence->ring->name;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* amdgpu_fence_enable_signaling - enable signalling on fence
|
|
|
|
* @fence: fence
|
|
|
|
*
|
|
|
|
* This function is called with fence_queue lock held, and adds a callback
|
|
|
|
* to fence_queue that checks if this fence is signaled, and if so it
|
|
|
|
* signals the fence and removes itself.
|
|
|
|
*/
|
2016-10-25 12:00:45 +00:00
|
|
|
static bool amdgpu_fence_enable_signaling(struct dma_fence *f)
|
2015-11-03 11:21:57 +00:00
|
|
|
{
|
|
|
|
struct amdgpu_fence *fence = to_amdgpu_fence(f);
|
|
|
|
struct amdgpu_ring *ring = fence->ring;
|
|
|
|
|
2015-11-03 12:27:39 +00:00
|
|
|
if (!timer_pending(&ring->fence_drv.fallback_timer))
|
|
|
|
amdgpu_fence_schedule_fallback(ring);
|
2016-03-14 13:29:46 +00:00
|
|
|
|
2016-10-25 12:00:45 +00:00
|
|
|
DMA_FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);
|
2016-03-14 13:29:46 +00:00
|
|
|
|
2015-11-03 11:21:57 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-03-15 12:40:17 +00:00
|
|
|
/**
|
|
|
|
* amdgpu_fence_free - free up the fence memory
|
|
|
|
*
|
|
|
|
* @rcu: RCU callback head
|
|
|
|
*
|
|
|
|
* Free up the fence memory after the RCU grace period.
|
|
|
|
*/
|
|
|
|
static void amdgpu_fence_free(struct rcu_head *rcu)
|
2015-11-05 03:28:28 +00:00
|
|
|
{
|
2016-10-25 12:00:45 +00:00
|
|
|
struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
|
2015-11-05 03:28:28 +00:00
|
|
|
struct amdgpu_fence *fence = to_amdgpu_fence(f);
|
|
|
|
kmem_cache_free(amdgpu_fence_slab, fence);
|
|
|
|
}
|
|
|
|
|
2016-03-15 12:40:17 +00:00
|
|
|
/**
|
|
|
|
* amdgpu_fence_release - callback that fence can be freed
|
|
|
|
*
|
|
|
|
* @fence: fence
|
|
|
|
*
|
|
|
|
* This function is called when the reference count becomes zero.
|
|
|
|
* It just RCU schedules freeing up the fence.
|
|
|
|
*/
|
2016-10-25 12:00:45 +00:00
|
|
|
static void amdgpu_fence_release(struct dma_fence *f)
|
2016-03-15 12:40:17 +00:00
|
|
|
{
|
|
|
|
call_rcu(&f->rcu, amdgpu_fence_free);
|
|
|
|
}
|
|
|
|
|
2016-10-25 12:00:45 +00:00
|
|
|
static const struct dma_fence_ops amdgpu_fence_ops = {
|
2015-11-03 11:21:57 +00:00
|
|
|
.get_driver_name = amdgpu_fence_get_driver_name,
|
|
|
|
.get_timeline_name = amdgpu_fence_get_timeline_name,
|
|
|
|
.enable_signaling = amdgpu_fence_enable_signaling,
|
2016-10-25 12:00:45 +00:00
|
|
|
.wait = dma_fence_default_wait,
|
2015-11-05 03:28:28 +00:00
|
|
|
.release = amdgpu_fence_release,
|
2015-11-03 11:21:57 +00:00
|
|
|
};
|
2015-04-20 20:55:21 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Fence debugfs
|
|
|
|
*/
|
|
|
|
#if defined(CONFIG_DEBUG_FS)
|
|
|
|
static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data)
|
|
|
|
{
|
|
|
|
struct drm_info_node *node = (struct drm_info_node *)m->private;
|
|
|
|
struct drm_device *dev = node->minor->dev;
|
|
|
|
struct amdgpu_device *adev = dev->dev_private;
|
2016-01-18 14:16:53 +00:00
|
|
|
int i;
|
2015-04-20 20:55:21 +00:00
|
|
|
|
|
|
|
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
|
|
|
|
struct amdgpu_ring *ring = adev->rings[i];
|
|
|
|
if (!ring || !ring->fence_drv.initialized)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
amdgpu_fence_process(ring);
|
|
|
|
|
2015-06-02 13:47:16 +00:00
|
|
|
seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);
|
2016-03-14 14:46:06 +00:00
|
|
|
seq_printf(m, "Last signaled fence 0x%08x\n",
|
|
|
|
atomic_read(&ring->fence_drv.last_seq));
|
|
|
|
seq_printf(m, "Last emitted 0x%08x\n",
|
2016-01-18 14:16:53 +00:00
|
|
|
ring->fence_drv.sync_seq);
|
2017-10-12 05:53:20 +00:00
|
|
|
|
|
|
|
if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* set in CP_VMID_PREEMPT and preemption occurred */
|
|
|
|
seq_printf(m, "Last preempted 0x%08x\n",
|
|
|
|
le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
|
|
|
|
/* set in CP_VMID_RESET and reset occurred */
|
|
|
|
seq_printf(m, "Last reset 0x%08x\n",
|
|
|
|
le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));
|
|
|
|
/* Both preemption and reset occurred */
|
|
|
|
seq_printf(m, "Last both 0x%08x\n",
|
|
|
|
le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));
|
2015-04-20 20:55:21 +00:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-01-14 15:25:22 +00:00
|
|
|
/**
|
2017-10-25 08:37:02 +00:00
|
|
|
* amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover
|
2016-01-14 15:25:22 +00:00
|
|
|
*
|
|
|
|
* Manually trigger a gpu reset at the next fence wait.
|
|
|
|
*/
|
2017-10-25 08:37:02 +00:00
|
|
|
static int amdgpu_debugfs_gpu_recover(struct seq_file *m, void *data)
|
2016-01-14 15:25:22 +00:00
|
|
|
{
|
|
|
|
struct drm_info_node *node = (struct drm_info_node *) m->private;
|
|
|
|
struct drm_device *dev = node->minor->dev;
|
|
|
|
struct amdgpu_device *adev = dev->dev_private;
|
|
|
|
|
2017-10-25 08:37:02 +00:00
|
|
|
seq_printf(m, "gpu recover\n");
|
2017-12-15 21:40:49 +00:00
|
|
|
amdgpu_device_gpu_recover(adev, NULL, true);
|
2016-01-14 15:25:22 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-05-02 16:46:15 +00:00
|
|
|
static const struct drm_info_list amdgpu_debugfs_fence_list[] = {
|
2015-04-20 20:55:21 +00:00
|
|
|
{"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL},
|
2017-10-25 08:37:02 +00:00
|
|
|
{"amdgpu_gpu_recover", &amdgpu_debugfs_gpu_recover, 0, NULL}
|
2015-04-20 20:55:21 +00:00
|
|
|
};
|
2017-05-05 07:09:42 +00:00
|
|
|
|
|
|
|
static const struct drm_info_list amdgpu_debugfs_fence_list_sriov[] = {
|
|
|
|
{"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL},
|
|
|
|
};
|
2015-04-20 20:55:21 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
int amdgpu_debugfs_fence_init(struct amdgpu_device *adev)
|
|
|
|
{
|
|
|
|
#if defined(CONFIG_DEBUG_FS)
|
2017-05-05 07:09:42 +00:00
|
|
|
if (amdgpu_sriov_vf(adev))
|
|
|
|
return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list_sriov, 1);
|
2016-01-14 15:25:22 +00:00
|
|
|
return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list, 2);
|
2015-04-20 20:55:21 +00:00
|
|
|
#else
|
|
|
|
return 0;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|