diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 55 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 2 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 4 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 6 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_memory.c | 1 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_object.c | 51 |
6 files changed, 45 insertions, 74 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index bf6e4b5a73b..2c54c3d414b 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -162,9 +162,9 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible) { if (interruptible) { return wait_event_interruptible(bo->event_queue, - atomic_read(&bo->reserved) == 0); + !ttm_bo_is_reserved(bo)); } else { - wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0); + wait_event(bo->event_queue, !ttm_bo_is_reserved(bo)); return 0; } } @@ -175,7 +175,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; - BUG_ON(!atomic_read(&bo->reserved)); + BUG_ON(!ttm_bo_is_reserved(bo)); if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { @@ -220,7 +220,7 @@ int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, struct ttm_bo_global *glob = bo->glob; int ret; - while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) { + while (unlikely(atomic_read(&bo->reserved) != 0)) { /** * Deadlock avoidance for multi-bo reserving. */ @@ -249,6 +249,7 @@ int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, return ret; } + atomic_set(&bo->reserved, 1); if (use_sequence) { /** * Wake up waiters that may need to recheck for deadlock, @@ -501,7 +502,6 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) struct ttm_bo_global *glob = bo->glob; struct ttm_bo_driver *driver; void *sync_obj = NULL; - void *sync_obj_arg; int put_count; int ret; @@ -537,7 +537,6 @@ queue: driver = bdev->driver; if (bo->sync_obj) sync_obj = driver->sync_obj_ref(bo->sync_obj); - sync_obj_arg = bo->sync_obj_arg; kref_get(&bo->list_kref); list_add_tail(&bo->ddestroy, &bdev->ddestroy); @@ -545,7 +544,7 @@ queue: spin_unlock(&bdev->fence_lock); if (sync_obj) { - driver->sync_obj_flush(sync_obj, sync_obj_arg); + driver->sync_obj_flush(sync_obj); driver->sync_obj_unref(&sync_obj); } schedule_delayed_work(&bdev->wq, @@ -697,6 +696,7 @@ static void ttm_bo_release(struct kref *kref) struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; + write_lock(&bdev->vm_lock); if (likely(bo->vm_node != NULL)) { rb_erase(&bo->vm_rb, &bdev->addr_space_rb); drm_mm_put_block(bo->vm_node); @@ -708,18 +708,14 @@ static void ttm_bo_release(struct kref *kref) ttm_mem_io_unlock(man); ttm_bo_cleanup_refs_or_queue(bo); kref_put(&bo->list_kref, ttm_bo_release_list); - write_lock(&bdev->vm_lock); } void ttm_bo_unref(struct ttm_buffer_object **p_bo) { struct ttm_buffer_object *bo = *p_bo; - struct ttm_bo_device *bdev = bo->bdev; *p_bo = NULL; - write_lock(&bdev->vm_lock); kref_put(&bo->kref, ttm_bo_release); - write_unlock(&bdev->vm_lock); } EXPORT_SYMBOL(ttm_bo_unref); @@ -756,7 +752,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, goto out; } - BUG_ON(!atomic_read(&bo->reserved)); + BUG_ON(!ttm_bo_is_reserved(bo)); evict_mem = bo->mem; evict_mem.mm_node = NULL; @@ -1054,16 +1050,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_mem_space); -int ttm_bo_wait_cpu(struct ttm_buffer_object *bo, bool no_wait) -{ - if ((atomic_read(&bo->cpu_writers) > 0) && no_wait) - return -EBUSY; - - return wait_event_interruptible(bo->event_queue, - atomic_read(&bo->cpu_writers) == 0); -} -EXPORT_SYMBOL(ttm_bo_wait_cpu); - int ttm_bo_move_buffer(struct ttm_buffer_object *bo, struct ttm_placement *placement, bool interruptible, bool no_wait_reserve, @@ -1073,7 +1059,7 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo, struct ttm_mem_reg mem; struct ttm_bo_device *bdev = bo->bdev; - BUG_ON(!atomic_read(&bo->reserved)); + BUG_ON(!ttm_bo_is_reserved(bo)); /* * FIXME: It's possible to pipeline buffer moves. @@ -1130,7 +1116,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, { int ret; - BUG_ON(!atomic_read(&bo->reserved)); + BUG_ON(!ttm_bo_is_reserved(bo)); /* Check that range is valid */ if (placement->lpfn || placement->fpfn) if (placement->fpfn > placement->lpfn || @@ -1179,7 +1165,6 @@ int ttm_bo_init(struct ttm_bo_device *bdev, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, - unsigned long buffer_start, bool interruptible, struct file *persistent_swap_storage, size_t acc_size, @@ -1200,7 +1185,6 @@ int ttm_bo_init(struct ttm_bo_device *bdev, return -ENOMEM; } - size += buffer_start & ~PAGE_MASK; num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; if (num_pages == 0) { pr_err("Illegal buffer object size\n"); @@ -1233,7 +1217,6 @@ int ttm_bo_init(struct ttm_bo_device *bdev, bo->mem.page_alignment = page_alignment; bo->mem.bus.io_reserved_vm = false; bo->mem.bus.io_reserved_count = 0; - bo->buffer_start = buffer_start & PAGE_MASK; bo->priv_flags = 0; bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); bo->seq_valid = false; @@ -1306,7 +1289,6 @@ int ttm_bo_create(struct ttm_bo_device *bdev, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, - unsigned long buffer_start, bool interruptible, struct file *persistent_swap_storage, struct ttm_buffer_object **p_bo) @@ -1321,8 +1303,8 @@ int ttm_bo_create(struct ttm_bo_device *bdev, acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, - buffer_start, interruptible, - persistent_swap_storage, acc_size, NULL, NULL); + interruptible, persistent_swap_storage, acc_size, + NULL, NULL); if (likely(ret == 0)) *p_bo = bo; @@ -1577,7 +1559,6 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, goto out_no_addr_mm; INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); - bdev->nice_mode = true; INIT_LIST_HEAD(&bdev->ddestroy); bdev->dev_mapping = NULL; bdev->glob = glob; @@ -1721,7 +1702,6 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, struct ttm_bo_driver *driver = bo->bdev->driver; struct ttm_bo_device *bdev = bo->bdev; void *sync_obj; - void *sync_obj_arg; int ret = 0; if (likely(bo->sync_obj == NULL)) @@ -1729,7 +1709,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, while (bo->sync_obj) { - if (driver->sync_obj_signaled(bo->sync_obj, bo->sync_obj_arg)) { + if (driver->sync_obj_signaled(bo->sync_obj)) { void *tmp_obj = bo->sync_obj; bo->sync_obj = NULL; clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); @@ -1743,9 +1723,8 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, return -EBUSY; sync_obj = driver->sync_obj_ref(bo->sync_obj); - sync_obj_arg = bo->sync_obj_arg; spin_unlock(&bdev->fence_lock); - ret = driver->sync_obj_wait(sync_obj, sync_obj_arg, + ret = driver->sync_obj_wait(sync_obj, lazy, interruptible); if (unlikely(ret != 0)) { driver->sync_obj_unref(&sync_obj); @@ -1753,8 +1732,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, return ret; } spin_lock(&bdev->fence_lock); - if (likely(bo->sync_obj == sync_obj && - bo->sync_obj_arg == sync_obj_arg)) { + if (likely(bo->sync_obj == sync_obj)) { void *tmp_obj = bo->sync_obj; bo->sync_obj = NULL; clear_bit(TTM_BO_PRIV_FLAG_MOVING, @@ -1797,8 +1775,7 @@ EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) { - if (atomic_dec_and_test(&bo->cpu_writers)) - wake_up_all(&bo->event_queue); + atomic_dec(&bo->cpu_writers); } EXPORT_SYMBOL(ttm_bo_synccpu_write_release); diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 2026060f03e..b9c4e515b1d 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -611,7 +611,6 @@ EXPORT_SYMBOL(ttm_bo_kunmap); int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, void *sync_obj, - void *sync_obj_arg, bool evict, bool no_wait_reserve, bool no_wait_gpu, struct ttm_mem_reg *new_mem) @@ -630,7 +629,6 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, bo->sync_obj = NULL; } bo->sync_obj = driver->sync_obj_ref(sync_obj); - bo->sync_obj_arg = sync_obj_arg; if (evict) { ret = ttm_bo_wait(bo, false, false, false); spin_unlock(&bdev->fence_lock); diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 3ba72dbdc4b..74705f329d9 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -259,8 +259,8 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, read_lock(&bdev->vm_lock); bo = ttm_bo_vm_lookup_rb(bdev, vma->vm_pgoff, (vma->vm_end - vma->vm_start) >> PAGE_SHIFT); - if (likely(bo != NULL)) - ttm_bo_reference(bo); + if (likely(bo != NULL) && !kref_get_unless_zero(&bo->kref)) + bo = NULL; read_unlock(&bdev->vm_lock); if (unlikely(bo == NULL)) { diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index 1937069432c..1986d006c26 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -185,10 +185,7 @@ retry_this_bo: ttm_eu_backoff_reservation_locked(list); spin_unlock(&glob->lru_lock); ttm_eu_list_ref_sub(list); - ret = ttm_bo_wait_cpu(bo, false); - if (ret) - return ret; - goto retry; + return -EBUSY; } } @@ -223,7 +220,6 @@ void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj) bo = entry->bo; entry->old_sync_obj = bo->sync_obj; bo->sync_obj = driver->sync_obj_ref(sync_obj); - bo->sync_obj_arg = entry->new_sync_obj_arg; ttm_bo_unreserve_locked(bo); entry->reserved = false; } diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index 479c6b0467c..dbc2def887c 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -367,7 +367,6 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) spin_lock_init(&glob->lock); glob->swap_queue = create_singlethread_workqueue("ttm_swap"); INIT_WORK(&glob->work, ttm_shrink_work); - init_waitqueue_head(&glob->queue); ret = kobject_init_and_add( &glob->kobj, &ttm_mem_glob_kobj_type, ttm_get_kobj(), "memory_accounting"); if (unlikely(ret != 0)) { diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c index c7857874956..58a5f3261c0 100644 --- a/drivers/gpu/drm/ttm/ttm_object.c +++ b/drivers/gpu/drm/ttm/ttm_object.c @@ -80,7 +80,7 @@ struct ttm_object_file { */ struct ttm_object_device { - rwlock_t object_lock; + spinlock_t object_lock; struct drm_open_hash object_hash; atomic_t object_count; struct ttm_mem_global *mem_glob; @@ -157,12 +157,12 @@ int ttm_base_object_init(struct ttm_object_file *tfile, base->refcount_release = refcount_release; base->ref_obj_release = ref_obj_release; base->object_type = object_type; - write_lock(&tdev->object_lock); kref_init(&base->refcount); - ret = drm_ht_just_insert_please(&tdev->object_hash, - &base->hash, - (unsigned long)base, 31, 0, 0); - write_unlock(&tdev->object_lock); + spin_lock(&tdev->object_lock); + ret = drm_ht_just_insert_please_rcu(&tdev->object_hash, + &base->hash, + (unsigned long)base, 31, 0, 0); + spin_unlock(&tdev->object_lock); if (unlikely(ret != 0)) goto out_err0; @@ -174,7 +174,9 @@ int ttm_base_object_init(struct ttm_object_file *tfile, return 0; out_err1: - (void)drm_ht_remove_item(&tdev->object_hash, &base->hash); + spin_lock(&tdev->object_lock); + (void)drm_ht_remove_item_rcu(&tdev->object_hash, &base->hash); + spin_unlock(&tdev->object_lock); out_err0: return ret; } @@ -186,30 +188,29 @@ static void ttm_release_base(struct kref *kref) container_of(kref, struct ttm_base_object, refcount); struct ttm_object_device *tdev = base->tfile->tdev; - (void)drm_ht_remove_item(&tdev->object_hash, &base->hash); - write_unlock(&tdev->object_lock); + spin_lock(&tdev->object_lock); + (void)drm_ht_remove_item_rcu(&tdev->object_hash, &base->hash); + spin_unlock(&tdev->object_lock); + + /* + * Note: We don't use synchronize_rcu() here because it's far + * too slow. It's up to the user to free the object using + * call_rcu() or ttm_base_object_kfree(). + */ + if (base->refcount_release) { ttm_object_file_unref(&base->tfile); base->refcount_release(&base); } - write_lock(&tdev->object_lock); } void ttm_base_object_unref(struct ttm_base_object **p_base) { struct ttm_base_object *base = *p_base; - struct ttm_object_device *tdev = base->tfile->tdev; *p_base = NULL; - /* - * Need to take the lock here to avoid racing with - * users trying to look up the object. - */ - - write_lock(&tdev->object_lock); kref_put(&base->refcount, ttm_release_base); - write_unlock(&tdev->object_lock); } EXPORT_SYMBOL(ttm_base_object_unref); @@ -221,14 +222,14 @@ struct ttm_base_object *ttm_base_object_lookup(struct ttm_object_file *tfile, struct drm_hash_item *hash; int ret; - read_lock(&tdev->object_lock); - ret = drm_ht_find_item(&tdev->object_hash, key, &hash); + rcu_read_lock(); + ret = drm_ht_find_item_rcu(&tdev->object_hash, key, &hash); if (likely(ret == 0)) { base = drm_hash_entry(hash, struct ttm_base_object, hash); - kref_get(&base->refcount); + ret = kref_get_unless_zero(&base->refcount) ? 0 : -EINVAL; } - read_unlock(&tdev->object_lock); + rcu_read_unlock(); if (unlikely(ret != 0)) return NULL; @@ -426,7 +427,7 @@ struct ttm_object_device *ttm_object_device_init(struct ttm_mem_global return NULL; tdev->mem_glob = mem_glob; - rwlock_init(&tdev->object_lock); + spin_lock_init(&tdev->object_lock); atomic_set(&tdev->object_count, 0); ret = drm_ht_create(&tdev->object_hash, hash_order); @@ -444,9 +445,9 @@ void ttm_object_device_release(struct ttm_object_device **p_tdev) *p_tdev = NULL; - write_lock(&tdev->object_lock); + spin_lock(&tdev->object_lock); drm_ht_remove(&tdev->object_hash); - write_unlock(&tdev->object_lock); + spin_unlock(&tdev->object_lock); kfree(tdev); } |