drm/amdgpu/amdgpu_bo.c

713 lines
18 KiB
C

/*
* Copyright © 2014 Advanced Micro Devices, Inc.
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/time.h>
#include "libdrm_macros.h"
#include "xf86drm.h"
#include "amdgpu_drm.h"
#include "amdgpu_internal.h"
#include "util_hash_table.h"
static void amdgpu_close_kms_handle(amdgpu_device_handle dev,
uint32_t handle)
{
struct drm_gem_close args = {};
args.handle = handle;
drmIoctl(dev->fd, DRM_IOCTL_GEM_CLOSE, &args);
}
void amdgpu_bo_free_internal(amdgpu_bo_handle bo)
{
/* Remove the buffer from the hash tables. */
pthread_mutex_lock(&bo->dev->bo_table_mutex);
util_hash_table_remove(bo->dev->bo_handles,
(void*)(uintptr_t)bo->handle);
if (bo->flink_name) {
util_hash_table_remove(bo->dev->bo_flink_names,
(void*)(uintptr_t)bo->flink_name);
}
pthread_mutex_unlock(&bo->dev->bo_table_mutex);
/* Release CPU access. */
if (bo->cpu_map_count > 0) {
bo->cpu_map_count = 1;
amdgpu_bo_cpu_unmap(bo);
}
amdgpu_close_kms_handle(bo->dev, bo->handle);
pthread_mutex_destroy(&bo->cpu_access_mutex);
amdgpu_vamgr_free_va(&bo->dev->vamgr, bo->virtual_mc_base_address, bo->alloc_size);
free(bo);
}
int amdgpu_bo_alloc(amdgpu_device_handle dev,
struct amdgpu_bo_alloc_request *alloc_buffer,
struct amdgpu_bo_alloc_result *info)
{
struct amdgpu_bo *bo;
union drm_amdgpu_gem_create args;
unsigned heap = alloc_buffer->preferred_heap;
int r = 0;
/* It's an error if the heap is not specified */
if (!(heap & (AMDGPU_GEM_DOMAIN_GTT | AMDGPU_GEM_DOMAIN_VRAM)))
return -EINVAL;
bo = calloc(1, sizeof(struct amdgpu_bo));
if (!bo)
return -ENOMEM;
atomic_set(&bo->refcount, 1);
bo->dev = dev;
bo->alloc_size = alloc_buffer->alloc_size;
memset(&args, 0, sizeof(args));
args.in.bo_size = alloc_buffer->alloc_size;
args.in.alignment = alloc_buffer->phys_alignment;
/* Set the placement. */
args.in.domains = heap & AMDGPU_GEM_DOMAIN_MASK;
args.in.domain_flags = alloc_buffer->flags & AMDGPU_GEM_CREATE_CPU_GTT_MASK;
/* Allocate the buffer with the preferred heap. */
r = drmCommandWriteRead(dev->fd, DRM_AMDGPU_GEM_CREATE,
&args, sizeof(args));
if (r) {
free(bo);
return r;
}
bo->handle = args.out.handle;
pthread_mutex_init(&bo->cpu_access_mutex, NULL);
/* map the buffer to the GPU virtual address space */
{
union drm_amdgpu_gem_va va;
memset(&va, 0, sizeof(va));
bo->virtual_mc_base_address = amdgpu_vamgr_find_va(&dev->vamgr,
alloc_buffer->alloc_size,
alloc_buffer->phys_alignment);
if (bo->virtual_mc_base_address == AMDGPU_INVALID_VA_ADDRESS) {
amdgpu_bo_free_internal(bo);
return -ENOSPC;
}
va.in.handle = bo->handle;
va.in.operation = AMDGPU_VA_OP_MAP;
va.in.flags = AMDGPU_VM_PAGE_READABLE |
AMDGPU_VM_PAGE_WRITEABLE |
AMDGPU_VM_PAGE_EXECUTABLE;
va.in.va_address = bo->virtual_mc_base_address;
r = drmCommandWriteRead(dev->fd, DRM_AMDGPU_GEM_VA, &va, sizeof(va));
if (r || va.out.result == AMDGPU_VA_RESULT_ERROR) {
amdgpu_bo_free_internal(bo);
return r;
}
pthread_mutex_lock(&dev->bo_table_mutex);
util_hash_table_set(dev->bo_vas,
(void*)(uintptr_t)bo->virtual_mc_base_address, bo);
pthread_mutex_unlock(&dev->bo_table_mutex);
}
info->buf_handle = bo;
info->virtual_mc_base_address = bo->virtual_mc_base_address;
return 0;
}
int amdgpu_bo_set_metadata(amdgpu_bo_handle bo,
struct amdgpu_bo_metadata *info)
{
struct drm_amdgpu_gem_metadata args = {};
args.handle = bo->handle;
args.op = AMDGPU_GEM_METADATA_OP_SET_METADATA;
args.data.flags = info->flags;
args.data.tiling_info = info->tiling_info;
if (info->size_metadata > sizeof(args.data.data))
return -EINVAL;
if (info->size_metadata) {
args.data.data_size_bytes = info->size_metadata;
memcpy(args.data.data, info->umd_metadata, info->size_metadata);
}
return drmCommandWriteRead(bo->dev->fd,
DRM_AMDGPU_GEM_METADATA,
&args, sizeof(args));
}
int amdgpu_bo_query_info(amdgpu_bo_handle bo,
struct amdgpu_bo_info *info)
{
struct drm_amdgpu_gem_metadata metadata = {};
struct drm_amdgpu_gem_create_in bo_info = {};
struct drm_amdgpu_gem_op gem_op = {};
int r;
/* Query metadata. */
metadata.handle = bo->handle;
metadata.op = AMDGPU_GEM_METADATA_OP_GET_METADATA;
r = drmCommandWriteRead(bo->dev->fd, DRM_AMDGPU_GEM_METADATA,
&metadata, sizeof(metadata));
if (r)
return r;
if (metadata.data.data_size_bytes >
sizeof(info->metadata.umd_metadata))
return -EINVAL;
/* Query buffer info. */
gem_op.handle = bo->handle;
gem_op.op = AMDGPU_GEM_OP_GET_GEM_CREATE_INFO;
gem_op.value = (uintptr_t)&bo_info;
r = drmCommandWriteRead(bo->dev->fd, DRM_AMDGPU_GEM_OP,
&gem_op, sizeof(gem_op));
if (r)
return r;
memset(info, 0, sizeof(*info));
info->alloc_size = bo_info.bo_size;
info->phys_alignment = bo_info.alignment;
info->virtual_mc_base_address = bo->virtual_mc_base_address;
info->preferred_heap = bo_info.domains;
info->alloc_flags = bo_info.domain_flags;
info->metadata.flags = metadata.data.flags;
info->metadata.tiling_info = metadata.data.tiling_info;
info->metadata.size_metadata = metadata.data.data_size_bytes;
if (metadata.data.data_size_bytes > 0)
memcpy(info->metadata.umd_metadata, metadata.data.data,
metadata.data.data_size_bytes);
return 0;
}
static void amdgpu_add_handle_to_table(amdgpu_bo_handle bo)
{
pthread_mutex_lock(&bo->dev->bo_table_mutex);
util_hash_table_set(bo->dev->bo_handles,
(void*)(uintptr_t)bo->handle, bo);
pthread_mutex_unlock(&bo->dev->bo_table_mutex);
}
static int amdgpu_bo_export_flink(amdgpu_bo_handle bo)
{
struct drm_gem_flink flink;
int fd, dma_fd;
uint32_t handle;
int r;
fd = bo->dev->fd;
handle = bo->handle;
if (bo->flink_name)
return 0;
if (bo->dev->flink_fd != bo->dev->fd) {
r = drmPrimeHandleToFD(bo->dev->fd, bo->handle, DRM_CLOEXEC,
&dma_fd);
if (!r) {
r = drmPrimeFDToHandle(bo->dev->flink_fd, dma_fd, &handle);
close(dma_fd);
}
if (r)
return r;
fd = bo->dev->flink_fd;
}
memset(&flink, 0, sizeof(flink));
flink.handle = handle;
r = drmIoctl(fd, DRM_IOCTL_GEM_FLINK, &flink);
if (r)
return r;
bo->flink_name = flink.name;
if (bo->dev->flink_fd != bo->dev->fd) {
struct drm_gem_close args = {};
args.handle = handle;
drmIoctl(bo->dev->flink_fd, DRM_IOCTL_GEM_CLOSE, &args);
}
pthread_mutex_lock(&bo->dev->bo_table_mutex);
util_hash_table_set(bo->dev->bo_flink_names,
(void*)(uintptr_t)bo->flink_name,
bo);
pthread_mutex_unlock(&bo->dev->bo_table_mutex);
return 0;
}
int amdgpu_bo_export(amdgpu_bo_handle bo,
enum amdgpu_bo_handle_type type,
uint32_t *shared_handle)
{
int r;
switch (type) {
case amdgpu_bo_handle_type_gem_flink_name:
r = amdgpu_bo_export_flink(bo);
if (r)
return r;
*shared_handle = bo->flink_name;
return 0;
case amdgpu_bo_handle_type_kms:
r = amdgpu_bo_export_flink(bo);
if (r)
return r;
amdgpu_add_handle_to_table(bo);
*shared_handle = bo->handle;
return 0;
case amdgpu_bo_handle_type_dma_buf_fd:
amdgpu_add_handle_to_table(bo);
return drmPrimeHandleToFD(bo->dev->fd, bo->handle, DRM_CLOEXEC,
(int*)shared_handle);
}
return -EINVAL;
}
int amdgpu_bo_import(amdgpu_device_handle dev,
enum amdgpu_bo_handle_type type,
uint32_t shared_handle,
struct amdgpu_bo_import_result *output)
{
struct drm_gem_open open_arg = {};
union drm_amdgpu_gem_va va;
struct amdgpu_bo *bo = NULL;
int r;
int dma_fd;
uint64_t dma_buf_size = 0;
/* Convert a DMA buf handle to a KMS handle now. */
if (type == amdgpu_bo_handle_type_dma_buf_fd) {
uint32_t handle;
off_t size;
/* Get a KMS handle. */
r = drmPrimeFDToHandle(dev->fd, shared_handle, &handle);
if (r) {
return r;
}
/* Query the buffer size. */
size = lseek(shared_handle, 0, SEEK_END);
if (size == (off_t)-1) {
amdgpu_close_kms_handle(dev, handle);
return -errno;
}
lseek(shared_handle, 0, SEEK_SET);
dma_buf_size = size;
shared_handle = handle;
}
/* We must maintain a list of pairs <handle, bo>, so that we always
* return the same amdgpu_bo instance for the same handle. */
pthread_mutex_lock(&dev->bo_table_mutex);
/* If we have already created a buffer with this handle, find it. */
switch (type) {
case amdgpu_bo_handle_type_gem_flink_name:
bo = util_hash_table_get(dev->bo_flink_names,
(void*)(uintptr_t)shared_handle);
break;
case amdgpu_bo_handle_type_dma_buf_fd:
bo = util_hash_table_get(dev->bo_handles,
(void*)(uintptr_t)shared_handle);
break;
case amdgpu_bo_handle_type_kms:
/* Importing a KMS handle in not allowed. */
pthread_mutex_unlock(&dev->bo_table_mutex);
return -EPERM;
default:
pthread_mutex_unlock(&dev->bo_table_mutex);
return -EINVAL;
}
if (bo) {
pthread_mutex_unlock(&dev->bo_table_mutex);
/* The buffer already exists, just bump the refcount. */
atomic_inc(&bo->refcount);
output->buf_handle = bo;
output->alloc_size = bo->alloc_size;
output->virtual_mc_base_address =
bo->virtual_mc_base_address;
return 0;
}
bo = calloc(1, sizeof(struct amdgpu_bo));
if (!bo) {
pthread_mutex_unlock(&dev->bo_table_mutex);
if (type == amdgpu_bo_handle_type_dma_buf_fd) {
amdgpu_close_kms_handle(dev, shared_handle);
}
return -ENOMEM;
}
/* Open the handle. */
switch (type) {
case amdgpu_bo_handle_type_gem_flink_name:
open_arg.name = shared_handle;
r = drmIoctl(dev->flink_fd, DRM_IOCTL_GEM_OPEN, &open_arg);
if (r) {
free(bo);
pthread_mutex_unlock(&dev->bo_table_mutex);
return r;
}
bo->handle = open_arg.handle;
if (dev->flink_fd != dev->fd) {
r = drmPrimeHandleToFD(dev->flink_fd, bo->handle, DRM_CLOEXEC, &dma_fd);
if (r) {
free(bo);
pthread_mutex_unlock(&dev->bo_table_mutex);
return r;
}
r = drmPrimeFDToHandle(dev->fd, dma_fd, &bo->handle );
close(dma_fd);
if (r) {
free(bo);
pthread_mutex_unlock(&dev->bo_table_mutex);
return r;
}
}
bo->flink_name = shared_handle;
bo->alloc_size = open_arg.size;
util_hash_table_set(dev->bo_flink_names,
(void*)(uintptr_t)bo->flink_name, bo);
break;
case amdgpu_bo_handle_type_dma_buf_fd:
bo->handle = shared_handle;
bo->alloc_size = dma_buf_size;
break;
case amdgpu_bo_handle_type_kms:
assert(0); /* unreachable */
}
/* Initialize it. */
atomic_set(&bo->refcount, 1);
bo->dev = dev;
pthread_mutex_init(&bo->cpu_access_mutex, NULL);
bo->virtual_mc_base_address = amdgpu_vamgr_find_va(&dev->vamgr, bo->alloc_size, 1 << 20);
if (bo->virtual_mc_base_address == AMDGPU_INVALID_VA_ADDRESS) {
pthread_mutex_unlock(&dev->bo_table_mutex);
amdgpu_bo_reference(&bo, NULL);
return -ENOSPC;
}
memset(&va, 0, sizeof(va));
va.in.handle = bo->handle;
va.in.operation = AMDGPU_VA_OP_MAP;
va.in.va_address = bo->virtual_mc_base_address;
va.in.flags = AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
AMDGPU_VM_PAGE_EXECUTABLE;
r = drmCommandWriteRead(dev->fd, DRM_AMDGPU_GEM_VA, &va, sizeof(va));
if (r || va.out.result == AMDGPU_VA_RESULT_ERROR) {
pthread_mutex_unlock(&dev->bo_table_mutex);
amdgpu_vamgr_free_va(&dev->vamgr, bo->virtual_mc_base_address, bo->alloc_size);
amdgpu_bo_reference(&bo, NULL);
return r;
}
util_hash_table_set(dev->bo_vas,
(void*)(uintptr_t)bo->virtual_mc_base_address, bo);
util_hash_table_set(dev->bo_handles, (void*)(uintptr_t)bo->handle, bo);
pthread_mutex_unlock(&dev->bo_table_mutex);
output->buf_handle = bo;
output->alloc_size = bo->alloc_size;
output->virtual_mc_base_address = bo->virtual_mc_base_address;
return 0;
}
int amdgpu_bo_free(amdgpu_bo_handle buf_handle)
{
/* Just drop the reference. */
amdgpu_bo_reference(&buf_handle, NULL);
return 0;
}
int amdgpu_bo_cpu_map(amdgpu_bo_handle bo, void **cpu)
{
union drm_amdgpu_gem_mmap args;
void *ptr;
int r;
pthread_mutex_lock(&bo->cpu_access_mutex);
if (bo->cpu_ptr) {
/* already mapped */
assert(bo->cpu_map_count > 0);
bo->cpu_map_count++;
*cpu = bo->cpu_ptr;
pthread_mutex_unlock(&bo->cpu_access_mutex);
return 0;
}
assert(bo->cpu_map_count == 0);
memset(&args, 0, sizeof(args));
/* Query the buffer address (args.addr_ptr).
* The kernel driver ignores the offset and size parameters. */
args.in.handle = bo->handle;
r = drmCommandWriteRead(bo->dev->fd, DRM_AMDGPU_GEM_MMAP, &args,
sizeof(args));
if (r) {
pthread_mutex_unlock(&bo->cpu_access_mutex);
return r;
}
/* Map the buffer. */
ptr = drm_mmap(NULL, bo->alloc_size, PROT_READ | PROT_WRITE, MAP_SHARED,
bo->dev->fd, args.out.addr_ptr);
if (ptr == MAP_FAILED) {
pthread_mutex_unlock(&bo->cpu_access_mutex);
return -errno;
}
bo->cpu_ptr = ptr;
bo->cpu_map_count = 1;
pthread_mutex_unlock(&bo->cpu_access_mutex);
*cpu = ptr;
return 0;
}
int amdgpu_bo_cpu_unmap(amdgpu_bo_handle bo)
{
int r;
pthread_mutex_lock(&bo->cpu_access_mutex);
assert(bo->cpu_map_count >= 0);
if (bo->cpu_map_count == 0) {
/* not mapped */
pthread_mutex_unlock(&bo->cpu_access_mutex);
return -EBADMSG;
}
bo->cpu_map_count--;
if (bo->cpu_map_count > 0) {
/* mapped multiple times */
pthread_mutex_unlock(&bo->cpu_access_mutex);
return 0;
}
r = drm_munmap(bo->cpu_ptr, bo->alloc_size) == 0 ? 0 : -errno;
bo->cpu_ptr = NULL;
pthread_mutex_unlock(&bo->cpu_access_mutex);
return r;
}
int amdgpu_query_buffer_size_alignment(amdgpu_device_handle dev,
struct amdgpu_buffer_size_alignments *info)
{
info->size_local = dev->dev_info.pte_fragment_size;
info->size_remote = dev->dev_info.gart_page_size;
return 0;
}
int amdgpu_bo_wait_for_idle(amdgpu_bo_handle bo,
uint64_t timeout_ns,
bool *busy)
{
union drm_amdgpu_gem_wait_idle args;
int r;
memset(&args, 0, sizeof(args));
args.in.handle = bo->handle;
args.in.timeout = amdgpu_cs_calculate_timeout(timeout_ns);
r = drmCommandWriteRead(bo->dev->fd, DRM_AMDGPU_GEM_WAIT_IDLE,
&args, sizeof(args));
if (r == 0) {
*busy = args.out.status;
return 0;
} else {
fprintf(stderr, "amdgpu: GEM_WAIT_IDLE failed with %i\n", r);
return r;
}
}
int amdgpu_create_bo_from_user_mem(amdgpu_device_handle dev,
void *cpu,
uint64_t size,
struct amdgpu_bo_alloc_result *info)
{
int r;
struct amdgpu_bo *bo;
struct drm_amdgpu_gem_userptr args;
union drm_amdgpu_gem_va va;
uintptr_t cpu0;
uint32_t ps, off;
memset(&args, 0, sizeof(args));
ps = getpagesize();
cpu0 = ROUND_DOWN((uintptr_t)cpu, ps);
off = (uintptr_t)cpu - cpu0;
size = ROUND_UP(size + off, ps);
args.addr = cpu0;
args.flags = AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_REGISTER;
args.size = size;
r = drmCommandWriteRead(dev->fd, DRM_AMDGPU_GEM_USERPTR,
&args, sizeof(args));
if (r)
return r;
bo = calloc(1, sizeof(struct amdgpu_bo));
if (!bo)
return -ENOMEM;
atomic_set(&bo->refcount, 1);
bo->dev = dev;
bo->alloc_size = size;
bo->handle = args.handle;
bo->virtual_mc_base_address = amdgpu_vamgr_find_va(&dev->vamgr, size, 4 * 1024);
if (bo->virtual_mc_base_address == AMDGPU_INVALID_VA_ADDRESS) {
amdgpu_bo_free_internal(bo);
return -ENOSPC;
}
memset(&va, 0, sizeof(va));
va.in.handle = bo->handle;
va.in.operation = AMDGPU_VA_OP_MAP;
va.in.flags = AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
AMDGPU_VM_PAGE_EXECUTABLE;
va.in.va_address = bo->virtual_mc_base_address;
r = drmCommandWriteRead(dev->fd, DRM_AMDGPU_GEM_VA, &va, sizeof(va));
if (r || va.out.result == AMDGPU_VA_RESULT_ERROR) {
amdgpu_bo_free_internal(bo);
return r;
}
pthread_mutex_lock(&dev->bo_table_mutex);
util_hash_table_set(dev->bo_vas,
(void*)(uintptr_t)bo->virtual_mc_base_address, bo);
pthread_mutex_unlock(&dev->bo_table_mutex);
info->buf_handle = bo;
info->virtual_mc_base_address = bo->virtual_mc_base_address;
info->virtual_mc_base_address += off;
return r;
}
int amdgpu_bo_list_create(amdgpu_device_handle dev,
uint32_t number_of_resources,
amdgpu_bo_handle *resources,
uint8_t *resource_prios,
amdgpu_bo_list_handle *result)
{
struct drm_amdgpu_bo_list_entry *list;
union drm_amdgpu_bo_list args;
unsigned i;
int r;
list = alloca(sizeof(struct drm_amdgpu_bo_list_entry) * number_of_resources);
memset(&args, 0, sizeof(args));
args.in.operation = AMDGPU_BO_LIST_OP_CREATE;
args.in.bo_number = number_of_resources;
args.in.bo_info_size = sizeof(struct drm_amdgpu_bo_list_entry);
args.in.bo_info_ptr = (uint64_t)(uintptr_t)list;
for (i = 0; i < number_of_resources; i++) {
list[i].bo_handle = resources[i]->handle;
if (resource_prios)
list[i].bo_priority = resource_prios[i];
else
list[i].bo_priority = 0;
}
r = drmCommandWriteRead(dev->fd, DRM_AMDGPU_BO_LIST,
&args, sizeof(args));
if (r)
return r;
*result = calloc(1, sizeof(struct amdgpu_bo_list));
(*result)->dev = dev;
(*result)->handle = args.out.list_handle;
return 0;
}
int amdgpu_bo_list_destroy(amdgpu_bo_list_handle list)
{
union drm_amdgpu_bo_list args;
int r;
memset(&args, 0, sizeof(args));
args.in.operation = AMDGPU_BO_LIST_OP_DESTROY;
args.in.list_handle = list->handle;
r = drmCommandWriteRead(list->dev->fd, DRM_AMDGPU_BO_LIST,
&args, sizeof(args));
if (!r)
free(list);
return r;
}