Use a nopage-based approach to fault in pfns.
Thomas Hellstrom
parent
cee659afb5
commit
c58574c605
|
|
@ -875,6 +875,7 @@ typedef struct drm_device {
|
|||
drm_mm_t offset_manager; /**< User token manager */
|
||||
drm_open_hash_t object_hash; /**< User token hash table for objects */
|
||||
struct address_space *dev_mapping; /**< For unmap_mapping_range() */
|
||||
struct page *ttm_dummy_page;
|
||||
|
||||
/** \name Context handle management */
|
||||
/*@{ */
|
||||
|
|
|
|||
|
|
@ -62,3 +62,82 @@ pgprot_t vm_get_page_prot(unsigned long vm_flags)
|
|||
return protection_map[vm_flags & 0x0F];
|
||||
#endif
|
||||
};
|
||||
|
||||
int drm_pte_is_clear(struct vm_area_struct *vma,
|
||||
unsigned long addr)
|
||||
{
|
||||
struct mm_struct *mm = vma->vm_mm;
|
||||
int ret = 1;
|
||||
pte_t *pte;
|
||||
pmd_t *pmd;
|
||||
pud_t *pud;
|
||||
pgd_t *pgd;
|
||||
|
||||
|
||||
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
|
||||
spin_lock(&mm->page_table_lock);
|
||||
#else
|
||||
spinlock_t ptl;
|
||||
#endif
|
||||
|
||||
pgd = pgd_offset(mm, addr);
|
||||
if (pgd_none(*pgd))
|
||||
goto unlock;
|
||||
pud = pud_offset(pgd, addr);
|
||||
if (pud_none(*pud))
|
||||
goto unlock;
|
||||
pmd = pmd_offset(pud, addr);
|
||||
if (pmd_none(*pmd))
|
||||
goto unlock;
|
||||
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
|
||||
pte = pte_offset_map(pmd, addr);
|
||||
#else
|
||||
pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
|
||||
#endif
|
||||
if (!pte)
|
||||
goto unlock;
|
||||
ret = pte_none(*pte);
|
||||
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
|
||||
pte_unmap(pte);
|
||||
unlock:
|
||||
spin_unlock(&mm->page_table_lock);
|
||||
#else
|
||||
pte_unmap_unlock(pte, ptl);
|
||||
unlock:
|
||||
#endif
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static struct {
|
||||
spinlock_t lock;
|
||||
struct page *dummy_page;
|
||||
atomic_t present;
|
||||
} drm_np_retry =
|
||||
{SPIN_LOCK_UNLOCKED, NOPAGE_OOM, ATOMIC_INIT(0)};
|
||||
|
||||
struct page * get_nopage_retry(void)
|
||||
{
|
||||
if (atomic_read(&drm_np_retry.present) == 0) {
|
||||
struct page *page = alloc_page(GFP_KERNEL);
|
||||
if (!page)
|
||||
return NOPAGE_OOM;
|
||||
spin_lock(&drm_np_retry.lock);
|
||||
drm_np_retry.dummy_page = page;
|
||||
atomic_set(&drm_np_retry.present,1);
|
||||
spin_unlock(&drm_np_retry.lock);
|
||||
}
|
||||
get_page(drm_np_retry.dummy_page);
|
||||
return drm_np_retry.dummy_page;
|
||||
}
|
||||
|
||||
void free_nopage_retry(void)
|
||||
{
|
||||
if (atomic_read(&drm_np_retry.present) == 1) {
|
||||
spin_lock(&drm_np_retry.lock);
|
||||
__free_page(drm_np_retry.dummy_page);
|
||||
drm_np_retry.dummy_page = NULL;
|
||||
atomic_set(&drm_np_retry.present, 0);
|
||||
spin_unlock(&drm_np_retry.lock);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -252,7 +252,9 @@ extern pgprot_t vm_get_page_prot(unsigned long vm_flags);
|
|||
* that are not in the kernel linear map.
|
||||
*/
|
||||
|
||||
#define drm_alloc_gatt_pages(order) virt_to_page(alloc_gatt_pages(order))
|
||||
#define drm_alloc_gatt_pages(order) ({ \
|
||||
void *_virt = alloc_gatt_pages(order); \
|
||||
((_virt) ? virt_to_page(_virt) : NULL);})
|
||||
#define drm_free_gatt_pages(pages, order) free_gatt_pages(page_address(pages), order)
|
||||
|
||||
#if defined(CONFIG_X86) && (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
|
||||
|
|
@ -268,4 +270,27 @@ extern int drm_map_page_into_agp(struct page *page);
|
|||
#define unmap_page_from_agp drm_unmap_page_from_agp
|
||||
#endif
|
||||
|
||||
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19))
|
||||
|
||||
/*
|
||||
* Hopefully, real NOPAGE_RETRY functionality will be in 2.6.19.
|
||||
* For now, just return a dummy page that we've allocated out of
|
||||
* static space. The page will be put by do_nopage() since we've already
|
||||
* filled out the pte.
|
||||
*/
|
||||
extern struct page * get_nopage_retry(void);
|
||||
extern void free_nopage_retry(void);
|
||||
|
||||
#define NOPAGE_RETRY get_nopage_retry()
|
||||
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Is the PTE for this address really clear so that we can use
|
||||
* io_remap_pfn_range?
|
||||
*/
|
||||
|
||||
int drm_pte_is_clear(struct vm_area_struct *vma,
|
||||
unsigned long addr);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -434,6 +434,9 @@ void drm_exit(struct drm_driver *driver)
|
|||
}
|
||||
} else
|
||||
pci_unregister_driver(&driver->pci_driver);
|
||||
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19))
|
||||
free_nopage_retry();
|
||||
#endif
|
||||
DRM_INFO("Module unloaded\n");
|
||||
}
|
||||
EXPORT_SYMBOL(drm_exit);
|
||||
|
|
|
|||
|
|
@ -79,10 +79,6 @@ static int drm_fill_in_dev(drm_device_t * dev, struct pci_dev *pdev,
|
|||
#endif
|
||||
dev->irq = pdev->irq;
|
||||
|
||||
dev->maplist = drm_calloc(1, sizeof(*dev->maplist), DRM_MEM_MAPS);
|
||||
if (dev->maplist == NULL)
|
||||
return -ENOMEM;
|
||||
INIT_LIST_HEAD(&dev->maplist->head);
|
||||
if (drm_ht_create(&dev->map_hash, DRM_MAP_HASH_ORDER)) {
|
||||
drm_free(dev->maplist, sizeof(*dev->maplist), DRM_MEM_MAPS);
|
||||
return -ENOMEM;
|
||||
|
|
@ -101,6 +97,11 @@ static int drm_fill_in_dev(drm_device_t * dev, struct pci_dev *pdev,
|
|||
return -ENOMEM;
|
||||
}
|
||||
|
||||
dev->maplist = drm_calloc(1, sizeof(*dev->maplist), DRM_MEM_MAPS);
|
||||
if (dev->maplist == NULL)
|
||||
return -ENOMEM;
|
||||
INIT_LIST_HEAD(&dev->maplist->head);
|
||||
|
||||
/* the DRM has 6 counters */
|
||||
dev->counters = 6;
|
||||
dev->types[0] = _DRM_STAT_LOCK;
|
||||
|
|
|
|||
|
|
@ -71,89 +71,6 @@ static void ttm_free(void *pointer, unsigned long size, int type)
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* We may be manipulating other processes page tables, so for each TTM, keep track of
|
||||
* which mm_structs are currently mapping the ttm so that we can take the appropriate
|
||||
* locks when we modify their page tables. A typical application is when we evict another
|
||||
* process' buffers.
|
||||
*/
|
||||
|
||||
int drm_ttm_add_mm_to_list(drm_ttm_t * ttm, struct mm_struct *mm)
|
||||
{
|
||||
p_mm_entry_t *entry, *n_entry;
|
||||
|
||||
list_for_each_entry(entry, &ttm->p_mm_list, head) {
|
||||
if (mm == entry->mm) {
|
||||
atomic_inc(&entry->refcount);
|
||||
return 0;
|
||||
} else if ((unsigned long)mm < (unsigned long)entry->mm) ;
|
||||
}
|
||||
|
||||
n_entry = drm_alloc(sizeof(*n_entry), DRM_MEM_TTM);
|
||||
if (!entry) {
|
||||
DRM_ERROR("Allocation of process mm pointer entry failed\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
INIT_LIST_HEAD(&n_entry->head);
|
||||
n_entry->mm = mm;
|
||||
atomic_set(&n_entry->refcount, 0);
|
||||
atomic_inc(&ttm->shared_count);
|
||||
ttm->mm_list_seq++;
|
||||
|
||||
list_add_tail(&n_entry->head, &entry->head);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void drm_ttm_delete_mm(drm_ttm_t * ttm, struct mm_struct *mm)
|
||||
{
|
||||
p_mm_entry_t *entry, *n;
|
||||
list_for_each_entry_safe(entry, n, &ttm->p_mm_list, head) {
|
||||
if (mm == entry->mm) {
|
||||
if (atomic_add_negative(-1, &entry->refcount)) {
|
||||
list_del(&entry->head);
|
||||
drm_free(entry, sizeof(*entry), DRM_MEM_TTM);
|
||||
atomic_dec(&ttm->shared_count);
|
||||
ttm->mm_list_seq++;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
BUG_ON(1);
|
||||
}
|
||||
|
||||
static void drm_ttm_unlock_mm(drm_ttm_t * ttm)
|
||||
{
|
||||
p_mm_entry_t *entry;
|
||||
|
||||
list_for_each_entry(entry, &ttm->p_mm_list, head) {
|
||||
up_write(&entry->mm->mmap_sem);
|
||||
}
|
||||
}
|
||||
|
||||
static int ioremap_vmas(drm_ttm_t * ttm, unsigned long page_offset,
|
||||
unsigned long num_pages, unsigned long aper_offset)
|
||||
{
|
||||
struct list_head *list;
|
||||
int ret = 0;
|
||||
|
||||
list_for_each(list, &ttm->vma_list->head) {
|
||||
drm_ttm_vma_list_t *entry =
|
||||
list_entry(list, drm_ttm_vma_list_t, head);
|
||||
|
||||
ret = io_remap_pfn_range(entry->vma,
|
||||
entry->vma->vm_start +
|
||||
(page_offset << PAGE_SHIFT),
|
||||
(ttm->aperture_base >> PAGE_SHIFT) +
|
||||
aper_offset, num_pages << PAGE_SHIFT,
|
||||
drm_io_prot(_DRM_AGP, entry->vma));
|
||||
if (ret)
|
||||
break;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* Unmap all vma pages from vmas mapping this ttm.
|
||||
*/
|
||||
|
|
@ -216,17 +133,15 @@ int drm_destroy_ttm(drm_ttm_t * ttm)
|
|||
do_tlbflush = 1;
|
||||
}
|
||||
if (*cur_page) {
|
||||
ClearPageLocked(*cur_page);
|
||||
|
||||
/*
|
||||
* Debugging code. Remove if the error message never
|
||||
* shows up.
|
||||
*/
|
||||
|
||||
unlock_page(*cur_page);
|
||||
if (page_count(*cur_page) != 1) {
|
||||
DRM_ERROR("Erroneous page count. "
|
||||
"Leaking pages.\n");
|
||||
}
|
||||
if (page_mapped(*cur_page)) {
|
||||
DRM_ERROR("Erroneous map count. "
|
||||
"Leaking page mappings.\n");
|
||||
}
|
||||
|
||||
/*
|
||||
* End debugging.
|
||||
|
|
@ -334,72 +249,6 @@ static drm_ttm_t *drm_init_ttm(struct drm_device *dev, unsigned long size)
|
|||
return ttm;
|
||||
}
|
||||
|
||||
/*
|
||||
* Lock the mmap_sems for processes that are mapping this ttm.
|
||||
* This looks a bit clumsy, since we need to maintain the correct
|
||||
* locking order
|
||||
* mm->mmap_sem
|
||||
* dev->struct_sem;
|
||||
* and while we release dev->struct_sem to lock the mmap_sems,
|
||||
* the mmap_sem list may have been updated. We need to revalidate
|
||||
* it after relocking dev->struc_sem.
|
||||
*/
|
||||
|
||||
static int drm_ttm_lock_mmap_sem(drm_ttm_t * ttm)
|
||||
{
|
||||
struct mm_struct **mm_list = NULL, **mm_list_p;
|
||||
uint32_t list_seq;
|
||||
uint32_t cur_count, shared_count;
|
||||
p_mm_entry_t *entry;
|
||||
unsigned i;
|
||||
|
||||
cur_count = 0;
|
||||
list_seq = ttm->mm_list_seq;
|
||||
shared_count = atomic_read(&ttm->shared_count);
|
||||
|
||||
do {
|
||||
if (shared_count > cur_count) {
|
||||
if (mm_list)
|
||||
drm_free(mm_list, sizeof(*mm_list) * cur_count,
|
||||
DRM_MEM_TTM);
|
||||
cur_count = shared_count + 10;
|
||||
mm_list =
|
||||
drm_alloc(sizeof(*mm_list) * cur_count,
|
||||
DRM_MEM_TTM);
|
||||
if (!mm_list)
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
mm_list_p = mm_list;
|
||||
list_for_each_entry(entry, &ttm->p_mm_list, head) {
|
||||
*mm_list_p++ = entry->mm;
|
||||
}
|
||||
|
||||
mutex_unlock(&ttm->dev->struct_mutex);
|
||||
mm_list_p = mm_list;
|
||||
for (i = 0; i < shared_count; ++i, ++mm_list_p) {
|
||||
down_write(&((*mm_list_p)->mmap_sem));
|
||||
}
|
||||
|
||||
mutex_lock(&ttm->dev->struct_mutex);
|
||||
|
||||
if (list_seq != ttm->mm_list_seq) {
|
||||
mm_list_p = mm_list;
|
||||
for (i = 0; i < shared_count; ++i, ++mm_list_p) {
|
||||
up_write(&((*mm_list_p)->mmap_sem));
|
||||
}
|
||||
|
||||
}
|
||||
shared_count = atomic_read(&ttm->shared_count);
|
||||
|
||||
} while (list_seq != ttm->mm_list_seq);
|
||||
|
||||
if (mm_list)
|
||||
drm_free(mm_list, sizeof(*mm_list) * cur_count, DRM_MEM_TTM);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Change caching policy for the linear kernel map
|
||||
* for range of pages in a ttm.
|
||||
|
|
@ -449,15 +298,11 @@ int drm_evict_ttm_region(drm_ttm_backend_list_t * entry)
|
|||
{
|
||||
drm_ttm_backend_t *be = entry->be;
|
||||
drm_ttm_t *ttm = entry->owner;
|
||||
int ret;
|
||||
|
||||
if (be) {
|
||||
switch (entry->state) {
|
||||
case ttm_bound:
|
||||
if (ttm && be->needs_cache_adjust(be)) {
|
||||
ret = drm_ttm_lock_mmap_sem(ttm);
|
||||
if (ret)
|
||||
return ret;
|
||||
unmap_vma_pages(ttm, entry->page_offset,
|
||||
entry->num_pages);
|
||||
}
|
||||
|
|
@ -465,7 +310,6 @@ int drm_evict_ttm_region(drm_ttm_backend_list_t * entry)
|
|||
if (ttm && be->needs_cache_adjust(be)) {
|
||||
drm_set_caching(ttm, entry->page_offset,
|
||||
entry->num_pages, 0);
|
||||
drm_ttm_unlock_mm(ttm);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
|
|
@ -613,7 +457,6 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region,
|
|||
ttm = region->owner;
|
||||
|
||||
if (ttm && be->needs_cache_adjust(be)) {
|
||||
ret = drm_ttm_lock_mmap_sem(ttm);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
|
|
@ -626,8 +469,6 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region,
|
|||
}
|
||||
|
||||
if ((ret = be->bind(be, aper_offset))) {
|
||||
if (ttm && be->needs_cache_adjust(be))
|
||||
drm_ttm_unlock_mm(ttm);
|
||||
drm_unbind_ttm_region(region);
|
||||
DRM_ERROR("Couldn't bind backend.\n");
|
||||
return ret;
|
||||
|
|
@ -640,12 +481,6 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region,
|
|||
cur_page_flag++;
|
||||
}
|
||||
|
||||
if (ttm && be->needs_cache_adjust(be)) {
|
||||
ioremap_vmas(ttm, region->page_offset, region->num_pages,
|
||||
aper_offset);
|
||||
drm_ttm_unlock_mm(ttm);
|
||||
}
|
||||
|
||||
region->state = ttm_bound;
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -164,8 +164,6 @@ int drm_rebind_ttm_region(drm_ttm_backend_list_t * entry,
|
|||
|
||||
extern int drm_destroy_ttm(drm_ttm_t * ttm);
|
||||
extern void drm_user_destroy_region(drm_ttm_backend_list_t * entry);
|
||||
extern int drm_ttm_add_mm_to_list(drm_ttm_t * ttm, struct mm_struct *mm);
|
||||
extern void drm_ttm_delete_mm(drm_ttm_t * ttm, struct mm_struct *mm);
|
||||
extern int drm_ttm_ioctl(DRM_IOCTL_ARGS);
|
||||
|
||||
static __inline__ drm_ttm_t *drm_ttm_from_object(drm_ttm_object_t * to)
|
||||
|
|
|
|||
|
|
@ -237,7 +237,7 @@ static int drm_ttm_remap_bound_pfn(struct vm_area_struct *vma,
|
|||
}
|
||||
|
||||
if (ret) {
|
||||
DRM_ERROR("Map returned %c\n", ret);
|
||||
DRM_ERROR("Map returned %c\n", ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -254,6 +254,7 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma,
|
|||
pgprot_t default_prot;
|
||||
uint32_t page_flags;
|
||||
drm_buffer_manager_t *bm;
|
||||
drm_device_t *dev;
|
||||
|
||||
if (address > vma->vm_end)
|
||||
return NOPAGE_SIGBUS; /* Disallow mremap */
|
||||
|
|
@ -262,7 +263,11 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma,
|
|||
|
||||
map = (drm_map_t *) entry->map;
|
||||
ttm = (drm_ttm_t *) map->offset;
|
||||
bm = &ttm->dev->bm;
|
||||
|
||||
dev = ttm->dev;
|
||||
mutex_lock(&dev->struct_mutex);
|
||||
|
||||
bm = &dev->bm;
|
||||
page_offset = (address - vma->vm_start) >> PAGE_SHIFT;
|
||||
page = ttm->pages[page_offset];
|
||||
|
||||
|
|
@ -270,22 +275,43 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma,
|
|||
|
||||
if (!page) {
|
||||
if (bm->cur_pages >= bm->max_pages) {
|
||||
DRM_ERROR("Maximum locked page count exceeded\n");
|
||||
return NOPAGE_OOM;
|
||||
DRM_ERROR("Maximum locked page count exceeded\n");
|
||||
page = NOPAGE_OOM;
|
||||
goto out;
|
||||
}
|
||||
++bm->cur_pages;
|
||||
page = ttm->pages[page_offset] = drm_alloc_gatt_pages(0);
|
||||
if (page) {
|
||||
SetPageLocked(page);
|
||||
} else {
|
||||
page = NOPAGE_OOM;
|
||||
}
|
||||
}
|
||||
if (!page)
|
||||
return NOPAGE_OOM;
|
||||
|
||||
SetPageLocked(page);
|
||||
if (page_flags & DRM_TTM_PAGE_UNCACHED) {
|
||||
|
||||
/*
|
||||
* This makes sure we don't race with another
|
||||
* drm_ttm_remap_bound_pfn();
|
||||
*/
|
||||
|
||||
if (!drm_pte_is_clear(vma, address)) {
|
||||
page = NOPAGE_RETRY;
|
||||
goto out1;
|
||||
}
|
||||
|
||||
drm_ttm_remap_bound_pfn(vma, address, PAGE_SIZE);
|
||||
page = NOPAGE_RETRY;
|
||||
goto out1;
|
||||
}
|
||||
get_page(page);
|
||||
|
||||
default_prot = vm_get_page_prot(vma->vm_flags);
|
||||
|
||||
BUG_ON(page_flags & DRM_TTM_PAGE_UNCACHED);
|
||||
|
||||
out1:
|
||||
default_prot = vm_get_page_prot(vma->vm_flags);
|
||||
vma->vm_page_prot = default_prot;
|
||||
|
||||
out:
|
||||
mutex_unlock(&dev->struct_mutex);
|
||||
return page;
|
||||
}
|
||||
|
||||
|
|
@ -645,7 +671,6 @@ static int drm_vm_ttm_open(struct vm_area_struct *vma) {
|
|||
*entry = *tmp_vma;
|
||||
map = (drm_map_t *) entry->map;
|
||||
ttm = (drm_ttm_t *) map->offset;
|
||||
ret = drm_ttm_add_mm_to_list(ttm, vma->vm_mm);
|
||||
if (!ret) {
|
||||
atomic_inc(&ttm->vma_count);
|
||||
INIT_LIST_HEAD(&entry->head);
|
||||
|
|
@ -717,7 +742,6 @@ static void drm_vm_ttm_close(struct vm_area_struct *vma)
|
|||
ttm = (drm_ttm_t *) map->offset;
|
||||
dev = ttm->dev;
|
||||
mutex_lock(&dev->struct_mutex);
|
||||
drm_ttm_delete_mm(ttm, vma->vm_mm);
|
||||
list_del(&ttm_vma->head);
|
||||
drm_free(ttm_vma, sizeof(*ttm_vma), DRM_MEM_VMAS);
|
||||
if (atomic_dec_and_test(&ttm->vma_count)) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue