diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/filemap.c | 94 | ||||
| -rw-r--r-- | mm/filemap_xip.c | 54 | ||||
| -rw-r--r-- | mm/fremap.c | 103 | ||||
| -rw-r--r-- | mm/memory.c | 132 | ||||
| -rw-r--r-- | mm/mmap.c | 8 | ||||
| -rw-r--r-- | mm/nommu.c | 3 | ||||
| -rw-r--r-- | mm/rmap.c | 4 | ||||
| -rw-r--r-- | mm/shmem.c | 82 | ||||
| -rw-r--r-- | mm/truncate.c | 2 |
9 files changed, 273 insertions, 209 deletions
diff --git a/mm/filemap.c b/mm/filemap.c index 462cda58a18..26b992d169e 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1301,40 +1301,38 @@ static int fastcall page_cache_read(struct file * file, unsigned long offset) #define MMAP_LOTSAMISS (100) /** - * filemap_nopage - read in file data for page fault handling - * @area: the applicable vm_area - * @address: target address to read in - * @type: returned with VM_FAULT_{MINOR,MAJOR} if not %NULL + * filemap_fault - read in file data for page fault handling + * @vma: user vma (not used) + * @fdata: the applicable fault_data * - * filemap_nopage() is invoked via the vma operations vector for a + * filemap_fault() is invoked via the vma operations vector for a * mapped memory region to read in file data during a page fault. * * The goto's are kind of ugly, but this streamlines the normal case of having * it in the page cache, and handles the special cases reasonably without * having a lot of duplicated code. */ -struct page *filemap_nopage(struct vm_area_struct *area, - unsigned long address, int *type) +struct page *filemap_fault(struct vm_area_struct *vma, struct fault_data *fdata) { int error; - struct file *file = area->vm_file; + struct file *file = vma->vm_file; struct address_space *mapping = file->f_mapping; struct file_ra_state *ra = &file->f_ra; struct inode *inode = mapping->host; struct page *page; - unsigned long size, pgoff; - int did_readaround = 0, majmin = VM_FAULT_MINOR; + unsigned long size; + int did_readaround = 0; - BUG_ON(!(area->vm_flags & VM_CAN_INVALIDATE)); + fdata->type = VM_FAULT_MINOR; - pgoff = ((address-area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff; + BUG_ON(!(vma->vm_flags & VM_CAN_INVALIDATE)); size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; - if (pgoff >= size) + if (fdata->pgoff >= size) goto outside_data_content; /* If we don't want any read-ahead, don't bother */ - if (VM_RandomReadHint(area)) + if (VM_RandomReadHint(vma)) goto no_cached_page; /* @@ -1343,19 +1341,19 @@ struct page *filemap_nopage(struct vm_area_struct *area, * * For sequential accesses, we use the generic readahead logic. */ - if (VM_SequentialReadHint(area)) - page_cache_readahead(mapping, ra, file, pgoff, 1); + if (VM_SequentialReadHint(vma)) + page_cache_readahead(mapping, ra, file, fdata->pgoff, 1); /* * Do we have something in the page cache already? */ retry_find: - page = find_lock_page(mapping, pgoff); + page = find_lock_page(mapping, fdata->pgoff); if (!page) { unsigned long ra_pages; - if (VM_SequentialReadHint(area)) { - handle_ra_miss(mapping, ra, pgoff); + if (VM_SequentialReadHint(vma)) { + handle_ra_miss(mapping, ra, fdata->pgoff); goto no_cached_page; } ra->mmap_miss++; @@ -1372,7 +1370,7 @@ retry_find: * check did_readaround, as this is an inner loop. */ if (!did_readaround) { - majmin = VM_FAULT_MAJOR; + fdata->type = VM_FAULT_MAJOR; count_vm_event(PGMAJFAULT); } did_readaround = 1; @@ -1380,11 +1378,11 @@ retry_find: if (ra_pages) { pgoff_t start = 0; - if (pgoff > ra_pages / 2) - start = pgoff - ra_pages / 2; + if (fdata->pgoff > ra_pages / 2) + start = fdata->pgoff - ra_pages / 2; do_page_cache_readahead(mapping, file, start, ra_pages); } - page = find_lock_page(mapping, pgoff); + page = find_lock_page(mapping, fdata->pgoff); if (!page) goto no_cached_page; } @@ -1401,7 +1399,7 @@ retry_find: /* Must recheck i_size under page lock */ size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; - if (unlikely(pgoff >= size)) { + if (unlikely(fdata->pgoff >= size)) { unlock_page(page); goto outside_data_content; } @@ -1410,8 +1408,6 @@ retry_find: * Found the page and have a reference on it. */ mark_page_accessed(page); - if (type) - *type = majmin; return page; outside_data_content: @@ -1419,15 +1415,17 @@ outside_data_content: * An external ptracer can access pages that normally aren't * accessible.. */ - if (area->vm_mm == current->mm) - return NOPAGE_SIGBUS; + if (vma->vm_mm == current->mm) { + fdata->type = VM_FAULT_SIGBUS; + return NULL; + } /* Fall through to the non-read-ahead case */ no_cached_page: /* * We're only likely to ever get here if MADV_RANDOM is in * effect. */ - error = page_cache_read(file, pgoff); + error = page_cache_read(file, fdata->pgoff); /* * The page we want has now been added to the page cache. @@ -1443,13 +1441,15 @@ no_cached_page: * to schedule I/O. */ if (error == -ENOMEM) - return NOPAGE_OOM; - return NOPAGE_SIGBUS; + fdata->type = VM_FAULT_OOM; + else + fdata->type = VM_FAULT_SIGBUS; + return NULL; page_not_uptodate: /* IO error path */ if (!did_readaround) { - majmin = VM_FAULT_MAJOR; + fdata->type = VM_FAULT_MAJOR; count_vm_event(PGMAJFAULT); } @@ -1468,7 +1468,30 @@ page_not_uptodate: /* Things didn't work out. Return zero to tell the mm layer so. */ shrink_readahead_size_eio(file, ra); - return NOPAGE_SIGBUS; + fdata->type = VM_FAULT_SIGBUS; + return NULL; +} +EXPORT_SYMBOL(filemap_fault); + +/* + * filemap_nopage and filemap_populate are legacy exports that are not used + * in tree. Scheduled for removal. + */ +struct page *filemap_nopage(struct vm_area_struct *area, + unsigned long address, int *type) +{ + struct page *page; + struct fault_data fdata; + fdata.address = address; + fdata.pgoff = ((address - area->vm_start) >> PAGE_CACHE_SHIFT) + + area->vm_pgoff; + fdata.flags = 0; + + page = filemap_fault(area, &fdata); + if (type) + *type = fdata.type; + + return page; } EXPORT_SYMBOL(filemap_nopage); @@ -1646,8 +1669,7 @@ repeat: EXPORT_SYMBOL(filemap_populate); struct vm_operations_struct generic_file_vm_ops = { - .nopage = filemap_nopage, - .populate = filemap_populate, + .fault = filemap_fault, }; /* This is used for a general mmap of a disk file */ @@ -1660,7 +1682,7 @@ int generic_file_mmap(struct file * file, struct vm_area_struct * vma) return -ENOEXEC; file_accessed(file); vma->vm_ops = &generic_file_vm_ops; - vma->vm_flags |= VM_CAN_INVALIDATE; + vma->vm_flags |= VM_CAN_INVALIDATE | VM_CAN_NONLINEAR; return 0; } diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c index 65ffc321f0c..82f4b8e9834 100644 --- a/mm/filemap_xip.c +++ b/mm/filemap_xip.c @@ -205,62 +205,67 @@ __xip_unmap (struct address_space * mapping, } /* - * xip_nopage() is invoked via the vma operations vector for a + * xip_fault() is invoked via the vma operations vector for a * mapped memory region to read in file data during a page fault. * - * This function is derived from filemap_nopage, but used for execute in place + * This function is derived from filemap_fault, but used for execute in place */ -static struct page * -xip_file_nopage(struct vm_area_struct * area, - unsigned long address, - int *type) +static struct page *xip_file_fault(struct vm_area_struct *area, + struct fault_data *fdata) { struct file *file = area->vm_file; struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; struct page *page; - unsigned long size, pgoff, endoff; + pgoff_t size; - pgoff = ((address - area->vm_start) >> PAGE_CACHE_SHIFT) - + area->vm_pgoff; - endoff = ((area->vm_end - area->vm_start) >> PAGE_CACHE_SHIFT) - + area->vm_pgoff; + /* XXX: are VM_FAULT_ codes OK? */ size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; - if (pgoff >= size) - return NOPAGE_SIGBUS; + if (fdata->pgoff >= size) { + fdata->type = VM_FAULT_SIGBUS; + return NULL; + } - page = mapping->a_ops->get_xip_page(mapping, pgoff*(PAGE_SIZE/512), 0); + page = mapping->a_ops->get_xip_page(mapping, + fdata->pgoff*(PAGE_SIZE/512), 0); if (!IS_ERR(page)) goto out; - if (PTR_ERR(page) != -ENODATA) - return NOPAGE_SIGBUS; + if (PTR_ERR(page) != -ENODATA) { + fdata->type = VM_FAULT_OOM; + return NULL; + } /* sparse block */ if ((area->vm_flags & (VM_WRITE | VM_MAYWRITE)) && (area->vm_flags & (VM_SHARED| VM_MAYSHARE)) && (!(mapping->host->i_sb->s_flags & MS_RDONLY))) { /* maybe shared writable, allocate new block */ - page = mapping->a_ops->get_xip_page (mapping, - pgoff*(PAGE_SIZE/512), 1); - if (IS_ERR(page)) - return NOPAGE_SIGBUS; + page = mapping->a_ops->get_xip_page(mapping, + fdata->pgoff*(PAGE_SIZE/512), 1); + if (IS_ERR(page)) { + fdata->type = VM_FAULT_SIGBUS; + return NULL; + } /* unmap page at pgoff from all other vmas */ - __xip_unmap(mapping, pgoff); + __xip_unmap(mapping, fdata->pgoff); } else { /* not shared and writable, use xip_sparse_page() */ page = xip_sparse_page(); - if (!page) - return NOPAGE_OOM; + if (!page) { + fdata->type = VM_FAULT_OOM; + return NULL; + } } out: + fdata->type = VM_FAULT_MINOR; page_cache_get(page); return page; } static struct vm_operations_struct xip_file_vm_ops = { - .nopage = xip_file_nopage, + .fault = xip_file_fault, }; int xip_file_mmap(struct file * file, struct vm_area_struct * vma) @@ -269,6 +274,7 @@ int xip_file_mmap(struct file * file, struct vm_area_struct * vma) file_accessed(file); vma->vm_ops = &xip_file_vm_ops; + vma->vm_flags |= VM_CAN_NONLINEAR; return 0; } EXPORT_SYMBOL_GPL(xip_file_mmap); diff --git a/mm/fremap.c b/mm/fremap.c index 4e3f53dd5fd..01e51f01b84 100644 --- a/mm/fremap.c +++ b/mm/fremap.c @@ -126,6 +126,25 @@ out: return err; } +static int populate_range(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, unsigned long size, pgoff_t pgoff) +{ + int err; + + do { + err = install_file_pte(mm, vma, addr, pgoff, vma->vm_page_prot); + if (err) + return err; + + size -= PAGE_SIZE; + addr += PAGE_SIZE; + pgoff++; + } while (size); + + return 0; + +} + /*** * sys_remap_file_pages - remap arbitrary pages of a shared backing store * file within an existing vma. @@ -183,41 +202,63 @@ asmlinkage long sys_remap_file_pages(unsigned long start, unsigned long size, * the single existing vma. vm_private_data is used as a * swapout cursor in a VM_NONLINEAR vma. */ - if (vma && (vma->vm_flags & VM_SHARED) && - (!vma->vm_private_data || (vma->vm_flags & VM_NONLINEAR)) && - vma->vm_ops && vma->vm_ops->populate && - end > start && start >= vma->vm_start && - end <= vma->vm_end) { - - /* Must set VM_NONLINEAR before any pages are populated. */ - if (pgoff != linear_page_index(vma, start) && - !(vma->vm_flags & VM_NONLINEAR)) { - if (!has_write_lock) { - up_read(&mm->mmap_sem); - down_write(&mm->mmap_sem); - has_write_lock = 1; - goto retry; + if (!vma || !(vma->vm_flags & VM_SHARED)) + goto out; + + if (vma->vm_private_data && !(vma->vm_flags & VM_NONLINEAR)) + goto out; + + if ((!vma->vm_ops || !vma->vm_ops->populate) && + !(vma->vm_flags & VM_CAN_NONLINEAR)) + goto out; + + if (end <= start || start < vma->vm_start || end > vma->vm_end) + goto out; + + /* Must set VM_NONLINEAR before any pages are populated. */ + if (!(vma->vm_flags & VM_NONLINEAR)) { + /* Don't need a nonlinear mapping, exit success */ + if (pgoff == linear_page_index(vma, start)) { + err = 0; + goto out; + } + + if (!has_write_lock) { + up_read(&mm->mmap_sem); + down_write(&mm->mmap_sem); + has_write_lock = 1; + goto retry; + } + mapping = vma->vm_file->f_mapping; + spin_lock(&mapping->i_mmap_lock); + flush_dcache_mmap_lock(mapping); + vma->vm_flags |= VM_NONLINEAR; + vma_prio_tree_remove(vma, &mapping->i_mmap); + vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear); + flush_dcache_mmap_unlock(mapping); + spin_unlock(&mapping->i_mmap_lock); + } + + if (vma->vm_flags & VM_CAN_NONLINEAR) { + err = populate_range(mm, vma, start, size, pgoff); + if (!err && !(flags & MAP_NONBLOCK)) { + if (unlikely(has_write_lock)) { + downgrade_write(&mm->mmap_sem); + has_write_lock = 0; } - mapping = vma->vm_file->f_mapping; - spin_lock(&mapping->i_mmap_lock); - flush_dcache_mmap_lock(mapping); - vma->vm_flags |= VM_NONLINEAR; - vma_prio_tree_remove(vma, &mapping->i_mmap); - vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear); - flush_dcache_mmap_unlock(mapping); - spin_unlock(&mapping->i_mmap_lock); + make_pages_present(start, start+size); } + } else + err = vma->vm_ops->populate(vma, start, size, vma->vm_page_prot, + pgoff, flags & MAP_NONBLOCK); - err = vma->vm_ops->populate(vma, start, size, - vma->vm_page_prot, - pgoff, flags & MAP_NONBLOCK); + /* + * We can't clear VM_NONLINEAR because we'd have to do + * it after ->populate completes, and that would prevent + * downgrading the lock. (Locks can't be upgraded). + */ - /* - * We can't clear VM_NONLINEAR because we'd have to do - * it after ->populate completes, and that would prevent - * downgrading the lock. (Locks can't be upgraded). - */ - } +out: if (likely(!has_write_lock)) up_read(&mm->mmap_sem); else diff --git a/mm/memory.c b/mm/memory.c index e6c99f6b564..eee7fec3ab5 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1047,7 +1047,8 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, if (pages) foll_flags |= FOLL_GET; if (!write && !(vma->vm_flags & VM_LOCKED) && - (!vma->vm_ops || !vma->vm_ops->nopage)) + (!vma->vm_ops || (!vma->vm_ops->nopage && + !vma->vm_ops->fault))) foll_flags |= FOLL_ANON; do { @@ -2288,10 +2289,10 @@ oom: } /* - * do_no_page() tries to create a new page mapping. It aggressively + * __do_fault() tries to create a new page mapping. It aggressively * tries to share with existing pages, but makes a separate copy if - * the "write_access" parameter is true in order to avoid the next - * page fault. + * the FAULT_FLAG_WRITE is set in the flags parameter in order to avoid + * the next page fault. * * As this is called only for pages that do not currently exist, we * do not need to flush old virtual caches or the TLB. @@ -2300,64 +2301,82 @@ oom: * but allow concurrent faults), and pte mapped but not yet locked. * We return with mmap_sem still held, but pte unmapped and unlocked. */ -static int do_no_page(struct mm_struct *mm, struct vm_area_struct *vma, +static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, pte_t *page_table, pmd_t *pmd, - int write_access) + pgoff_t pgoff, unsigned int flags, pte_t orig_pte) { spinlock_t *ptl; - struct page *page, *nopage_page; + struct page *page, *faulted_page; pte_t entry; - int ret = VM_FAULT_MINOR; int anon = 0; struct page *dirty_page = NULL; + struct fault_data fdata; + + fdata.address = address & PAGE_MASK; + fdata.pgoff = pgoff; + fdata.flags = flags; pte_unmap(page_table); BUG_ON(vma->vm_flags & VM_PFNMAP); - nopage_page = vma->vm_ops->nopage(vma, address & PAGE_MASK, &ret); - /* no page was available -- either SIGBUS, OOM or REFAULT */ - if (unlikely(nopage_page == NOPAGE_SIGBUS)) - return VM_FAULT_SIGBUS; - else if (unlikely(nopage_page == NOPAGE_OOM)) - return VM_FAULT_OOM; - else if (unlikely(nopage_page == NOPAGE_REFAULT)) - return VM_FAULT_MINOR; + if (likely(vma->vm_ops->fault)) { + fdata.type = -1; + faulted_page = vma->vm_ops->fault(vma, &fdata); + WARN_ON(fdata.type == -1); + if (unlikely(!faulted_page)) + return fdata.type; + } else { + /* Legacy ->nopage path */ + fdata.type = VM_FAULT_MINOR; + faulted_page = vma->vm_ops->nopage(vma, address & PAGE_MASK, + &fdata.type); + /* no page was available -- either SIGBUS or OOM */ + if (unlikely(faulted_page == NOPAGE_SIGBUS)) + return VM_FAULT_SIGBUS; + else if (unlikely(faulted_page == NOPAGE_OOM)) + return VM_FAULT_OOM; + } - BUG_ON(vma->vm_flags & VM_CAN_INVALIDATE && !PageLocked(nopage_page)); /* - * For consistency in subsequent calls, make the nopage_page always + * For consistency in subsequent calls, make the faulted_page always * locked. */ if (unlikely(!(vma->vm_flags & VM_CAN_INVALIDATE))) - lock_page(nopage_page); + lock_page(faulted_page); + else + BUG_ON(!PageLocked(faulted_page)); /* * Should we do an early C-O-W break? */ - page = nopage_page; - if (write_access) { + page = faulted_page; + if (flags & FAULT_FLAG_WRITE) { if (!(vma->vm_flags & VM_SHARED)) { + anon = 1; if (unlikely(anon_vma_prepare(vma))) { - ret = VM_FAULT_OOM; - goto out_error; + fdata.type = VM_FAULT_OOM; + goto out; } page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); if (!page) { - ret = VM_FAULT_OOM; - goto out_error; + fdata.type = VM_FAULT_OOM; + goto out; } - copy_user_highpage(page, nopage_page, address, vma); - anon = 1; + copy_user_highpage(page, faulted_page, address, vma); } else { - /* if the page will be shareable, see if the backing + /* + * If the page will be shareable, see if the backing * address space wants to know that the page is about - * to become writable */ + * to become writable + */ if (vma->vm_ops->page_mkwrite && vma->vm_ops->page_mkwrite(vma, page) < 0) { - ret = VM_FAULT_SIGBUS; - goto out_error; + fdata.type = VM_FAULT_SIGBUS; + anon = 1; /* no anon but release faulted_page */ + goto out; } } + } page_table = pte_offset_map_lock(mm, pmd, address, &ptl); @@ -2373,10 +2392,10 @@ static int do_no_page(struct mm_struct *mm, struct vm_area_struct *vma, * handle that later. */ /* Only go through if we didn't race with anybody else... */ - if (likely(pte_none(*page_table))) { + if (likely(pte_same(*page_table, orig_pte))) { flush_icache_page(vma, page); entry = mk_pte(page, vma->vm_page_prot); - if (write_access) + if (flags & FAULT_FLAG_WRITE) entry = maybe_mkwrite(pte_mkdirty(entry), vma); set_pte_at(mm, address, page_table, entry); if (anon) { @@ -2386,7 +2405,7 @@ static int do_no_page(struct mm_struct *mm, struct vm_area_struct *vma, } else { inc_mm_counter(mm, file_rss); page_add_file_rmap(page); - if (write_access) { + if (flags & FAULT_FLAG_WRITE) { dirty_page = page; get_page(dirty_page); } @@ -2399,25 +2418,42 @@ static int do_no_page(struct mm_struct *mm, struct vm_area_struct *vma, if (anon) page_cache_release(page); else - anon = 1; /* not anon, but release nopage_page */ + anon = 1; /* no anon but release faulted_page */ } pte_unmap_unlock(page_table, ptl); out: - unlock_page(nopage_page); + unlock_page(faulted_page); if (anon) - page_cache_release(nopage_page); + page_cache_release(faulted_page); else if (dirty_page) { set_page_dirty_balance(dirty_page); put_page(dirty_page); } - return ret; + return fdata.type; +} -out_error: - anon = 1; /* relase nopage_page */ - goto out; +static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long address, pte_t *page_table, pmd_t *pmd, + int write_access, pte_t orig_pte) +{ + pgoff_t pgoff = (((address & PAGE_MASK) + - vma->vm_start) >> PAGE_CACHE_SHIFT) + vma->vm_pgoff; + unsigned int flags = (write_access ? FAULT_FLAG_WRITE : 0); + + return __do_fault(mm, vma, address, page_table, pmd, pgoff, flags, orig_pte); +} + +static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long address, pte_t *page_table, pmd_t *pmd, + int write_access, pgoff_t pgoff, pte_t orig_pte) +{ + unsigned int flags = FAULT_FLAG_NONLINEAR | + (write_access ? FAULT_FLAG_WRITE : 0); + + return __do_fault(mm, vma, address, page_table, pmd, pgoff, flags, orig_pte); } /* @@ -2496,9 +2532,14 @@ static int do_file_page(struct mm_struct *mm, struct vm_area_struct *vma, print_bad_pte(vma, orig_pte, address); return VM_FAULT_OOM; } - /* We can then assume vm->vm_ops && vma->vm_ops->populate */ pgoff = pte_to_pgoff(orig_pte); + + if (vma->vm_ops && vma->vm_ops->fault) + return do_nonlinear_fault(mm, vma, address, page_table, pmd, + write_access, pgoff, orig_pte); + + /* We can then assume vm->vm_ops && vma->vm_ops->populate */ err = vma->vm_ops->populate(vma, address & PAGE_MASK, PAGE_SIZE, vma->vm_page_prot, pgoff, 0); if (err == -ENOMEM) @@ -2532,10 +2573,9 @@ static inline int handle_pte_fault(struct mm_struct *mm, if (!pte_present(entry)) { if (pte_none(entry)) { if (vma->vm_ops) { - if (vma->vm_ops->nopage) - return do_no_page(mm, vma, address, - pte, pmd, - write_access); + if (vma->vm_ops->fault || vma->vm_ops->nopage) + return do_linear_fault(mm, vma, address, + pte, pmd, write_access, entry); if (unlikely(vma->vm_ops->nopfn)) return do_no_pfn(mm, vma, address, pte, pmd, write_access); diff --git a/mm/mmap.c b/mm/mmap.c index 144b4a290f2..724f342bcf8 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1165,12 +1165,8 @@ out: mm->locked_vm += len >> PAGE_SHIFT; make_pages_present(addr, addr + len); } - if (flags & MAP_POPULATE) { - up_write(&mm->mmap_sem); - sys_remap_file_pages(addr, len, 0, - pgoff, flags & MAP_NONBLOCK); - down_write(&mm->mmap_sem); - } + if ((flags & MAP_POPULATE) && !(flags & MAP_NONBLOCK)) + make_pages_present(addr, addr + len); return addr; unmap_and_free_vma: diff --git a/mm/nommu.c b/mm/nommu.c index 8bbbf147a79..aee0e1b0ebe 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1341,8 +1341,7 @@ int in_gate_area_no_task(unsigned long addr) return 0; } -struct page *filemap_nopage(struct vm_area_struct *area, - unsigned long address, int *type) +struct page *filemap_fault(struct vm_area_struct *vma, struct fault_data *fdata) { BUG(); return NULL; diff --git a/mm/rmap.c b/mm/rmap.c index 61e492597a0..fede5c7910b 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -621,8 +621,10 @@ void page_remove_rmap(struct page *page, struct vm_area_struct *vma) printk (KERN_EMERG " page->count = %x\n", page_count(page)); printk (KERN_EMERG " page->mapping = %p\n", page->mapping); print_symbol (KERN_EMERG " vma->vm_ops = %s\n", (unsigned long)vma->vm_ops); - if (vma->vm_ops) + if (vma->vm_ops) { print_symbol (KERN_EMERG " vma->vm_ops->nopage = %s\n", (unsigned long)vma->vm_ops->nopage); + print_symbol (KERN_EMERG " vma->vm_ops->fault = %s\n", (unsigned long)vma->vm_ops->fault); + } if (vma->vm_file && vma->vm_file->f_op) print_symbol (KERN_EMERG " vma->vm_file->f_op->mmap = %s\n", (unsigned long)vma->vm_file->f_op->mmap); BUG(); diff --git a/mm/shmem.c b/mm/shmem.c index 5808fadd394..6b44440f1b2 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -83,7 +83,7 @@ enum sgp_type { SGP_READ, /* don't exceed i_size, don't allocate page */ SGP_CACHE, /* don't exceed i_size, may allocate page */ SGP_WRITE, /* may exceed i_size, may allocate page */ - SGP_NOPAGE, /* same as SGP_CACHE, return with page locked */ + SGP_FAULT, /* same as SGP_CACHE, return with page locked */ }; static int shmem_getpage(struct inode *inode, unsigned long idx, @@ -1101,6 +1101,10 @@ static int shmem_getpage(struct inode *inode, unsigned long idx, if (idx >= SHMEM_MAX_INDEX) return -EFBIG; + + if (type) + *type = VM_FAULT_MINOR; + /* * Normally, filepage is NULL on entry, and either found * uptodate immediately, or allocated and zeroed, or read @@ -1291,7 +1295,7 @@ repeat: done: if (*pagep != filepage) { *pagep = filepage; - if (sgp != SGP_NOPAGE) + if (sgp != SGP_FAULT) unlock_page(filepage); } @@ -1305,76 +1309,31 @@ failed: return error; } -static struct page *shmem_nopage(struct vm_area_struct *vma, - unsigned long address, int *type) +static struct page *shmem_fault(struct vm_area_struct *vma, + struct fault_data *fdata) { struct inode *inode = vma->vm_file->f_path.dentry->d_inode; struct page *page = NULL; - unsigned long idx; int error; BUG_ON(!(vma->vm_flags & VM_CAN_INVALIDATE)); - idx = (address - vma->vm_start) >> PAGE_SHIFT; - idx += vma->vm_pgoff; - idx >>= PAGE_CACHE_SHIFT - PAGE_SHIFT; - if (((loff_t) idx << PAGE_CACHE_SHIFT) >= i_size_read(inode)) - return NOPAGE_SIGBUS; + if (((loff_t)fdata->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode)) { + fdata->type = VM_FAULT_SIGBUS; + return NULL; + } - error = shmem_getpage(inode, idx, &page, SGP_NOPAGE, type); - if (error) - return (error == -ENOMEM)? NOPAGE_OOM: NOPAGE_SIGBUS; + error = shmem_getpage(inode, fdata->pgoff, &page, + SGP_FAULT, &fdata->type); + if (error) { + fdata->type = ((error == -ENOMEM)?VM_FAULT_OOM:VM_FAULT_SIGBUS); + return NULL; + } mark_page_accessed(page); return page; } -static int shmem_populate(struct vm_area_struct *vma, - unsigned long addr, unsigned long len, - pgprot_t prot, unsigned long pgoff, int nonblock) -{ - struct inode *inode = vma->vm_file->f_path.dentry->d_inode; - struct mm_struct *mm = vma->vm_mm; - enum sgp_type sgp = nonblock? SGP_QUICK: SGP_CACHE; - unsigned long size; - - size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; - if (pgoff >= size || pgoff + (len >> PAGE_SHIFT) > size) - return -EINVAL; - - while ((long) len > 0) { - struct page *page = NULL; - int err; - /* - * Will need changing if PAGE_CACHE_SIZE != PAGE_SIZE - */ - err = shmem_getpage(inode, pgoff, &page, sgp, NULL); - if (err) - return err; - /* Page may still be null, but only if nonblock was set. */ - if (page) { - mark_page_accessed(page); - err = install_page(mm, vma, addr, page, prot); - if (err) { - page_cache_release(page); - return err; - } - } else if (vma->vm_flags & VM_NONLINEAR) { - /* No page was found just because we can't read it in - * now (being here implies nonblock != 0), but the page - * may exist, so set the PTE to fault it in later. */ - err = install_file_pte(mm, vma, addr, pgoff, prot); - if (err) - return err; - } - - len -= PAGE_SIZE; - addr += PAGE_SIZE; - pgoff++; - } - return 0; -} - #ifdef CONFIG_NUMA int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new) { @@ -1419,7 +1378,7 @@ static int shmem_mmap(struct file *file, struct vm_area_struct *vma) { file_accessed(file); vma->vm_ops = &shmem_vm_ops; - vma->vm_flags |= VM_CAN_INVALIDATE; + vma->vm_flags |= VM_CAN_INVALIDATE | VM_CAN_NONLINEAR; return 0; } @@ -2465,8 +2424,7 @@ static const struct super_operations shmem_ops = { }; static struct vm_operations_struct shmem_vm_ops = { - .nopage = shmem_nopage, - .populate = shmem_populate, + .fault = shmem_fault, #ifdef CONFIG_NUMA .set_policy = shmem_set_policy, .get_policy = shmem_get_policy, diff --git a/mm/truncate.c b/mm/truncate.c index aed85f0b707..5cdfbc1a59f 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -82,7 +82,7 @@ EXPORT_SYMBOL(cancel_dirty_page); /* * If truncate cannot remove the fs-private metadata from the page, the page * becomes anonymous. It will be left on the LRU and may even be mapped into - * user pagetables if we're racing with filemap_nopage(). + * user pagetables if we're racing with filemap_fault(). * * We need to bale out if page->mapping is no longer equal to the original * mapping. This happens a) when the VM reclaimed the page while we waited on |