Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mason/linux...
[~shefty/rdma-dev.git] / fs / btrfs / inode.c
1 /*
2  * Copyright (C) 2007 Oracle.  All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public
6  * License v2 as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11  * General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public
14  * License along with this program; if not, write to the
15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16  * Boston, MA 021110-1307, USA.
17  */
18
19 #include <linux/kernel.h>
20 #include <linux/bio.h>
21 #include <linux/buffer_head.h>
22 #include <linux/file.h>
23 #include <linux/fs.h>
24 #include <linux/pagemap.h>
25 #include <linux/highmem.h>
26 #include <linux/time.h>
27 #include <linux/init.h>
28 #include <linux/string.h>
29 #include <linux/backing-dev.h>
30 #include <linux/mpage.h>
31 #include <linux/swap.h>
32 #include <linux/writeback.h>
33 #include <linux/statfs.h>
34 #include <linux/compat.h>
35 #include <linux/bit_spinlock.h>
36 #include <linux/xattr.h>
37 #include <linux/posix_acl.h>
38 #include <linux/falloc.h>
39 #include <linux/slab.h>
40 #include <linux/ratelimit.h>
41 #include <linux/mount.h>
42 #include <linux/btrfs.h>
43 #include <linux/blkdev.h>
44 #include "compat.h"
45 #include "ctree.h"
46 #include "disk-io.h"
47 #include "transaction.h"
48 #include "btrfs_inode.h"
49 #include "print-tree.h"
50 #include "ordered-data.h"
51 #include "xattr.h"
52 #include "tree-log.h"
53 #include "volumes.h"
54 #include "compression.h"
55 #include "locking.h"
56 #include "free-space-cache.h"
57 #include "inode-map.h"
58 #include "backref.h"
59
60 struct btrfs_iget_args {
61         u64 ino;
62         struct btrfs_root *root;
63 };
64
65 static const struct inode_operations btrfs_dir_inode_operations;
66 static const struct inode_operations btrfs_symlink_inode_operations;
67 static const struct inode_operations btrfs_dir_ro_inode_operations;
68 static const struct inode_operations btrfs_special_inode_operations;
69 static const struct inode_operations btrfs_file_inode_operations;
70 static const struct address_space_operations btrfs_aops;
71 static const struct address_space_operations btrfs_symlink_aops;
72 static const struct file_operations btrfs_dir_file_operations;
73 static struct extent_io_ops btrfs_extent_io_ops;
74
75 static struct kmem_cache *btrfs_inode_cachep;
76 static struct kmem_cache *btrfs_delalloc_work_cachep;
77 struct kmem_cache *btrfs_trans_handle_cachep;
78 struct kmem_cache *btrfs_transaction_cachep;
79 struct kmem_cache *btrfs_path_cachep;
80 struct kmem_cache *btrfs_free_space_cachep;
81
82 #define S_SHIFT 12
83 static unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = {
84         [S_IFREG >> S_SHIFT]    = BTRFS_FT_REG_FILE,
85         [S_IFDIR >> S_SHIFT]    = BTRFS_FT_DIR,
86         [S_IFCHR >> S_SHIFT]    = BTRFS_FT_CHRDEV,
87         [S_IFBLK >> S_SHIFT]    = BTRFS_FT_BLKDEV,
88         [S_IFIFO >> S_SHIFT]    = BTRFS_FT_FIFO,
89         [S_IFSOCK >> S_SHIFT]   = BTRFS_FT_SOCK,
90         [S_IFLNK >> S_SHIFT]    = BTRFS_FT_SYMLINK,
91 };
92
93 static int btrfs_setsize(struct inode *inode, struct iattr *attr);
94 static int btrfs_truncate(struct inode *inode);
95 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
96 static noinline int cow_file_range(struct inode *inode,
97                                    struct page *locked_page,
98                                    u64 start, u64 end, int *page_started,
99                                    unsigned long *nr_written, int unlock);
100 static struct extent_map *create_pinned_em(struct inode *inode, u64 start,
101                                            u64 len, u64 orig_start,
102                                            u64 block_start, u64 block_len,
103                                            u64 orig_block_len, int type);
104
105 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
106                                      struct inode *inode,  struct inode *dir,
107                                      const struct qstr *qstr)
108 {
109         int err;
110
111         err = btrfs_init_acl(trans, inode, dir);
112         if (!err)
113                 err = btrfs_xattr_security_init(trans, inode, dir, qstr);
114         return err;
115 }
116
117 /*
118  * this does all the hard work for inserting an inline extent into
119  * the btree.  The caller should have done a btrfs_drop_extents so that
120  * no overlapping inline items exist in the btree
121  */
122 static noinline int insert_inline_extent(struct btrfs_trans_handle *trans,
123                                 struct btrfs_root *root, struct inode *inode,
124                                 u64 start, size_t size, size_t compressed_size,
125                                 int compress_type,
126                                 struct page **compressed_pages)
127 {
128         struct btrfs_key key;
129         struct btrfs_path *path;
130         struct extent_buffer *leaf;
131         struct page *page = NULL;
132         char *kaddr;
133         unsigned long ptr;
134         struct btrfs_file_extent_item *ei;
135         int err = 0;
136         int ret;
137         size_t cur_size = size;
138         size_t datasize;
139         unsigned long offset;
140
141         if (compressed_size && compressed_pages)
142                 cur_size = compressed_size;
143
144         path = btrfs_alloc_path();
145         if (!path)
146                 return -ENOMEM;
147
148         path->leave_spinning = 1;
149
150         key.objectid = btrfs_ino(inode);
151         key.offset = start;
152         btrfs_set_key_type(&key, BTRFS_EXTENT_DATA_KEY);
153         datasize = btrfs_file_extent_calc_inline_size(cur_size);
154
155         inode_add_bytes(inode, size);
156         ret = btrfs_insert_empty_item(trans, root, path, &key,
157                                       datasize);
158         if (ret) {
159                 err = ret;
160                 goto fail;
161         }
162         leaf = path->nodes[0];
163         ei = btrfs_item_ptr(leaf, path->slots[0],
164                             struct btrfs_file_extent_item);
165         btrfs_set_file_extent_generation(leaf, ei, trans->transid);
166         btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
167         btrfs_set_file_extent_encryption(leaf, ei, 0);
168         btrfs_set_file_extent_other_encoding(leaf, ei, 0);
169         btrfs_set_file_extent_ram_bytes(leaf, ei, size);
170         ptr = btrfs_file_extent_inline_start(ei);
171
172         if (compress_type != BTRFS_COMPRESS_NONE) {
173                 struct page *cpage;
174                 int i = 0;
175                 while (compressed_size > 0) {
176                         cpage = compressed_pages[i];
177                         cur_size = min_t(unsigned long, compressed_size,
178                                        PAGE_CACHE_SIZE);
179
180                         kaddr = kmap_atomic(cpage);
181                         write_extent_buffer(leaf, kaddr, ptr, cur_size);
182                         kunmap_atomic(kaddr);
183
184                         i++;
185                         ptr += cur_size;
186                         compressed_size -= cur_size;
187                 }
188                 btrfs_set_file_extent_compression(leaf, ei,
189                                                   compress_type);
190         } else {
191                 page = find_get_page(inode->i_mapping,
192                                      start >> PAGE_CACHE_SHIFT);
193                 btrfs_set_file_extent_compression(leaf, ei, 0);
194                 kaddr = kmap_atomic(page);
195                 offset = start & (PAGE_CACHE_SIZE - 1);
196                 write_extent_buffer(leaf, kaddr + offset, ptr, size);
197                 kunmap_atomic(kaddr);
198                 page_cache_release(page);
199         }
200         btrfs_mark_buffer_dirty(leaf);
201         btrfs_free_path(path);
202
203         /*
204          * we're an inline extent, so nobody can
205          * extend the file past i_size without locking
206          * a page we already have locked.
207          *
208          * We must do any isize and inode updates
209          * before we unlock the pages.  Otherwise we
210          * could end up racing with unlink.
211          */
212         BTRFS_I(inode)->disk_i_size = inode->i_size;
213         ret = btrfs_update_inode(trans, root, inode);
214
215         return ret;
216 fail:
217         btrfs_free_path(path);
218         return err;
219 }
220
221
222 /*
223  * conditionally insert an inline extent into the file.  This
224  * does the checks required to make sure the data is small enough
225  * to fit as an inline extent.
226  */
227 static noinline int cow_file_range_inline(struct btrfs_trans_handle *trans,
228                                  struct btrfs_root *root,
229                                  struct inode *inode, u64 start, u64 end,
230                                  size_t compressed_size, int compress_type,
231                                  struct page **compressed_pages)
232 {
233         u64 isize = i_size_read(inode);
234         u64 actual_end = min(end + 1, isize);
235         u64 inline_len = actual_end - start;
236         u64 aligned_end = ALIGN(end, root->sectorsize);
237         u64 data_len = inline_len;
238         int ret;
239
240         if (compressed_size)
241                 data_len = compressed_size;
242
243         if (start > 0 ||
244             actual_end >= PAGE_CACHE_SIZE ||
245             data_len >= BTRFS_MAX_INLINE_DATA_SIZE(root) ||
246             (!compressed_size &&
247             (actual_end & (root->sectorsize - 1)) == 0) ||
248             end + 1 < isize ||
249             data_len > root->fs_info->max_inline) {
250                 return 1;
251         }
252
253         ret = btrfs_drop_extents(trans, root, inode, start, aligned_end, 1);
254         if (ret)
255                 return ret;
256
257         if (isize > actual_end)
258                 inline_len = min_t(u64, isize, actual_end);
259         ret = insert_inline_extent(trans, root, inode, start,
260                                    inline_len, compressed_size,
261                                    compress_type, compressed_pages);
262         if (ret && ret != -ENOSPC) {
263                 btrfs_abort_transaction(trans, root, ret);
264                 return ret;
265         } else if (ret == -ENOSPC) {
266                 return 1;
267         }
268
269         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
270         btrfs_delalloc_release_metadata(inode, end + 1 - start);
271         btrfs_drop_extent_cache(inode, start, aligned_end - 1, 0);
272         return 0;
273 }
274
275 struct async_extent {
276         u64 start;
277         u64 ram_size;
278         u64 compressed_size;
279         struct page **pages;
280         unsigned long nr_pages;
281         int compress_type;
282         struct list_head list;
283 };
284
285 struct async_cow {
286         struct inode *inode;
287         struct btrfs_root *root;
288         struct page *locked_page;
289         u64 start;
290         u64 end;
291         struct list_head extents;
292         struct btrfs_work work;
293 };
294
295 static noinline int add_async_extent(struct async_cow *cow,
296                                      u64 start, u64 ram_size,
297                                      u64 compressed_size,
298                                      struct page **pages,
299                                      unsigned long nr_pages,
300                                      int compress_type)
301 {
302         struct async_extent *async_extent;
303
304         async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
305         BUG_ON(!async_extent); /* -ENOMEM */
306         async_extent->start = start;
307         async_extent->ram_size = ram_size;
308         async_extent->compressed_size = compressed_size;
309         async_extent->pages = pages;
310         async_extent->nr_pages = nr_pages;
311         async_extent->compress_type = compress_type;
312         list_add_tail(&async_extent->list, &cow->extents);
313         return 0;
314 }
315
316 /*
317  * we create compressed extents in two phases.  The first
318  * phase compresses a range of pages that have already been
319  * locked (both pages and state bits are locked).
320  *
321  * This is done inside an ordered work queue, and the compression
322  * is spread across many cpus.  The actual IO submission is step
323  * two, and the ordered work queue takes care of making sure that
324  * happens in the same order things were put onto the queue by
325  * writepages and friends.
326  *
327  * If this code finds it can't get good compression, it puts an
328  * entry onto the work queue to write the uncompressed bytes.  This
329  * makes sure that both compressed inodes and uncompressed inodes
330  * are written in the same order that the flusher thread sent them
331  * down.
332  */
333 static noinline int compress_file_range(struct inode *inode,
334                                         struct page *locked_page,
335                                         u64 start, u64 end,
336                                         struct async_cow *async_cow,
337                                         int *num_added)
338 {
339         struct btrfs_root *root = BTRFS_I(inode)->root;
340         struct btrfs_trans_handle *trans;
341         u64 num_bytes;
342         u64 blocksize = root->sectorsize;
343         u64 actual_end;
344         u64 isize = i_size_read(inode);
345         int ret = 0;
346         struct page **pages = NULL;
347         unsigned long nr_pages;
348         unsigned long nr_pages_ret = 0;
349         unsigned long total_compressed = 0;
350         unsigned long total_in = 0;
351         unsigned long max_compressed = 128 * 1024;
352         unsigned long max_uncompressed = 128 * 1024;
353         int i;
354         int will_compress;
355         int compress_type = root->fs_info->compress_type;
356
357         /* if this is a small write inside eof, kick off a defrag */
358         if ((end - start + 1) < 16 * 1024 &&
359             (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
360                 btrfs_add_inode_defrag(NULL, inode);
361
362         actual_end = min_t(u64, isize, end + 1);
363 again:
364         will_compress = 0;
365         nr_pages = (end >> PAGE_CACHE_SHIFT) - (start >> PAGE_CACHE_SHIFT) + 1;
366         nr_pages = min(nr_pages, (128 * 1024UL) / PAGE_CACHE_SIZE);
367
368         /*
369          * we don't want to send crud past the end of i_size through
370          * compression, that's just a waste of CPU time.  So, if the
371          * end of the file is before the start of our current
372          * requested range of bytes, we bail out to the uncompressed
373          * cleanup code that can deal with all of this.
374          *
375          * It isn't really the fastest way to fix things, but this is a
376          * very uncommon corner.
377          */
378         if (actual_end <= start)
379                 goto cleanup_and_bail_uncompressed;
380
381         total_compressed = actual_end - start;
382
383         /* we want to make sure that amount of ram required to uncompress
384          * an extent is reasonable, so we limit the total size in ram
385          * of a compressed extent to 128k.  This is a crucial number
386          * because it also controls how easily we can spread reads across
387          * cpus for decompression.
388          *
389          * We also want to make sure the amount of IO required to do
390          * a random read is reasonably small, so we limit the size of
391          * a compressed extent to 128k.
392          */
393         total_compressed = min(total_compressed, max_uncompressed);
394         num_bytes = ALIGN(end - start + 1, blocksize);
395         num_bytes = max(blocksize,  num_bytes);
396         total_in = 0;
397         ret = 0;
398
399         /*
400          * we do compression for mount -o compress and when the
401          * inode has not been flagged as nocompress.  This flag can
402          * change at any time if we discover bad compression ratios.
403          */
404         if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS) &&
405             (btrfs_test_opt(root, COMPRESS) ||
406              (BTRFS_I(inode)->force_compress) ||
407              (BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS))) {
408                 WARN_ON(pages);
409                 pages = kzalloc(sizeof(struct page *) * nr_pages, GFP_NOFS);
410                 if (!pages) {
411                         /* just bail out to the uncompressed code */
412                         goto cont;
413                 }
414
415                 if (BTRFS_I(inode)->force_compress)
416                         compress_type = BTRFS_I(inode)->force_compress;
417
418                 ret = btrfs_compress_pages(compress_type,
419                                            inode->i_mapping, start,
420                                            total_compressed, pages,
421                                            nr_pages, &nr_pages_ret,
422                                            &total_in,
423                                            &total_compressed,
424                                            max_compressed);
425
426                 if (!ret) {
427                         unsigned long offset = total_compressed &
428                                 (PAGE_CACHE_SIZE - 1);
429                         struct page *page = pages[nr_pages_ret - 1];
430                         char *kaddr;
431
432                         /* zero the tail end of the last page, we might be
433                          * sending it down to disk
434                          */
435                         if (offset) {
436                                 kaddr = kmap_atomic(page);
437                                 memset(kaddr + offset, 0,
438                                        PAGE_CACHE_SIZE - offset);
439                                 kunmap_atomic(kaddr);
440                         }
441                         will_compress = 1;
442                 }
443         }
444 cont:
445         if (start == 0) {
446                 trans = btrfs_join_transaction(root);
447                 if (IS_ERR(trans)) {
448                         ret = PTR_ERR(trans);
449                         trans = NULL;
450                         goto cleanup_and_out;
451                 }
452                 trans->block_rsv = &root->fs_info->delalloc_block_rsv;
453
454                 /* lets try to make an inline extent */
455                 if (ret || total_in < (actual_end - start)) {
456                         /* we didn't compress the entire range, try
457                          * to make an uncompressed inline extent.
458                          */
459                         ret = cow_file_range_inline(trans, root, inode,
460                                                     start, end, 0, 0, NULL);
461                 } else {
462                         /* try making a compressed inline extent */
463                         ret = cow_file_range_inline(trans, root, inode,
464                                                     start, end,
465                                                     total_compressed,
466                                                     compress_type, pages);
467                 }
468                 if (ret <= 0) {
469                         /*
470                          * inline extent creation worked or returned error,
471                          * we don't need to create any more async work items.
472                          * Unlock and free up our temp pages.
473                          */
474                         extent_clear_unlock_delalloc(inode,
475                              &BTRFS_I(inode)->io_tree,
476                              start, end, NULL,
477                              EXTENT_CLEAR_UNLOCK_PAGE | EXTENT_CLEAR_DIRTY |
478                              EXTENT_CLEAR_DELALLOC |
479                              EXTENT_SET_WRITEBACK | EXTENT_END_WRITEBACK);
480
481                         btrfs_end_transaction(trans, root);
482                         goto free_pages_out;
483                 }
484                 btrfs_end_transaction(trans, root);
485         }
486
487         if (will_compress) {
488                 /*
489                  * we aren't doing an inline extent round the compressed size
490                  * up to a block size boundary so the allocator does sane
491                  * things
492                  */
493                 total_compressed = ALIGN(total_compressed, blocksize);
494
495                 /*
496                  * one last check to make sure the compression is really a
497                  * win, compare the page count read with the blocks on disk
498                  */
499                 total_in = ALIGN(total_in, PAGE_CACHE_SIZE);
500                 if (total_compressed >= total_in) {
501                         will_compress = 0;
502                 } else {
503                         num_bytes = total_in;
504                 }
505         }
506         if (!will_compress && pages) {
507                 /*
508                  * the compression code ran but failed to make things smaller,
509                  * free any pages it allocated and our page pointer array
510                  */
511                 for (i = 0; i < nr_pages_ret; i++) {
512                         WARN_ON(pages[i]->mapping);
513                         page_cache_release(pages[i]);
514                 }
515                 kfree(pages);
516                 pages = NULL;
517                 total_compressed = 0;
518                 nr_pages_ret = 0;
519
520                 /* flag the file so we don't compress in the future */
521                 if (!btrfs_test_opt(root, FORCE_COMPRESS) &&
522                     !(BTRFS_I(inode)->force_compress)) {
523                         BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
524                 }
525         }
526         if (will_compress) {
527                 *num_added += 1;
528
529                 /* the async work queues will take care of doing actual
530                  * allocation on disk for these compressed pages,
531                  * and will submit them to the elevator.
532                  */
533                 add_async_extent(async_cow, start, num_bytes,
534                                  total_compressed, pages, nr_pages_ret,
535                                  compress_type);
536
537                 if (start + num_bytes < end) {
538                         start += num_bytes;
539                         pages = NULL;
540                         cond_resched();
541                         goto again;
542                 }
543         } else {
544 cleanup_and_bail_uncompressed:
545                 /*
546                  * No compression, but we still need to write the pages in
547                  * the file we've been given so far.  redirty the locked
548                  * page if it corresponds to our extent and set things up
549                  * for the async work queue to run cow_file_range to do
550                  * the normal delalloc dance
551                  */
552                 if (page_offset(locked_page) >= start &&
553                     page_offset(locked_page) <= end) {
554                         __set_page_dirty_nobuffers(locked_page);
555                         /* unlocked later on in the async handlers */
556                 }
557                 add_async_extent(async_cow, start, end - start + 1,
558                                  0, NULL, 0, BTRFS_COMPRESS_NONE);
559                 *num_added += 1;
560         }
561
562 out:
563         return ret;
564
565 free_pages_out:
566         for (i = 0; i < nr_pages_ret; i++) {
567                 WARN_ON(pages[i]->mapping);
568                 page_cache_release(pages[i]);
569         }
570         kfree(pages);
571
572         goto out;
573
574 cleanup_and_out:
575         extent_clear_unlock_delalloc(inode, &BTRFS_I(inode)->io_tree,
576                                      start, end, NULL,
577                                      EXTENT_CLEAR_UNLOCK_PAGE |
578                                      EXTENT_CLEAR_DIRTY |
579                                      EXTENT_CLEAR_DELALLOC |
580                                      EXTENT_SET_WRITEBACK |
581                                      EXTENT_END_WRITEBACK);
582         if (!trans || IS_ERR(trans))
583                 btrfs_error(root->fs_info, ret, "Failed to join transaction");
584         else
585                 btrfs_abort_transaction(trans, root, ret);
586         goto free_pages_out;
587 }
588
589 /*
590  * phase two of compressed writeback.  This is the ordered portion
591  * of the code, which only gets called in the order the work was
592  * queued.  We walk all the async extents created by compress_file_range
593  * and send them down to the disk.
594  */
595 static noinline int submit_compressed_extents(struct inode *inode,
596                                               struct async_cow *async_cow)
597 {
598         struct async_extent *async_extent;
599         u64 alloc_hint = 0;
600         struct btrfs_trans_handle *trans;
601         struct btrfs_key ins;
602         struct extent_map *em;
603         struct btrfs_root *root = BTRFS_I(inode)->root;
604         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
605         struct extent_io_tree *io_tree;
606         int ret = 0;
607
608         if (list_empty(&async_cow->extents))
609                 return 0;
610
611 again:
612         while (!list_empty(&async_cow->extents)) {
613                 async_extent = list_entry(async_cow->extents.next,
614                                           struct async_extent, list);
615                 list_del(&async_extent->list);
616
617                 io_tree = &BTRFS_I(inode)->io_tree;
618
619 retry:
620                 /* did the compression code fall back to uncompressed IO? */
621                 if (!async_extent->pages) {
622                         int page_started = 0;
623                         unsigned long nr_written = 0;
624
625                         lock_extent(io_tree, async_extent->start,
626                                          async_extent->start +
627                                          async_extent->ram_size - 1);
628
629                         /* allocate blocks */
630                         ret = cow_file_range(inode, async_cow->locked_page,
631                                              async_extent->start,
632                                              async_extent->start +
633                                              async_extent->ram_size - 1,
634                                              &page_started, &nr_written, 0);
635
636                         /* JDM XXX */
637
638                         /*
639                          * if page_started, cow_file_range inserted an
640                          * inline extent and took care of all the unlocking
641                          * and IO for us.  Otherwise, we need to submit
642                          * all those pages down to the drive.
643                          */
644                         if (!page_started && !ret)
645                                 extent_write_locked_range(io_tree,
646                                                   inode, async_extent->start,
647                                                   async_extent->start +
648                                                   async_extent->ram_size - 1,
649                                                   btrfs_get_extent,
650                                                   WB_SYNC_ALL);
651                         else if (ret)
652                                 unlock_page(async_cow->locked_page);
653                         kfree(async_extent);
654                         cond_resched();
655                         continue;
656                 }
657
658                 lock_extent(io_tree, async_extent->start,
659                             async_extent->start + async_extent->ram_size - 1);
660
661                 trans = btrfs_join_transaction(root);
662                 if (IS_ERR(trans)) {
663                         ret = PTR_ERR(trans);
664                 } else {
665                         trans->block_rsv = &root->fs_info->delalloc_block_rsv;
666                         ret = btrfs_reserve_extent(trans, root,
667                                            async_extent->compressed_size,
668                                            async_extent->compressed_size,
669                                            0, alloc_hint, &ins, 1);
670                         if (ret && ret != -ENOSPC)
671                                 btrfs_abort_transaction(trans, root, ret);
672                         btrfs_end_transaction(trans, root);
673                 }
674
675                 if (ret) {
676                         int i;
677
678                         for (i = 0; i < async_extent->nr_pages; i++) {
679                                 WARN_ON(async_extent->pages[i]->mapping);
680                                 page_cache_release(async_extent->pages[i]);
681                         }
682                         kfree(async_extent->pages);
683                         async_extent->nr_pages = 0;
684                         async_extent->pages = NULL;
685
686                         if (ret == -ENOSPC)
687                                 goto retry;
688                         goto out_free;
689                 }
690
691                 /*
692                  * here we're doing allocation and writeback of the
693                  * compressed pages
694                  */
695                 btrfs_drop_extent_cache(inode, async_extent->start,
696                                         async_extent->start +
697                                         async_extent->ram_size - 1, 0);
698
699                 em = alloc_extent_map();
700                 if (!em)
701                         goto out_free_reserve;
702                 em->start = async_extent->start;
703                 em->len = async_extent->ram_size;
704                 em->orig_start = em->start;
705                 em->mod_start = em->start;
706                 em->mod_len = em->len;
707
708                 em->block_start = ins.objectid;
709                 em->block_len = ins.offset;
710                 em->orig_block_len = ins.offset;
711                 em->bdev = root->fs_info->fs_devices->latest_bdev;
712                 em->compress_type = async_extent->compress_type;
713                 set_bit(EXTENT_FLAG_PINNED, &em->flags);
714                 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
715                 em->generation = -1;
716
717                 while (1) {
718                         write_lock(&em_tree->lock);
719                         ret = add_extent_mapping(em_tree, em);
720                         if (!ret)
721                                 list_move(&em->list,
722                                           &em_tree->modified_extents);
723                         write_unlock(&em_tree->lock);
724                         if (ret != -EEXIST) {
725                                 free_extent_map(em);
726                                 break;
727                         }
728                         btrfs_drop_extent_cache(inode, async_extent->start,
729                                                 async_extent->start +
730                                                 async_extent->ram_size - 1, 0);
731                 }
732
733                 if (ret)
734                         goto out_free_reserve;
735
736                 ret = btrfs_add_ordered_extent_compress(inode,
737                                                 async_extent->start,
738                                                 ins.objectid,
739                                                 async_extent->ram_size,
740                                                 ins.offset,
741                                                 BTRFS_ORDERED_COMPRESSED,
742                                                 async_extent->compress_type);
743                 if (ret)
744                         goto out_free_reserve;
745
746                 /*
747                  * clear dirty, set writeback and unlock the pages.
748                  */
749                 extent_clear_unlock_delalloc(inode,
750                                 &BTRFS_I(inode)->io_tree,
751                                 async_extent->start,
752                                 async_extent->start +
753                                 async_extent->ram_size - 1,
754                                 NULL, EXTENT_CLEAR_UNLOCK_PAGE |
755                                 EXTENT_CLEAR_UNLOCK |
756                                 EXTENT_CLEAR_DELALLOC |
757                                 EXTENT_CLEAR_DIRTY | EXTENT_SET_WRITEBACK);
758
759                 ret = btrfs_submit_compressed_write(inode,
760                                     async_extent->start,
761                                     async_extent->ram_size,
762                                     ins.objectid,
763                                     ins.offset, async_extent->pages,
764                                     async_extent->nr_pages);
765                 alloc_hint = ins.objectid + ins.offset;
766                 kfree(async_extent);
767                 if (ret)
768                         goto out;
769                 cond_resched();
770         }
771         ret = 0;
772 out:
773         return ret;
774 out_free_reserve:
775         btrfs_free_reserved_extent(root, ins.objectid, ins.offset);
776 out_free:
777         extent_clear_unlock_delalloc(inode, &BTRFS_I(inode)->io_tree,
778                                      async_extent->start,
779                                      async_extent->start +
780                                      async_extent->ram_size - 1,
781                                      NULL, EXTENT_CLEAR_UNLOCK_PAGE |
782                                      EXTENT_CLEAR_UNLOCK |
783                                      EXTENT_CLEAR_DELALLOC |
784                                      EXTENT_CLEAR_DIRTY |
785                                      EXTENT_SET_WRITEBACK |
786                                      EXTENT_END_WRITEBACK);
787         kfree(async_extent);
788         goto again;
789 }
790
791 static u64 get_extent_allocation_hint(struct inode *inode, u64 start,
792                                       u64 num_bytes)
793 {
794         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
795         struct extent_map *em;
796         u64 alloc_hint = 0;
797
798         read_lock(&em_tree->lock);
799         em = search_extent_mapping(em_tree, start, num_bytes);
800         if (em) {
801                 /*
802                  * if block start isn't an actual block number then find the
803                  * first block in this inode and use that as a hint.  If that
804                  * block is also bogus then just don't worry about it.
805                  */
806                 if (em->block_start >= EXTENT_MAP_LAST_BYTE) {
807                         free_extent_map(em);
808                         em = search_extent_mapping(em_tree, 0, 0);
809                         if (em && em->block_start < EXTENT_MAP_LAST_BYTE)
810                                 alloc_hint = em->block_start;
811                         if (em)
812                                 free_extent_map(em);
813                 } else {
814                         alloc_hint = em->block_start;
815                         free_extent_map(em);
816                 }
817         }
818         read_unlock(&em_tree->lock);
819
820         return alloc_hint;
821 }
822
823 /*
824  * when extent_io.c finds a delayed allocation range in the file,
825  * the call backs end up in this code.  The basic idea is to
826  * allocate extents on disk for the range, and create ordered data structs
827  * in ram to track those extents.
828  *
829  * locked_page is the page that writepage had locked already.  We use
830  * it to make sure we don't do extra locks or unlocks.
831  *
832  * *page_started is set to one if we unlock locked_page and do everything
833  * required to start IO on it.  It may be clean and already done with
834  * IO when we return.
835  */
836 static noinline int __cow_file_range(struct btrfs_trans_handle *trans,
837                                      struct inode *inode,
838                                      struct btrfs_root *root,
839                                      struct page *locked_page,
840                                      u64 start, u64 end, int *page_started,
841                                      unsigned long *nr_written,
842                                      int unlock)
843 {
844         u64 alloc_hint = 0;
845         u64 num_bytes;
846         unsigned long ram_size;
847         u64 disk_num_bytes;
848         u64 cur_alloc_size;
849         u64 blocksize = root->sectorsize;
850         struct btrfs_key ins;
851         struct extent_map *em;
852         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
853         int ret = 0;
854
855         BUG_ON(btrfs_is_free_space_inode(inode));
856
857         num_bytes = ALIGN(end - start + 1, blocksize);
858         num_bytes = max(blocksize,  num_bytes);
859         disk_num_bytes = num_bytes;
860
861         /* if this is a small write inside eof, kick off defrag */
862         if (num_bytes < 64 * 1024 &&
863             (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
864                 btrfs_add_inode_defrag(trans, inode);
865
866         if (start == 0) {
867                 /* lets try to make an inline extent */
868                 ret = cow_file_range_inline(trans, root, inode,
869                                             start, end, 0, 0, NULL);
870                 if (ret == 0) {
871                         extent_clear_unlock_delalloc(inode,
872                                      &BTRFS_I(inode)->io_tree,
873                                      start, end, NULL,
874                                      EXTENT_CLEAR_UNLOCK_PAGE |
875                                      EXTENT_CLEAR_UNLOCK |
876                                      EXTENT_CLEAR_DELALLOC |
877                                      EXTENT_CLEAR_DIRTY |
878                                      EXTENT_SET_WRITEBACK |
879                                      EXTENT_END_WRITEBACK);
880
881                         *nr_written = *nr_written +
882                              (end - start + PAGE_CACHE_SIZE) / PAGE_CACHE_SIZE;
883                         *page_started = 1;
884                         goto out;
885                 } else if (ret < 0) {
886                         btrfs_abort_transaction(trans, root, ret);
887                         goto out_unlock;
888                 }
889         }
890
891         BUG_ON(disk_num_bytes >
892                btrfs_super_total_bytes(root->fs_info->super_copy));
893
894         alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
895         btrfs_drop_extent_cache(inode, start, start + num_bytes - 1, 0);
896
897         while (disk_num_bytes > 0) {
898                 unsigned long op;
899
900                 cur_alloc_size = disk_num_bytes;
901                 ret = btrfs_reserve_extent(trans, root, cur_alloc_size,
902                                            root->sectorsize, 0, alloc_hint,
903                                            &ins, 1);
904                 if (ret < 0) {
905                         btrfs_abort_transaction(trans, root, ret);
906                         goto out_unlock;
907                 }
908
909                 em = alloc_extent_map();
910                 BUG_ON(!em); /* -ENOMEM */
911                 em->start = start;
912                 em->orig_start = em->start;
913                 ram_size = ins.offset;
914                 em->len = ins.offset;
915                 em->mod_start = em->start;
916                 em->mod_len = em->len;
917
918                 em->block_start = ins.objectid;
919                 em->block_len = ins.offset;
920                 em->orig_block_len = ins.offset;
921                 em->bdev = root->fs_info->fs_devices->latest_bdev;
922                 set_bit(EXTENT_FLAG_PINNED, &em->flags);
923                 em->generation = -1;
924
925                 while (1) {
926                         write_lock(&em_tree->lock);
927                         ret = add_extent_mapping(em_tree, em);
928                         if (!ret)
929                                 list_move(&em->list,
930                                           &em_tree->modified_extents);
931                         write_unlock(&em_tree->lock);
932                         if (ret != -EEXIST) {
933                                 free_extent_map(em);
934                                 break;
935                         }
936                         btrfs_drop_extent_cache(inode, start,
937                                                 start + ram_size - 1, 0);
938                 }
939
940                 cur_alloc_size = ins.offset;
941                 ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
942                                                ram_size, cur_alloc_size, 0);
943                 BUG_ON(ret); /* -ENOMEM */
944
945                 if (root->root_key.objectid ==
946                     BTRFS_DATA_RELOC_TREE_OBJECTID) {
947                         ret = btrfs_reloc_clone_csums(inode, start,
948                                                       cur_alloc_size);
949                         if (ret) {
950                                 btrfs_abort_transaction(trans, root, ret);
951                                 goto out_unlock;
952                         }
953                 }
954
955                 if (disk_num_bytes < cur_alloc_size)
956                         break;
957
958                 /* we're not doing compressed IO, don't unlock the first
959                  * page (which the caller expects to stay locked), don't
960                  * clear any dirty bits and don't set any writeback bits
961                  *
962                  * Do set the Private2 bit so we know this page was properly
963                  * setup for writepage
964                  */
965                 op = unlock ? EXTENT_CLEAR_UNLOCK_PAGE : 0;
966                 op |= EXTENT_CLEAR_UNLOCK | EXTENT_CLEAR_DELALLOC |
967                         EXTENT_SET_PRIVATE2;
968
969                 extent_clear_unlock_delalloc(inode, &BTRFS_I(inode)->io_tree,
970                                              start, start + ram_size - 1,
971                                              locked_page, op);
972                 disk_num_bytes -= cur_alloc_size;
973                 num_bytes -= cur_alloc_size;
974                 alloc_hint = ins.objectid + ins.offset;
975                 start += cur_alloc_size;
976         }
977 out:
978         return ret;
979
980 out_unlock:
981         extent_clear_unlock_delalloc(inode,
982                      &BTRFS_I(inode)->io_tree,
983                      start, end, locked_page,
984                      EXTENT_CLEAR_UNLOCK_PAGE |
985                      EXTENT_CLEAR_UNLOCK |
986                      EXTENT_CLEAR_DELALLOC |
987                      EXTENT_CLEAR_DIRTY |
988                      EXTENT_SET_WRITEBACK |
989                      EXTENT_END_WRITEBACK);
990
991         goto out;
992 }
993
994 static noinline int cow_file_range(struct inode *inode,
995                                    struct page *locked_page,
996                                    u64 start, u64 end, int *page_started,
997                                    unsigned long *nr_written,
998                                    int unlock)
999 {
1000         struct btrfs_trans_handle *trans;
1001         struct btrfs_root *root = BTRFS_I(inode)->root;
1002         int ret;
1003
1004         trans = btrfs_join_transaction(root);
1005         if (IS_ERR(trans)) {
1006                 extent_clear_unlock_delalloc(inode,
1007                              &BTRFS_I(inode)->io_tree,
1008                              start, end, locked_page,
1009                              EXTENT_CLEAR_UNLOCK_PAGE |
1010                              EXTENT_CLEAR_UNLOCK |
1011                              EXTENT_CLEAR_DELALLOC |
1012                              EXTENT_CLEAR_DIRTY |
1013                              EXTENT_SET_WRITEBACK |
1014                              EXTENT_END_WRITEBACK);
1015                 return PTR_ERR(trans);
1016         }
1017         trans->block_rsv = &root->fs_info->delalloc_block_rsv;
1018
1019         ret = __cow_file_range(trans, inode, root, locked_page, start, end,
1020                                page_started, nr_written, unlock);
1021
1022         btrfs_end_transaction(trans, root);
1023
1024         return ret;
1025 }
1026
1027 /*
1028  * work queue call back to started compression on a file and pages
1029  */
1030 static noinline void async_cow_start(struct btrfs_work *work)
1031 {
1032         struct async_cow *async_cow;
1033         int num_added = 0;
1034         async_cow = container_of(work, struct async_cow, work);
1035
1036         compress_file_range(async_cow->inode, async_cow->locked_page,
1037                             async_cow->start, async_cow->end, async_cow,
1038                             &num_added);
1039         if (num_added == 0) {
1040                 btrfs_add_delayed_iput(async_cow->inode);
1041                 async_cow->inode = NULL;
1042         }
1043 }
1044
1045 /*
1046  * work queue call back to submit previously compressed pages
1047  */
1048 static noinline void async_cow_submit(struct btrfs_work *work)
1049 {
1050         struct async_cow *async_cow;
1051         struct btrfs_root *root;
1052         unsigned long nr_pages;
1053
1054         async_cow = container_of(work, struct async_cow, work);
1055
1056         root = async_cow->root;
1057         nr_pages = (async_cow->end - async_cow->start + PAGE_CACHE_SIZE) >>
1058                 PAGE_CACHE_SHIFT;
1059
1060         if (atomic_sub_return(nr_pages, &root->fs_info->async_delalloc_pages) <
1061             5 * 1024 * 1024 &&
1062             waitqueue_active(&root->fs_info->async_submit_wait))
1063                 wake_up(&root->fs_info->async_submit_wait);
1064
1065         if (async_cow->inode)
1066                 submit_compressed_extents(async_cow->inode, async_cow);
1067 }
1068
1069 static noinline void async_cow_free(struct btrfs_work *work)
1070 {
1071         struct async_cow *async_cow;
1072         async_cow = container_of(work, struct async_cow, work);
1073         if (async_cow->inode)
1074                 btrfs_add_delayed_iput(async_cow->inode);
1075         kfree(async_cow);
1076 }
1077
1078 static int cow_file_range_async(struct inode *inode, struct page *locked_page,
1079                                 u64 start, u64 end, int *page_started,
1080                                 unsigned long *nr_written)
1081 {
1082         struct async_cow *async_cow;
1083         struct btrfs_root *root = BTRFS_I(inode)->root;
1084         unsigned long nr_pages;
1085         u64 cur_end;
1086         int limit = 10 * 1024 * 1024;
1087
1088         clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED,
1089                          1, 0, NULL, GFP_NOFS);
1090         while (start < end) {
1091                 async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS);
1092                 BUG_ON(!async_cow); /* -ENOMEM */
1093                 async_cow->inode = igrab(inode);
1094                 async_cow->root = root;
1095                 async_cow->locked_page = locked_page;
1096                 async_cow->start = start;
1097
1098                 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS)
1099                         cur_end = end;
1100                 else
1101                         cur_end = min(end, start + 512 * 1024 - 1);
1102
1103                 async_cow->end = cur_end;
1104                 INIT_LIST_HEAD(&async_cow->extents);
1105
1106                 async_cow->work.func = async_cow_start;
1107                 async_cow->work.ordered_func = async_cow_submit;
1108                 async_cow->work.ordered_free = async_cow_free;
1109                 async_cow->work.flags = 0;
1110
1111                 nr_pages = (cur_end - start + PAGE_CACHE_SIZE) >>
1112                         PAGE_CACHE_SHIFT;
1113                 atomic_add(nr_pages, &root->fs_info->async_delalloc_pages);
1114
1115                 btrfs_queue_worker(&root->fs_info->delalloc_workers,
1116                                    &async_cow->work);
1117
1118                 if (atomic_read(&root->fs_info->async_delalloc_pages) > limit) {
1119                         wait_event(root->fs_info->async_submit_wait,
1120                            (atomic_read(&root->fs_info->async_delalloc_pages) <
1121                             limit));
1122                 }
1123
1124                 while (atomic_read(&root->fs_info->async_submit_draining) &&
1125                       atomic_read(&root->fs_info->async_delalloc_pages)) {
1126                         wait_event(root->fs_info->async_submit_wait,
1127                           (atomic_read(&root->fs_info->async_delalloc_pages) ==
1128                            0));
1129                 }
1130
1131                 *nr_written += nr_pages;
1132                 start = cur_end + 1;
1133         }
1134         *page_started = 1;
1135         return 0;
1136 }
1137
1138 static noinline int csum_exist_in_range(struct btrfs_root *root,
1139                                         u64 bytenr, u64 num_bytes)
1140 {
1141         int ret;
1142         struct btrfs_ordered_sum *sums;
1143         LIST_HEAD(list);
1144
1145         ret = btrfs_lookup_csums_range(root->fs_info->csum_root, bytenr,
1146                                        bytenr + num_bytes - 1, &list, 0);
1147         if (ret == 0 && list_empty(&list))
1148                 return 0;
1149
1150         while (!list_empty(&list)) {
1151                 sums = list_entry(list.next, struct btrfs_ordered_sum, list);
1152                 list_del(&sums->list);
1153                 kfree(sums);
1154         }
1155         return 1;
1156 }
1157
1158 /*
1159  * when nowcow writeback call back.  This checks for snapshots or COW copies
1160  * of the extents that exist in the file, and COWs the file as required.
1161  *
1162  * If no cow copies or snapshots exist, we write directly to the existing
1163  * blocks on disk
1164  */
1165 static noinline int run_delalloc_nocow(struct inode *inode,
1166                                        struct page *locked_page,
1167                               u64 start, u64 end, int *page_started, int force,
1168                               unsigned long *nr_written)
1169 {
1170         struct btrfs_root *root = BTRFS_I(inode)->root;
1171         struct btrfs_trans_handle *trans;
1172         struct extent_buffer *leaf;
1173         struct btrfs_path *path;
1174         struct btrfs_file_extent_item *fi;
1175         struct btrfs_key found_key;
1176         u64 cow_start;
1177         u64 cur_offset;
1178         u64 extent_end;
1179         u64 extent_offset;
1180         u64 disk_bytenr;
1181         u64 num_bytes;
1182         u64 disk_num_bytes;
1183         int extent_type;
1184         int ret, err;
1185         int type;
1186         int nocow;
1187         int check_prev = 1;
1188         bool nolock;
1189         u64 ino = btrfs_ino(inode);
1190
1191         path = btrfs_alloc_path();
1192         if (!path) {
1193                 extent_clear_unlock_delalloc(inode,
1194                              &BTRFS_I(inode)->io_tree,
1195                              start, end, locked_page,
1196                              EXTENT_CLEAR_UNLOCK_PAGE |
1197                              EXTENT_CLEAR_UNLOCK |
1198                              EXTENT_CLEAR_DELALLOC |
1199                              EXTENT_CLEAR_DIRTY |
1200                              EXTENT_SET_WRITEBACK |
1201                              EXTENT_END_WRITEBACK);
1202                 return -ENOMEM;
1203         }
1204
1205         nolock = btrfs_is_free_space_inode(inode);
1206
1207         if (nolock)
1208                 trans = btrfs_join_transaction_nolock(root);
1209         else
1210                 trans = btrfs_join_transaction(root);
1211
1212         if (IS_ERR(trans)) {
1213                 extent_clear_unlock_delalloc(inode,
1214                              &BTRFS_I(inode)->io_tree,
1215                              start, end, locked_page,
1216                              EXTENT_CLEAR_UNLOCK_PAGE |
1217                              EXTENT_CLEAR_UNLOCK |
1218                              EXTENT_CLEAR_DELALLOC |
1219                              EXTENT_CLEAR_DIRTY |
1220                              EXTENT_SET_WRITEBACK |
1221                              EXTENT_END_WRITEBACK);
1222                 btrfs_free_path(path);
1223                 return PTR_ERR(trans);
1224         }
1225
1226         trans->block_rsv = &root->fs_info->delalloc_block_rsv;
1227
1228         cow_start = (u64)-1;
1229         cur_offset = start;
1230         while (1) {
1231                 ret = btrfs_lookup_file_extent(trans, root, path, ino,
1232                                                cur_offset, 0);
1233                 if (ret < 0) {
1234                         btrfs_abort_transaction(trans, root, ret);
1235                         goto error;
1236                 }
1237                 if (ret > 0 && path->slots[0] > 0 && check_prev) {
1238                         leaf = path->nodes[0];
1239                         btrfs_item_key_to_cpu(leaf, &found_key,
1240                                               path->slots[0] - 1);
1241                         if (found_key.objectid == ino &&
1242                             found_key.type == BTRFS_EXTENT_DATA_KEY)
1243                                 path->slots[0]--;
1244                 }
1245                 check_prev = 0;
1246 next_slot:
1247                 leaf = path->nodes[0];
1248                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
1249                         ret = btrfs_next_leaf(root, path);
1250                         if (ret < 0) {
1251                                 btrfs_abort_transaction(trans, root, ret);
1252                                 goto error;
1253                         }
1254                         if (ret > 0)
1255                                 break;
1256                         leaf = path->nodes[0];
1257                 }
1258
1259                 nocow = 0;
1260                 disk_bytenr = 0;
1261                 num_bytes = 0;
1262                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1263
1264                 if (found_key.objectid > ino ||
1265                     found_key.type > BTRFS_EXTENT_DATA_KEY ||
1266                     found_key.offset > end)
1267                         break;
1268
1269                 if (found_key.offset > cur_offset) {
1270                         extent_end = found_key.offset;
1271                         extent_type = 0;
1272                         goto out_check;
1273                 }
1274
1275                 fi = btrfs_item_ptr(leaf, path->slots[0],
1276                                     struct btrfs_file_extent_item);
1277                 extent_type = btrfs_file_extent_type(leaf, fi);
1278
1279                 if (extent_type == BTRFS_FILE_EXTENT_REG ||
1280                     extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1281                         disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1282                         extent_offset = btrfs_file_extent_offset(leaf, fi);
1283                         extent_end = found_key.offset +
1284                                 btrfs_file_extent_num_bytes(leaf, fi);
1285                         disk_num_bytes =
1286                                 btrfs_file_extent_disk_num_bytes(leaf, fi);
1287                         if (extent_end <= start) {
1288                                 path->slots[0]++;
1289                                 goto next_slot;
1290                         }
1291                         if (disk_bytenr == 0)
1292                                 goto out_check;
1293                         if (btrfs_file_extent_compression(leaf, fi) ||
1294                             btrfs_file_extent_encryption(leaf, fi) ||
1295                             btrfs_file_extent_other_encoding(leaf, fi))
1296                                 goto out_check;
1297                         if (extent_type == BTRFS_FILE_EXTENT_REG && !force)
1298                                 goto out_check;
1299                         if (btrfs_extent_readonly(root, disk_bytenr))
1300                                 goto out_check;
1301                         if (btrfs_cross_ref_exist(trans, root, ino,
1302                                                   found_key.offset -
1303                                                   extent_offset, disk_bytenr))
1304                                 goto out_check;
1305                         disk_bytenr += extent_offset;
1306                         disk_bytenr += cur_offset - found_key.offset;
1307                         num_bytes = min(end + 1, extent_end) - cur_offset;
1308                         /*
1309                          * force cow if csum exists in the range.
1310                          * this ensure that csum for a given extent are
1311                          * either valid or do not exist.
1312                          */
1313                         if (csum_exist_in_range(root, disk_bytenr, num_bytes))
1314                                 goto out_check;
1315                         nocow = 1;
1316                 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
1317                         extent_end = found_key.offset +
1318                                 btrfs_file_extent_inline_len(leaf, fi);
1319                         extent_end = ALIGN(extent_end, root->sectorsize);
1320                 } else {
1321                         BUG_ON(1);
1322                 }
1323 out_check:
1324                 if (extent_end <= start) {
1325                         path->slots[0]++;
1326                         goto next_slot;
1327                 }
1328                 if (!nocow) {
1329                         if (cow_start == (u64)-1)
1330                                 cow_start = cur_offset;
1331                         cur_offset = extent_end;
1332                         if (cur_offset > end)
1333                                 break;
1334                         path->slots[0]++;
1335                         goto next_slot;
1336                 }
1337
1338                 btrfs_release_path(path);
1339                 if (cow_start != (u64)-1) {
1340                         ret = __cow_file_range(trans, inode, root, locked_page,
1341                                                cow_start, found_key.offset - 1,
1342                                                page_started, nr_written, 1);
1343                         if (ret) {
1344                                 btrfs_abort_transaction(trans, root, ret);
1345                                 goto error;
1346                         }
1347                         cow_start = (u64)-1;
1348                 }
1349
1350                 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1351                         struct extent_map *em;
1352                         struct extent_map_tree *em_tree;
1353                         em_tree = &BTRFS_I(inode)->extent_tree;
1354                         em = alloc_extent_map();
1355                         BUG_ON(!em); /* -ENOMEM */
1356                         em->start = cur_offset;
1357                         em->orig_start = found_key.offset - extent_offset;
1358                         em->len = num_bytes;
1359                         em->block_len = num_bytes;
1360                         em->block_start = disk_bytenr;
1361                         em->orig_block_len = disk_num_bytes;
1362                         em->bdev = root->fs_info->fs_devices->latest_bdev;
1363                         em->mod_start = em->start;
1364                         em->mod_len = em->len;
1365                         set_bit(EXTENT_FLAG_PINNED, &em->flags);
1366                         set_bit(EXTENT_FLAG_FILLING, &em->flags);
1367                         em->generation = -1;
1368                         while (1) {
1369                                 write_lock(&em_tree->lock);
1370                                 ret = add_extent_mapping(em_tree, em);
1371                                 if (!ret)
1372                                         list_move(&em->list,
1373                                                   &em_tree->modified_extents);
1374                                 write_unlock(&em_tree->lock);
1375                                 if (ret != -EEXIST) {
1376                                         free_extent_map(em);
1377                                         break;
1378                                 }
1379                                 btrfs_drop_extent_cache(inode, em->start,
1380                                                 em->start + em->len - 1, 0);
1381                         }
1382                         type = BTRFS_ORDERED_PREALLOC;
1383                 } else {
1384                         type = BTRFS_ORDERED_NOCOW;
1385                 }
1386
1387                 ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr,
1388                                                num_bytes, num_bytes, type);
1389                 BUG_ON(ret); /* -ENOMEM */
1390
1391                 if (root->root_key.objectid ==
1392                     BTRFS_DATA_RELOC_TREE_OBJECTID) {
1393                         ret = btrfs_reloc_clone_csums(inode, cur_offset,
1394                                                       num_bytes);
1395                         if (ret) {
1396                                 btrfs_abort_transaction(trans, root, ret);
1397                                 goto error;
1398                         }
1399                 }
1400
1401                 extent_clear_unlock_delalloc(inode, &BTRFS_I(inode)->io_tree,
1402                                 cur_offset, cur_offset + num_bytes - 1,
1403                                 locked_page, EXTENT_CLEAR_UNLOCK_PAGE |
1404                                 EXTENT_CLEAR_UNLOCK | EXTENT_CLEAR_DELALLOC |
1405                                 EXTENT_SET_PRIVATE2);
1406                 cur_offset = extent_end;
1407                 if (cur_offset > end)
1408                         break;
1409         }
1410         btrfs_release_path(path);
1411
1412         if (cur_offset <= end && cow_start == (u64)-1) {
1413                 cow_start = cur_offset;
1414                 cur_offset = end;
1415         }
1416
1417         if (cow_start != (u64)-1) {
1418                 ret = __cow_file_range(trans, inode, root, locked_page,
1419                                        cow_start, end,
1420                                        page_started, nr_written, 1);
1421                 if (ret) {
1422                         btrfs_abort_transaction(trans, root, ret);
1423                         goto error;
1424                 }
1425         }
1426
1427 error:
1428         err = btrfs_end_transaction(trans, root);
1429         if (!ret)
1430                 ret = err;
1431
1432         if (ret && cur_offset < end)
1433                 extent_clear_unlock_delalloc(inode,
1434                              &BTRFS_I(inode)->io_tree,
1435                              cur_offset, end, locked_page,
1436                              EXTENT_CLEAR_UNLOCK_PAGE |
1437                              EXTENT_CLEAR_UNLOCK |
1438                              EXTENT_CLEAR_DELALLOC |
1439                              EXTENT_CLEAR_DIRTY |
1440                              EXTENT_SET_WRITEBACK |
1441                              EXTENT_END_WRITEBACK);
1442
1443         btrfs_free_path(path);
1444         return ret;
1445 }
1446
1447 /*
1448  * extent_io.c call back to do delayed allocation processing
1449  */
1450 static int run_delalloc_range(struct inode *inode, struct page *locked_page,
1451                               u64 start, u64 end, int *page_started,
1452                               unsigned long *nr_written)
1453 {
1454         int ret;
1455         struct btrfs_root *root = BTRFS_I(inode)->root;
1456
1457         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) {
1458                 ret = run_delalloc_nocow(inode, locked_page, start, end,
1459                                          page_started, 1, nr_written);
1460         } else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC) {
1461                 ret = run_delalloc_nocow(inode, locked_page, start, end,
1462                                          page_started, 0, nr_written);
1463         } else if (!btrfs_test_opt(root, COMPRESS) &&
1464                    !(BTRFS_I(inode)->force_compress) &&
1465                    !(BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS)) {
1466                 ret = cow_file_range(inode, locked_page, start, end,
1467                                       page_started, nr_written, 1);
1468         } else {
1469                 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
1470                         &BTRFS_I(inode)->runtime_flags);
1471                 ret = cow_file_range_async(inode, locked_page, start, end,
1472                                            page_started, nr_written);
1473         }
1474         return ret;
1475 }
1476
1477 static void btrfs_split_extent_hook(struct inode *inode,
1478                                     struct extent_state *orig, u64 split)
1479 {
1480         /* not delalloc, ignore it */
1481         if (!(orig->state & EXTENT_DELALLOC))
1482                 return;
1483
1484         spin_lock(&BTRFS_I(inode)->lock);
1485         BTRFS_I(inode)->outstanding_extents++;
1486         spin_unlock(&BTRFS_I(inode)->lock);
1487 }
1488
1489 /*
1490  * extent_io.c merge_extent_hook, used to track merged delayed allocation
1491  * extents so we can keep track of new extents that are just merged onto old
1492  * extents, such as when we are doing sequential writes, so we can properly
1493  * account for the metadata space we'll need.
1494  */
1495 static void btrfs_merge_extent_hook(struct inode *inode,
1496                                     struct extent_state *new,
1497                                     struct extent_state *other)
1498 {
1499         /* not delalloc, ignore it */
1500         if (!(other->state & EXTENT_DELALLOC))
1501                 return;
1502
1503         spin_lock(&BTRFS_I(inode)->lock);
1504         BTRFS_I(inode)->outstanding_extents--;
1505         spin_unlock(&BTRFS_I(inode)->lock);
1506 }
1507
1508 /*
1509  * extent_io.c set_bit_hook, used to track delayed allocation
1510  * bytes in this file, and to maintain the list of inodes that
1511  * have pending delalloc work to be done.
1512  */
1513 static void btrfs_set_bit_hook(struct inode *inode,
1514                                struct extent_state *state, int *bits)
1515 {
1516
1517         /*
1518          * set_bit and clear bit hooks normally require _irqsave/restore
1519          * but in this case, we are only testing for the DELALLOC
1520          * bit, which is only set or cleared with irqs on
1521          */
1522         if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1523                 struct btrfs_root *root = BTRFS_I(inode)->root;
1524                 u64 len = state->end + 1 - state->start;
1525                 bool do_list = !btrfs_is_free_space_inode(inode);
1526
1527                 if (*bits & EXTENT_FIRST_DELALLOC) {
1528                         *bits &= ~EXTENT_FIRST_DELALLOC;
1529                 } else {
1530                         spin_lock(&BTRFS_I(inode)->lock);
1531                         BTRFS_I(inode)->outstanding_extents++;
1532                         spin_unlock(&BTRFS_I(inode)->lock);
1533                 }
1534
1535                 __percpu_counter_add(&root->fs_info->delalloc_bytes, len,
1536                                      root->fs_info->delalloc_batch);
1537                 spin_lock(&BTRFS_I(inode)->lock);
1538                 BTRFS_I(inode)->delalloc_bytes += len;
1539                 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1540                                          &BTRFS_I(inode)->runtime_flags)) {
1541                         spin_lock(&root->fs_info->delalloc_lock);
1542                         if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
1543                                 list_add_tail(&BTRFS_I(inode)->delalloc_inodes,
1544                                               &root->fs_info->delalloc_inodes);
1545                                 set_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1546                                         &BTRFS_I(inode)->runtime_flags);
1547                         }
1548                         spin_unlock(&root->fs_info->delalloc_lock);
1549                 }
1550                 spin_unlock(&BTRFS_I(inode)->lock);
1551         }
1552 }
1553
1554 /*
1555  * extent_io.c clear_bit_hook, see set_bit_hook for why
1556  */
1557 static void btrfs_clear_bit_hook(struct inode *inode,
1558                                  struct extent_state *state, int *bits)
1559 {
1560         /*
1561          * set_bit and clear bit hooks normally require _irqsave/restore
1562          * but in this case, we are only testing for the DELALLOC
1563          * bit, which is only set or cleared with irqs on
1564          */
1565         if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1566                 struct btrfs_root *root = BTRFS_I(inode)->root;
1567                 u64 len = state->end + 1 - state->start;
1568                 bool do_list = !btrfs_is_free_space_inode(inode);
1569
1570                 if (*bits & EXTENT_FIRST_DELALLOC) {
1571                         *bits &= ~EXTENT_FIRST_DELALLOC;
1572                 } else if (!(*bits & EXTENT_DO_ACCOUNTING)) {
1573                         spin_lock(&BTRFS_I(inode)->lock);
1574                         BTRFS_I(inode)->outstanding_extents--;
1575                         spin_unlock(&BTRFS_I(inode)->lock);
1576                 }
1577
1578                 if (*bits & EXTENT_DO_ACCOUNTING)
1579                         btrfs_delalloc_release_metadata(inode, len);
1580
1581                 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
1582                     && do_list)
1583                         btrfs_free_reserved_data_space(inode, len);
1584
1585                 __percpu_counter_add(&root->fs_info->delalloc_bytes, -len,
1586                                      root->fs_info->delalloc_batch);
1587                 spin_lock(&BTRFS_I(inode)->lock);
1588                 BTRFS_I(inode)->delalloc_bytes -= len;
1589                 if (do_list && BTRFS_I(inode)->delalloc_bytes == 0 &&
1590                     test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1591                              &BTRFS_I(inode)->runtime_flags)) {
1592                         spin_lock(&root->fs_info->delalloc_lock);
1593                         if (!list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
1594                                 list_del_init(&BTRFS_I(inode)->delalloc_inodes);
1595                                 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1596                                           &BTRFS_I(inode)->runtime_flags);
1597                         }
1598                         spin_unlock(&root->fs_info->delalloc_lock);
1599                 }
1600                 spin_unlock(&BTRFS_I(inode)->lock);
1601         }
1602 }
1603
1604 /*
1605  * extent_io.c merge_bio_hook, this must check the chunk tree to make sure
1606  * we don't create bios that span stripes or chunks
1607  */
1608 int btrfs_merge_bio_hook(int rw, struct page *page, unsigned long offset,
1609                          size_t size, struct bio *bio,
1610                          unsigned long bio_flags)
1611 {
1612         struct btrfs_root *root = BTRFS_I(page->mapping->host)->root;
1613         u64 logical = (u64)bio->bi_sector << 9;
1614         u64 length = 0;
1615         u64 map_length;
1616         int ret;
1617
1618         if (bio_flags & EXTENT_BIO_COMPRESSED)
1619                 return 0;
1620
1621         length = bio->bi_size;
1622         map_length = length;
1623         ret = btrfs_map_block(root->fs_info, rw, logical,
1624                               &map_length, NULL, 0);
1625         /* Will always return 0 with map_multi == NULL */
1626         BUG_ON(ret < 0);
1627         if (map_length < length + size)
1628                 return 1;
1629         return 0;
1630 }
1631
1632 /*
1633  * in order to insert checksums into the metadata in large chunks,
1634  * we wait until bio submission time.   All the pages in the bio are
1635  * checksummed and sums are attached onto the ordered extent record.
1636  *
1637  * At IO completion time the cums attached on the ordered extent record
1638  * are inserted into the btree
1639  */
1640 static int __btrfs_submit_bio_start(struct inode *inode, int rw,
1641                                     struct bio *bio, int mirror_num,
1642                                     unsigned long bio_flags,
1643                                     u64 bio_offset)
1644 {
1645         struct btrfs_root *root = BTRFS_I(inode)->root;
1646         int ret = 0;
1647
1648         ret = btrfs_csum_one_bio(root, inode, bio, 0, 0);
1649         BUG_ON(ret); /* -ENOMEM */
1650         return 0;
1651 }
1652
1653 /*
1654  * in order to insert checksums into the metadata in large chunks,
1655  * we wait until bio submission time.   All the pages in the bio are
1656  * checksummed and sums are attached onto the ordered extent record.
1657  *
1658  * At IO completion time the cums attached on the ordered extent record
1659  * are inserted into the btree
1660  */
1661 static int __btrfs_submit_bio_done(struct inode *inode, int rw, struct bio *bio,
1662                           int mirror_num, unsigned long bio_flags,
1663                           u64 bio_offset)
1664 {
1665         struct btrfs_root *root = BTRFS_I(inode)->root;
1666         int ret;
1667
1668         ret = btrfs_map_bio(root, rw, bio, mirror_num, 1);
1669         if (ret)
1670                 bio_endio(bio, ret);
1671         return ret;
1672 }
1673
1674 /*
1675  * extent_io.c submission hook. This does the right thing for csum calculation
1676  * on write, or reading the csums from the tree before a read
1677  */
1678 static int btrfs_submit_bio_hook(struct inode *inode, int rw, struct bio *bio,
1679                           int mirror_num, unsigned long bio_flags,
1680                           u64 bio_offset)
1681 {
1682         struct btrfs_root *root = BTRFS_I(inode)->root;
1683         int ret = 0;
1684         int skip_sum;
1685         int metadata = 0;
1686         int async = !atomic_read(&BTRFS_I(inode)->sync_writers);
1687
1688         skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
1689
1690         if (btrfs_is_free_space_inode(inode))
1691                 metadata = 2;
1692
1693         if (!(rw & REQ_WRITE)) {
1694                 ret = btrfs_bio_wq_end_io(root->fs_info, bio, metadata);
1695                 if (ret)
1696                         goto out;
1697
1698                 if (bio_flags & EXTENT_BIO_COMPRESSED) {
1699                         ret = btrfs_submit_compressed_read(inode, bio,
1700                                                            mirror_num,
1701                                                            bio_flags);
1702                         goto out;
1703                 } else if (!skip_sum) {
1704                         ret = btrfs_lookup_bio_sums(root, inode, bio, NULL);
1705                         if (ret)
1706                                 goto out;
1707                 }
1708                 goto mapit;
1709         } else if (async && !skip_sum) {
1710                 /* csum items have already been cloned */
1711                 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
1712                         goto mapit;
1713                 /* we're doing a write, do the async checksumming */
1714                 ret = btrfs_wq_submit_bio(BTRFS_I(inode)->root->fs_info,
1715                                    inode, rw, bio, mirror_num,
1716                                    bio_flags, bio_offset,
1717                                    __btrfs_submit_bio_start,
1718                                    __btrfs_submit_bio_done);
1719                 goto out;
1720         } else if (!skip_sum) {
1721                 ret = btrfs_csum_one_bio(root, inode, bio, 0, 0);
1722                 if (ret)
1723                         goto out;
1724         }
1725
1726 mapit:
1727         ret = btrfs_map_bio(root, rw, bio, mirror_num, 0);
1728
1729 out:
1730         if (ret < 0)
1731                 bio_endio(bio, ret);
1732         return ret;
1733 }
1734
1735 /*
1736  * given a list of ordered sums record them in the inode.  This happens
1737  * at IO completion time based on sums calculated at bio submission time.
1738  */
1739 static noinline int add_pending_csums(struct btrfs_trans_handle *trans,
1740                              struct inode *inode, u64 file_offset,
1741                              struct list_head *list)
1742 {
1743         struct btrfs_ordered_sum *sum;
1744
1745         list_for_each_entry(sum, list, list) {
1746                 btrfs_csum_file_blocks(trans,
1747                        BTRFS_I(inode)->root->fs_info->csum_root, sum);
1748         }
1749         return 0;
1750 }
1751
1752 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end,
1753                               struct extent_state **cached_state)
1754 {
1755         WARN_ON((end & (PAGE_CACHE_SIZE - 1)) == 0);
1756         return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end,
1757                                    cached_state, GFP_NOFS);
1758 }
1759
1760 /* see btrfs_writepage_start_hook for details on why this is required */
1761 struct btrfs_writepage_fixup {
1762         struct page *page;
1763         struct btrfs_work work;
1764 };
1765
1766 static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
1767 {
1768         struct btrfs_writepage_fixup *fixup;
1769         struct btrfs_ordered_extent *ordered;
1770         struct extent_state *cached_state = NULL;
1771         struct page *page;
1772         struct inode *inode;
1773         u64 page_start;
1774         u64 page_end;
1775         int ret;
1776
1777         fixup = container_of(work, struct btrfs_writepage_fixup, work);
1778         page = fixup->page;
1779 again:
1780         lock_page(page);
1781         if (!page->mapping || !PageDirty(page) || !PageChecked(page)) {
1782                 ClearPageChecked(page);
1783                 goto out_page;
1784         }
1785
1786         inode = page->mapping->host;
1787         page_start = page_offset(page);
1788         page_end = page_offset(page) + PAGE_CACHE_SIZE - 1;
1789
1790         lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end, 0,
1791                          &cached_state);
1792
1793         /* already ordered? We're done */
1794         if (PagePrivate2(page))
1795                 goto out;
1796
1797         ordered = btrfs_lookup_ordered_extent(inode, page_start);
1798         if (ordered) {
1799                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start,
1800                                      page_end, &cached_state, GFP_NOFS);
1801                 unlock_page(page);
1802                 btrfs_start_ordered_extent(inode, ordered, 1);
1803                 btrfs_put_ordered_extent(ordered);
1804                 goto again;
1805         }
1806
1807         ret = btrfs_delalloc_reserve_space(inode, PAGE_CACHE_SIZE);
1808         if (ret) {
1809                 mapping_set_error(page->mapping, ret);
1810                 end_extent_writepage(page, ret, page_start, page_end);
1811                 ClearPageChecked(page);
1812                 goto out;
1813          }
1814
1815         btrfs_set_extent_delalloc(inode, page_start, page_end, &cached_state);
1816         ClearPageChecked(page);
1817         set_page_dirty(page);
1818 out:
1819         unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end,
1820                              &cached_state, GFP_NOFS);
1821 out_page:
1822         unlock_page(page);
1823         page_cache_release(page);
1824         kfree(fixup);
1825 }
1826
1827 /*
1828  * There are a few paths in the higher layers of the kernel that directly
1829  * set the page dirty bit without asking the filesystem if it is a
1830  * good idea.  This causes problems because we want to make sure COW
1831  * properly happens and the data=ordered rules are followed.
1832  *
1833  * In our case any range that doesn't have the ORDERED bit set
1834  * hasn't been properly setup for IO.  We kick off an async process
1835  * to fix it up.  The async helper will wait for ordered extents, set
1836  * the delalloc bit and make it safe to write the page.
1837  */
1838 static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end)
1839 {
1840         struct inode *inode = page->mapping->host;
1841         struct btrfs_writepage_fixup *fixup;
1842         struct btrfs_root *root = BTRFS_I(inode)->root;
1843
1844         /* this page is properly in the ordered list */
1845         if (TestClearPagePrivate2(page))
1846                 return 0;
1847
1848         if (PageChecked(page))
1849                 return -EAGAIN;
1850
1851         fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
1852         if (!fixup)
1853                 return -EAGAIN;
1854
1855         SetPageChecked(page);
1856         page_cache_get(page);
1857         fixup->work.func = btrfs_writepage_fixup_worker;
1858         fixup->page = page;
1859         btrfs_queue_worker(&root->fs_info->fixup_workers, &fixup->work);
1860         return -EBUSY;
1861 }
1862
1863 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
1864                                        struct inode *inode, u64 file_pos,
1865                                        u64 disk_bytenr, u64 disk_num_bytes,
1866                                        u64 num_bytes, u64 ram_bytes,
1867                                        u8 compression, u8 encryption,
1868                                        u16 other_encoding, int extent_type)
1869 {
1870         struct btrfs_root *root = BTRFS_I(inode)->root;
1871         struct btrfs_file_extent_item *fi;
1872         struct btrfs_path *path;
1873         struct extent_buffer *leaf;
1874         struct btrfs_key ins;
1875         int ret;
1876
1877         path = btrfs_alloc_path();
1878         if (!path)
1879                 return -ENOMEM;
1880
1881         path->leave_spinning = 1;
1882
1883         /*
1884          * we may be replacing one extent in the tree with another.
1885          * The new extent is pinned in the extent map, and we don't want
1886          * to drop it from the cache until it is completely in the btree.
1887          *
1888          * So, tell btrfs_drop_extents to leave this extent in the cache.
1889          * the caller is expected to unpin it and allow it to be merged
1890          * with the others.
1891          */
1892         ret = btrfs_drop_extents(trans, root, inode, file_pos,
1893                                  file_pos + num_bytes, 0);
1894         if (ret)
1895                 goto out;
1896
1897         ins.objectid = btrfs_ino(inode);
1898         ins.offset = file_pos;
1899         ins.type = BTRFS_EXTENT_DATA_KEY;
1900         ret = btrfs_insert_empty_item(trans, root, path, &ins, sizeof(*fi));
1901         if (ret)
1902                 goto out;
1903         leaf = path->nodes[0];
1904         fi = btrfs_item_ptr(leaf, path->slots[0],
1905                             struct btrfs_file_extent_item);
1906         btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1907         btrfs_set_file_extent_type(leaf, fi, extent_type);
1908         btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr);
1909         btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes);
1910         btrfs_set_file_extent_offset(leaf, fi, 0);
1911         btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
1912         btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes);
1913         btrfs_set_file_extent_compression(leaf, fi, compression);
1914         btrfs_set_file_extent_encryption(leaf, fi, encryption);
1915         btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding);
1916
1917         btrfs_mark_buffer_dirty(leaf);
1918         btrfs_release_path(path);
1919
1920         inode_add_bytes(inode, num_bytes);
1921
1922         ins.objectid = disk_bytenr;
1923         ins.offset = disk_num_bytes;
1924         ins.type = BTRFS_EXTENT_ITEM_KEY;
1925         ret = btrfs_alloc_reserved_file_extent(trans, root,
1926                                         root->root_key.objectid,
1927                                         btrfs_ino(inode), file_pos, &ins);
1928 out:
1929         btrfs_free_path(path);
1930
1931         return ret;
1932 }
1933
1934 /* snapshot-aware defrag */
1935 struct sa_defrag_extent_backref {
1936         struct rb_node node;
1937         struct old_sa_defrag_extent *old;
1938         u64 root_id;
1939         u64 inum;
1940         u64 file_pos;
1941         u64 extent_offset;
1942         u64 num_bytes;
1943         u64 generation;
1944 };
1945
1946 struct old_sa_defrag_extent {
1947         struct list_head list;
1948         struct new_sa_defrag_extent *new;
1949
1950         u64 extent_offset;
1951         u64 bytenr;
1952         u64 offset;
1953         u64 len;
1954         int count;
1955 };
1956
1957 struct new_sa_defrag_extent {
1958         struct rb_root root;
1959         struct list_head head;
1960         struct btrfs_path *path;
1961         struct inode *inode;
1962         u64 file_pos;
1963         u64 len;
1964         u64 bytenr;
1965         u64 disk_len;
1966         u8 compress_type;
1967 };
1968
1969 static int backref_comp(struct sa_defrag_extent_backref *b1,
1970                         struct sa_defrag_extent_backref *b2)
1971 {
1972         if (b1->root_id < b2->root_id)
1973                 return -1;
1974         else if (b1->root_id > b2->root_id)
1975                 return 1;
1976
1977         if (b1->inum < b2->inum)
1978                 return -1;
1979         else if (b1->inum > b2->inum)
1980                 return 1;
1981
1982         if (b1->file_pos < b2->file_pos)
1983                 return -1;
1984         else if (b1->file_pos > b2->file_pos)
1985                 return 1;
1986
1987         /*
1988          * [------------------------------] ===> (a range of space)
1989          *     |<--->|   |<---->| =============> (fs/file tree A)
1990          * |<---------------------------->| ===> (fs/file tree B)
1991          *
1992          * A range of space can refer to two file extents in one tree while
1993          * refer to only one file extent in another tree.
1994          *
1995          * So we may process a disk offset more than one time(two extents in A)
1996          * and locate at the same extent(one extent in B), then insert two same
1997          * backrefs(both refer to the extent in B).
1998          */
1999         return 0;
2000 }
2001
2002 static void backref_insert(struct rb_root *root,
2003                            struct sa_defrag_extent_backref *backref)
2004 {
2005         struct rb_node **p = &root->rb_node;
2006         struct rb_node *parent = NULL;
2007         struct sa_defrag_extent_backref *entry;
2008         int ret;
2009
2010         while (*p) {
2011                 parent = *p;
2012                 entry = rb_entry(parent, struct sa_defrag_extent_backref, node);
2013
2014                 ret = backref_comp(backref, entry);
2015                 if (ret < 0)
2016                         p = &(*p)->rb_left;
2017                 else
2018                         p = &(*p)->rb_right;
2019         }
2020
2021         rb_link_node(&backref->node, parent, p);
2022         rb_insert_color(&backref->node, root);
2023 }
2024
2025 /*
2026  * Note the backref might has changed, and in this case we just return 0.
2027  */
2028 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id,
2029                                        void *ctx)
2030 {
2031         struct btrfs_file_extent_item *extent;
2032         struct btrfs_fs_info *fs_info;
2033         struct old_sa_defrag_extent *old = ctx;
2034         struct new_sa_defrag_extent *new = old->new;
2035         struct btrfs_path *path = new->path;
2036         struct btrfs_key key;
2037         struct btrfs_root *root;
2038         struct sa_defrag_extent_backref *backref;
2039         struct extent_buffer *leaf;
2040         struct inode *inode = new->inode;
2041         int slot;
2042         int ret;
2043         u64 extent_offset;
2044         u64 num_bytes;
2045
2046         if (BTRFS_I(inode)->root->root_key.objectid == root_id &&
2047             inum == btrfs_ino(inode))
2048                 return 0;
2049
2050         key.objectid = root_id;
2051         key.type = BTRFS_ROOT_ITEM_KEY;
2052         key.offset = (u64)-1;
2053
2054         fs_info = BTRFS_I(inode)->root->fs_info;
2055         root = btrfs_read_fs_root_no_name(fs_info, &key);
2056         if (IS_ERR(root)) {
2057                 if (PTR_ERR(root) == -ENOENT)
2058                         return 0;
2059                 WARN_ON(1);
2060                 pr_debug("inum=%llu, offset=%llu, root_id=%llu\n",
2061                          inum, offset, root_id);
2062                 return PTR_ERR(root);
2063         }
2064
2065         key.objectid = inum;
2066         key.type = BTRFS_EXTENT_DATA_KEY;
2067         if (offset > (u64)-1 << 32)
2068                 key.offset = 0;
2069         else
2070                 key.offset = offset;
2071
2072         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2073         if (ret < 0) {
2074                 WARN_ON(1);
2075                 return ret;
2076         }
2077
2078         while (1) {
2079                 cond_resched();
2080
2081                 leaf = path->nodes[0];
2082                 slot = path->slots[0];
2083
2084                 if (slot >= btrfs_header_nritems(leaf)) {
2085                         ret = btrfs_next_leaf(root, path);
2086                         if (ret < 0) {
2087                                 goto out;
2088                         } else if (ret > 0) {
2089                                 ret = 0;
2090                                 goto out;
2091                         }
2092                         continue;
2093                 }
2094
2095                 path->slots[0]++;
2096
2097                 btrfs_item_key_to_cpu(leaf, &key, slot);
2098
2099                 if (key.objectid > inum)
2100                         goto out;
2101
2102                 if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY)
2103                         continue;
2104
2105                 extent = btrfs_item_ptr(leaf, slot,
2106                                         struct btrfs_file_extent_item);
2107
2108                 if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr)
2109                         continue;
2110
2111                 extent_offset = btrfs_file_extent_offset(leaf, extent);
2112                 if (key.offset - extent_offset != offset)
2113                         continue;
2114
2115                 num_bytes = btrfs_file_extent_num_bytes(leaf, extent);
2116                 if (extent_offset >= old->extent_offset + old->offset +
2117                     old->len || extent_offset + num_bytes <=
2118                     old->extent_offset + old->offset)
2119                         continue;
2120
2121                 break;
2122         }
2123
2124         backref = kmalloc(sizeof(*backref), GFP_NOFS);
2125         if (!backref) {
2126                 ret = -ENOENT;
2127                 goto out;
2128         }
2129
2130         backref->root_id = root_id;
2131         backref->inum = inum;
2132         backref->file_pos = offset + extent_offset;
2133         backref->num_bytes = num_bytes;
2134         backref->extent_offset = extent_offset;
2135         backref->generation = btrfs_file_extent_generation(leaf, extent);
2136         backref->old = old;
2137         backref_insert(&new->root, backref);
2138         old->count++;
2139 out:
2140         btrfs_release_path(path);
2141         WARN_ON(ret);
2142         return ret;
2143 }
2144
2145 static noinline bool record_extent_backrefs(struct btrfs_path *path,
2146                                    struct new_sa_defrag_extent *new)
2147 {
2148         struct btrfs_fs_info *fs_info = BTRFS_I(new->inode)->root->fs_info;
2149         struct old_sa_defrag_extent *old, *tmp;
2150         int ret;
2151
2152         new->path = path;
2153
2154         list_for_each_entry_safe(old, tmp, &new->head, list) {
2155                 ret = iterate_inodes_from_logical(old->bytenr, fs_info,
2156                                                   path, record_one_backref,
2157                                                   old);
2158                 BUG_ON(ret < 0 && ret != -ENOENT);
2159
2160                 /* no backref to be processed for this extent */
2161                 if (!old->count) {
2162                         list_del(&old->list);
2163                         kfree(old);
2164                 }
2165         }
2166
2167         if (list_empty(&new->head))
2168                 return false;
2169
2170         return true;
2171 }
2172
2173 static int relink_is_mergable(struct extent_buffer *leaf,
2174                               struct btrfs_file_extent_item *fi,
2175                               u64 disk_bytenr)
2176 {
2177         if (btrfs_file_extent_disk_bytenr(leaf, fi) != disk_bytenr)
2178                 return 0;
2179
2180         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2181                 return 0;
2182
2183         if (btrfs_file_extent_compression(leaf, fi) ||
2184             btrfs_file_extent_encryption(leaf, fi) ||
2185             btrfs_file_extent_other_encoding(leaf, fi))
2186                 return 0;
2187
2188         return 1;
2189 }
2190
2191 /*
2192  * Note the backref might has changed, and in this case we just return 0.
2193  */
2194 static noinline int relink_extent_backref(struct btrfs_path *path,
2195                                  struct sa_defrag_extent_backref *prev,
2196                                  struct sa_defrag_extent_backref *backref)
2197 {
2198         struct btrfs_file_extent_item *extent;
2199         struct btrfs_file_extent_item *item;
2200         struct btrfs_ordered_extent *ordered;
2201         struct btrfs_trans_handle *trans;
2202         struct btrfs_fs_info *fs_info;
2203         struct btrfs_root *root;
2204         struct btrfs_key key;
2205         struct extent_buffer *leaf;
2206         struct old_sa_defrag_extent *old = backref->old;
2207         struct new_sa_defrag_extent *new = old->new;
2208         struct inode *src_inode = new->inode;
2209         struct inode *inode;
2210         struct extent_state *cached = NULL;
2211         int ret = 0;
2212         u64 start;
2213         u64 len;
2214         u64 lock_start;
2215         u64 lock_end;
2216         bool merge = false;
2217         int index;
2218
2219         if (prev && prev->root_id == backref->root_id &&
2220             prev->inum == backref->inum &&
2221             prev->file_pos + prev->num_bytes == backref->file_pos)
2222                 merge = true;
2223
2224         /* step 1: get root */
2225         key.objectid = backref->root_id;
2226         key.type = BTRFS_ROOT_ITEM_KEY;
2227         key.offset = (u64)-1;
2228
2229         fs_info = BTRFS_I(src_inode)->root->fs_info;
2230         index = srcu_read_lock(&fs_info->subvol_srcu);
2231
2232         root = btrfs_read_fs_root_no_name(fs_info, &key);
2233         if (IS_ERR(root)) {
2234                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2235                 if (PTR_ERR(root) == -ENOENT)
2236                         return 0;
2237                 return PTR_ERR(root);
2238         }
2239         if (btrfs_root_refs(&root->root_item) == 0) {
2240                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2241                 /* parse ENOENT to 0 */
2242                 return 0;
2243         }
2244
2245         /* step 2: get inode */
2246         key.objectid = backref->inum;
2247         key.type = BTRFS_INODE_ITEM_KEY;
2248         key.offset = 0;
2249
2250         inode = btrfs_iget(fs_info->sb, &key, root, NULL);
2251         if (IS_ERR(inode)) {
2252                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2253                 return 0;
2254         }
2255
2256         srcu_read_unlock(&fs_info->subvol_srcu, index);
2257
2258         /* step 3: relink backref */
2259         lock_start = backref->file_pos;
2260         lock_end = backref->file_pos + backref->num_bytes - 1;
2261         lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2262                          0, &cached);
2263
2264         ordered = btrfs_lookup_first_ordered_extent(inode, lock_end);
2265         if (ordered) {
2266                 btrfs_put_ordered_extent(ordered);
2267                 goto out_unlock;
2268         }
2269
2270         trans = btrfs_join_transaction(root);
2271         if (IS_ERR(trans)) {
2272                 ret = PTR_ERR(trans);
2273                 goto out_unlock;
2274         }
2275
2276         key.objectid = backref->inum;
2277         key.type = BTRFS_EXTENT_DATA_KEY;
2278         key.offset = backref->file_pos;
2279
2280         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2281         if (ret < 0) {
2282                 goto out_free_path;
2283         } else if (ret > 0) {
2284                 ret = 0;
2285                 goto out_free_path;
2286         }
2287
2288         extent = btrfs_item_ptr(path->nodes[0], path->slots[0],
2289                                 struct btrfs_file_extent_item);
2290
2291         if (btrfs_file_extent_generation(path->nodes[0], extent) !=
2292             backref->generation)
2293                 goto out_free_path;
2294
2295         btrfs_release_path(path);
2296
2297         start = backref->file_pos;
2298         if (backref->extent_offset < old->extent_offset + old->offset)
2299                 start += old->extent_offset + old->offset -
2300                          backref->extent_offset;
2301
2302         len = min(backref->extent_offset + backref->num_bytes,
2303                   old->extent_offset + old->offset + old->len);
2304         len -= max(backref->extent_offset, old->extent_offset + old->offset);
2305
2306         ret = btrfs_drop_extents(trans, root, inode, start,
2307                                  start + len, 1);
2308         if (ret)
2309                 goto out_free_path;
2310 again:
2311         key.objectid = btrfs_ino(inode);
2312         key.type = BTRFS_EXTENT_DATA_KEY;
2313         key.offset = start;
2314
2315         path->leave_spinning = 1;
2316         if (merge) {
2317                 struct btrfs_file_extent_item *fi;
2318                 u64 extent_len;
2319                 struct btrfs_key found_key;
2320
2321                 ret = btrfs_search_slot(trans, root, &key, path, 1, 1);
2322                 if (ret < 0)
2323                         goto out_free_path;
2324
2325                 path->slots[0]--;
2326                 leaf = path->nodes[0];
2327                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
2328
2329                 fi = btrfs_item_ptr(leaf, path->slots[0],
2330                                     struct btrfs_file_extent_item);
2331                 extent_len = btrfs_file_extent_num_bytes(leaf, fi);
2332
2333                 if (relink_is_mergable(leaf, fi, new->bytenr) &&
2334                     extent_len + found_key.offset == start) {
2335                         btrfs_set_file_extent_num_bytes(leaf, fi,
2336                                                         extent_len + len);
2337                         btrfs_mark_buffer_dirty(leaf);
2338                         inode_add_bytes(inode, len);
2339
2340                         ret = 1;
2341                         goto out_free_path;
2342                 } else {
2343                         merge = false;
2344                         btrfs_release_path(path);
2345                         goto again;
2346                 }
2347         }
2348
2349         ret = btrfs_insert_empty_item(trans, root, path, &key,
2350                                         sizeof(*extent));
2351         if (ret) {
2352                 btrfs_abort_transaction(trans, root, ret);
2353                 goto out_free_path;
2354         }
2355
2356         leaf = path->nodes[0];
2357         item = btrfs_item_ptr(leaf, path->slots[0],
2358                                 struct btrfs_file_extent_item);
2359         btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr);
2360         btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len);
2361         btrfs_set_file_extent_offset(leaf, item, start - new->file_pos);
2362         btrfs_set_file_extent_num_bytes(leaf, item, len);
2363         btrfs_set_file_extent_ram_bytes(leaf, item, new->len);
2364         btrfs_set_file_extent_generation(leaf, item, trans->transid);
2365         btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
2366         btrfs_set_file_extent_compression(leaf, item, new->compress_type);
2367         btrfs_set_file_extent_encryption(leaf, item, 0);
2368         btrfs_set_file_extent_other_encoding(leaf, item, 0);
2369
2370         btrfs_mark_buffer_dirty(leaf);
2371         inode_add_bytes(inode, len);
2372         btrfs_release_path(path);
2373
2374         ret = btrfs_inc_extent_ref(trans, root, new->bytenr,
2375                         new->disk_len, 0,
2376                         backref->root_id, backref->inum,
2377                         new->file_pos, 0);      /* start - extent_offset */
2378         if (ret) {
2379                 btrfs_abort_transaction(trans, root, ret);
2380                 goto out_free_path;
2381         }
2382
2383         ret = 1;
2384 out_free_path:
2385         btrfs_release_path(path);
2386         path->leave_spinning = 0;
2387         btrfs_end_transaction(trans, root);
2388 out_unlock:
2389         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2390                              &cached, GFP_NOFS);
2391         iput(inode);
2392         return ret;
2393 }
2394
2395 static void relink_file_extents(struct new_sa_defrag_extent *new)
2396 {
2397         struct btrfs_path *path;
2398         struct old_sa_defrag_extent *old, *tmp;
2399         struct sa_defrag_extent_backref *backref;
2400         struct sa_defrag_extent_backref *prev = NULL;
2401         struct inode *inode;
2402         struct btrfs_root *root;
2403         struct rb_node *node;
2404         int ret;
2405
2406         inode = new->inode;
2407         root = BTRFS_I(inode)->root;
2408
2409         path = btrfs_alloc_path();
2410         if (!path)
2411                 return;
2412
2413         if (!record_extent_backrefs(path, new)) {
2414                 btrfs_free_path(path);
2415                 goto out;
2416         }
2417         btrfs_release_path(path);
2418
2419         while (1) {
2420                 node = rb_first(&new->root);
2421                 if (!node)
2422                         break;
2423                 rb_erase(node, &new->root);
2424
2425                 backref = rb_entry(node, struct sa_defrag_extent_backref, node);
2426
2427                 ret = relink_extent_backref(path, prev, backref);
2428                 WARN_ON(ret < 0);
2429
2430                 kfree(prev);
2431
2432                 if (ret == 1)
2433                         prev = backref;
2434                 else
2435                         prev = NULL;
2436                 cond_resched();
2437         }
2438         kfree(prev);
2439
2440         btrfs_free_path(path);
2441
2442         list_for_each_entry_safe(old, tmp, &new->head, list) {
2443                 list_del(&old->list);
2444                 kfree(old);
2445         }
2446 out:
2447         atomic_dec(&root->fs_info->defrag_running);
2448         wake_up(&root->fs_info->transaction_wait);
2449
2450         kfree(new);
2451 }
2452
2453 static struct new_sa_defrag_extent *
2454 record_old_file_extents(struct inode *inode,
2455                         struct btrfs_ordered_extent *ordered)
2456 {
2457         struct btrfs_root *root = BTRFS_I(inode)->root;
2458         struct btrfs_path *path;
2459         struct btrfs_key key;
2460         struct old_sa_defrag_extent *old, *tmp;
2461         struct new_sa_defrag_extent *new;
2462         int ret;
2463
2464         new = kmalloc(sizeof(*new), GFP_NOFS);
2465         if (!new)
2466                 return NULL;
2467
2468         new->inode = inode;
2469         new->file_pos = ordered->file_offset;
2470         new->len = ordered->len;
2471         new->bytenr = ordered->start;
2472         new->disk_len = ordered->disk_len;
2473         new->compress_type = ordered->compress_type;
2474         new->root = RB_ROOT;
2475         INIT_LIST_HEAD(&new->head);
2476
2477         path = btrfs_alloc_path();
2478         if (!path)
2479                 goto out_kfree;
2480
2481         key.objectid = btrfs_ino(inode);
2482         key.type = BTRFS_EXTENT_DATA_KEY;
2483         key.offset = new->file_pos;
2484
2485         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2486         if (ret < 0)
2487                 goto out_free_path;
2488         if (ret > 0 && path->slots[0] > 0)
2489                 path->slots[0]--;
2490
2491         /* find out all the old extents for the file range */
2492         while (1) {
2493                 struct btrfs_file_extent_item *extent;
2494                 struct extent_buffer *l;
2495                 int slot;
2496                 u64 num_bytes;
2497                 u64 offset;
2498                 u64 end;
2499                 u64 disk_bytenr;
2500                 u64 extent_offset;
2501
2502                 l = path->nodes[0];
2503                 slot = path->slots[0];
2504
2505                 if (slot >= btrfs_header_nritems(l)) {
2506                         ret = btrfs_next_leaf(root, path);
2507                         if (ret < 0)
2508                                 goto out_free_list;
2509                         else if (ret > 0)
2510                                 break;
2511                         continue;
2512                 }
2513
2514                 btrfs_item_key_to_cpu(l, &key, slot);
2515
2516                 if (key.objectid != btrfs_ino(inode))
2517                         break;
2518                 if (key.type != BTRFS_EXTENT_DATA_KEY)
2519                         break;
2520                 if (key.offset >= new->file_pos + new->len)
2521                         break;
2522
2523                 extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item);
2524
2525                 num_bytes = btrfs_file_extent_num_bytes(l, extent);
2526                 if (key.offset + num_bytes < new->file_pos)
2527                         goto next;
2528
2529                 disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent);
2530                 if (!disk_bytenr)
2531                         goto next;
2532
2533                 extent_offset = btrfs_file_extent_offset(l, extent);
2534
2535                 old = kmalloc(sizeof(*old), GFP_NOFS);
2536                 if (!old)
2537                         goto out_free_list;
2538
2539                 offset = max(new->file_pos, key.offset);
2540                 end = min(new->file_pos + new->len, key.offset + num_bytes);
2541
2542                 old->bytenr = disk_bytenr;
2543                 old->extent_offset = extent_offset;
2544                 old->offset = offset - key.offset;
2545                 old->len = end - offset;
2546                 old->new = new;
2547                 old->count = 0;
2548                 list_add_tail(&old->list, &new->head);
2549 next:
2550                 path->slots[0]++;
2551                 cond_resched();
2552         }
2553
2554         btrfs_free_path(path);
2555         atomic_inc(&root->fs_info->defrag_running);
2556
2557         return new;
2558
2559 out_free_list:
2560         list_for_each_entry_safe(old, tmp, &new->head, list) {
2561                 list_del(&old->list);
2562                 kfree(old);
2563         }
2564 out_free_path:
2565         btrfs_free_path(path);
2566 out_kfree:
2567         kfree(new);
2568         return NULL;
2569 }
2570
2571 /*
2572  * helper function for btrfs_finish_ordered_io, this
2573  * just reads in some of the csum leaves to prime them into ram
2574  * before we start the transaction.  It limits the amount of btree
2575  * reads required while inside the transaction.
2576  */
2577 /* as ordered data IO finishes, this gets called so we can finish
2578  * an ordered extent if the range of bytes in the file it covers are
2579  * fully written.
2580  */
2581 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
2582 {
2583         struct inode *inode = ordered_extent->inode;
2584         struct btrfs_root *root = BTRFS_I(inode)->root;
2585         struct btrfs_trans_handle *trans = NULL;
2586         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
2587         struct extent_state *cached_state = NULL;
2588         struct new_sa_defrag_extent *new = NULL;
2589         int compress_type = 0;
2590         int ret;
2591         bool nolock;
2592
2593         nolock = btrfs_is_free_space_inode(inode);
2594
2595         if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
2596                 ret = -EIO;
2597                 goto out;
2598         }
2599
2600         if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
2601                 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */
2602                 btrfs_ordered_update_i_size(inode, 0, ordered_extent);
2603                 if (nolock)
2604                         trans = btrfs_join_transaction_nolock(root);
2605                 else
2606                         trans = btrfs_join_transaction(root);
2607                 if (IS_ERR(trans)) {
2608                         ret = PTR_ERR(trans);
2609                         trans = NULL;
2610                         goto out;
2611                 }
2612                 trans->block_rsv = &root->fs_info->delalloc_block_rsv;
2613                 ret = btrfs_update_inode_fallback(trans, root, inode);
2614                 if (ret) /* -ENOMEM or corruption */
2615                         btrfs_abort_transaction(trans, root, ret);
2616                 goto out;
2617         }
2618
2619         lock_extent_bits(io_tree, ordered_extent->file_offset,
2620                          ordered_extent->file_offset + ordered_extent->len - 1,
2621                          0, &cached_state);
2622
2623         ret = test_range_bit(io_tree, ordered_extent->file_offset,
2624                         ordered_extent->file_offset + ordered_extent->len - 1,
2625                         EXTENT_DEFRAG, 1, cached_state);
2626         if (ret) {
2627                 u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item);
2628                 if (last_snapshot >= BTRFS_I(inode)->generation)
2629                         /* the inode is shared */
2630                         new = record_old_file_extents(inode, ordered_extent);
2631
2632                 clear_extent_bit(io_tree, ordered_extent->file_offset,
2633                         ordered_extent->file_offset + ordered_extent->len - 1,
2634                         EXTENT_DEFRAG, 0, 0, &cached_state, GFP_NOFS);
2635         }
2636
2637         if (nolock)
2638                 trans = btrfs_join_transaction_nolock(root);
2639         else
2640                 trans = btrfs_join_transaction(root);
2641         if (IS_ERR(trans)) {
2642                 ret = PTR_ERR(trans);
2643                 trans = NULL;
2644                 goto out_unlock;
2645         }
2646         trans->block_rsv = &root->fs_info->delalloc_block_rsv;
2647
2648         if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
2649                 compress_type = ordered_extent->compress_type;
2650         if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
2651                 BUG_ON(compress_type);
2652                 ret = btrfs_mark_extent_written(trans, inode,
2653                                                 ordered_extent->file_offset,
2654                                                 ordered_extent->file_offset +
2655                                                 ordered_extent->len);
2656         } else {
2657                 BUG_ON(root == root->fs_info->tree_root);
2658                 ret = insert_reserved_file_extent(trans, inode,
2659                                                 ordered_extent->file_offset,
2660                                                 ordered_extent->start,
2661                                                 ordered_extent->disk_len,
2662                                                 ordered_extent->len,
2663                                                 ordered_extent->len,
2664                                                 compress_type, 0, 0,
2665                                                 BTRFS_FILE_EXTENT_REG);
2666         }
2667         unpin_extent_cache(&BTRFS_I(inode)->extent_tree,
2668                            ordered_extent->file_offset, ordered_extent->len,
2669                            trans->transid);
2670         if (ret < 0) {
2671                 btrfs_abort_transaction(trans, root, ret);
2672                 goto out_unlock;
2673         }
2674
2675         add_pending_csums(trans, inode, ordered_extent->file_offset,
2676                           &ordered_extent->list);
2677
2678         btrfs_ordered_update_i_size(inode, 0, ordered_extent);
2679         ret = btrfs_update_inode_fallback(trans, root, inode);
2680         if (ret) { /* -ENOMEM or corruption */
2681                 btrfs_abort_transaction(trans, root, ret);
2682                 goto out_unlock;
2683         }
2684         ret = 0;
2685 out_unlock:
2686         unlock_extent_cached(io_tree, ordered_extent->file_offset,
2687                              ordered_extent->file_offset +
2688                              ordered_extent->len - 1, &cached_state, GFP_NOFS);
2689 out:
2690         if (root != root->fs_info->tree_root)
2691                 btrfs_delalloc_release_metadata(inode, ordered_extent->len);
2692         if (trans)
2693                 btrfs_end_transaction(trans, root);
2694
2695         if (ret) {
2696                 clear_extent_uptodate(io_tree, ordered_extent->file_offset,
2697                                       ordered_extent->file_offset +
2698                                       ordered_extent->len - 1, NULL, GFP_NOFS);
2699
2700                 /*
2701                  * If the ordered extent had an IOERR or something else went
2702                  * wrong we need to return the space for this ordered extent
2703                  * back to the allocator.
2704                  */
2705                 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
2706                     !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags))
2707                         btrfs_free_reserved_extent(root, ordered_extent->start,
2708                                                    ordered_extent->disk_len);
2709         }
2710
2711
2712         /*
2713          * This needs to be done to make sure anybody waiting knows we are done
2714          * updating everything for this ordered extent.
2715          */
2716         btrfs_remove_ordered_extent(inode, ordered_extent);
2717
2718         /* for snapshot-aware defrag */
2719         if (new)
2720                 relink_file_extents(new);
2721
2722         /* once for us */
2723         btrfs_put_ordered_extent(ordered_extent);
2724         /* once for the tree */
2725         btrfs_put_ordered_extent(ordered_extent);
2726
2727         return ret;
2728 }
2729
2730 static void finish_ordered_fn(struct btrfs_work *work)
2731 {
2732         struct btrfs_ordered_extent *ordered_extent;
2733         ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
2734         btrfs_finish_ordered_io(ordered_extent);
2735 }
2736
2737 static int btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end,
2738                                 struct extent_state *state, int uptodate)
2739 {
2740         struct inode *inode = page->mapping->host;
2741         struct btrfs_root *root = BTRFS_I(inode)->root;
2742         struct btrfs_ordered_extent *ordered_extent = NULL;
2743         struct btrfs_workers *workers;
2744
2745         trace_btrfs_writepage_end_io_hook(page, start, end, uptodate);
2746
2747         ClearPagePrivate2(page);
2748         if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start,
2749                                             end - start + 1, uptodate))
2750                 return 0;
2751
2752         ordered_extent->work.func = finish_ordered_fn;
2753         ordered_extent->work.flags = 0;
2754
2755         if (btrfs_is_free_space_inode(inode))
2756                 workers = &root->fs_info->endio_freespace_worker;
2757         else
2758                 workers = &root->fs_info->endio_write_workers;
2759         btrfs_queue_worker(workers, &ordered_extent->work);
2760
2761         return 0;
2762 }
2763
2764 /*
2765  * when reads are done, we need to check csums to verify the data is correct
2766  * if there's a match, we allow the bio to finish.  If not, the code in
2767  * extent_io.c will try to find good copies for us.
2768  */
2769 static int btrfs_readpage_end_io_hook(struct page *page, u64 start, u64 end,
2770                                struct extent_state *state, int mirror)
2771 {
2772         size_t offset = start - page_offset(page);
2773         struct inode *inode = page->mapping->host;
2774         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
2775         char *kaddr;
2776         u64 private = ~(u32)0;
2777         int ret;
2778         struct btrfs_root *root = BTRFS_I(inode)->root;
2779         u32 csum = ~(u32)0;
2780
2781         if (PageChecked(page)) {
2782                 ClearPageChecked(page);
2783                 goto good;
2784         }
2785
2786         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
2787                 goto good;
2788
2789         if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID &&
2790             test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) {
2791                 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM,
2792                                   GFP_NOFS);
2793                 return 0;
2794         }
2795
2796         if (state && state->start == start) {
2797                 private = state->private;
2798                 ret = 0;
2799         } else {
2800                 ret = get_state_private(io_tree, start, &private);
2801         }
2802         kaddr = kmap_atomic(page);
2803         if (ret)
2804                 goto zeroit;
2805
2806         csum = btrfs_csum_data(root, kaddr + offset, csum,  end - start + 1);
2807         btrfs_csum_final(csum, (char *)&csum);
2808         if (csum != private)
2809                 goto zeroit;
2810
2811         kunmap_atomic(kaddr);
2812 good:
2813         return 0;
2814
2815 zeroit:
2816         printk_ratelimited(KERN_INFO "btrfs csum failed ino %llu off %llu csum %u "
2817                        "private %llu\n",
2818                        (unsigned long long)btrfs_ino(page->mapping->host),
2819                        (unsigned long long)start, csum,
2820                        (unsigned long long)private);
2821         memset(kaddr + offset, 1, end - start + 1);
2822         flush_dcache_page(page);
2823         kunmap_atomic(kaddr);
2824         if (private == 0)
2825                 return 0;
2826         return -EIO;
2827 }
2828
2829 struct delayed_iput {
2830         struct list_head list;
2831         struct inode *inode;
2832 };
2833
2834 /* JDM: If this is fs-wide, why can't we add a pointer to
2835  * btrfs_inode instead and avoid the allocation? */
2836 void btrfs_add_delayed_iput(struct inode *inode)
2837 {
2838         struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2839         struct delayed_iput *delayed;
2840
2841         if (atomic_add_unless(&inode->i_count, -1, 1))
2842                 return;
2843
2844         delayed = kmalloc(sizeof(*delayed), GFP_NOFS | __GFP_NOFAIL);
2845         delayed->inode = inode;
2846
2847         spin_lock(&fs_info->delayed_iput_lock);
2848         list_add_tail(&delayed->list, &fs_info->delayed_iputs);
2849         spin_unlock(&fs_info->delayed_iput_lock);
2850 }
2851
2852 void btrfs_run_delayed_iputs(struct btrfs_root *root)
2853 {
2854         LIST_HEAD(list);
2855         struct btrfs_fs_info *fs_info = root->fs_info;
2856         struct delayed_iput *delayed;
2857         int empty;
2858
2859         spin_lock(&fs_info->delayed_iput_lock);
2860         empty = list_empty(&fs_info->delayed_iputs);
2861         spin_unlock(&fs_info->delayed_iput_lock);
2862         if (empty)
2863                 return;
2864
2865         spin_lock(&fs_info->delayed_iput_lock);
2866         list_splice_init(&fs_info->delayed_iputs, &list);
2867         spin_unlock(&fs_info->delayed_iput_lock);
2868
2869         while (!list_empty(&list)) {
2870                 delayed = list_entry(list.next, struct delayed_iput, list);
2871                 list_del(&delayed->list);
2872                 iput(delayed->inode);
2873                 kfree(delayed);
2874         }
2875 }
2876
2877 /*
2878  * This is called in transaction commit time. If there are no orphan
2879  * files in the subvolume, it removes orphan item and frees block_rsv
2880  * structure.
2881  */
2882 void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans,
2883                               struct btrfs_root *root)
2884 {
2885         struct btrfs_block_rsv *block_rsv;
2886         int ret;
2887
2888         if (atomic_read(&root->orphan_inodes) ||
2889             root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE)
2890                 return;
2891
2892         spin_lock(&root->orphan_lock);
2893         if (atomic_read(&root->orphan_inodes)) {
2894                 spin_unlock(&root->orphan_lock);
2895                 return;
2896         }
2897
2898         if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) {
2899                 spin_unlock(&root->orphan_lock);
2900                 return;
2901         }
2902
2903         block_rsv = root->orphan_block_rsv;
2904         root->orphan_block_rsv = NULL;
2905         spin_unlock(&root->orphan_lock);
2906
2907         if (root->orphan_item_inserted &&
2908             btrfs_root_refs(&root->root_item) > 0) {
2909                 ret = btrfs_del_orphan_item(trans, root->fs_info->tree_root,
2910                                             root->root_key.objectid);
2911                 BUG_ON(ret);
2912                 root->orphan_item_inserted = 0;
2913         }
2914
2915         if (block_rsv) {
2916                 WARN_ON(block_rsv->size > 0);
2917                 btrfs_free_block_rsv(root, block_rsv);
2918         }
2919 }
2920
2921 /*
2922  * This creates an orphan entry for the given inode in case something goes
2923  * wrong in the middle of an unlink/truncate.
2924  *
2925  * NOTE: caller of this function should reserve 5 units of metadata for
2926  *       this function.
2927  */
2928 int btrfs_orphan_add(struct btrfs_trans_handle *trans, struct inode *inode)
2929 {
2930         struct btrfs_root *root = BTRFS_I(inode)->root;
2931         struct btrfs_block_rsv *block_rsv = NULL;
2932         int reserve = 0;
2933         int insert = 0;
2934         int ret;
2935
2936         if (!root->orphan_block_rsv) {
2937                 block_rsv = btrfs_alloc_block_rsv(root, BTRFS_BLOCK_RSV_TEMP);
2938                 if (!block_rsv)
2939                         return -ENOMEM;
2940         }
2941
2942         spin_lock(&root->orphan_lock);
2943         if (!root->orphan_block_rsv) {
2944                 root->orphan_block_rsv = block_rsv;
2945         } else if (block_rsv) {
2946                 btrfs_free_block_rsv(root, block_rsv);
2947                 block_rsv = NULL;
2948         }
2949
2950         if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
2951                               &BTRFS_I(inode)->runtime_flags)) {
2952 #if 0
2953                 /*
2954                  * For proper ENOSPC handling, we should do orphan
2955                  * cleanup when mounting. But this introduces backward
2956                  * compatibility issue.
2957                  */
2958                 if (!xchg(&root->orphan_item_inserted, 1))
2959                         insert = 2;
2960                 else
2961                         insert = 1;
2962 #endif
2963                 insert = 1;
2964                 atomic_inc(&root->orphan_inodes);
2965         }
2966
2967         if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
2968                               &BTRFS_I(inode)->runtime_flags))
2969                 reserve = 1;
2970         spin_unlock(&root->orphan_lock);
2971
2972         /* grab metadata reservation from transaction handle */
2973         if (reserve) {
2974                 ret = btrfs_orphan_reserve_metadata(trans, inode);
2975                 BUG_ON(ret); /* -ENOSPC in reservation; Logic error? JDM */
2976         }
2977
2978         /* insert an orphan item to track this unlinked/truncated file */
2979         if (insert >= 1) {
2980                 ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode));
2981                 if (ret && ret != -EEXIST) {
2982                         clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
2983                                   &BTRFS_I(inode)->runtime_flags);
2984                         btrfs_abort_transaction(trans, root, ret);
2985                         return ret;
2986                 }
2987                 ret = 0;
2988         }
2989
2990         /* insert an orphan item to track subvolume contains orphan files */
2991         if (insert >= 2) {
2992                 ret = btrfs_insert_orphan_item(trans, root->fs_info->tree_root,
2993                                                root->root_key.objectid);
2994                 if (ret && ret != -EEXIST) {
2995                         btrfs_abort_transaction(trans, root, ret);
2996                         return ret;
2997                 }
2998         }
2999         return 0;
3000 }
3001
3002 /*
3003  * We have done the truncate/delete so we can go ahead and remove the orphan
3004  * item for this particular inode.
3005  */
3006 int btrfs_orphan_del(struct btrfs_trans_handle *trans, struct inode *inode)
3007 {
3008         struct btrfs_root *root = BTRFS_I(inode)->root;
3009         int delete_item = 0;
3010         int release_rsv = 0;
3011         int ret = 0;
3012
3013         spin_lock(&root->orphan_lock);
3014         if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3015                                &BTRFS_I(inode)->runtime_flags))
3016                 delete_item = 1;
3017
3018         if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3019                                &BTRFS_I(inode)->runtime_flags))
3020                 release_rsv = 1;
3021         spin_unlock(&root->orphan_lock);
3022
3023         if (trans && delete_item) {
3024                 ret = btrfs_del_orphan_item(trans, root, btrfs_ino(inode));
3025                 BUG_ON(ret); /* -ENOMEM or corruption (JDM: Recheck) */
3026         }
3027
3028         if (release_rsv) {
3029                 btrfs_orphan_release_metadata(inode);
3030                 atomic_dec(&root->orphan_inodes);
3031         }
3032
3033         return 0;
3034 }
3035
3036 /*
3037  * this cleans up any orphans that may be left on the list from the last use
3038  * of this root.
3039  */
3040 int btrfs_orphan_cleanup(struct btrfs_root *root)
3041 {
3042         struct btrfs_path *path;
3043         struct extent_buffer *leaf;
3044         struct btrfs_key key, found_key;
3045         struct btrfs_trans_handle *trans;
3046         struct inode *inode;
3047         u64 last_objectid = 0;
3048         int ret = 0, nr_unlink = 0, nr_truncate = 0;
3049
3050         if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED))
3051                 return 0;
3052
3053         path = btrfs_alloc_path();
3054         if (!path) {
3055                 ret = -ENOMEM;
3056                 goto out;
3057         }
3058         path->reada = -1;
3059
3060         key.objectid = BTRFS_ORPHAN_OBJECTID;
3061         btrfs_set_key_type(&key, BTRFS_ORPHAN_ITEM_KEY);
3062         key.offset = (u64)-1;
3063
3064         while (1) {
3065                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3066                 if (ret < 0)
3067                         goto out;
3068
3069                 /*
3070                  * if ret == 0 means we found what we were searching for, which
3071                  * is weird, but possible, so only screw with path if we didn't
3072                  * find the key and see if we have stuff that matches
3073                  */
3074                 if (ret > 0) {
3075                         ret = 0;
3076                         if (path->slots[0] == 0)
3077                                 break;
3078                         path->slots[0]--;
3079                 }
3080
3081                 /* pull out the item */
3082                 leaf = path->nodes[0];
3083                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3084
3085                 /* make sure the item matches what we want */
3086                 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
3087                         break;
3088                 if (btrfs_key_type(&found_key) != BTRFS_ORPHAN_ITEM_KEY)
3089                         break;
3090
3091                 /* release the path since we're done with it */
3092                 btrfs_release_path(path);
3093
3094                 /*
3095                  * this is where we are basically btrfs_lookup, without the
3096                  * crossing root thing.  we store the inode number in the
3097                  * offset of the orphan item.
3098                  */
3099
3100                 if (found_key.offset == last_objectid) {
3101                         printk(KERN_ERR "btrfs: Error removing orphan entry, "
3102                                "stopping orphan cleanup\n");
3103                         ret = -EINVAL;
3104                         goto out;
3105                 }
3106
3107                 last_objectid = found_key.offset;
3108
3109                 found_key.objectid = found_key.offset;
3110                 found_key.type = BTRFS_INODE_ITEM_KEY;
3111                 found_key.offset = 0;
3112                 inode = btrfs_iget(root->fs_info->sb, &found_key, root, NULL);
3113                 ret = PTR_RET(inode);
3114                 if (ret && ret != -ESTALE)
3115                         goto out;
3116
3117                 if (ret == -ESTALE && root == root->fs_info->tree_root) {
3118                         struct btrfs_root *dead_root;
3119                         struct btrfs_fs_info *fs_info = root->fs_info;
3120                         int is_dead_root = 0;
3121
3122                         /*
3123                          * this is an orphan in the tree root. Currently these
3124                          * could come from 2 sources:
3125                          *  a) a snapshot deletion in progress
3126                          *  b) a free space cache inode
3127                          * We need to distinguish those two, as the snapshot
3128                          * orphan must not get deleted.
3129                          * find_dead_roots already ran before us, so if this
3130                          * is a snapshot deletion, we should find the root
3131                          * in the dead_roots list
3132                          */
3133                         spin_lock(&fs_info->trans_lock);
3134                         list_for_each_entry(dead_root, &fs_info->dead_roots,
3135                                             root_list) {
3136                                 if (dead_root->root_key.objectid ==
3137                                     found_key.objectid) {
3138                                         is_dead_root = 1;
3139                                         break;
3140                                 }
3141                         }
3142                         spin_unlock(&fs_info->trans_lock);
3143                         if (is_dead_root) {
3144                                 /* prevent this orphan from being found again */
3145                                 key.offset = found_key.objectid - 1;
3146                                 continue;
3147                         }
3148                 }
3149                 /*
3150                  * Inode is already gone but the orphan item is still there,
3151                  * kill the orphan item.
3152                  */
3153                 if (ret == -ESTALE) {
3154                         trans = btrfs_start_transaction(root, 1);
3155                         if (IS_ERR(trans)) {
3156                                 ret = PTR_ERR(trans);
3157                                 goto out;
3158                         }
3159                         printk(KERN_ERR "auto deleting %Lu\n",
3160                                found_key.objectid);
3161                         ret = btrfs_del_orphan_item(trans, root,
3162                                                     found_key.objectid);
3163                         BUG_ON(ret); /* -ENOMEM or corruption (JDM: Recheck) */
3164                         btrfs_end_transaction(trans, root);
3165                         continue;
3166                 }
3167
3168                 /*
3169                  * add this inode to the orphan list so btrfs_orphan_del does
3170                  * the proper thing when we hit it
3171                  */
3172                 set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3173                         &BTRFS_I(inode)->runtime_flags);
3174                 atomic_inc(&root->orphan_inodes);
3175
3176                 /* if we have links, this was a truncate, lets do that */
3177                 if (inode->i_nlink) {
3178                         if (!S_ISREG(inode->i_mode)) {
3179                                 WARN_ON(1);
3180                                 iput(inode);
3181                                 continue;
3182                         }
3183                         nr_truncate++;
3184
3185                         /* 1 for the orphan item deletion. */
3186                         trans = btrfs_start_transaction(root, 1);
3187                         if (IS_ERR(trans)) {
3188                                 ret = PTR_ERR(trans);
3189                                 goto out;
3190                         }
3191                         ret = btrfs_orphan_add(trans, inode);
3192                         btrfs_end_transaction(trans, root);
3193                         if (ret)
3194                                 goto out;
3195
3196                         ret = btrfs_truncate(inode);
3197                         if (ret)
3198                                 btrfs_orphan_del(NULL, inode);
3199                 } else {
3200                         nr_unlink++;
3201                 }