Pileus Git - ~andy/linux/blob - fs/btrfs/scrub.c

   1 /*
   2  * Copyright (C) 2011, 2012 STRATO.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18
  19 #include <linux/blkdev.h>
  20 #include <linux/ratelimit.h>
  21 #include "ctree.h"
  22 #include "volumes.h"
  23 #include "disk-io.h"
  24 #include "ordered-data.h"
  25 #include "transaction.h"
  26 #include "backref.h"
  27 #include "extent_io.h"
  28 #include "dev-replace.h"
  29 #include "check-integrity.h"
  30 #include "rcu-string.h"
  31
  32 /*
  33  * This is only the first step towards a full-features scrub. It reads all
  34  * extent and super block and verifies the checksums. In case a bad checksum
  35  * is found or the extent cannot be read, good data will be written back if
  36  * any can be found.
  37  *
  38  * Future enhancements:
  39  *  - In case an unrepairable extent is encountered, track which files are
  40  *    affected and report them
  41  *  - track and record media errors, throw out bad devices
  42  *  - add a mode to also read unallocated space
  43  */
  44
  45 struct scrub_block;
  46 struct scrub_ctx;
  47
  48 /*
  49  * the following three values only influence the performance.
  50  * The last one configures the number of parallel and outstanding I/O
  51  * operations. The first two values configure an upper limit for the number
  52  * of (dynamically allocated) pages that are added to a bio.
  53  */
  54 #define SCRUB_PAGES_PER_RD_BIO  32      /* 128k per bio */
  55 #define SCRUB_PAGES_PER_WR_BIO  32      /* 128k per bio */
  56 #define SCRUB_BIOS_PER_SCTX     64      /* 8MB per device in flight */
  57
  58 /*
  59  * the following value times PAGE_SIZE needs to be large enough to match the
  60  * largest node/leaf/sector size that shall be supported.
  61  * Values larger than BTRFS_STRIPE_LEN are not supported.
  62  */
  63 #define SCRUB_MAX_PAGES_PER_BLOCK       16      /* 64k per node/leaf/sector */
  64
  65 struct scrub_page {
  66         struct scrub_block      *sblock;
  67         struct page             *page;
  68         struct btrfs_device     *dev;
  69         u64                     flags;  /* extent flags */
  70         u64                     generation;
  71         u64                     logical;
  72         u64                     physical;
  73         u64                     physical_for_dev_replace;
  74         atomic_t                ref_count;
  75         struct {
  76                 unsigned int    mirror_num:8;
  77                 unsigned int    have_csum:1;
  78                 unsigned int    io_error:1;
  79         };
  80         u8                      csum[BTRFS_CSUM_SIZE];
  81 };
  82
  83 struct scrub_bio {
  84         int                     index;
  85         struct scrub_ctx        *sctx;
  86         struct btrfs_device     *dev;
  87         struct bio              *bio;
  88         int                     err;
  89         u64                     logical;
  90         u64                     physical;
  91 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
  92         struct scrub_page       *pagev[SCRUB_PAGES_PER_WR_BIO];
  93 #else
  94         struct scrub_page       *pagev[SCRUB_PAGES_PER_RD_BIO];
  95 #endif
  96         int                     page_count;
  97         int                     next_free;
  98         struct btrfs_work       work;
  99 };
 100
 101 struct scrub_block {
 102         struct scrub_page       *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
 103         int                     page_count;
 104         atomic_t                outstanding_pages;
 105         atomic_t                ref_count; /* free mem on transition to zero */
 106         struct scrub_ctx        *sctx;
 107         struct {
 108                 unsigned int    header_error:1;
 109                 unsigned int    checksum_error:1;
 110                 unsigned int    no_io_error_seen:1;
 111                 unsigned int    generation_error:1; /* also sets header_error */
 112         };
 113 };
 114
 115 struct scrub_wr_ctx {
 116         struct scrub_bio *wr_curr_bio;
 117         struct btrfs_device *tgtdev;
 118         int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
 119         atomic_t flush_all_writes;
 120         struct mutex wr_lock;
 121 };
 122
 123 struct scrub_ctx {
 124         struct scrub_bio        *bios[SCRUB_BIOS_PER_SCTX];
 125         struct btrfs_root       *dev_root;
 126         int                     first_free;
 127         int                     curr;
 128         atomic_t                bios_in_flight;
 129         atomic_t                workers_pending;
 130         spinlock_t              list_lock;
 131         wait_queue_head_t       list_wait;
 132         u16                     csum_size;
 133         struct list_head        csum_list;
 134         atomic_t                cancel_req;
 135         int                     readonly;
 136         int                     pages_per_rd_bio;
 137         u32                     sectorsize;
 138         u32                     nodesize;
 139         u32                     leafsize;
 140
 141         int                     is_dev_replace;
 142         struct scrub_wr_ctx     wr_ctx;
 143
 144         /*
 145          * statistics
 146          */
 147         struct btrfs_scrub_progress stat;
 148         spinlock_t              stat_lock;
 149 };
 150
 151 struct scrub_fixup_nodatasum {
 152         struct scrub_ctx        *sctx;
 153         struct btrfs_device     *dev;
 154         u64                     logical;
 155         struct btrfs_root       *root;
 156         struct btrfs_work       work;
 157         int                     mirror_num;
 158 };
 159
 160 struct scrub_copy_nocow_ctx {
 161         struct scrub_ctx        *sctx;
 162         u64                     logical;
 163         u64                     len;
 164         int                     mirror_num;
 165         u64                     physical_for_dev_replace;
 166         struct btrfs_work       work;
 167 };
 168
 169 struct scrub_warning {
 170         struct btrfs_path       *path;
 171         u64                     extent_item_size;
 172         char                    *scratch_buf;
 173         char                    *msg_buf;
 174         const char              *errstr;
 175         sector_t                sector;
 176         u64                     logical;
 177         struct btrfs_device     *dev;
 178         int                     msg_bufsize;
 179         int                     scratch_bufsize;
 180 };
 181
 182
 183 static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
 184 static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
 185 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
 186 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
 187 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
 188 static int scrub_setup_recheck_block(struct scrub_ctx *sctx,
 189                                      struct btrfs_fs_info *fs_info,
 190                                      struct scrub_block *original_sblock,
 191                                      u64 length, u64 logical,
 192                                      struct scrub_block *sblocks_for_recheck);
 193 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
 194                                 struct scrub_block *sblock, int is_metadata,
 195                                 int have_csum, u8 *csum, u64 generation,
 196                                 u16 csum_size);
 197 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
 198                                          struct scrub_block *sblock,
 199                                          int is_metadata, int have_csum,
 200                                          const u8 *csum, u64 generation,
 201                                          u16 csum_size);
 202 static void scrub_complete_bio_end_io(struct bio *bio, int err);
 203 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
 204                                              struct scrub_block *sblock_good,
 205                                              int force_write);
 206 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
 207                                             struct scrub_block *sblock_good,
 208                                             int page_num, int force_write);
 209 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
 210 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
 211                                            int page_num);
 212 static int scrub_checksum_data(struct scrub_block *sblock);
 213 static int scrub_checksum_tree_block(struct scrub_block *sblock);
 214 static int scrub_checksum_super(struct scrub_block *sblock);
 215 static void scrub_block_get(struct scrub_block *sblock);
 216 static void scrub_block_put(struct scrub_block *sblock);
 217 static void scrub_page_get(struct scrub_page *spage);
 218 static void scrub_page_put(struct scrub_page *spage);
 219 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
 220                                     struct scrub_page *spage);
 221 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
 222                        u64 physical, struct btrfs_device *dev, u64 flags,
 223                        u64 gen, int mirror_num, u8 *csum, int force,
 224                        u64 physical_for_dev_replace);
 225 static void scrub_bio_end_io(struct bio *bio, int err);
 226 static void scrub_bio_end_io_worker(struct btrfs_work *work);
 227 static void scrub_block_complete(struct scrub_block *sblock);
 228 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
 229                                u64 extent_logical, u64 extent_len,
 230                                u64 *extent_physical,
 231                                struct btrfs_device **extent_dev,
 232                                int *extent_mirror_num);
 233 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
 234                               struct scrub_wr_ctx *wr_ctx,
 235                               struct btrfs_fs_info *fs_info,
 236                               struct btrfs_device *dev,
 237                               int is_dev_replace);
 238 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx);
 239 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
 240                                     struct scrub_page *spage);
 241 static void scrub_wr_submit(struct scrub_ctx *sctx);
 242 static void scrub_wr_bio_end_io(struct bio *bio, int err);
 243 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
 244 static int write_page_nocow(struct scrub_ctx *sctx,
 245                             u64 physical_for_dev_replace, struct page *page);
 246 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
 247                                       void *ctx);
 248 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
 249                             int mirror_num, u64 physical_for_dev_replace);
 250 static void copy_nocow_pages_worker(struct btrfs_work *work);
 251
 252
 253 static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
 254 {
 255         atomic_inc(&sctx->bios_in_flight);
 256 }
 257
 258 static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
 259 {
 260         atomic_dec(&sctx->bios_in_flight);
 261         wake_up(&sctx->list_wait);
 262 }
 263
 264 /*
 265  * used for workers that require transaction commits (i.e., for the
 266  * NOCOW case)
 267  */
 268 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
 269 {
 270         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
 271
 272         /*
 273          * increment scrubs_running to prevent cancel requests from
 274          * completing as long as a worker is running. we must also
 275          * increment scrubs_paused to prevent deadlocking on pause
 276          * requests used for transactions commits (as the worker uses a
 277          * transaction context). it is safe to regard the worker
 278          * as paused for all matters practical. effectively, we only
 279          * avoid cancellation requests from completing.
 280          */
 281         mutex_lock(&fs_info->scrub_lock);
 282         atomic_inc(&fs_info->scrubs_running);
 283         atomic_inc(&fs_info->scrubs_paused);
 284         mutex_unlock(&fs_info->scrub_lock);
 285         atomic_inc(&sctx->workers_pending);
 286 }
 287
 288 /* used for workers that require transaction commits */
 289 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
 290 {
 291         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
 292
 293         /*
 294          * see scrub_pending_trans_workers_inc() why we're pretending
 295          * to be paused in the scrub counters
 296          */
 297         mutex_lock(&fs_info->scrub_lock);
 298         atomic_dec(&fs_info->scrubs_running);
 299         atomic_dec(&fs_info->scrubs_paused);
 300         mutex_unlock(&fs_info->scrub_lock);
 301         atomic_dec(&sctx->workers_pending);
 302         wake_up(&fs_info->scrub_pause_wait);
 303         wake_up(&sctx->list_wait);
 304 }
 305
 306 static void scrub_free_csums(struct scrub_ctx *sctx)
 307 {
 308         while (!list_empty(&sctx->csum_list)) {
 309                 struct btrfs_ordered_sum *sum;
 310                 sum = list_first_entry(&sctx->csum_list,
 311                                        struct btrfs_ordered_sum, list);
 312                 list_del(&sum->list);
 313                 kfree(sum);
 314         }
 315 }
 316
 317 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
 318 {
 319         int i;
 320
 321         if (!sctx)
 322                 return;
 323
 324         scrub_free_wr_ctx(&sctx->wr_ctx);
 325
 326         /* this can happen when scrub is cancelled */
 327         if (sctx->curr != -1) {
 328                 struct scrub_bio *sbio = sctx->bios[sctx->curr];
 329
 330                 for (i = 0; i < sbio->page_count; i++) {
 331                         WARN_ON(!sbio->pagev[i]->page);
 332                         scrub_block_put(sbio->pagev[i]->sblock);
 333                 }
 334                 bio_put(sbio->bio);
 335         }
 336
 337         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 338                 struct scrub_bio *sbio = sctx->bios[i];
 339
 340                 if (!sbio)
 341                         break;
 342                 kfree(sbio);
 343         }
 344
 345         scrub_free_csums(sctx);
 346         kfree(sctx);
 347 }
 348
 349 static noinline_for_stack
 350 struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
 351 {
 352         struct scrub_ctx *sctx;
 353         int             i;
 354         struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
 355         int pages_per_rd_bio;
 356         int ret;
 357
 358         /*
 359          * the setting of pages_per_rd_bio is correct for scrub but might
 360          * be wrong for the dev_replace code where we might read from
 361          * different devices in the initial huge bios. However, that
 362          * code is able to correctly handle the case when adding a page
 363          * to a bio fails.
 364          */
 365         if (dev->bdev)
 366                 pages_per_rd_bio = min_t(int, SCRUB_PAGES_PER_RD_BIO,
 367                                          bio_get_nr_vecs(dev->bdev));
 368         else
 369                 pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
 370         sctx = kzalloc(sizeof(*sctx), GFP_NOFS);
 371         if (!sctx)
 372                 goto nomem;
 373         sctx->is_dev_replace = is_dev_replace;
 374         sctx->pages_per_rd_bio = pages_per_rd_bio;
 375         sctx->curr = -1;
 376         sctx->dev_root = dev->dev_root;
 377         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 378                 struct scrub_bio *sbio;
 379
 380                 sbio = kzalloc(sizeof(*sbio), GFP_NOFS);
 381                 if (!sbio)
 382                         goto nomem;
 383                 sctx->bios[i] = sbio;
 384
 385                 sbio->index = i;
 386                 sbio->sctx = sctx;
 387                 sbio->page_count = 0;
 388                 sbio->work.func = scrub_bio_end_io_worker;
 389
 390                 if (i != SCRUB_BIOS_PER_SCTX - 1)
 391                         sctx->bios[i]->next_free = i + 1;
 392                 else
 393                         sctx->bios[i]->next_free = -1;
 394         }
 395         sctx->first_free = 0;
 396         sctx->nodesize = dev->dev_root->nodesize;
 397         sctx->leafsize = dev->dev_root->leafsize;
 398         sctx->sectorsize = dev->dev_root->sectorsize;
 399         atomic_set(&sctx->bios_in_flight, 0);
 400         atomic_set(&sctx->workers_pending, 0);
 401         atomic_set(&sctx->cancel_req, 0);
 402         sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
 403         INIT_LIST_HEAD(&sctx->csum_list);
 404
 405         spin_lock_init(&sctx->list_lock);
 406         spin_lock_init(&sctx->stat_lock);
 407         init_waitqueue_head(&sctx->list_wait);
 408
 409         ret = scrub_setup_wr_ctx(sctx, &sctx->wr_ctx, fs_info,
 410                                  fs_info->dev_replace.tgtdev, is_dev_replace);
 411         if (ret) {
 412                 scrub_free_ctx(sctx);
 413                 return ERR_PTR(ret);
 414         }
 415         return sctx;
 416
 417 nomem:
 418         scrub_free_ctx(sctx);
 419         return ERR_PTR(-ENOMEM);
 420 }
 421
 422 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
 423                                      void *warn_ctx)
 424 {
 425         u64 isize;
 426         u32 nlink;
 427         int ret;
 428         int i;
 429         struct extent_buffer *eb;
 430         struct btrfs_inode_item *inode_item;
 431         struct scrub_warning *swarn = warn_ctx;
 432         struct btrfs_fs_info *fs_info = swarn->dev->dev_root->fs_info;
 433         struct inode_fs_paths *ipath = NULL;
 434         struct btrfs_root *local_root;
 435         struct btrfs_key root_key;
 436
 437         root_key.objectid = root;
 438         root_key.type = BTRFS_ROOT_ITEM_KEY;
 439         root_key.offset = (u64)-1;
 440         local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
 441         if (IS_ERR(local_root)) {
 442                 ret = PTR_ERR(local_root);
 443                 goto err;
 444         }
 445
 446         ret = inode_item_info(inum, 0, local_root, swarn->path);
 447         if (ret) {
 448                 btrfs_release_path(swarn->path);
 449                 goto err;
 450         }
 451
 452         eb = swarn->path->nodes[0];
 453         inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
 454                                         struct btrfs_inode_item);
 455         isize = btrfs_inode_size(eb, inode_item);
 456         nlink = btrfs_inode_nlink(eb, inode_item);
 457         btrfs_release_path(swarn->path);
 458
 459         ipath = init_ipath(4096, local_root, swarn->path);
 460         if (IS_ERR(ipath)) {
 461                 ret = PTR_ERR(ipath);
 462                 ipath = NULL;
 463                 goto err;
 464         }
 465         ret = paths_from_inode(inum, ipath);
 466
 467         if (ret < 0)
 468                 goto err;
 469
 470         /*
 471          * we deliberately ignore the bit ipath might have been too small to
 472          * hold all of the paths here
 473          */
 474         for (i = 0; i < ipath->fspath->elem_cnt; ++i)
 475                 printk_in_rcu(KERN_WARNING "btrfs: %s at logical %llu on dev "
 476                         "%s, sector %llu, root %llu, inode %llu, offset %llu, "
 477                         "length %llu, links %u (path: %s)\n", swarn->errstr,
 478                         swarn->logical, rcu_str_deref(swarn->dev->name),
 479                         (unsigned long long)swarn->sector, root, inum, offset,
 480                         min(isize - offset, (u64)PAGE_SIZE), nlink,
 481                         (char *)(unsigned long)ipath->fspath->val[i]);
 482
 483         free_ipath(ipath);
 484         return 0;
 485
 486 err:
 487         printk_in_rcu(KERN_WARNING "btrfs: %s at logical %llu on dev "
 488                 "%s, sector %llu, root %llu, inode %llu, offset %llu: path "
 489                 "resolving failed with ret=%d\n", swarn->errstr,
 490                 swarn->logical, rcu_str_deref(swarn->dev->name),
 491                 (unsigned long long)swarn->sector, root, inum, offset, ret);
 492
 493         free_ipath(ipath);
 494         return 0;
 495 }
 496
 497 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
 498 {
 499         struct btrfs_device *dev;
 500         struct btrfs_fs_info *fs_info;
 501         struct btrfs_path *path;
 502         struct btrfs_key found_key;
 503         struct extent_buffer *eb;
 504         struct btrfs_extent_item *ei;
 505         struct scrub_warning swarn;
 506         unsigned long ptr = 0;
 507         u64 extent_item_pos;
 508         u64 flags = 0;
 509         u64 ref_root;
 510         u32 item_size;
 511         u8 ref_level;
 512         const int bufsize = 4096;
 513         int ret;
 514
 515         WARN_ON(sblock->page_count < 1);
 516         dev = sblock->pagev[0]->dev;
 517         fs_info = sblock->sctx->dev_root->fs_info;
 518
 519         path = btrfs_alloc_path();
 520
 521         swarn.scratch_buf = kmalloc(bufsize, GFP_NOFS);
 522         swarn.msg_buf = kmalloc(bufsize, GFP_NOFS);
 523         swarn.sector = (sblock->pagev[0]->physical) >> 9;
 524         swarn.logical = sblock->pagev[0]->logical;
 525         swarn.errstr = errstr;
 526         swarn.dev = NULL;
 527         swarn.msg_bufsize = bufsize;
 528         swarn.scratch_bufsize = bufsize;
 529
 530         if (!path || !swarn.scratch_buf || !swarn.msg_buf)
 531                 goto out;
 532
 533         ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
 534                                   &flags);
 535         if (ret < 0)
 536                 goto out;
 537
 538         extent_item_pos = swarn.logical - found_key.objectid;
 539         swarn.extent_item_size = found_key.offset;
 540
 541         eb = path->nodes[0];
 542         ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
 543         item_size = btrfs_item_size_nr(eb, path->slots[0]);
 544         btrfs_release_path(path);
 545
 546         if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
 547                 do {
 548                         ret = tree_backref_for_extent(&ptr, eb, ei, item_size,
 549                                                         &ref_root, &ref_level);
 550                         printk_in_rcu(KERN_WARNING
 551                                 "btrfs: %s at logical %llu on dev %s, "
 552                                 "sector %llu: metadata %s (level %d) in tree "
 553                                 "%llu\n", errstr, swarn.logical,
 554                                 rcu_str_deref(dev->name),
 555                                 (unsigned long long)swarn.sector,
 556                                 ref_level ? "node" : "leaf",
 557                                 ret < 0 ? -1 : ref_level,
 558                                 ret < 0 ? -1 : ref_root);
 559                 } while (ret != 1);
 560         } else {
 561                 swarn.path = path;
 562                 swarn.dev = dev;
 563                 iterate_extent_inodes(fs_info, found_key.objectid,
 564                                         extent_item_pos, 1,
 565                                         scrub_print_warning_inode, &swarn);
 566         }
 567
 568 out:
 569         btrfs_free_path(path);
 570         kfree(swarn.scratch_buf);
 571         kfree(swarn.msg_buf);
 572 }
 573
 574 static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
 575 {
 576         struct page *page = NULL;
 577         unsigned long index;
 578         struct scrub_fixup_nodatasum *fixup = fixup_ctx;
 579         int ret;
 580         int corrected = 0;
 581         struct btrfs_key key;
 582         struct inode *inode = NULL;
 583         u64 end = offset + PAGE_SIZE - 1;
 584         struct btrfs_root *local_root;
 585
 586         key.objectid = root;
 587         key.type = BTRFS_ROOT_ITEM_KEY;
 588         key.offset = (u64)-1;
 589         local_root = btrfs_read_fs_root_no_name(fixup->root->fs_info, &key);
 590         if (IS_ERR(local_root))
 591                 return PTR_ERR(local_root);
 592
 593         key.type = BTRFS_INODE_ITEM_KEY;
 594         key.objectid = inum;
 595         key.offset = 0;
 596         inode = btrfs_iget(fixup->root->fs_info->sb, &key, local_root, NULL);
 597         if (IS_ERR(inode))
 598                 return PTR_ERR(inode);
 599
 600         index = offset >> PAGE_CACHE_SHIFT;
 601
 602         page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
 603         if (!page) {
 604                 ret = -ENOMEM;
 605                 goto out;
 606         }
 607
 608         if (PageUptodate(page)) {
 609                 struct btrfs_fs_info *fs_info;
 610                 if (PageDirty(page)) {
 611                         /*
 612                          * we need to write the data to the defect sector. the
 613                          * data that was in that sector is not in memory,
 614                          * because the page was modified. we must not write the
 615                          * modified page to that sector.
 616                          *
 617                          * TODO: what could be done here: wait for the delalloc
 618                          *       runner to write out that page (might involve
 619                          *       COW) and see whether the sector is still
 620                          *       referenced afterwards.
 621                          *
 622                          * For the meantime, we'll treat this error
 623                          * incorrectable, although there is a chance that a
 624                          * later scrub will find the bad sector again and that
 625                          * there's no dirty page in memory, then.
 626                          */
 627                         ret = -EIO;
 628                         goto out;
 629                 }
 630                 fs_info = BTRFS_I(inode)->root->fs_info;
 631                 ret = repair_io_failure(fs_info, offset, PAGE_SIZE,
 632                                         fixup->logical, page,
 633                                         fixup->mirror_num);
 634                 unlock_page(page);
 635                 corrected = !ret;
 636         } else {
 637                 /*
 638                  * we need to get good data first. the general readpage path
 639                  * will call repair_io_failure for us, we just have to make
 640                  * sure we read the bad mirror.
 641                  */
 642                 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
 643                                         EXTENT_DAMAGED, GFP_NOFS);
 644                 if (ret) {
 645                         /* set_extent_bits should give proper error */
 646                         WARN_ON(ret > 0);
 647                         if (ret > 0)
 648                                 ret = -EFAULT;
 649                         goto out;
 650                 }
 651
 652                 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
 653                                                 btrfs_get_extent,
 654                                                 fixup->mirror_num);
 655                 wait_on_page_locked(page);
 656
 657                 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
 658                                                 end, EXTENT_DAMAGED, 0, NULL);
 659                 if (!corrected)
 660                         clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
 661                                                 EXTENT_DAMAGED, GFP_NOFS);
 662         }
 663
 664 out:
 665         if (page)
 666                 put_page(page);
 667         if (inode)
 668                 iput(inode);
 669
 670         if (ret < 0)
 671                 return ret;
 672
 673         if (ret == 0 && corrected) {
 674                 /*
 675                  * we only need to call readpage for one of the inodes belonging
 676                  * to this extent. so make iterate_extent_inodes stop
 677                  */
 678                 return 1;
 679         }
 680
 681         return -EIO;
 682 }
 683
 684 static void scrub_fixup_nodatasum(struct btrfs_work *work)
 685 {
 686         int ret;
 687         struct scrub_fixup_nodatasum *fixup;
 688         struct scrub_ctx *sctx;
 689         struct btrfs_trans_handle *trans = NULL;
 690         struct btrfs_fs_info *fs_info;
 691         struct btrfs_path *path;
 692         int uncorrectable = 0;
 693
 694         fixup = container_of(work, struct scrub_fixup_nodatasum, work);
 695         sctx = fixup->sctx;
 696         fs_info = fixup->root->fs_info;
 697
 698         path = btrfs_alloc_path();
 699         if (!path) {
 700                 spin_lock(&sctx->stat_lock);
 701                 ++sctx->stat.malloc_errors;
 702                 spin_unlock(&sctx->stat_lock);
 703                 uncorrectable = 1;
 704                 goto out;
 705         }
 706
 707         trans = btrfs_join_transaction(fixup->root);
 708         if (IS_ERR(trans)) {
 709                 uncorrectable = 1;
 710                 goto out;
 711         }
 712
 713         /*
 714          * the idea is to trigger a regular read through the standard path. we
 715          * read a page from the (failed) logical address by specifying the
 716          * corresponding copynum of the failed sector. thus, that readpage is
 717          * expected to fail.
 718          * that is the point where on-the-fly error correction will kick in
 719          * (once it's finished) and rewrite the failed sector if a good copy
 720          * can be found.
 721          */
 722         ret = iterate_inodes_from_logical(fixup->logical, fixup->root->fs_info,
 723                                                 path, scrub_fixup_readpage,
 724                                                 fixup);
 725         if (ret < 0) {
 726                 uncorrectable = 1;
 727                 goto out;
 728         }
 729         WARN_ON(ret != 1);
 730
 731         spin_lock(&sctx->stat_lock);
 732         ++sctx->stat.corrected_errors;
 733         spin_unlock(&sctx->stat_lock);
 734
 735 out:
 736         if (trans && !IS_ERR(trans))
 737                 btrfs_end_transaction(trans, fixup->root);
 738         if (uncorrectable) {
 739                 spin_lock(&sctx->stat_lock);
 740                 ++sctx->stat.uncorrectable_errors;
 741                 spin_unlock(&sctx->stat_lock);
 742                 btrfs_dev_replace_stats_inc(
 743                         &sctx->dev_root->fs_info->dev_replace.
 744                         num_uncorrectable_read_errors);
 745                 printk_ratelimited_in_rcu(KERN_ERR
 746                         "btrfs: unable to fixup (nodatasum) error at logical %llu on dev %s\n",
 747                         (unsigned long long)fixup->logical,
 748                         rcu_str_deref(fixup->dev->name));
 749         }
 750
 751         btrfs_free_path(path);
 752         kfree(fixup);
 753
 754         scrub_pending_trans_workers_dec(sctx);
 755 }
 756
 757 /*
 758  * scrub_handle_errored_block gets called when either verification of the
 759  * pages failed or the bio failed to read, e.g. with EIO. In the latter
 760  * case, this function handles all pages in the bio, even though only one
 761  * may be bad.
 762  * The goal of this function is to repair the errored block by using the
 763  * contents of one of the mirrors.
 764  */
 765 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
 766 {
 767         struct scrub_ctx *sctx = sblock_to_check->sctx;
 768         struct btrfs_device *dev;
 769         struct btrfs_fs_info *fs_info;
 770         u64 length;
 771         u64 logical;
 772         u64 generation;
 773         unsigned int failed_mirror_index;
 774         unsigned int is_metadata;
 775         unsigned int have_csum;
 776         u8 *csum;
 777         struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
 778         struct scrub_block *sblock_bad;
 779         int ret;
 780         int mirror_index;
 781         int page_num;
 782         int success;
 783         static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
 784                                       DEFAULT_RATELIMIT_BURST);
 785
 786         BUG_ON(sblock_to_check->page_count < 1);
 787         fs_info = sctx->dev_root->fs_info;
 788         if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
 789                 /*
 790                  * if we find an error in a super block, we just report it.
 791                  * They will get written with the next transaction commit
 792                  * anyway
 793                  */
 794                 spin_lock(&sctx->stat_lock);
 795                 ++sctx->stat.super_errors;
 796                 spin_unlock(&sctx->stat_lock);
 797                 return 0;
 798         }
 799         length = sblock_to_check->page_count * PAGE_SIZE;
 800         logical = sblock_to_check->pagev[0]->logical;
 801         generation = sblock_to_check->pagev[0]->generation;
 802         BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
 803         failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
 804         is_metadata = !(sblock_to_check->pagev[0]->flags &
 805                         BTRFS_EXTENT_FLAG_DATA);
 806         have_csum = sblock_to_check->pagev[0]->have_csum;
 807         csum = sblock_to_check->pagev[0]->csum;
 808         dev = sblock_to_check->pagev[0]->dev;
 809
 810         if (sctx->is_dev_replace && !is_metadata && !have_csum) {
 811                 sblocks_for_recheck = NULL;
 812                 goto nodatasum_case;
 813         }
 814
 815         /*
 816          * read all mirrors one after the other. This includes to
 817          * re-read the extent or metadata block that failed (that was
 818          * the cause that this fixup code is called) another time,
 819          * page by page this time in order to know which pages
 820          * caused I/O errors and which ones are good (for all mirrors).
 821          * It is the goal to handle the situation when more than one
 822          * mirror contains I/O errors, but the errors do not
 823          * overlap, i.e. the data can be repaired by selecting the
 824          * pages from those mirrors without I/O error on the
 825          * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
 826          * would be that mirror #1 has an I/O error on the first page,
 827          * the second page is good, and mirror #2 has an I/O error on
 828          * the second page, but the first page is good.
 829          * Then the first page of the first mirror can be repaired by
 830          * taking the first page of the second mirror, and the
 831          * second page of the second mirror can be repaired by
 832          * copying the contents of the 2nd page of the 1st mirror.
 833          * One more note: if the pages of one mirror contain I/O
 834          * errors, the checksum cannot be verified. In order to get
 835          * the best data for repairing, the first attempt is to find
 836          * a mirror without I/O errors and with a validated checksum.
 837          * Only if this is not possible, the pages are picked from
 838          * mirrors with I/O errors without considering the checksum.
 839          * If the latter is the case, at the end, the checksum of the
 840          * repaired area is verified in order to correctly maintain
 841          * the statistics.
 842          */
 843
 844         sblocks_for_recheck = kzalloc(BTRFS_MAX_MIRRORS *
 845                                      sizeof(*sblocks_for_recheck),
 846                                      GFP_NOFS);
 847         if (!sblocks_for_recheck) {
 848                 spin_lock(&sctx->stat_lock);
 849                 sctx->stat.malloc_errors++;
 850                 sctx->stat.read_errors++;
 851                 sctx->stat.uncorrectable_errors++;
 852                 spin_unlock(&sctx->stat_lock);
 853                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 854                 goto out;
 855         }
 856
 857         /* setup the context, map the logical blocks and alloc the pages */
 858         ret = scrub_setup_recheck_block(sctx, fs_info, sblock_to_check, length,
 859                                         logical, sblocks_for_recheck);
 860         if (ret) {
 861                 spin_lock(&sctx->stat_lock);
 862                 sctx->stat.read_errors++;
 863                 sctx->stat.uncorrectable_errors++;
 864                 spin_unlock(&sctx->stat_lock);
 865                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 866                 goto out;
 867         }
 868         BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
 869         sblock_bad = sblocks_for_recheck + failed_mirror_index;
 870
 871         /* build and submit the bios for the failed mirror, check checksums */
 872         scrub_recheck_block(fs_info, sblock_bad, is_metadata, have_csum,
 873                             csum, generation, sctx->csum_size);
 874
 875         if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
 876             sblock_bad->no_io_error_seen) {
 877                 /*
 878                  * the error disappeared after reading page by page, or
 879                  * the area was part of a huge bio and other parts of the
 880                  * bio caused I/O errors, or the block layer merged several
 881                  * read requests into one and the error is caused by a
 882                  * different bio (usually one of the two latter cases is
 883                  * the cause)
 884                  */
 885                 spin_lock(&sctx->stat_lock);
 886                 sctx->stat.unverified_errors++;
 887                 spin_unlock(&sctx->stat_lock);
 888
 889                 if (sctx->is_dev_replace)
 890                         scrub_write_block_to_dev_replace(sblock_bad);
 891                 goto out;
 892         }
 893
 894         if (!sblock_bad->no_io_error_seen) {
 895                 spin_lock(&sctx->stat_lock);
 896                 sctx->stat.read_errors++;
 897                 spin_unlock(&sctx->stat_lock);
 898                 if (__ratelimit(&_rs))
 899                         scrub_print_warning("i/o error", sblock_to_check);
 900                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 901         } else if (sblock_bad->checksum_error) {
 902                 spin_lock(&sctx->stat_lock);
 903                 sctx->stat.csum_errors++;
 904                 spin_unlock(&sctx->stat_lock);
 905                 if (__ratelimit(&_rs))
 906                         scrub_print_warning("checksum error", sblock_to_check);
 907                 btrfs_dev_stat_inc_and_print(dev,
 908                                              BTRFS_DEV_STAT_CORRUPTION_ERRS);
 909         } else if (sblock_bad->header_error) {
 910                 spin_lock(&sctx->stat_lock);
 911                 sctx->stat.verify_errors++;
 912                 spin_unlock(&sctx->stat_lock);
 913                 if (__ratelimit(&_rs))
 914                         scrub_print_warning("checksum/header error",
 915                                             sblock_to_check);
 916                 if (sblock_bad->generation_error)
 917                         btrfs_dev_stat_inc_and_print(dev,
 918                                 BTRFS_DEV_STAT_GENERATION_ERRS);
 919                 else
 920                         btrfs_dev_stat_inc_and_print(dev,
 921                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
 922         }
 923
 924         if (sctx->readonly && !sctx->is_dev_replace)
 925                 goto did_not_correct_error;
 926
 927         if (!is_metadata && !have_csum) {
 928                 struct scrub_fixup_nodatasum *fixup_nodatasum;
 929
 930 nodatasum_case:
 931                 WARN_ON(sctx->is_dev_replace);
 932
 933                 /*
 934                  * !is_metadata and !have_csum, this means that the data
 935                  * might not be COW'ed, that it might be modified
 936                  * concurrently. The general strategy to work on the
 937                  * commit root does not help in the case when COW is not
 938                  * used.
 939                  */
 940                 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
 941                 if (!fixup_nodatasum)
 942                         goto did_not_correct_error;
 943                 fixup_nodatasum->sctx = sctx;
 944                 fixup_nodatasum->dev = dev;
 945                 fixup_nodatasum->logical = logical;
 946                 fixup_nodatasum->root = fs_info->extent_root;
 947                 fixup_nodatasum->mirror_num = failed_mirror_index + 1;
 948                 scrub_pending_trans_workers_inc(sctx);
 949                 fixup_nodatasum->work.func = scrub_fixup_nodatasum;
 950                 btrfs_queue_worker(&fs_info->scrub_workers,
 951                                    &fixup_nodatasum->work);
 952                 goto out;
 953         }
 954
 955         /*
 956          * now build and submit the bios for the other mirrors, check
 957          * checksums.
 958          * First try to pick the mirror which is completely without I/O
 959          * errors and also does not have a checksum error.
 960          * If one is found, and if a checksum is present, the full block
 961          * that is known to contain an error is rewritten. Afterwards
 962          * the block is known to be corrected.
 963          * If a mirror is found which is completely correct, and no
 964          * checksum is present, only those pages are rewritten that had
 965          * an I/O error in the block to be repaired, since it cannot be
 966          * determined, which copy of the other pages is better (and it
 967          * could happen otherwise that a correct page would be
 968          * overwritten by a bad one).
 969          */
 970         for (mirror_index = 0;
 971              mirror_index < BTRFS_MAX_MIRRORS &&
 972              sblocks_for_recheck[mirror_index].page_count > 0;
 973              mirror_index++) {
 974                 struct scrub_block *sblock_other;
 975
 976                 if (mirror_index == failed_mirror_index)
 977                         continue;
 978                 sblock_other = sblocks_for_recheck + mirror_index;
 979
 980                 /* build and submit the bios, check checksums */
 981                 scrub_recheck_block(fs_info, sblock_other, is_metadata,
 982                                     have_csum, csum, generation,
 983                                     sctx->csum_size);
 984
 985                 if (!sblock_other->header_error &&
 986                     !sblock_other->checksum_error &&
 987                     sblock_other->no_io_error_seen) {
 988                         if (sctx->is_dev_replace) {
 989                                 scrub_write_block_to_dev_replace(sblock_other);
 990                         } else {
 991                                 int force_write = is_metadata || have_csum;
 992
 993                                 ret = scrub_repair_block_from_good_copy(
 994                                                 sblock_bad, sblock_other,
 995                                                 force_write);
 996                         }
 997                         if (0 == ret)
 998                                 goto corrected_error;
 999                 }
1000         }
1001
1002         /*
1003          * for dev_replace, pick good pages and write to the target device.
1004          */
1005         if (sctx->is_dev_replace) {
1006                 success = 1;
1007                 for (page_num = 0; page_num < sblock_bad->page_count;
1008                      page_num++) {
1009                         int sub_success;
1010
1011                         sub_success = 0;
1012                         for (mirror_index = 0;
1013                              mirror_index < BTRFS_MAX_MIRRORS &&
1014                              sblocks_for_recheck[mirror_index].page_count > 0;
1015                              mirror_index++) {
1016                                 struct scrub_block *sblock_other =
1017                                         sblocks_for_recheck + mirror_index;
1018                                 struct scrub_page *page_other =
1019                                         sblock_other->pagev[page_num];
1020
1021                                 if (!page_other->io_error) {
1022                                         ret = scrub_write_page_to_dev_replace(
1023                                                         sblock_other, page_num);
1024                                         if (ret == 0) {
1025                                                 /* succeeded for this page */
1026                                                 sub_success = 1;
1027                                                 break;
1028                                         } else {
1029                                                 btrfs_dev_replace_stats_inc(
1030                                                         &sctx->dev_root->
1031                                                         fs_info->dev_replace.
1032                                                         num_write_errors);
1033                                         }
1034                                 }
1035                         }
1036
1037                         if (!sub_success) {
1038                                 /*
1039                                  * did not find a mirror to fetch the page
1040                                  * from. scrub_write_page_to_dev_replace()
1041                                  * handles this case (page->io_error), by
1042                                  * filling the block with zeros before
1043                                  * submitting the write request
1044                                  */
1045                                 success = 0;
1046                                 ret = scrub_write_page_to_dev_replace(
1047                                                 sblock_bad, page_num);
1048                                 if (ret)
1049                                         btrfs_dev_replace_stats_inc(
1050                                                 &sctx->dev_root->fs_info->
1051                                                 dev_replace.num_write_errors);
1052                         }
1053                 }
1054
1055                 goto out;
1056         }
1057
1058         /*
1059          * for regular scrub, repair those pages that are errored.
1060          * In case of I/O errors in the area that is supposed to be
1061          * repaired, continue by picking good copies of those pages.
1062          * Select the good pages from mirrors to rewrite bad pages from
1063          * the area to fix. Afterwards verify the checksum of the block
1064          * that is supposed to be repaired. This verification step is
1065          * only done for the purpose of statistic counting and for the
1066          * final scrub report, whether errors remain.
1067          * A perfect algorithm could make use of the checksum and try
1068          * all possible combinations of pages from the different mirrors
1069          * until the checksum verification succeeds. For example, when
1070          * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1071          * of mirror #2 is readable but the final checksum test fails,
1072          * then the 2nd page of mirror #3 could be tried, whether now
1073          * the final checksum succeedes. But this would be a rare
1074          * exception and is therefore not implemented. At least it is
1075          * avoided that the good copy is overwritten.
1076          * A more useful improvement would be to pick the sectors
1077          * without I/O error based on sector sizes (512 bytes on legacy
1078          * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1079          * mirror could be repaired by taking 512 byte of a different
1080          * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1081          * area are unreadable.
1082          */
1083
1084         /* can only fix I/O errors from here on */
1085         if (sblock_bad->no_io_error_seen)
1086                 goto did_not_correct_error;
1087
1088         success = 1;
1089         for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1090                 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1091
1092                 if (!page_bad->io_error)
1093                         continue;
1094
1095                 for (mirror_index = 0;
1096                      mirror_index < BTRFS_MAX_MIRRORS &&
1097                      sblocks_for_recheck[mirror_index].page_count > 0;
1098                      mirror_index++) {
1099                         struct scrub_block *sblock_other = sblocks_for_recheck +
1100                                                            mirror_index;
1101                         struct scrub_page *page_other = sblock_other->pagev[
1102                                                         page_num];
1103
1104                         if (!page_other->io_error) {
1105                                 ret = scrub_repair_page_from_good_copy(
1106                                         sblock_bad, sblock_other, page_num, 0);
1107                                 if (0 == ret) {
1108                                         page_bad->io_error = 0;
1109                                         break; /* succeeded for this page */
1110                                 }
1111                         }
1112                 }
1113
1114                 if (page_bad->io_error) {
1115                         /* did not find a mirror to copy the page from */
1116                         success = 0;
1117                 }
1118         }
1119
1120         if (success) {
1121                 if (is_metadata || have_csum) {
1122                         /*
1123                          * need to verify the checksum now that all
1124                          * sectors on disk are repaired (the write
1125                          * request for data to be repaired is on its way).
1126                          * Just be lazy and use scrub_recheck_block()
1127                          * which re-reads the data before the checksum
1128                          * is verified, but most likely the data comes out
1129                          * of the page cache.
1130                          */
1131                         scrub_recheck_block(fs_info, sblock_bad,
1132                                             is_metadata, have_csum, csum,
1133                                             generation, sctx->csum_size);
1134                         if (!sblock_bad->header_error &&
1135                             !sblock_bad->checksum_error &&
1136                             sblock_bad->no_io_error_seen)
1137                                 goto corrected_error;
1138                         else
1139                                 goto did_not_correct_error;
1140                 } else {
1141 corrected_error:
1142                         spin_lock(&sctx->stat_lock);
1143                         sctx->stat.corrected_errors++;
1144                         spin_unlock(&sctx->stat_lock);
1145                         printk_ratelimited_in_rcu(KERN_ERR
1146                                 "btrfs: fixed up error at logical %llu on dev %s\n",
1147                                 (unsigned long long)logical,
1148                                 rcu_str_deref(dev->name));
1149                 }
1150         } else {
1151 did_not_correct_error:
1152                 spin_lock(&sctx->stat_lock);
1153                 sctx->stat.uncorrectable_errors++;
1154                 spin_unlock(&sctx->stat_lock);
1155                 printk_ratelimited_in_rcu(KERN_ERR
1156                         "btrfs: unable to fixup (regular) error at logical %llu on dev %s\n",
1157                         (unsigned long long)logical,
1158                         rcu_str_deref(dev->name));
1159         }
1160
1161 out:
1162         if (sblocks_for_recheck) {
1163                 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1164                      mirror_index++) {
1165                         struct scrub_block *sblock = sblocks_for_recheck +
1166                                                      mirror_index;
1167                         int page_index;
1168
1169                         for (page_index = 0; page_index < sblock->page_count;
1170                              page_index++) {
1171                                 sblock->pagev[page_index]->sblock = NULL;
1172                                 scrub_page_put(sblock->pagev[page_index]);
1173                         }
1174                 }
1175                 kfree(sblocks_for_recheck);
1176         }
1177
1178         return 0;
1179 }
1180
1181 static int scrub_setup_recheck_block(struct scrub_ctx *sctx,
1182                                      struct btrfs_fs_info *fs_info,
1183                                      struct scrub_block *original_sblock,
1184                                      u64 length, u64 logical,
1185                                      struct scrub_block *sblocks_for_recheck)
1186 {
1187         int page_index;
1188         int mirror_index;
1189         int ret;
1190
1191         /*
1192          * note: the two members ref_count and outstanding_pages
1193          * are not used (and not set) in the blocks that are used for
1194          * the recheck procedure
1195          */
1196
1197         page_index = 0;
1198         while (length > 0) {
1199                 u64 sublen = min_t(u64, length, PAGE_SIZE);
1200                 u64 mapped_length = sublen;
1201                 struct btrfs_bio *bbio = NULL;
1202
1203                 /*
1204                  * with a length of PAGE_SIZE, each returned stripe
1205                  * represents one mirror
1206                  */
1207                 ret = btrfs_map_block(fs_info, REQ_GET_READ_MIRRORS, logical,
1208                                       &mapped_length, &bbio, 0);
1209                 if (ret || !bbio || mapped_length < sublen) {
1210                         kfree(bbio);
1211                         return -EIO;
1212                 }
1213
1214                 BUG_ON(page_index >= SCRUB_PAGES_PER_RD_BIO);
1215                 for (mirror_index = 0; mirror_index < (int)bbio->num_stripes;
1216                      mirror_index++) {
1217                         struct scrub_block *sblock;
1218                         struct scrub_page *page;
1219
1220                         if (mirror_index >= BTRFS_MAX_MIRRORS)
1221                                 continue;
1222
1223                         sblock = sblocks_for_recheck + mirror_index;
1224                         sblock->sctx = sctx;
1225                         page = kzalloc(sizeof(*page), GFP_NOFS);
1226                         if (!page) {
1227 leave_nomem:
1228                                 spin_lock(&sctx->stat_lock);
1229                                 sctx->stat.malloc_errors++;
1230                                 spin_unlock(&sctx->stat_lock);
1231                                 kfree(bbio);
1232                                 return -ENOMEM;
1233                         }
1234                         scrub_page_get(page);
1235                         sblock->pagev[page_index] = page;
1236                         page->logical = logical;
1237                         page->physical = bbio->stripes[mirror_index].physical;
1238                         BUG_ON(page_index >= original_sblock->page_count);
1239                         page->physical_for_dev_replace =
1240                                 original_sblock->pagev[page_index]->
1241                                 physical_for_dev_replace;
1242                         /* for missing devices, dev->bdev is NULL */
1243                         page->dev = bbio->stripes[mirror_index].dev;
1244                         page->mirror_num = mirror_index + 1;
1245                         sblock->page_count++;
1246                         page->page = alloc_page(GFP_NOFS);
1247                         if (!page->page)
1248                                 goto leave_nomem;
1249                 }
1250                 kfree(bbio);
1251                 length -= sublen;
1252                 logical += sublen;
1253                 page_index++;
1254         }
1255
1256         return 0;
1257 }
1258
1259 /*
1260  * this function will check the on disk data for checksum errors, header
1261  * errors and read I/O errors. If any I/O errors happen, the exact pages
1262  * which are errored are marked as being bad. The goal is to enable scrub
1263  * to take those pages that are not errored from all the mirrors so that
1264  * the pages that are errored in the just handled mirror can be repaired.
1265  */
1266 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1267                                 struct scrub_block *sblock, int is_metadata,
1268                                 int have_csum, u8 *csum, u64 generation,
1269                                 u16 csum_size)
1270 {
1271         int page_num;
1272
1273         sblock->no_io_error_seen = 1;
1274         sblock->header_error = 0;
1275         sblock->checksum_error = 0;
1276
1277         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1278                 struct bio *bio;
1279                 struct scrub_page *page = sblock->pagev[page_num];
1280                 DECLARE_COMPLETION_ONSTACK(complete);
1281
1282                 if (page->dev->bdev == NULL) {
1283                         page->io_error = 1;
1284                         sblock->no_io_error_seen = 0;
1285                         continue;
1286                 }
1287
1288                 WARN_ON(!page->page);
1289                 bio = bio_alloc(GFP_NOFS, 1);
1290                 if (!bio) {
1291                         page->io_error = 1;
1292                         sblock->no_io_error_seen = 0;
1293                         continue;
1294                 }
1295                 bio->bi_bdev = page->dev->bdev;
1296                 bio->bi_sector = page->physical >> 9;
1297                 bio->bi_end_io = scrub_complete_bio_end_io;
1298                 bio->bi_private = &complete;
1299
1300                 bio_add_page(bio, page->page, PAGE_SIZE, 0);
1301                 btrfsic_submit_bio(READ, bio);
1302
1303                 /* this will also unplug the queue */
1304                 wait_for_completion(&complete);
1305
1306                 page->io_error = !test_bit(BIO_UPTODATE, &bio->bi_flags);
1307                 if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1308                         sblock->no_io_error_seen = 0;
1309                 bio_put(bio);
1310         }
1311
1312         if (sblock->no_io_error_seen)
1313                 scrub_recheck_block_checksum(fs_info, sblock, is_metadata,
1314                                              have_csum, csum, generation,
1315                                              csum_size);
1316
1317         return;
1318 }
1319
1320 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
1321                                          struct scrub_block *sblock,
1322                                          int is_metadata, int have_csum,
1323                                          const u8 *csum, u64 generation,
1324                                          u16 csum_size)
1325 {
1326         int page_num;
1327         u8 calculated_csum[BTRFS_CSUM_SIZE];
1328         u32 crc = ~(u32)0;
1329         struct btrfs_root *root = fs_info->extent_root;
1330         void *mapped_buffer;
1331
1332         WARN_ON(!sblock->pagev[0]->page);
1333         if (is_metadata) {
1334                 struct btrfs_header *h;
1335
1336                 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
1337                 h = (struct btrfs_header *)mapped_buffer;
1338
1339                 if (sblock->pagev[0]->logical != le64_to_cpu(h->bytenr) ||
1340                     memcmp(h->fsid, fs_info->fsid, BTRFS_UUID_SIZE) ||
1341                     memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1342                            BTRFS_UUID_SIZE)) {
1343                         sblock->header_error = 1;
1344                 } else if (generation != le64_to_cpu(h->generation)) {
1345                         sblock->header_error = 1;
1346                         sblock->generation_error = 1;
1347                 }
1348                 csum = h->csum;
1349         } else {
1350                 if (!have_csum)
1351                         return;
1352
1353                 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
1354         }
1355
1356         for (page_num = 0;;) {
1357                 if (page_num == 0 && is_metadata)
1358                         crc = btrfs_csum_data(root,
1359                                 ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE,
1360                                 crc, PAGE_SIZE - BTRFS_CSUM_SIZE);
1361                 else
1362                         crc = btrfs_csum_data(root, mapped_buffer, crc,
1363                                               PAGE_SIZE);
1364
1365                 kunmap_atomic(mapped_buffer);
1366                 page_num++;
1367                 if (page_num >= sblock->page_count)
1368                         break;
1369                 WARN_ON(!sblock->pagev[page_num]->page);
1370
1371                 mapped_buffer = kmap_atomic(sblock->pagev[page_num]->page);
1372         }
1373
1374         btrfs_csum_final(crc, calculated_csum);
1375         if (memcmp(calculated_csum, csum, csum_size))
1376                 sblock->checksum_error = 1;
1377 }
1378
1379 static void scrub_complete_bio_end_io(struct bio *bio, int err)
1380 {
1381         complete((struct completion *)bio->bi_private);
1382 }
1383
1384 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1385                                              struct scrub_block *sblock_good,
1386                                              int force_write)
1387 {
1388         int page_num;
1389         int ret = 0;
1390
1391         for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1392                 int ret_sub;
1393
1394                 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1395                                                            sblock_good,
1396                                                            page_num,
1397                                                            force_write);
1398                 if (ret_sub)
1399                         ret = ret_sub;
1400         }
1401
1402         return ret;
1403 }
1404
1405 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1406                                             struct scrub_block *sblock_good,
1407                                             int page_num, int force_write)
1408 {
1409         struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1410         struct scrub_page *page_good = sblock_good->pagev[page_num];
1411
1412         BUG_ON(page_bad->page == NULL);
1413         BUG_ON(page_good->page == NULL);
1414         if (force_write || sblock_bad->header_error ||
1415             sblock_bad->checksum_error || page_bad->io_error) {
1416                 struct bio *bio;
1417                 int ret;
1418                 DECLARE_COMPLETION_ONSTACK(complete);
1419
1420                 if (!page_bad->dev->bdev) {
1421                         printk_ratelimited(KERN_WARNING
1422                                 "btrfs: scrub_repair_page_from_good_copy(bdev == NULL) is unexpected!\n");
1423                         return -EIO;
1424                 }
1425
1426                 bio = bio_alloc(GFP_NOFS, 1);
1427                 if (!bio)
1428                         return -EIO;
1429                 bio->bi_bdev = page_bad->dev->bdev;
1430                 bio->bi_sector = page_bad->physical >> 9;
1431                 bio->bi_end_io = scrub_complete_bio_end_io;
1432                 bio->bi_private = &complete;
1433
1434                 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1435                 if (PAGE_SIZE != ret) {
1436                         bio_put(bio);
1437                         return -EIO;
1438                 }
1439                 btrfsic_submit_bio(WRITE, bio);
1440
1441                 /* this will also unplug the queue */
1442                 wait_for_completion(&complete);
1443                 if (!bio_flagged(bio, BIO_UPTODATE)) {
1444                         btrfs_dev_stat_inc_and_print(page_bad->dev,
1445                                 BTRFS_DEV_STAT_WRITE_ERRS);
1446                         btrfs_dev_replace_stats_inc(
1447                                 &sblock_bad->sctx->dev_root->fs_info->
1448                                 dev_replace.num_write_errors);
1449                         bio_put(bio);
1450                         return -EIO;
1451                 }
1452                 bio_put(bio);
1453         }
1454
1455         return 0;
1456 }
1457
1458 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1459 {
1460         int page_num;
1461
1462         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1463                 int ret;
1464
1465                 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1466                 if (ret)
1467                         btrfs_dev_replace_stats_inc(
1468                                 &sblock->sctx->dev_root->fs_info->dev_replace.
1469                                 num_write_errors);
1470         }
1471 }
1472
1473 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1474                                            int page_num)
1475 {
1476         struct scrub_page *spage = sblock->pagev[page_num];
1477
1478         BUG_ON(spage->page == NULL);
1479         if (spage->io_error) {
1480                 void *mapped_buffer = kmap_atomic(spage->page);
1481
1482                 memset(mapped_buffer, 0, PAGE_CACHE_SIZE);
1483                 flush_dcache_page(spage->page);
1484                 kunmap_atomic(mapped_buffer);
1485         }
1486         return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1487 }
1488
1489 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1490                                     struct scrub_page *spage)
1491 {
1492         struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1493         struct scrub_bio *sbio;
1494         int ret;
1495
1496         mutex_lock(&wr_ctx->wr_lock);
1497 again:
1498         if (!wr_ctx->wr_curr_bio) {
1499                 wr_ctx->wr_curr_bio = kzalloc(sizeof(*wr_ctx->wr_curr_bio),
1500                                               GFP_NOFS);
1501                 if (!wr_ctx->wr_curr_bio) {
1502                         mutex_unlock(&wr_ctx->wr_lock);
1503                         return -ENOMEM;
1504                 }
1505                 wr_ctx->wr_curr_bio->sctx = sctx;
1506                 wr_ctx->wr_curr_bio->page_count = 0;
1507         }
1508         sbio = wr_ctx->wr_curr_bio;
1509         if (sbio->page_count == 0) {
1510                 struct bio *bio;
1511
1512                 sbio->physical = spage->physical_for_dev_replace;
1513                 sbio->logical = spage->logical;
1514                 sbio->dev = wr_ctx->tgtdev;
1515                 bio = sbio->bio;
1516                 if (!bio) {
1517                         bio = bio_alloc(GFP_NOFS, wr_ctx->pages_per_wr_bio);
1518                         if (!bio) {
1519                                 mutex_unlock(&wr_ctx->wr_lock);
1520                                 return -ENOMEM;
1521                         }
1522                         sbio->bio = bio;
1523                 }
1524
1525                 bio->bi_private = sbio;
1526                 bio->bi_end_io = scrub_wr_bio_end_io;
1527                 bio->bi_bdev = sbio->dev->bdev;
1528                 bio->bi_sector = sbio->physical >> 9;
1529                 sbio->err = 0;
1530         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1531                    spage->physical_for_dev_replace ||
1532                    sbio->logical + sbio->page_count * PAGE_SIZE !=
1533                    spage->logical) {
1534                 scrub_wr_submit(sctx);
1535                 goto again;
1536         }
1537
1538         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1539         if (ret != PAGE_SIZE) {
1540                 if (sbio->page_count < 1) {
1541                         bio_put(sbio->bio);
1542                         sbio->bio = NULL;
1543                         mutex_unlock(&wr_ctx->wr_lock);
1544                         return -EIO;
1545                 }
1546                 scrub_wr_submit(sctx);
1547                 goto again;
1548         }
1549
1550         sbio->pagev[sbio->page_count] = spage;
1551         scrub_page_get(spage);
1552         sbio->page_count++;
1553         if (sbio->page_count == wr_ctx->pages_per_wr_bio)
1554                 scrub_wr_submit(sctx);
1555         mutex_unlock(&wr_ctx->wr_lock);
1556
1557         return 0;
1558 }
1559
1560 static void scrub_wr_submit(struct scrub_ctx *sctx)
1561 {
1562         struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1563         struct scrub_bio *sbio;
1564
1565         if (!wr_ctx->wr_curr_bio)
1566                 return;
1567
1568         sbio = wr_ctx->wr_curr_bio;
1569         wr_ctx->wr_curr_bio = NULL;
1570         WARN_ON(!sbio->bio->bi_bdev);
1571         scrub_pending_bio_inc(sctx);
1572         /* process all writes in a single worker thread. Then the block layer
1573          * orders the requests before sending them to the driver which
1574          * doubled the write performance on spinning disks when measured
1575          * with Linux 3.5 */
1576         btrfsic_submit_bio(WRITE, sbio->bio);
1577 }
1578
1579 static void scrub_wr_bio_end_io(struct bio *bio, int err)
1580 {
1581         struct scrub_bio *sbio = bio->bi_private;
1582         struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
1583
1584         sbio->err = err;
1585         sbio->bio = bio;
1586
1587         sbio->work.func = scrub_wr_bio_end_io_worker;
1588         btrfs_queue_worker(&fs_info->scrub_wr_completion_workers, &sbio->work);
1589 }
1590
1591 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
1592 {
1593         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
1594         struct scrub_ctx *sctx = sbio->sctx;
1595         int i;
1596
1597         WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
1598         if (sbio->err) {
1599                 struct btrfs_dev_replace *dev_replace =
1600                         &sbio->sctx->dev_root->fs_info->dev_replace;
1601
1602                 for (i = 0; i < sbio->page_count; i++) {
1603                         struct scrub_page *spage = sbio->pagev[i];
1604
1605                         spage->io_error = 1;
1606                         btrfs_dev_replace_stats_inc(&dev_replace->
1607                                                     num_write_errors);
1608                 }
1609         }
1610
1611         for (i = 0; i < sbio->page_count; i++)
1612                 scrub_page_put(sbio->pagev[i]);
1613
1614         bio_put(sbio->bio);
1615         kfree(sbio);
1616         scrub_pending_bio_dec(sctx);
1617 }
1618
1619 static int scrub_checksum(struct scrub_block *sblock)
1620 {
1621         u64 flags;
1622         int ret;
1623
1624         WARN_ON(sblock->page_count < 1);
1625         flags = sblock->pagev[0]->flags;
1626         ret = 0;
1627         if (flags & BTRFS_EXTENT_FLAG_DATA)
1628                 ret = scrub_checksum_data(sblock);
1629         else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1630                 ret = scrub_checksum_tree_block(sblock);
1631         else if (flags & BTRFS_EXTENT_FLAG_SUPER)
1632                 (void)scrub_checksum_super(sblock);
1633         else
1634                 WARN_ON(1);
1635         if (ret)
1636                 scrub_handle_errored_block(sblock);
1637
1638         return ret;
1639 }
1640
1641 static int scrub_checksum_data(struct scrub_block *sblock)
1642 {
1643         struct scrub_ctx *sctx = sblock->sctx;
1644         u8 csum[BTRFS_CSUM_SIZE];
1645         u8 *on_disk_csum;
1646         struct page *page;
1647         void *buffer;
1648         u32 crc = ~(u32)0;
1649         int fail = 0;
1650         struct btrfs_root *root = sctx->dev_root;
1651         u64 len;
1652         int index;
1653
1654         BUG_ON(sblock->page_count < 1);
1655         if (!sblock->pagev[0]->have_csum)
1656                 return 0;
1657
1658         on_disk_csum = sblock->pagev[0]->csum;
1659         page = sblock->pagev[0]->page;
1660         buffer = kmap_atomic(page);
1661
1662         len = sctx->sectorsize;
1663         index = 0;
1664         for (;;) {
1665                 u64 l = min_t(u64, len, PAGE_SIZE);
1666
1667                 crc = btrfs_csum_data(root, buffer, crc, l);
1668                 kunmap_atomic(buffer);
1669                 len -= l;
1670                 if (len == 0)
1671                         break;
1672                 index++;
1673                 BUG_ON(index >= sblock->page_count);
1674                 BUG_ON(!sblock->pagev[index]->page);
1675                 page = sblock->pagev[index]->page;
1676                 buffer = kmap_atomic(page);
1677         }
1678
1679         btrfs_csum_final(crc, csum);
1680         if (memcmp(csum, on_disk_csum, sctx->csum_size))
1681                 fail = 1;
1682
1683         return fail;
1684 }
1685
1686 static int scrub_checksum_tree_block(struct scrub_block *sblock)
1687 {
1688         struct scrub_ctx *sctx = sblock->sctx;
1689         struct btrfs_header *h;
1690         struct btrfs_root *root = sctx->dev_root;
1691         struct btrfs_fs_info *fs_info = root->fs_info;
1692         u8 calculated_csum[BTRFS_CSUM_SIZE];
1693         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1694         struct page *page;
1695         void *mapped_buffer;
1696         u64 mapped_size;
1697         void *p;
1698         u32 crc = ~(u32)0;
1699         int fail = 0;
1700         int crc_fail = 0;
1701         u64 len;
1702         int index;
1703
1704         BUG_ON(sblock->page_count < 1);
1705         page = sblock->pagev[0]->page;
1706         mapped_buffer = kmap_atomic(page);
1707         h = (struct btrfs_header *)mapped_buffer;
1708         memcpy(on_disk_csum, h->csum, sctx->csum_size);
1709
1710         /*
1711          * we don't use the getter functions here, as we
1712          * a) don't have an extent buffer and
1713          * b) the page is already kmapped
1714          */
1715
1716         if (sblock->pagev[0]->logical != le64_to_cpu(h->bytenr))
1717                 ++fail;
1718
1719         if (sblock->pagev[0]->generation != le64_to_cpu(h->generation))
1720                 ++fail;
1721
1722         if (memcmp(h->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
1723                 ++fail;
1724
1725         if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1726                    BTRFS_UUID_SIZE))
1727                 ++fail;
1728
1729         WARN_ON(sctx->nodesize != sctx->leafsize);
1730         len = sctx->nodesize - BTRFS_CSUM_SIZE;
1731         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1732         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1733         index = 0;
1734         for (;;) {
1735                 u64 l = min_t(u64, len, mapped_size);
1736
1737                 crc = btrfs_csum_data(root, p, crc, l);
1738                 kunmap_atomic(mapped_buffer);
1739                 len -= l;
1740                 if (len == 0)
1741                         break;
1742                 index++;
1743                 BUG_ON(index >= sblock->page_count);
1744                 BUG_ON(!sblock->pagev[index]->page);
1745                 page = sblock->pagev[index]->page;
1746                 mapped_buffer = kmap_atomic(page);
1747                 mapped_size = PAGE_SIZE;
1748                 p = mapped_buffer;
1749         }
1750
1751         btrfs_csum_final(crc, calculated_csum);
1752         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1753                 ++crc_fail;
1754
1755         return fail || crc_fail;
1756 }
1757
1758 static int scrub_checksum_super(struct scrub_block *sblock)
1759 {
1760         struct btrfs_super_block *s;
1761         struct scrub_ctx *sctx = sblock->sctx;
1762         struct btrfs_root *root = sctx->dev_root;
1763         struct btrfs_fs_info *fs_info = root->fs_info;
1764         u8 calculated_csum[BTRFS_CSUM_SIZE];
1765         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1766         struct page *page;
1767         void *mapped_buffer;
1768         u64 mapped_size;
1769         void *p;
1770         u32 crc = ~(u32)0;
1771         int fail_gen = 0;
1772         int fail_cor = 0;
1773         u64 len;
1774         int index;
1775
1776         BUG_ON(sblock->page_count < 1);
1777         page = sblock->pagev[0]->page;
1778         mapped_buffer = kmap_atomic(page);
1779         s = (struct btrfs_super_block *)mapped_buffer;
1780         memcpy(on_disk_csum, s->csum, sctx->csum_size);
1781
1782         if (sblock->pagev[0]->logical != le64_to_cpu(s->bytenr))
1783                 ++fail_cor;
1784
1785         if (sblock->pagev[0]->generation != le64_to_cpu(s->generation))
1786                 ++fail_gen;
1787
1788         if (memcmp(s->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
1789                 ++fail_cor;
1790
1791         len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
1792         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1793         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1794         index = 0;
1795         for (;;) {
1796                 u64 l = min_t(u64, len, mapped_size);
1797
1798                 crc = btrfs_csum_data(root, p, crc, l);
1799                 kunmap_atomic(mapped_buffer);
1800                 len -= l;
1801                 if (len == 0)
1802                         break;
1803                 index++;
1804                 BUG_ON(index >= sblock->page_count);
1805                 BUG_ON(!sblock->pagev[index]->page);
1806                 page = sblock->pagev[index]->page;
1807                 mapped_buffer = kmap_atomic(page);
1808                 mapped_size = PAGE_SIZE;
1809                 p = mapped_buffer;
1810         }
1811
1812         btrfs_csum_final(crc, calculated_csum);
1813         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1814                 ++fail_cor;
1815
1816         if (fail_cor + fail_gen) {
1817                 /*
1818                  * if we find an error in a super block, we just report it.
1819                  * They will get written with the next transaction commit
1820                  * anyway
1821                  */
1822                 spin_lock(&sctx->stat_lock);
1823                 ++sctx->stat.super_errors;
1824                 spin_unlock(&sctx->stat_lock);
1825                 if (fail_cor)
1826                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
1827                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1828                 else
1829                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
1830                                 BTRFS_DEV_STAT_GENERATION_ERRS);
1831         }
1832
1833         return fail_cor + fail_gen;
1834 }
1835
1836 static void scrub_block_get(struct scrub_block *sblock)
1837 {
1838         atomic_inc(&sblock->ref_count);
1839 }
1840
1841 static void scrub_block_put(struct scrub_block *sblock)
1842 {
1843         if (atomic_dec_and_test(&sblock->ref_count)) {
1844                 int i;
1845
1846                 for (i = 0; i < sblock->page_count; i++)
1847                         scrub_page_put(sblock->pagev[i]);
1848                 kfree(sblock);
1849         }
1850 }
1851
1852 static void scrub_page_get(struct scrub_page *spage)
1853 {
1854         atomic_inc(&spage->ref_count);
1855 }
1856
1857 static void scrub_page_put(struct scrub_page *spage)
1858 {
1859         if (atomic_dec_and_test(&spage->ref_count)) {
1860                 if (spage->page)
1861                         __free_page(spage->page);
1862                 kfree(spage);
1863         }
1864 }
1865
1866 static void scrub_submit(struct scrub_ctx *sctx)
1867 {
1868         struct scrub_bio *sbio;
1869
1870         if (sctx->curr == -1)
1871                 return;
1872
1873         sbio = sctx->bios[sctx->curr];
1874         sctx->curr = -1;
1875         scrub_pending_bio_inc(sctx);
1876
1877         if (!sbio->bio->bi_bdev) {
1878                 /*
1879                  * this case should not happen. If btrfs_map_block() is
1880                  * wrong, it could happen for dev-replace operations on
1881                  * missing devices when no mirrors are available, but in
1882                  * this case it should already fail the mount.
1883                  * This case is handled correctly (but _very_ slowly).
1884                  */
1885                 printk_ratelimited(KERN_WARNING
1886                         "btrfs: scrub_submit(bio bdev == NULL) is unexpected!\n");
1887                 bio_endio(sbio->bio, -EIO);
1888         } else {
1889                 btrfsic_submit_bio(READ, sbio->bio);
1890         }
1891 }
1892
1893 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
1894                                     struct scrub_page *spage)
1895 {
1896         struct scrub_block *sblock = spage->sblock;
1897         struct scrub_bio *sbio;
1898         int ret;
1899
1900 again:
1901         /*
1902          * grab a fresh bio or wait for one to become available
1903          */
1904         while (sctx->curr == -1) {
1905                 spin_lock(&sctx->list_lock);
1906                 sctx->curr = sctx->first_free;
1907                 if (sctx->curr != -1) {
1908                         sctx->first_free = sctx->bios[sctx->curr]->next_free;
1909                         sctx->bios[sctx->curr]->next_free = -1;
1910                         sctx->bios[sctx->curr]->page_count = 0;
1911                         spin_unlock(&sctx->list_lock);
1912                 } else {
1913                         spin_unlock(&sctx->list_lock);
1914                         wait_event(sctx->list_wait, sctx->first_free != -1);
1915                 }
1916         }
1917         sbio = sctx->bios[sctx->curr];
1918         if (sbio->page_count == 0) {
1919                 struct bio *bio;
1920
1921                 sbio->physical = spage->physical;
1922                 sbio->logical = spage->logical;
1923                 sbio->dev = spage->dev;
1924                 bio = sbio->bio;
1925                 if (!bio) {
1926                         bio = bio_alloc(GFP_NOFS, sctx->pages_per_rd_bio);
1927                         if (!bio)
1928                                 return -ENOMEM;
1929                         sbio->bio = bio;
1930                 }
1931
1932                 bio->bi_private = sbio;
1933                 bio->bi_end_io = scrub_bio_end_io;
1934                 bio->bi_bdev = sbio->dev->bdev;
1935                 bio->bi_sector = sbio->physical >> 9;
1936                 sbio->err = 0;
1937         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1938                    spage->physical ||
1939                    sbio->logical + sbio->page_count * PAGE_SIZE !=
1940                    spage->logical ||
1941                    sbio->dev != spage->dev) {
1942                 scrub_submit(sctx);
1943                 goto again;
1944         }
1945
1946         sbio->pagev[sbio->page_count] = spage;
1947         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1948         if (ret != PAGE_SIZE) {
1949                 if (sbio->page_count < 1) {
1950                         bio_put(sbio->bio);
1951                         sbio->bio = NULL;
1952                         return -EIO;
1953                 }
1954                 scrub_submit(sctx);
1955                 goto again;
1956         }
1957
1958         scrub_block_get(sblock); /* one for the page added to the bio */
1959         atomic_inc(&sblock->outstanding_pages);
1960         sbio->page_count++;
1961         if (sbio->page_count == sctx->pages_per_rd_bio)
1962                 scrub_submit(sctx);
1963
1964         return 0;
1965 }
1966
1967 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
1968                        u64 physical, struct btrfs_device *dev, u64 flags,
1969                        u64 gen, int mirror_num, u8 *csum, int force,
1970                        u64 physical_for_dev_replace)
1971 {
1972         struct scrub_block *sblock;
1973         int index;
1974
1975         sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
1976         if (!sblock) {
1977                 spin_lock(&sctx->stat_lock);
1978                 sctx->stat.malloc_errors++;
1979                 spin_unlock(&sctx->stat_lock);
1980                 return -ENOMEM;
1981         }
1982
1983         /* one ref inside this function, plus one for each page added to
1984          * a bio later on */
1985         atomic_set(&sblock->ref_count, 1);
1986         sblock->sctx = sctx;
1987         sblock->no_io_error_seen = 1;
1988
1989         for (index = 0; len > 0; index++) {
1990                 struct scrub_page *spage;
1991                 u64 l = min_t(u64, len, PAGE_SIZE);
1992
1993                 spage = kzalloc(sizeof(*spage), GFP_NOFS);
1994                 if (!spage) {
1995 leave_nomem:
1996                         spin_lock(&sctx->stat_lock);
1997                         sctx->stat.malloc_errors++;
1998                         spin_unlock(&sctx->stat_lock);
1999                         scrub_block_put(sblock);
2000                         return -ENOMEM;
2001                 }
2002                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2003                 scrub_page_get(spage);
2004                 sblock->pagev[index] = spage;
2005                 spage->sblock = sblock;
2006                 spage->dev = dev;
2007                 spage->flags = flags;
2008                 spage->generation = gen;
2009                 spage->logical = logical;
2010                 spage->physical = physical;
2011                 spage->physical_for_dev_replace = physical_for_dev_replace;
2012                 spage->mirror_num = mirror_num;
2013                 if (csum) {
2014                         spage->have_csum = 1;
2015                         memcpy(spage->csum, csum, sctx->csum_size);
2016                 } else {
2017                         spage->have_csum = 0;
2018                 }
2019                 sblock->page_count++;
2020                 spage->page = alloc_page(GFP_NOFS);
2021                 if (!spage->page)
2022                         goto leave_nomem;
2023                 len -= l;
2024                 logical += l;
2025                 physical += l;
2026                 physical_for_dev_replace += l;
2027         }
2028
2029         WARN_ON(sblock->page_count == 0);
2030         for (index = 0; index < sblock->page_count; index++) {
2031                 struct scrub_page *spage = sblock->pagev[index];
2032                 int ret;
2033
2034                 ret = scrub_add_page_to_rd_bio(sctx, spage);
2035                 if (ret) {
2036                         scrub_block_put(sblock);
2037                         return ret;
2038                 }
2039         }
2040
2041         if (force)
2042                 scrub_submit(sctx);
2043
2044         /* last one frees, either here or in bio completion for last page */
2045         scrub_block_put(sblock);
2046         return 0;
2047 }
2048
2049 static void scrub_bio_end_io(struct bio *bio, int err)
2050 {
2051         struct scrub_bio *sbio = bio->bi_private;
2052         struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
2053
2054         sbio->err = err;
2055         sbio->bio = bio;
2056
2057         btrfs_queue_worker(&fs_info->scrub_workers, &sbio->work);
2058 }
2059
2060 static void scrub_bio_end_io_worker(struct btrfs_work *work)
2061 {
2062         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2063         struct scrub_ctx *sctx = sbio->sctx;
2064         int i;
2065
2066         BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
2067         if (sbio->err) {
2068                 for (i = 0; i < sbio->page_count; i++) {
2069                         struct scrub_page *spage = sbio->pagev[i];
2070
2071                         spage->io_error = 1;
2072                         spage->sblock->no_io_error_seen = 0;
2073                 }
2074         }
2075
2076         /* now complete the scrub_block items that have all pages completed */
2077         for (i = 0; i < sbio->page_count; i++) {
2078                 struct scrub_page *spage = sbio->pagev[i];
2079                 struct scrub_block *sblock = spage->sblock;
2080
2081                 if (atomic_dec_and_test(&sblock->outstanding_pages))
2082                         scrub_block_complete(sblock);
2083                 scrub_block_put(sblock);
2084         }
2085
2086         bio_put(sbio->bio);
2087         sbio->bio = NULL;
2088         spin_lock(&sctx->list_lock);
2089         sbio->next_free = sctx->first_free;
2090         sctx->first_free = sbio->index;
2091         spin_unlock(&sctx->list_lock);
2092
2093         if (sctx->is_dev_replace &&
2094             atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2095                 mutex_lock(&sctx->wr_ctx.wr_lock);
2096                 scrub_wr_submit(sctx);
2097                 mutex_unlock(&sctx->wr_ctx.wr_lock);
2098         }
2099
2100         scrub_pending_bio_dec(sctx);
2101 }
2102
2103 static void scrub_block_complete(struct scrub_block *sblock)
2104 {
2105         if (!sblock->no_io_error_seen) {
2106                 scrub_handle_errored_block(sblock);
2107         } else {
2108                 /*
2109                  * if has checksum error, write via repair mechanism in
2110                  * dev replace case, otherwise write here in dev replace
2111                  * case.
2112                  */
2113                 if (!scrub_checksum(sblock) && sblock->sctx->is_dev_replace)
2114                         scrub_write_block_to_dev_replace(sblock);
2115         }
2116 }
2117
2118 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u64 len,
2119                            u8 *csum)
2120 {
2121         struct btrfs_ordered_sum *sum = NULL;
2122         int ret = 0;
2123         unsigned long i;
2124         unsigned long num_sectors;
2125
2126         while (!list_empty(&sctx->csum_list)) {
2127                 sum = list_first_entry(&sctx->csum_list,
2128                                        struct btrfs_ordered_sum, list);
2129                 if (sum->bytenr > logical)
2130                         return 0;
2131                 if (sum->bytenr + sum->len > logical)
2132                         break;
2133
2134                 ++sctx->stat.csum_discards;
2135                 list_del(&sum->list);
2136                 kfree(sum);
2137                 sum = NULL;
2138         }
2139         if (!sum)
2140                 return 0;
2141
2142         num_sectors = sum->len / sctx->sectorsize;
2143         for (i = 0; i < num_sectors; ++i) {
2144                 if (sum->sums[i].bytenr == logical) {
2145                         memcpy(csum, &sum->sums[i].sum, sctx->csum_size);
2146                         ret = 1;
2147                         break;
2148                 }
2149         }
2150         if (ret && i == num_sectors - 1) {
2151                 list_del(&sum->list);
2152                 kfree(sum);
2153         }
2154         return ret;
2155 }
2156
2157 /* scrub extent tries to collect up to 64 kB for each bio */
2158 static int scrub_extent(struct scrub_ctx *sctx, u64 logical, u64 len,
2159                         u64 physical, struct btrfs_device *dev, u64 flags,
2160                         u64 gen, int mirror_num, u64 physical_for_dev_replace)
2161 {
2162         int ret;
2163         u8 csum[BTRFS_CSUM_SIZE];
2164         u32 blocksize;
2165
2166         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2167                 blocksize = sctx->sectorsize;
2168                 spin_lock(&sctx->stat_lock);
2169                 sctx->stat.data_extents_scrubbed++;
2170                 sctx->stat.data_bytes_scrubbed += len;
2171                 spin_unlock(&sctx->stat_lock);
2172         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2173                 WARN_ON(sctx->nodesize != sctx->leafsize);
2174                 blocksize = sctx->nodesize;
2175                 spin_lock(&sctx->stat_lock);
2176                 sctx->stat.tree_extents_scrubbed++;
2177                 sctx->stat.tree_bytes_scrubbed += len;
2178                 spin_unlock(&sctx->stat_lock);
2179         } else {
2180                 blocksize = sctx->sectorsize;
2181                 WARN_ON(1);
2182         }
2183
2184         while (len) {
2185                 u64 l = min_t(u64, len, blocksize);
2186                 int have_csum = 0;
2187
2188                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2189                         /* push csums to sbio */
2190                         have_csum = scrub_find_csum(sctx, logical, l, csum);
2191                         if (have_csum == 0)
2192                                 ++sctx->stat.no_csum;
2193                         if (sctx->is_dev_replace && !have_csum) {
2194                                 ret = copy_nocow_pages(sctx, logical, l,
2195                                                        mirror_num,
2196                                                       physical_for_dev_replace);
2197                                 goto behind_scrub_pages;
2198                         }
2199                 }
2200                 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
2201                                   mirror_num, have_csum ? csum : NULL, 0,
2202                                   physical_for_dev_replace);
2203 behind_scrub_pages:
2204                 if (ret)
2205                         return ret;
2206                 len -= l;
2207                 logical += l;
2208                 physical += l;
2209                 physical_for_dev_replace += l;
2210         }
2211         return 0;
2212 }
2213
2214 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
2215                                            struct map_lookup *map,
2216                                            struct btrfs_device *scrub_dev,
2217                                            int num, u64 base, u64 length,
2218                                            int is_dev_replace)
2219 {
2220         struct btrfs_path *path;
2221         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2222         struct btrfs_root *root = fs_info->extent_root;
2223         struct btrfs_root *csum_root = fs_info->csum_root;
2224         struct btrfs_extent_item *extent;
2225         struct blk_plug plug;
2226         u64 flags;
2227         int ret;
2228         int slot;
2229         int i;
2230         u64 nstripes;
2231         struct extent_buffer *l;
2232         struct btrfs_key key;
2233         u64 physical;
2234         u64 logical;
2235         u64 generation;
2236         int mirror_num;
2237         struct reada_control *reada1;
2238         struct reada_control *reada2;
2239         struct btrfs_key key_start;
2240         struct btrfs_key key_end;
2241         u64 increment = map->stripe_len;
2242         u64 offset;
2243         u64 extent_logical;
2244         u64 extent_physical;
2245         u64 extent_len;
2246         struct btrfs_device *extent_dev;
2247         int extent_mirror_num;
2248
2249         nstripes = length;
2250         offset = 0;
2251         do_div(nstripes, map->stripe_len);
2252         if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
2253                 offset = map->stripe_len * num;
2254                 increment = map->stripe_len * map->num_stripes;
2255                 mirror_num = 1;
2256         } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
2257                 int factor = map->num_stripes / map->sub_stripes;
2258                 offset = map->stripe_len * (num / map->sub_stripes);
2259                 increment = map->stripe_len * factor;
2260                 mirror_num = num % map->sub_stripes + 1;
2261         } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
2262                 increment = map->stripe_len;
2263                 mirror_num = num % map->num_stripes + 1;
2264         } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
2265                 increment = map->stripe_len;
2266                 mirror_num = num % map->num_stripes + 1;
2267         } else {
2268                 increment = map->stripe_len;
2269                 mirror_num = 1;
2270         }
2271
2272         path = btrfs_alloc_path();
2273         if (!path)
2274                 return -ENOMEM;
2275
2276         /*
2277          * work on commit root. The related disk blocks are static as
2278          * long as COW is applied. This means, it is save to rewrite
2279          * them to repair disk errors without any race conditions
2280          */
2281         path->search_commit_root = 1;
2282         path->skip_locking = 1;
2283
2284         /*
2285          * trigger the readahead for extent tree csum tree and wait for
2286          * completion. During readahead, the scrub is officially paused
2287          * to not hold off transaction commits
2288          */
2289         logical = base + offset;
2290
2291         wait_event(sctx->list_wait,
2292                    atomic_read(&sctx->bios_in_flight) == 0);
2293         atomic_inc(&fs_info->scrubs_paused);
2294         wake_up(&fs_info->scrub_pause_wait);
2295
2296         /* FIXME it might be better to start readahead at commit root */
2297         key_start.objectid = logical;
2298         key_start.type = BTRFS_EXTENT_ITEM_KEY;
2299         key_start.offset = (u64)0;
2300         key_end.objectid = base + offset + nstripes * increment;
2301         key_end.type = BTRFS_EXTENT_ITEM_KEY;
2302         key_end.offset = (u64)0;
2303         reada1 = btrfs_reada_add(root, &key_start, &key_end);
2304
2305         key_start.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
2306         key_start.type = BTRFS_EXTENT_CSUM_KEY;
2307         key_start.offset = logical;
2308         key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
2309         key_end.type = BTRFS_EXTENT_CSUM_KEY;
2310         key_end.offset = base + offset + nstripes * increment;
2311         reada2 = btrfs_reada_add(csum_root, &key_start, &key_end);
2312
2313         if (!IS_ERR(reada1))
2314                 btrfs_reada_wait(reada1);
2315         if (!IS_ERR(reada2))
2316                 btrfs_reada_wait(reada2);
2317
2318         mutex_lock(&fs_info->scrub_lock);
2319         while (atomic_read(&fs_info->scrub_pause_req)) {
2320                 mutex_unlock(&fs_info->scrub_lock);
2321                 wait_event(fs_info->scrub_pause_wait,
2322                    atomic_read(&fs_info->scrub_pause_req) == 0);
2323                 mutex_lock(&fs_info->scrub_lock);
2324         }
2325         atomic_dec(&fs_info->scrubs_paused);
2326         mutex_unlock(&fs_info->scrub_lock);
2327         wake_up(&fs_info->scrub_pause_wait);
2328
2329         /*
2330          * collect all data csums for the stripe to avoid seeking during
2331          * the scrub. This might currently (crc32) end up to be about 1MB
2332          */
2333         blk_start_plug(&plug);
2334
2335         /*
2336          * now find all extents for each stripe and scrub them
2337          */
2338         logical = base + offset;
2339         physical = map->stripes[num].physical;
2340         ret = 0;
2341         for (i = 0; i < nstripes; ++i) {
2342                 /*
2343                  * canceled?
2344                  */
2345                 if (atomic_read(&fs_info->scrub_cancel_req) ||
2346                     atomic_read(&sctx->cancel_req)) {
2347                         ret = -ECANCELED;
2348                         goto out;
2349                 }
2350                 /*
2351                  * check to see if we have to pause
2352                  */
2353                 if (atomic_read(&fs_info->scrub_pause_req)) {
2354                         /* push queued extents */
2355                         atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
2356                         scrub_submit(sctx);
2357                         mutex_lock(&sctx->wr_ctx.wr_lock);
2358                         scrub_wr_submit(sctx);
2359                         mutex_unlock(&sctx->wr_ctx.wr_lock);
2360                         wait_event(sctx->list_wait,
2361                                    atomic_read(&sctx->bios_in_flight) == 0);
2362                         atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
2363                         atomic_inc(&fs_info->scrubs_paused);
2364                         wake_up(&fs_info->scrub_pause_wait);
2365                         mutex_lock(&fs_info->scrub_lock);
2366                         while (atomic_read(&fs_info->scrub_pause_req)) {
2367                                 mutex_unlock(&fs_info->scrub_lock);
2368                                 wait_event(fs_info->scrub_pause_wait,
2369                                    atomic_read(&fs_info->scrub_pause_req) == 0);
2370                                 mutex_lock(&fs_info->scrub_lock);
2371                         }
2372                         atomic_dec(&fs_info->scrubs_paused);
2373                         mutex_unlock(&fs_info->scrub_lock);
2374                         wake_up(&fs_info->scrub_pause_wait);
2375                 }
2376
2377                 ret = btrfs_lookup_csums_range(csum_root, logical,
2378                                                logical + map->stripe_len - 1,
2379                                                &sctx->csum_list, 1);
2380                 if (ret)
2381                         goto out;
2382
2383                 key.objectid = logical;
2384                 key.type = BTRFS_EXTENT_ITEM_KEY;
2385                 key.offset = (u64)0;
2386
2387                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2388                 if (ret < 0)
2389                         goto out;
2390                 if (ret > 0) {
2391                         ret = btrfs_previous_item(root, path, 0,
2392                                                   BTRFS_EXTENT_ITEM_KEY);
2393                         if (ret < 0)
2394                                 goto out;
2395                         if (ret > 0) {
2396                                 /* there's no smaller item, so stick with the
2397                                  * larger one */
2398                                 btrfs_release_path(path);
2399                                 ret = btrfs_search_slot(NULL, root, &key,
2400                                                         path, 0, 0);
2401                                 if (ret < 0)
2402                                         goto out;
2403                         }
2404                 }
2405
2406                 while (1) {
2407                         l = path->nodes[0];
2408                         slot = path->slots[0];
2409                         if (slot >= btrfs_header_nritems(l)) {
2410                                 ret = btrfs_next_leaf(root, path);
2411                                 if (ret == 0)
2412                                         continue;
2413                                 if (ret < 0)
2414                                         goto out;
2415
2416                                 break;
2417                         }
2418                         btrfs_item_key_to_cpu(l, &key, slot);
2419
2420                         if (key.objectid + key.offset <= logical)
2421                                 goto next;
2422
2423                         if (key.objectid >= logical + map->stripe_len)
2424                                 break;
2425
2426                         if (btrfs_key_type(&key) != BTRFS_EXTENT_ITEM_KEY)
2427                                 goto next;
2428
2429                         extent = btrfs_item_ptr(l, slot,
2430                                                 struct btrfs_extent_item);
2431                         flags = btrfs_extent_flags(l, extent);
2432                         generation = btrfs_extent_generation(l, extent);
2433
2434                         if (key.objectid < logical &&
2435                             (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)) {
2436                                 printk(KERN_ERR
2437                                        "btrfs scrub: tree block %llu spanning "
2438                                        "stripes, ignored. logical=%llu\n",
2439                                        (unsigned long long)key.objectid,
2440                                        (unsigned long long)logical);
2441                                 goto next;
2442                         }
2443
2444                         /*
2445                          * trim extent to this stripe
2446                          */
2447                         if (key.objectid < logical) {
2448                                 key.offset -= logical - key.objectid;
2449                                 key.objectid = logical;
2450                         }
2451                         if (key.objectid + key.offset >
2452                             logical + map->stripe_len) {
2453                                 key.offset = logical + map->stripe_len -
2454                                              key.objectid;
2455                         }
2456
2457                         extent_logical = key.objectid;
2458                         extent_physical = key.objectid - logical + physical;
2459                         extent_len = key.offset;
2460                         extent_dev = scrub_dev;
2461                         extent_mirror_num = mirror_num;
2462                         if (is_dev_replace)
2463                                 scrub_remap_extent(fs_info, extent_logical,
2464                                                    extent_len, &extent_physical,
2465                                                    &extent_dev,
2466                                                    &extent_mirror_num);
2467                         ret = scrub_extent(sctx, extent_logical, extent_len,
2468                                            extent_physical, extent_dev, flags,
2469                                            generation, extent_mirror_num,
2470                                            key.objectid - logical + physical);
2471                         if (ret)
2472                                 goto out;
2473
2474 next:
2475                         path->slots[0]++;
2476                 }
2477                 btrfs_release_path(path);
2478                 logical += increment;
2479                 physical += map->stripe_len;
2480                 spin_lock(&sctx->stat_lock);
2481                 sctx->stat.last_physical = physical;
2482                 spin_unlock(&sctx->stat_lock);
2483         }
2484 out:
2485         /* push queued extents */
2486         scrub_submit(sctx);
2487         mutex_lock(&sctx->wr_ctx.wr_lock);
2488         scrub_wr_submit(sctx);
2489         mutex_unlock(&sctx->wr_ctx.wr_lock);
2490
2491         blk_finish_plug(&plug);
2492         btrfs_free_path(path);
2493         return ret < 0 ? ret : 0;
2494 }
2495
2496 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
2497                                           struct btrfs_device *scrub_dev,
2498                                           u64 chunk_tree, u64 chunk_objectid,
2499                                           u64 chunk_offset, u64 length,
2500                                           u64 dev_offset, int is_dev_replace)
2501 {
2502         struct btrfs_mapping_tree *map_tree =
2503                 &sctx->dev_root->fs_info->mapping_tree;
2504         struct map_lookup *map;
2505         struct extent_map *em;
2506         int i;
2507         int ret = 0;
2508
2509         read_lock(&map_tree->map_tree.lock);
2510         em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
2511         read_unlock(&map_tree->map_tree.lock);
2512
2513         if (!em)
2514                 return -EINVAL;
2515
2516         map = (struct map_lookup *)em->bdev;
2517         if (em->start != chunk_offset)
2518                 goto out;
2519
2520         if (em->len < length)
2521                 goto out;
2522
2523         for (i = 0; i < map->num_stripes; ++i) {
2524                 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
2525                     map->stripes[i].physical == dev_offset) {
2526                         ret = scrub_stripe(sctx, map, scrub_dev, i,
2527                                            chunk_offset, length,
2528                                            is_dev_replace);
2529                         if (ret)
2530                                 goto out;
2531                 }
2532         }
2533 out:
2534         free_extent_map(em);
2535
2536         return ret;
2537 }
2538
2539 static noinline_for_stack
2540 int scrub_enumerate_chunks(struct scrub_ctx *sctx,
2541                            struct btrfs_device *scrub_dev, u64 start, u64 end,
2542                            int is_dev_replace)
2543 {
2544         struct btrfs_dev_extent *dev_extent = NULL;
2545         struct btrfs_path *path;
2546         struct btrfs_root *root = sctx->dev_root;
2547         struct btrfs_fs_info *fs_info = root->fs_info;
2548         u64 length;
2549         u64 chunk_tree;
2550         u64 chunk_objectid;
2551         u64 chunk_offset;
2552         int ret;
2553         int slot;
2554         struct extent_buffer *l;
2555         struct btrfs_key key;
2556         struct btrfs_key found_key;
2557         struct btrfs_block_group_cache *cache;
2558         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
2559
2560         path = btrfs_alloc_path();
2561         if (!path)
2562                 return -ENOMEM;
2563
2564         path->reada = 2;
2565         path->search_commit_root = 1;
2566         path->skip_locking = 1;
2567
2568         key.objectid = scrub_dev->devid;
2569         key.offset = 0ull;
2570         key.type = BTRFS_DEV_EXTENT_KEY;
2571
2572         while (1) {
2573                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2574                 if (ret < 0)
2575                         break;
2576                 if (ret > 0) {
2577                         if (path->slots[0] >=
2578                             btrfs_header_nritems(path->nodes[0])) {
2579                                 ret = btrfs_next_leaf(root, path);
2580                                 if (ret)
2581                                         break;
2582                         }
2583                 }
2584
2585                 l = path->nodes[0];
2586                 slot = path->slots[0];
2587
2588                 btrfs_item_key_to_cpu(l, &found_key, slot);
2589
2590                 if (found_key.objectid != scrub_dev->devid)
2591                         break;
2592
2593                 if (btrfs_key_type(&found_key) != BTRFS_DEV_EXTENT_KEY)
2594                         break;
2595
2596                 if (found_key.offset >= end)
2597                         break;
2598
2599                 if (found_key.offset < key.offset)
2600                         break;
2601
2602                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
2603                 length = btrfs_dev_extent_length(l, dev_extent);
2604
2605                 if (found_key.offset + length <= start) {
2606                         key.offset = found_key.offset + length;
2607                         btrfs_release_path(path);
2608                         continue;
2609                 }
2610
2611                 chunk_tree = btrfs_dev_extent_chunk_tree(l, dev_extent);
2612                 chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
2613                 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
2614
2615                 /*
2616                  * get a reference on the corresponding block group to prevent
2617                  * the chunk from going away while we scrub it
2618                  */
2619                 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
2620                 if (!cache) {
2621                         ret = -ENOENT;
2622                         break;
2623                 }
2624                 dev_replace->cursor_right = found_key.offset + length;
2625                 dev_replace->cursor_left = found_key.offset;
2626                 dev_replace->item_needs_writeback = 1;
2627                 ret = scrub_chunk(sctx, scrub_dev, chunk_tree, chunk_objectid,
2628                                   chunk_offset, length, found_key.offset,
2629                                   is_dev_replace);
2630
2631                 /*
2632                  * flush, submit all pending read and write bios, afterwards
2633                  * wait for them.
2634                  * Note that in the dev replace case, a read request causes
2635                  * write requests that are submitted in the read completion
2636                  * worker. Therefore in the current situation, it is required
2637                  * that all write requests are flushed, so that all read and
2638                  * write requests are really completed when bios_in_flight
2639                  * changes to 0.
2640                  */
2641                 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
2642                 scrub_submit(sctx);
2643                 mutex_lock(&sctx->wr_ctx.wr_lock);
2644                 scrub_wr_submit(sctx);
2645                 mutex_unlock(&sctx->wr_ctx.wr_lock);
2646
2647                 wait_event(sctx->list_wait,
2648                            atomic_read(&sctx->bios_in_flight) == 0);
2649                 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
2650                 atomic_inc(&fs_info->scrubs_paused);
2651                 wake_up(&fs_info->scrub_pause_wait);
2652                 wait_event(sctx->list_wait,
2653                            atomic_read(&sctx->workers_pending) == 0);
2654
2655                 mutex_lock(&fs_info->scrub_lock);
2656                 while (atomic_read(&fs_info->scrub_pause_req)) {
2657                         mutex_unlock(&fs_info->scrub_lock);
2658                         wait_event(fs_info->scrub_pause_wait,
2659                            atomic_read(&fs_info->scrub_pause_req) == 0);
2660                         mutex_lock(&fs_info->scrub_lock);
2661                 }
2662                 atomic_dec(&fs_info->scrubs_paused);
2663                 mutex_unlock(&fs_info->scrub_lock);
2664                 wake_up(&fs_info->scrub_pause_wait);
2665
2666                 dev_replace->cursor_left = dev_replace->cursor_right;
2667                 dev_replace->item_needs_writeback = 1;
2668                 btrfs_put_block_group(cache);
2669                 if (ret)
2670                         break;
2671                 if (is_dev_replace &&
2672                     atomic64_read(&dev_replace->num_write_errors) > 0) {
2673                         ret = -EIO;
2674                         break;
2675                 }
2676                 if (sctx->stat.malloc_errors > 0) {
2677                         ret = -ENOMEM;
2678                         break;
2679                 }
2680
2681                 key.offset = found_key.offset + length;
2682                 btrfs_release_path(path);
2683         }
2684
2685         btrfs_free_path(path);
2686
2687         /*
2688          * ret can still be 1 from search_slot or next_leaf,
2689          * that's not an error
2690          */
2691         return ret < 0 ? ret : 0;
2692 }
2693
2694 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
2695                                            struct btrfs_device *scrub_dev)
2696 {
2697         int     i;
2698         u64     bytenr;
2699         u64     gen;
2700         int     ret;
2701         struct btrfs_root *root = sctx->dev_root;
2702
2703         if (root->fs_info->fs_state & BTRFS_SUPER_FLAG_ERROR)
2704                 return -EIO;
2705
2706         gen = root->fs_info->last_trans_committed;
2707
2708         for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
2709                 bytenr = btrfs_sb_offset(i);
2710                 if (bytenr + BTRFS_SUPER_INFO_SIZE > scrub_dev->total_bytes)
2711                         break;
2712
2713                 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
2714                                   scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
2715                                   NULL, 1, bytenr);
2716                 if (ret)
2717                         return ret;
2718         }
2719         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
2720
2721         return 0;
2722 }
2723
2724 /*
2725  * get a reference count on fs_info->scrub_workers. start worker if necessary
2726  */
2727 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
2728                                                 int is_dev_replace)
2729 {
2730         int ret = 0;
2731
2732         mutex_lock(&fs_info->scrub_lock);
2733         if (fs_info->scrub_workers_refcnt == 0) {
2734                 if (is_dev_replace)
2735                         btrfs_init_workers(&fs_info->scrub_workers, "scrub", 1,
2736                                         &fs_info->generic_worker);
2737                 else
2738                         btrfs_init_workers(&fs_info->scrub_workers, "scrub",
2739                                         fs_info->thread_pool_size,
2740                                         &fs_info->generic_worker);
2741                 fs_info->scrub_workers.idle_thresh = 4;
2742                 ret = btrfs_start_workers(&fs_info->scrub_workers);
2743                 if (ret)
2744                         goto out;
2745                 btrfs_init_workers(&fs_info->scrub_wr_completion_workers,
2746                                    "scrubwrc",
2747                                    fs_info->thread_pool_size,
2748                                    &fs_info->generic_worker);
2749                 fs_info->scrub_wr_completion_workers.idle_thresh = 2;
2750                 ret = btrfs_start_workers(
2751                                 &fs_info->scrub_wr_completion_workers);
2752                 if (ret)
2753                         goto out;
2754                 btrfs_init_workers(&fs_info->scrub_nocow_workers, "scrubnc", 1,
2755                                    &fs_info->generic_worker);
2756                 ret = btrfs_start_workers(&fs_info->scrub_nocow_workers);
2757                 if (ret)
2758                         goto out;
2759         }
2760         ++fs_info->scrub_workers_refcnt;
2761 out:
2762         mutex_unlock(&fs_info->scrub_lock);
2763
2764         return ret;
2765 }
2766
2767 static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info)
2768 {
2769         mutex_lock(&fs_info->scrub_lock);
2770         if (--fs_info->scrub_workers_refcnt == 0) {
2771                 btrfs_stop_workers(&fs_info->scrub_workers);
2772                 btrfs_stop_workers(&fs_info->scrub_wr_completion_workers);
2773                 btrfs_stop_workers(&fs_info->scrub_nocow_workers);
2774         }
2775         WARN_ON(fs_info->scrub_workers_refcnt < 0);
2776         mutex_unlock(&fs_info->scrub_lock);
2777 }
2778
2779 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
2780                     u64 end, struct btrfs_scrub_progress *progress,
2781                     int readonly, int is_dev_replace)
2782 {
2783         struct scrub_ctx *sctx;
2784         int ret;
2785         struct btrfs_device *dev;
2786
2787         if (btrfs_fs_closing(fs_info))
2788                 return -EINVAL;
2789
2790         /*
2791          * check some assumptions
2792          */
2793         if (fs_info->chunk_root->nodesize != fs_info->chunk_root->leafsize) {
2794                 printk(KERN_ERR
2795                        "btrfs_scrub: size assumption nodesize == leafsize (%d == %d) fails\n",
2796                        fs_info->chunk_root->nodesize,
2797                        fs_info->chunk_root->leafsize);
2798                 return -EINVAL;
2799         }
2800
2801         if (fs_info->chunk_root->nodesize > BTRFS_STRIPE_LEN) {
2802                 /*
2803                  * in this case scrub is unable to calculate the checksum
2804                  * the way scrub is implemented. Do not handle this
2805                  * situation at all because it won't ever happen.
2806                  */
2807                 printk(KERN_ERR
2808                        "btrfs_scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails\n",
2809                        fs_info->chunk_root->nodesize, BTRFS_STRIPE_LEN);
2810                 return -EINVAL;
2811         }
2812
2813         if (fs_info->chunk_root->sectorsize != PAGE_SIZE) {
2814                 /* not supported for data w/o checksums */
2815                 printk(KERN_ERR
2816                        "btrfs_scrub: size assumption sectorsize != PAGE_SIZE (%d != %lld) fails\n",
2817                        fs_info->chunk_root->sectorsize,
2818                        (unsigned long long)PAGE_SIZE);
2819                 return -EINVAL;
2820         }
2821
2822         if (fs_info->chunk_root->nodesize >
2823             PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
2824             fs_info->chunk_root->sectorsize >
2825             PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
2826                 /*
2827                  * would exhaust the array bounds of pagev member in
2828                  * struct scrub_block
2829                  */
2830                 pr_err("btrfs_scrub: size assumption nodesize and sectorsize <= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails\n",
2831                        fs_info->chunk_root->nodesize,
2832                        SCRUB_MAX_PAGES_PER_BLOCK,
2833                        fs_info->chunk_root->sectorsize,
2834                        SCRUB_MAX_PAGES_PER_BLOCK);
2835                 return -EINVAL;
2836         }
2837
2838         ret = scrub_workers_get(fs_info, is_dev_replace);
2839         if (ret)
2840                 return ret;
2841
2842         mutex_lock(&fs_info->fs_devices->device_list_mutex);
2843         dev = btrfs_find_device(fs_info, devid, NULL, NULL);
2844         if (!dev || (dev->missing && !is_dev_replace)) {
2845                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2846                 scrub_workers_put(fs_info);
2847                 return -ENODEV;
2848         }
2849         mutex_lock(&fs_info->scrub_lock);
2850
2851         if (!dev->in_fs_metadata || dev->is_tgtdev_for_dev_replace) {
2852                 mutex_unlock(&fs_info->scrub_lock);
2853                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2854                 scrub_workers_put(fs_info);
2855                 return -EIO;
2856         }
2857
2858         btrfs_dev_replace_lock(&fs_info->dev_replace);
2859         if (dev->scrub_device ||
2860             (!is_dev_replace &&
2861              btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
2862                 btrfs_dev_replace_unlock(&fs_info->dev_replace);
2863                 mutex_unlock(&fs_info->scrub_lock);
2864                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2865                 scrub_workers_put(fs_info);
2866                 return -EINPROGRESS;
2867         }
2868         btrfs_dev_replace_unlock(&fs_info->dev_replace);
2869         sctx = scrub_setup_ctx(dev, is_dev_replace);
2870         if (IS_ERR(sctx)) {
2871                 mutex_unlock(&fs_info->scrub_lock);
2872                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2873                 scrub_workers_put(fs_info);
2874                 return PTR_ERR(sctx);
2875         }
2876         sctx->readonly = readonly;
2877         dev->scrub_device = sctx;
2878
2879         atomic_inc(&fs_info->scrubs_running);
2880         mutex_unlock(&fs_info->scrub_lock);
2881         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2882
2883         if (!is_dev_replace) {
2884                 down_read(&fs_info->scrub_super_lock);
2885                 ret = scrub_supers(sctx, dev);
2886                 up_read(&fs_info->scrub_super_lock);
2887         }
2888
2889         if (!ret)
2890                 ret = scrub_enumerate_chunks(sctx, dev, start, end,
2891                                              is_dev_replace);
2892
2893         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
2894         atomic_dec(&fs_info->scrubs_running);
2895         wake_up(&fs_info->scrub_pause_wait);
2896
2897         wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
2898
2899         if (progress)
2900                 memcpy(progress, &sctx->stat, sizeof(*progress));
2901
2902         mutex_lock(&fs_info->scrub_lock);
2903         dev->scrub_device = NULL;
2904         mutex_unlock(&fs_info->scrub_lock);
2905
2906         scrub_free_ctx(sctx);
2907         scrub_workers_put(fs_info);
2908
2909         return ret;
2910 }
2911
2912 void btrfs_scrub_pause(struct btrfs_root *root)
2913 {
2914         struct btrfs_fs_info *fs_info = root->fs_info;
2915
2916         mutex_lock(&fs_info->scrub_lock);
2917         atomic_inc(&fs_info->scrub_pause_req);
2918         while (atomic_read(&fs_info->scrubs_paused) !=
2919                atomic_read(&fs_info->scrubs_running)) {
2920                 mutex_unlock(&fs_info->scrub_lock);
2921                 wait_event(fs_info->scrub_pause_wait,
2922                            atomic_read(&fs_info->scrubs_paused) ==
2923                            atomic_read(&fs_info->scrubs_running));
2924                 mutex_lock(&fs_info->scrub_lock);
2925         }
2926         mutex_unlock(&fs_info->scrub_lock);
2927 }
2928
2929 void btrfs_scrub_continue(struct btrfs_root *root)
2930 {
2931         struct btrfs_fs_info *fs_info = root->fs_info;
2932
2933         atomic_dec(&fs_info->scrub_pause_req);
2934         wake_up(&fs_info->scrub_pause_wait);
2935 }
2936
2937 void btrfs_scrub_pause_super(struct btrfs_root *root)
2938 {
2939         down_write(&root->fs_info->scrub_super_lock);
2940 }
2941
2942 void btrfs_scrub_continue_super(struct btrfs_root *root)
2943 {
2944         up_write(&root->fs_info->scrub_super_lock);
2945 }
2946
2947 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
2948 {
2949         mutex_lock(&fs_info->scrub_lock);
2950         if (!atomic_read(&fs_info->scrubs_running)) {
2951                 mutex_unlock(&fs_info->scrub_lock);
2952                 return -ENOTCONN;
2953         }
2954
2955         atomic_inc(&fs_info->scrub_cancel_req);
2956         while (atomic_read(&fs_info->scrubs_running)) {
2957                 mutex_unlock(&fs_info->scrub_lock);
2958                 wait_event(fs_info->scrub_pause_wait,
2959                            atomic_read(&fs_info->scrubs_running) == 0);
2960                 mutex_lock(&fs_info->scrub_lock);
2961         }
2962         atomic_dec(&fs_info->scrub_cancel_req);
2963         mutex_unlock(&fs_info->scrub_lock);
2964
2965         return 0;
2966 }
2967
2968 int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info,
2969                            struct btrfs_device *dev)
2970 {
2971         struct scrub_ctx *sctx;
2972
2973         mutex_lock(&fs_info->scrub_lock);
2974         sctx = dev->scrub_device;
2975         if (!sctx) {
2976                 mutex_unlock(&fs_info->scrub_lock);
2977                 return -ENOTCONN;
2978         }
2979         atomic_inc(&sctx->cancel_req);
2980         while (dev->scrub_device) {
2981                 mutex_unlock(&fs_info->scrub_lock);
2982                 wait_event(fs_info->scrub_pause_wait,
2983                            dev->scrub_device == NULL);
2984                 mutex_lock(&fs_info->scrub_lock);
2985         }
2986         mutex_unlock(&fs_info->scrub_lock);
2987
2988         return 0;
2989 }
2990
2991 int btrfs_scrub_cancel_devid(struct btrfs_root *root, u64 devid)
2992 {
2993         struct btrfs_fs_info *fs_info = root->fs_info;
2994         struct btrfs_device *dev;
2995         int ret;
2996
2997         /*
2998          * we have to hold the device_list_mutex here so the device
2999          * does not go away in cancel_dev. FIXME: find a better solution
3000          */
3001         mutex_lock(&fs_info->fs_devices->device_list_mutex);
3002         dev = btrfs_find_device(fs_info, devid, NULL, NULL);
3003         if (!dev) {
3004                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3005                 return -ENODEV;
3006         }
3007         ret = btrfs_scrub_cancel_dev(fs_info, dev);
3008         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3009
3010         return ret;
3011 }
3012
3013 int btrfs_scrub_progress(struct btrfs_root *root, u64 devid,
3014                          struct btrfs_scrub_progress *progress)
3015 {
3016         struct btrfs_device *dev;
3017         struct scrub_ctx *sctx = NULL;
3018
3019         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
3020         dev = btrfs_find_device(root->fs_info, devid, NULL, NULL);
3021         if (dev)
3022                 sctx = dev->scrub_device;
3023         if (sctx)
3024                 memcpy(progress, &sctx->stat, sizeof(*progress));
3025         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
3026
3027         return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
3028 }
3029
3030 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
3031                                u64 extent_logical, u64 extent_len,
3032                                u64 *extent_physical,
3033                                struct btrfs_device **extent_dev,
3034                                int *extent_mirror_num)
3035 {
3036         u64 mapped_length;
3037         struct btrfs_bio *bbio = NULL;
3038         int ret;
3039
3040         mapped_length = extent_len;
3041         ret = btrfs_map_block(fs_info, READ, extent_logical,
3042                               &mapped_length, &bbio, 0);
3043         if (ret || !bbio || mapped_length < extent_len ||
3044             !bbio->stripes[0].dev->bdev) {
3045                 kfree(bbio);
3046                 return;
3047         }
3048
3049         *extent_physical = bbio->stripes[0].physical;
3050         *extent_mirror_num = bbio->mirror_num;
3051         *extent_dev = bbio->stripes[0].dev;
3052         kfree(bbio);
3053 }
3054
3055 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
3056                               struct scrub_wr_ctx *wr_ctx,
3057                               struct btrfs_fs_info *fs_info,
3058                               struct btrfs_device *dev,
3059                               int is_dev_replace)
3060 {
3061         WARN_ON(wr_ctx->wr_curr_bio != NULL);
3062
3063         mutex_init(&wr_ctx->wr_lock);
3064         wr_ctx->wr_curr_bio = NULL;
3065         if (!is_dev_replace)
3066                 return 0;
3067
3068         WARN_ON(!dev->bdev);
3069         wr_ctx->pages_per_wr_bio = min_t(int, SCRUB_PAGES_PER_WR_BIO,
3070                                          bio_get_nr_vecs(dev->bdev));
3071         wr_ctx->tgtdev = dev;
3072         atomic_set(&wr_ctx->flush_all_writes, 0);
3073         return 0;
3074 }
3075
3076 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx)
3077 {
3078         mutex_lock(&wr_ctx->wr_lock);
3079         kfree(wr_ctx->wr_curr_bio);
3080         wr_ctx->wr_curr_bio = NULL;
3081         mutex_unlock(&wr_ctx->wr_lock);
3082 }
3083
3084 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
3085                             int mirror_num, u64 physical_for_dev_replace)
3086 {
3087         struct scrub_copy_nocow_ctx *nocow_ctx;
3088         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
3089
3090         nocow_ctx = kzalloc(sizeof(*nocow_ctx), GFP_NOFS);
3091         if (!nocow_ctx) {
3092                 spin_lock(&sctx->stat_lock);
3093                 sctx->stat.malloc_errors++;
3094                 spin_unlock(&sctx->stat_lock);
3095                 return -ENOMEM;
3096         }
3097
3098         scrub_pending_trans_workers_inc(sctx);
3099
3100         nocow_ctx->sctx = sctx;
3101         nocow_ctx->logical = logical;
3102         nocow_ctx->len = len;
3103         nocow_ctx->mirror_num = mirror_num;
3104         nocow_ctx->physical_for_dev_replace = physical_for_dev_replace;
3105         nocow_ctx->work.func = copy_nocow_pages_worker;
3106         btrfs_queue_worker(&fs_info->scrub_nocow_workers,
3107                            &nocow_ctx->work);
3108
3109         return 0;
3110 }
3111
3112 static void copy_nocow_pages_worker(struct btrfs_work *work)
3113 {
3114         struct scrub_copy_nocow_ctx *nocow_ctx =
3115                 container_of(work, struct scrub_copy_nocow_ctx, work);
3116         struct scrub_ctx *sctx = nocow_ctx->sctx;
3117         u64 logical = nocow_ctx->logical;
3118         u64 len = nocow_ctx->len;
3119         int mirror_num = nocow_ctx->mirror_num;
3120         u64 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
3121         int ret;
3122         struct btrfs_trans_handle *trans = NULL;
3123         struct btrfs_fs_info *fs_info;
3124         struct btrfs_path *path;
3125         struct btrfs_root *root;
3126         int not_written = 0;
3127
3128         fs_info = sctx->dev_root->fs_info;
3129         root = fs_info->extent_root;
3130
3131         path = btrfs_alloc_path();
3132         if (!path) {
3133                 spin_lock(&sctx->stat_lock);
3134                 sctx->stat.malloc_errors++;
3135                 spin_unlock(&sctx->stat_lock);
3136                 not_written = 1;
3137                 goto out;
3138         }
3139
3140         trans = btrfs_join_transaction(root);
3141         if (IS_ERR(trans)) {
3142                 not_written = 1;
3143                 goto out;
3144         }
3145
3146         ret = iterate_inodes_from_logical(logical, fs_info, path,
3147                                           copy_nocow_pages_for_inode,
3148                                           nocow_ctx);
3149         if (ret != 0 && ret != -ENOENT) {
3150                 pr_warn("iterate_inodes_from_logical() failed: log %llu, phys %llu, len %llu, mir %llu, ret %d\n",
3151                         (unsigned long long)logical,
3152                         (unsigned long long)physical_for_dev_replace,
3153                         (unsigned long long)len,
3154                         (unsigned long long)mirror_num, ret);
3155                 not_written = 1;
3156                 goto out;
3157         }
3158
3159 out:
3160         if (trans && !IS_ERR(trans))
3161                 btrfs_end_transaction(trans, root);
3162         if (not_written)
3163                 btrfs_dev_replace_stats_inc(&fs_info->dev_replace.
3164                                             num_uncorrectable_read_errors);
3165
3166         btrfs_free_path(path);
3167         kfree(nocow_ctx);
3168
3169         scrub_pending_trans_workers_dec(sctx);
3170 }
3171
3172 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root, void *ctx)
3173 {
3174         unsigned long index;
3175         struct scrub_copy_nocow_ctx *nocow_ctx = ctx;
3176         int ret = 0;
3177         struct btrfs_key key;
3178         struct inode *inode = NULL;
3179         struct btrfs_root *local_root;
3180         u64 physical_for_dev_replace;
3181         u64 len;
3182         struct btrfs_fs_info *fs_info = nocow_ctx->sctx->dev_root->fs_info;
3183
3184         key.objectid = root;
3185         key.type = BTRFS_ROOT_ITEM_KEY;
3186         key.offset = (u64)-1;
3187         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
3188         if (IS_ERR(local_root))
3189                 return PTR_ERR(local_root);
3190
3191         key.type = BTRFS_INODE_ITEM_KEY;
3192         key.objectid = inum;
3193         key.offset = 0;
3194         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
3195         if (IS_ERR(inode))
3196                 return PTR_ERR(inode);
3197
3198         physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
3199         len = nocow_ctx->len;
3200         while (len >= PAGE_CACHE_SIZE) {
3201                 struct page *page = NULL;
3202                 int ret_sub;
3203
3204                 index = offset >> PAGE_CACHE_SHIFT;
3205
3206                 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
3207                 if (!page) {
3208                         pr_err("find_or_create_page() failed\n");
3209                         ret = -ENOMEM;
3210                         goto next_page;
3211                 }
3212
3213                 if (PageUptodate(page)) {
3214                         if (PageDirty(page))
3215                                 goto next_page;
3216                 } else {
3217                         ClearPageError(page);
3218                         ret_sub = extent_read_full_page(&BTRFS_I(inode)->
3219                                                          io_tree,
3220                                                         page, btrfs_get_extent,
3221                                                         nocow_ctx->mirror_num);
3222                         if (ret_sub) {
3223                                 ret = ret_sub;
3224                                 goto next_page;
3225                         }
3226                         wait_on_page_locked(page);
3227                         if (!PageUptodate(page)) {
3228                                 ret = -EIO;
3229                                 goto next_page;
3230                         }
3231                 }
3232                 ret_sub = write_page_nocow(nocow_ctx->sctx,
3233                                            physical_for_dev_replace, page);
3234                 if (ret_sub) {
3235                         ret = ret_sub;
3236                         goto next_page;
3237                 }
3238
3239 next_page:
3240                 if (page) {
3241                         unlock_page(page);
3242                         put_page(page);
3243                 }
3244                 offset += PAGE_CACHE_SIZE;
3245                 physical_for_dev_replace += PAGE_CACHE_SIZE;
3246                 len -= PAGE_CACHE_SIZE;
3247         }
3248
3249         if (inode)
3250                 iput(inode);
3251         return ret;
3252 }
3253
3254 static int write_page_nocow(struct scrub_ctx *sctx,
3255                             u64 physical_for_dev_replace, struct page *page)
3256 {
3257         struct bio *bio;
3258         struct btrfs_device *dev;
3259         int ret;
3260         DECLARE_COMPLETION_ONSTACK(compl);
3261
3262         dev = sctx->wr_ctx.tgtdev;
3263         if (!dev)
3264                 return -EIO;
3265         if (!dev->bdev) {
3266                 printk_ratelimited(KERN_WARNING
3267                         "btrfs: scrub write_page_nocow(bdev == NULL) is unexpected!\n");
3268                 return -EIO;
3269         }
3270         bio = bio_alloc(GFP_NOFS, 1);
3271         if (!bio) {
3272                 spin_lock(&sctx->stat_lock);
3273                 sctx->stat.malloc_errors++;
3274                 spin_unlock(&sctx->stat_lock);
3275                 return -ENOMEM;
3276         }
3277         bio->bi_private = &compl;
3278         bio->bi_end_io = scrub_complete_bio_end_io;
3279         bio->bi_size = 0;
3280         bio->bi_sector = physical_for_dev_replace >> 9;
3281         bio->bi_bdev = dev->bdev;
3282         ret = bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
3283         if (ret != PAGE_CACHE_SIZE) {
3284 leave_with_eio:
3285                 bio_put(bio);
3286                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
3287                 return -EIO;
3288         }
3289         btrfsic_submit_bio(WRITE_SYNC, bio);
3290         wait_for_completion(&compl);
3291
3292         if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
3293                 goto leave_with_eio;
3294
3295         bio_put(bio);
3296         return 0;
3297 }