mirrors/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2024-09-24 19:35:58 +00:00
Code Issues Releases Wiki Activity

for-6.4-tag

Browse source 
-----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEE8rQSAMVO+zA4DBdWxWXV+ddtWDsFAmRHC3gACgkQxWXV+ddt
 WDvI/A//ZzREEE0wNexbuidoTacDVXVJ6LBb2K1eP+HUKfsmd6GYWQDJ9x/ExpKb
 T1ehLibCYWLeYxEREFbjXI3x9G8mrvLzvzsqXs/MzJPkmEF1igPddFztidBwvLQH
 ey/Bh+cra2bpVhRhkX0Cf09/q/YWp17/d14ZxxW60PMfyhx8RWXejXhHkulOPVv8
 +3FL8E0kc2Zjx9ioUwOy/i18LR6YzsCNVXoHzUZuWyWM4A7NG2TZR6FhuLSjlWSZ
 3RAnROwr+8i5nR0xchcyYaVMO2LMbqH6mBtHnXCtxCr+4pFrfrvKym+CQco/Xriz
 v1y/xDc23XeYXLCVhb0beJ6uRcjaM9+gvDF1oVBSJEv6V7sQr/tEGo/8QRehfEfT
 FTro7Lf89R1GOa1IBSkv/T5S25d9LlIID3/g7PbcUBtXNKvLAjDAGTH9bzL4HS5x
 /MKwN80GvaGs1KyEfUndbVPIpAwNFDYZPHM7nw1x+JTkIBcHgfjRyAMAC9jrJd0D
 730W04c+0nXZtQGtKKsxc3U8y4ewzSJAKx9t7Vgo7+1P6dSRnzvJee3x/5kXV9Yn
 MhxxzYDfIN9EcWbASdSm11gY5WZdG3an609pO7nc1T2K4Tuo0SPs4xOR7c3xuZrY
 MN5z3QFWyI2ustUuTG+nsd5J81j76DEmj5ymWQfG3SBplTneDM0=
 =Jt7p
 -----END PGP SIGNATURE-----

Merge tag 'for-6.4-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux

Pull btrfs updates from David Sterba:
 "Mostly core changes and cleanups, some notable fixes and two
  performance improvements in directory logging.

  The IO path cleanups are removing or refactoring old code, scrub main
  loop has been completely rewritten also refactoring old code.

  There are some changes to non-btrfs code, mostly trivial, the cgroup
  punt bio logic is only moved from generic code.

  Performance improvements:

   - improve logging changes in a directory during one transaction,
     avoid iterating over items and reduce lock contention (fsync time
     4x lower)

   - when logging directory entries during one transaction, reduce
     locking of subvolume trees by checking tree-log instead
     (improvement in throughput and latency for concurrent access to a
     subvolume)

  Notable fixes:

   - dev-replace:
      - properly honor read mode when requested to avoid reading from
        source device
      - target device won't be used for eventual read repair, this is
        unreliable for NODATASUM files
      - when there are unpaired (and unrepairable) metadata during
        replace, exit early with error and don't try to finish whole
        operation

   - scrub ioctl properly rejects unknown flags

   - fix global block reserve calculations

   - fix partial direct io write when there's a page fault in the
     middle, iomap will try to continue with partial request but the
     btrfs part did not match that, this can lead to zeros written
     instead of data

  Core changes:

   - io path:
      - continued cleanups and refactoring around bio handling
      - extent io submit path simplifications and cleanups
      - flush write path simplifications and cleanups
      - rework logic of passing sync mode of bio, with further cleanups

   - rewrite scrub code flow, restructure how the stripes are enumerated
     and verified in a more unified way

   - allow to set lower threshold for block group reclaim in debug mode
     to aid zoned mode testing

   - remove obsolete time-based delayed ref throttling logic when
     truncating items

   - DREW locks are not using percpu variables anymore

   - more warning fixes (-Wmaybe-uninitialized)

   - u64 division simplifications

   - error handling improvements

  Non-btrfs code changes:

   - push cgroup punt bio logic to btrfs code (there was no other user
     of that), the functionality can be now selected separately by
     BLK_CGROUP_PUNT_BIO

   - crc32c_impl removed after removing last uses in btrfs code

   - add btrfs_assertfail() to objtool table"

* tag 'for-6.4-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux: (147 commits)
  btrfs: mark btrfs_assertfail() __noreturn
  btrfs: fix uninitialized variable warnings
  btrfs: use log root when iterating over index keys when logging directory
  btrfs: avoid iterating over all indexes when logging directory
  btrfs: dev-replace: error out if we have unrepaired metadata error during
  btrfs: remove pointless loop at btrfs_get_next_valid_item()
  btrfs: scrub: reject unsupported scrub flags
  btrfs: reinterpret async discard iops_limit=0 as no delay
  btrfs: set default discard iops_limit to 1000
  btrfs: remove unused raid56 functions which were dedicated for scrub
  btrfs: scrub: remove scrub_bio structure
  btrfs: scrub: remove scrub_block and scrub_sector structures
  btrfs: scrub: remove the old scrub recheck code
  btrfs: scrub: remove the old writeback infrastructure
  btrfs: scrub: remove scrub_parity structure
  btrfs: scrub: use scrub_stripe to implement RAID56 P/Q scrub
  btrfs: scrub: switch scrub_simple_mirror() to scrub_stripe infrastructure
  btrfs: scrub: introduce helper to queue a stripe for scrub
  btrfs: scrub: introduce error reporting functionality for scrub_stripe
  btrfs: scrub: introduce a writeback helper for scrub_stripe
  ...
This commit is contained in:
Linus Torvalds 2023-04-26 09:13:44 -07:00
commit 85d7ab2463
62 changed files with 2912 additions and 4889 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
block/Kconfig
View file

@ -41,6 +41,9 @@ config BLK_RQ_ALLOC_TIME
config BLK_CGROUP_RWSTAT
bool
config BLK_CGROUP_PUNT_BIO
bool
config BLK_DEV_BSG_COMMON
tristate

78
block/blk-cgroup.c
View file

@ -56,7 +56,6 @@ static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
bool blkcg_debug_stats = false;
static struct workqueue_struct *blkcg_punt_bio_wq;
#define BLKG_DESTROY_BATCH_SIZE 64
@ -166,7 +165,9 @@ static void __blkg_release(struct rcu_head *rcu)
{
struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
WARN_ON(!bio_list_empty(&blkg->async_bios));
#endif
/* release the blkcg and parent blkg refs this blkg has been holding */
css_put(&blkg->blkcg->css);
@ -188,6 +189,9 @@ static void blkg_release(struct percpu_ref *ref)
call_rcu(&blkg->rcu_head, __blkg_release);
}
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
static struct workqueue_struct *blkcg_punt_bio_wq;
static void blkg_async_bio_workfn(struct work_struct *work)
{
struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
@ -198,10 +202,10 @@ static void blkg_async_bio_workfn(struct work_struct *work)
bool need_plug = false;
/* as long as there are pending bios, @blkg can't go away */
spin_lock_bh(&blkg->async_bio_lock);
spin_lock(&blkg->async_bio_lock);
bio_list_merge(&bios, &blkg->async_bios);
bio_list_init(&blkg->async_bios);
spin_unlock_bh(&blkg->async_bio_lock);
spin_unlock(&blkg->async_bio_lock);
/* start plug only when bio_list contains at least 2 bios */
if (bios.head && bios.head->bi_next) {
@ -214,6 +218,40 @@ static void blkg_async_bio_workfn(struct work_struct *work)
blk_finish_plug(&plug);
}
/*
* When a shared kthread issues a bio for a cgroup, doing so synchronously can
* lead to priority inversions as the kthread can be trapped waiting for that
* cgroup. Use this helper instead of submit_bio to punt the actual issuing to
* a dedicated per-blkcg work item to avoid such priority inversions.
*/
void blkcg_punt_bio_submit(struct bio *bio)
{
struct blkcg_gq *blkg = bio->bi_blkg;
if (blkg->parent) {
spin_lock(&blkg->async_bio_lock);
bio_list_add(&blkg->async_bios, bio);
spin_unlock(&blkg->async_bio_lock);
queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);
} else {
/* never bounce for the root cgroup */
submit_bio(bio);
}
}
EXPORT_SYMBOL_GPL(blkcg_punt_bio_submit);
static int __init blkcg_punt_bio_init(void)
{
blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio",
WQ_MEM_RECLAIM | WQ_FREEZABLE |
WQ_UNBOUND | WQ_SYSFS, 0);
if (!blkcg_punt_bio_wq)
return -ENOMEM;
return 0;
}
subsys_initcall(blkcg_punt_bio_init);
#endif /* CONFIG_BLK_CGROUP_PUNT_BIO */
/**
* bio_blkcg_css - return the blkcg CSS associated with a bio
* @bio: target bio
@ -269,10 +307,12 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,
blkg->q = disk->queue;
INIT_LIST_HEAD(&blkg->q_node);
blkg->blkcg = blkcg;
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
spin_lock_init(&blkg->async_bio_lock);
bio_list_init(&blkg->async_bios);
INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
blkg->blkcg = blkcg;
#endif
u64_stats_init(&blkg->iostat.sync);
for_each_possible_cpu(cpu) {
@ -1688,25 +1728,6 @@ void blkcg_policy_unregister(struct blkcg_policy *pol)
}
EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
bool __blkcg_punt_bio_submit(struct bio *bio)
{
struct blkcg_gq *blkg = bio->bi_blkg;
/* consume the flag first */
bio->bi_opf &= ~REQ_CGROUP_PUNT;
/* never bounce for the root cgroup */
if (!blkg->parent)
return false;
spin_lock_bh(&blkg->async_bio_lock);
bio_list_add(&blkg->async_bios, bio);
spin_unlock_bh(&blkg->async_bio_lock);
queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);
return true;
}
/*
* Scale the accumulated delay based on how long it has been since we updated
* the delay. We only call this when we are adding delay, in case it's been a
@ -2085,16 +2106,5 @@ bool blk_cgroup_congested(void)
return ret;
}
static int __init blkcg_init(void)
{
blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio",
WQ_MEM_RECLAIM | WQ_FREEZABLE |
WQ_UNBOUND | WQ_SYSFS, 0);
if (!blkcg_punt_bio_wq)
return -ENOMEM;
return 0;
}
subsys_initcall(blkcg_init);
module_param(blkcg_debug_stats, bool, 0644);
MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not");

15
block/blk-cgroup.h
View file

@ -72,9 +72,10 @@ struct blkcg_gq {
struct blkg_iostat_set iostat;
struct blkg_policy_data *pd[BLKCG_MAX_POLS];
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
spinlock_t async_bio_lock;
struct bio_list async_bios;
#endif
union {
struct work_struct async_bio_work;
struct work_struct free_work;
@ -375,16 +376,6 @@ static inline void blkg_put(struct blkcg_gq *blkg)
if (((d_blkg) = blkg_lookup(css_to_blkcg(pos_css), \
(p_blkg)->q)))
bool __blkcg_punt_bio_submit(struct bio *bio);
static inline bool blkcg_punt_bio_submit(struct bio *bio)
{
if (bio->bi_opf & REQ_CGROUP_PUNT)
return __blkcg_punt_bio_submit(bio);
else
return false;
}
static inline void blkcg_bio_issue_init(struct bio *bio)
{
bio_issue_init(&bio->bi_issue, bio_sectors(bio));
@ -506,8 +497,6 @@ static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return
static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
static inline void blkg_get(struct blkcg_gq *blkg) { }
static inline void blkg_put(struct blkcg_gq *blkg) { }
static inline bool blkcg_punt_bio_submit(struct bio *bio) { return false; }
static inline void blkcg_bio_issue_init(struct bio *bio) { }
static inline void blk_cgroup_bio_start(struct bio *bio) { }
static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio) { return true; }

3
block/blk-core.c
View file

@ -830,9 +830,6 @@ EXPORT_SYMBOL(submit_bio_noacct);
*/
void submit_bio(struct bio *bio)
{
if (blkcg_punt_bio_submit(bio))
return;
if (bio_op(bio) == REQ_OP_READ) {
task_io_account_read(bio->bi_iter.bi_size);
count_vm_events(PGPGIN, bio_sectors(bio));

1
fs/btrfs/Kconfig
View file

@ -2,6 +2,7 @@
config BTRFS_FS
tristate "Btrfs filesystem support"
select BLK_CGROUP_PUNT_BIO
select CRYPTO
select CRYPTO_CRC32C
select LIBCRC32C

211
fs/btrfs/bio.c
View file

@ -31,11 +31,11 @@ struct btrfs_failed_bio {
* Initialize a btrfs_bio structure. This skips the embedded bio itself as it
* is already initialized by the block layer.
*/
void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode,
void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_fs_info *fs_info,
btrfs_bio_end_io_t end_io, void *private)
{
memset(bbio, 0, offsetof(struct btrfs_bio, bio));
bbio->inode = inode;
bbio->fs_info = fs_info;
bbio->end_io = end_io;
bbio->private = private;
atomic_set(&bbio->pending_ios, 1);
@ -48,41 +48,58 @@ void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode,
* Just like the underlying bio_alloc_bioset it will not fail as it is backed by
* a mempool.
*/
struct bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf,
struct btrfs_inode *inode,
btrfs_bio_end_io_t end_io, void *private)
struct btrfs_bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf,
struct btrfs_fs_info *fs_info,
btrfs_bio_end_io_t end_io, void *private)
{
struct btrfs_bio *bbio;
struct bio *bio;
bio = bio_alloc_bioset(NULL, nr_vecs, opf, GFP_NOFS, &btrfs_bioset);
btrfs_bio_init(btrfs_bio(bio), inode, end_io, private);
return bio;
bbio = btrfs_bio(bio);
btrfs_bio_init(bbio, fs_info, end_io, private);
return bbio;
}
static struct bio *btrfs_split_bio(struct btrfs_fs_info *fs_info,
struct bio *orig, u64 map_length,
bool use_append)
static blk_status_t btrfs_bio_extract_ordered_extent(struct btrfs_bio *bbio)
{
struct btrfs_bio *orig_bbio = btrfs_bio(orig);
struct btrfs_ordered_extent *ordered;
int ret;
ordered = btrfs_lookup_ordered_extent(bbio->inode, bbio->file_offset);
if (WARN_ON_ONCE(!ordered))
return BLK_STS_IOERR;
ret = btrfs_extract_ordered_extent(bbio, ordered);
btrfs_put_ordered_extent(ordered);
return errno_to_blk_status(ret);
}
static struct btrfs_bio *btrfs_split_bio(struct btrfs_fs_info *fs_info,
struct btrfs_bio *orig_bbio,
u64 map_length, bool use_append)
{
struct btrfs_bio *bbio;
struct bio *bio;
if (use_append) {
unsigned int nr_segs;
bio = bio_split_rw(orig, &fs_info->limits, &nr_segs,
bio = bio_split_rw(&orig_bbio->bio, &fs_info->limits, &nr_segs,
&btrfs_clone_bioset, map_length);
} else {
bio = bio_split(orig, map_length >> SECTOR_SHIFT, GFP_NOFS,
&btrfs_clone_bioset);
bio = bio_split(&orig_bbio->bio, map_length >> SECTOR_SHIFT,
GFP_NOFS, &btrfs_clone_bioset);
}
btrfs_bio_init(btrfs_bio(bio), orig_bbio->inode, NULL, orig_bbio);
btrfs_bio(bio)->file_offset = orig_bbio->file_offset;
if (!(orig->bi_opf & REQ_BTRFS_ONE_ORDERED))
bbio = btrfs_bio(bio);
btrfs_bio_init(bbio, fs_info, NULL, orig_bbio);
bbio->inode = orig_bbio->inode;
bbio->file_offset = orig_bbio->file_offset;
if (!(orig_bbio->bio.bi_opf & REQ_BTRFS_ONE_ORDERED))
orig_bbio->file_offset += map_length;
atomic_inc(&orig_bbio->pending_ios);
return bio;
return bbio;
}
static void btrfs_orig_write_end_io(struct bio *bio);
@ -164,7 +181,7 @@ static void btrfs_end_repair_bio(struct btrfs_bio *repair_bbio,
goto done;
}
btrfs_submit_bio(&repair_bbio->bio, mirror);
btrfs_submit_bio(repair_bbio, mirror);
return;
}
@ -224,15 +241,16 @@ static struct btrfs_failed_bio *repair_one_sector(struct btrfs_bio *failed_bbio,
repair_bio = bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_NOFS,
&btrfs_repair_bioset);
repair_bio->bi_iter.bi_sector = failed_bbio->saved_iter.bi_sector;
bio_add_page(repair_bio, bv->bv_page, bv->bv_len, bv->bv_offset);
__bio_add_page(repair_bio, bv->bv_page, bv->bv_len, bv->bv_offset);
repair_bbio = btrfs_bio(repair_bio);
btrfs_bio_init(repair_bbio, failed_bbio->inode, NULL, fbio);
btrfs_bio_init(repair_bbio, fs_info, NULL, fbio);
repair_bbio->inode = failed_bbio->inode;
repair_bbio->file_offset = failed_bbio->file_offset + bio_offset;
mirror = next_repair_mirror(fbio, failed_bbio->mirror_num);
btrfs_debug(fs_info, "submitting repair read to mirror %d", mirror);
btrfs_submit_bio(repair_bio, mirror);
btrfs_submit_bio(repair_bbio, mirror);
return fbio;
}
@ -246,6 +264,9 @@ static void btrfs_check_read_bio(struct btrfs_bio *bbio, struct btrfs_device *de
struct btrfs_failed_bio *fbio = NULL;
u32 offset = 0;
/* Read-repair requires the inode field to be set by the submitter. */
ASSERT(inode);
/*
* Hand off repair bios to the repair code as there is no upper level
* submitter for them.
@ -306,17 +327,17 @@ static void btrfs_end_bio_work(struct work_struct *work)
struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work);
/* Metadata reads are checked and repaired by the submitter. */
if (bbio->bio.bi_opf & REQ_META)
bbio->end_io(bbio);
else
if (bbio->inode && !(bbio->bio.bi_opf & REQ_META))
btrfs_check_read_bio(bbio, bbio->bio.bi_private);
else
bbio->end_io(bbio);
}
static void btrfs_simple_end_io(struct bio *bio)
{
struct btrfs_bio *bbio = btrfs_bio(bio);
struct btrfs_device *dev = bio->bi_private;
struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;
struct btrfs_fs_info *fs_info = bbio->fs_info;
btrfs_bio_counter_dec(fs_info);
@ -340,7 +361,8 @@ static void btrfs_raid56_end_io(struct bio *bio)
btrfs_bio_counter_dec(bioc->fs_info);
bbio->mirror_num = bioc->mirror_num;
if (bio_op(bio) == REQ_OP_READ && !(bbio->bio.bi_opf & REQ_META))
if (bio_op(bio) == REQ_OP_READ && bbio->inode &&
!(bbio->bio.bi_opf & REQ_META))
btrfs_check_read_bio(bbio, NULL);
else
btrfs_orig_bbio_end_io(bbio);
@ -418,7 +440,11 @@ static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio)
dev->devid, bio->bi_iter.bi_size);
btrfsic_check_bio(bio);
submit_bio(bio);
if (bio->bi_opf & REQ_BTRFS_CGROUP_PUNT)
blkcg_punt_bio_submit(bio);
else
submit_bio(bio);
}
static void btrfs_submit_mirrored_bio(struct btrfs_io_context *bioc, int dev_nr)
@ -534,10 +560,10 @@ static void run_one_async_done(struct btrfs_work *work)
/*
* All of the bios that pass through here are from async helpers.
* Use REQ_CGROUP_PUNT to issue them from the owning cgroup's context.
* This changes nothing when cgroups aren't in use.
* Use REQ_BTRFS_CGROUP_PUNT to issue them from the owning cgroup's
* context. This changes nothing when cgroups aren't in use.
*/
bio->bi_opf |= REQ_CGROUP_PUNT;
bio->bi_opf |= REQ_BTRFS_CGROUP_PUNT;
__btrfs_submit_bio(bio, async->bioc, &async->smap, async->mirror_num);
}
@ -562,7 +588,7 @@ static bool should_async_write(struct btrfs_bio *bbio)
* in order.
*/
if (bbio->bio.bi_opf & REQ_META) {
struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;
struct btrfs_fs_info *fs_info = bbio->fs_info;
if (btrfs_is_zoned(fs_info))
return false;
@ -582,7 +608,7 @@ static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio,
struct btrfs_io_context *bioc,
struct btrfs_io_stripe *smap, int mirror_num)
{
struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;
struct btrfs_fs_info *fs_info = bbio->fs_info;
struct async_submit_bio *async;
async = kmalloc(sizeof(*async), GFP_NOFS);
@ -603,12 +629,12 @@ static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio,
return true;
}
static bool btrfs_submit_chunk(struct bio *bio, int mirror_num)
static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num)
{
struct btrfs_bio *bbio = btrfs_bio(bio);
struct btrfs_inode *inode = bbio->inode;
struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct btrfs_fs_info *fs_info = bbio->fs_info;
struct btrfs_bio *orig_bbio = bbio;
struct bio *bio = &bbio->bio;
u64 logical = bio->bi_iter.bi_sector << 9;
u64 length = bio->bi_iter.bi_size;
u64 map_length = length;
@ -631,15 +657,15 @@ static bool btrfs_submit_chunk(struct bio *bio, int mirror_num)
map_length = min(map_length, fs_info->max_zone_append_size);
if (map_length < length) {
bio = btrfs_split_bio(fs_info, bio, map_length, use_append);
bbio = btrfs_bio(bio);
bbio = btrfs_split_bio(fs_info, bbio, map_length, use_append);
bio = &bbio->bio;
}
/*
* Save the iter for the end_io handler and preload the checksums for
* data reads.
*/
if (bio_op(bio) == REQ_OP_READ && !(bio->bi_opf & REQ_META)) {
if (bio_op(bio) == REQ_OP_READ && inode && !(bio->bi_opf & REQ_META)) {
bbio->saved_iter = bio->bi_iter;
ret = btrfs_lookup_bio_sums(bbio);
if (ret)
@ -650,7 +676,7 @@ static bool btrfs_submit_chunk(struct bio *bio, int mirror_num)
if (use_append) {
bio->bi_opf &= ~REQ_OP_WRITE;
bio->bi_opf |= REQ_OP_ZONE_APPEND;
ret = btrfs_extract_ordered_extent(btrfs_bio(bio));
ret = btrfs_bio_extract_ordered_extent(bbio);
if (ret)
goto fail_put_bio;
}
@ -659,7 +685,7 @@ static bool btrfs_submit_chunk(struct bio *bio, int mirror_num)
* Csum items for reloc roots have already been cloned at this
* point, so they are handled as part of the no-checksum case.
*/
if (!(inode->flags & BTRFS_INODE_NODATASUM) &&
if (inode && !(inode->flags & BTRFS_INODE_NODATASUM) &&
!test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state) &&
!btrfs_is_data_reloc_root(inode->root)) {
if (should_async_write(bbio) &&
@ -686,9 +712,12 @@ static bool btrfs_submit_chunk(struct bio *bio, int mirror_num)
return true;
}
void btrfs_submit_bio(struct bio *bio, int mirror_num)
void btrfs_submit_bio(struct btrfs_bio *bbio, int mirror_num)
{
while (!btrfs_submit_chunk(bio, mirror_num))
/* If bbio->inode is not populated, its file_offset must be 0. */
ASSERT(bbio->inode || bbio->file_offset == 0);
while (!btrfs_submit_chunk(bbio, mirror_num))
;
}
@ -706,12 +735,9 @@ int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
u64 length, u64 logical, struct page *page,
unsigned int pg_offset, int mirror_num)
{
struct btrfs_device *dev;
struct btrfs_io_stripe smap = { 0 };
struct bio_vec bvec;
struct bio bio;
u64 map_length = 0;
u64 sector;
struct btrfs_io_context *bioc = NULL;
int ret = 0;
ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));
@ -720,68 +746,38 @@ int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
if (btrfs_repair_one_zone(fs_info, logical))
return 0;
map_length = length;
/*
* Avoid races with device replace and make sure our bioc has devices
* associated to its stripes that don't go away while we are doing the
* read repair operation.
*/
btrfs_bio_counter_inc_blocked(fs_info);
if (btrfs_is_parity_mirror(fs_info, logical, length)) {
/*
* Note that we don't use BTRFS_MAP_WRITE because it's supposed
* to update all raid stripes, but here we just want to correct
* bad stripe, thus BTRFS_MAP_READ is abused to only get the bad
* stripe's dev and sector.
*/
ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,
&map_length, &bioc, 0);
if (ret)
goto out_counter_dec;
ASSERT(bioc->mirror_num == 1);
} else {
ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical,
&map_length, &bioc, mirror_num);
if (ret)
goto out_counter_dec;
/*
* This happens when dev-replace is also running, and the
* mirror_num indicates the dev-replace target.
*
* In this case, we don't need to do anything, as the read
* error just means the replace progress hasn't reached our
* read range, and later replace routine would handle it well.
*/
if (mirror_num != bioc->mirror_num)
goto out_counter_dec;
}
ret = btrfs_map_repair_block(fs_info, &smap, logical, length, mirror_num);
if (ret < 0)
goto out_counter_dec;
sector = bioc->stripes[bioc->mirror_num - 1].physical >> 9;
dev = bioc->stripes[bioc->mirror_num - 1].dev;
btrfs_put_bioc(bioc);
if (!dev || !dev->bdev ||
!test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
if (!smap.dev->bdev ||
!test_bit(BTRFS_DEV_STATE_WRITEABLE, &smap.dev->dev_state)) {
ret = -EIO;
goto out_counter_dec;
}
bio_init(&bio, dev->bdev, &bvec, 1, REQ_OP_WRITE | REQ_SYNC);
bio.bi_iter.bi_sector = sector;
bio_init(&bio, smap.dev->bdev, &bvec, 1, REQ_OP_WRITE | REQ_SYNC);
bio.bi_iter.bi_sector = smap.physical >> SECTOR_SHIFT;
__bio_add_page(&bio, page, length, pg_offset);
btrfsic_check_bio(&bio);
ret = submit_bio_wait(&bio);
if (ret) {
/* try to remap that extent elsewhere? */
btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
btrfs_dev_stat_inc_and_print(smap.dev, BTRFS_DEV_STAT_WRITE_ERRS);
goto out_bio_uninit;
}
btrfs_info_rl_in_rcu(fs_info,
"read error corrected: ino %llu off %llu (dev %s sector %llu)",
ino, start, btrfs_dev_name(dev), sector);
ino, start, btrfs_dev_name(smap.dev),
smap.physical >> SECTOR_SHIFT);
ret = 0;
out_bio_uninit:
@ -791,6 +787,45 @@ int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
return ret;
}
/*
* Submit a btrfs_bio based repair write.
*
* If @dev_replace is true, the write would be submitted to dev-replace target.
*/
void btrfs_submit_repair_write(struct btrfs_bio *bbio, int mirror_num, bool dev_replace)
{
struct btrfs_fs_info *fs_info = bbio->fs_info;
u64 logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT;
u64 length = bbio->bio.bi_iter.bi_size;
struct btrfs_io_stripe smap = { 0 };
int ret;
ASSERT(fs_info);
ASSERT(mirror_num > 0);
ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE);
ASSERT(!bbio->inode);
btrfs_bio_counter_inc_blocked(fs_info);
ret = btrfs_map_repair_block(fs_info, &smap, logical, length, mirror_num);
if (ret < 0)
goto fail;
if (dev_replace) {
if (btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE && btrfs_is_zoned(fs_info)) {
bbio->bio.bi_opf &= ~REQ_OP_WRITE;
bbio->bio.bi_opf |= REQ_OP_ZONE_APPEND;
}
ASSERT(smap.dev == fs_info->dev_replace.srcdev);
smap.dev = fs_info->dev_replace.tgtdev;
}
__btrfs_submit_bio(&bbio->bio, NULL, &smap, mirror_num);
return;
fail:
btrfs_bio_counter_dec(fs_info);
btrfs_bio_end_io(bbio, errno_to_blk_status(ret));
}
int __init btrfs_bioset_init(void)
{
if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE,

22
fs/btrfs/bio.h
View file

@ -30,7 +30,10 @@ typedef void (*btrfs_bio_end_io_t)(struct btrfs_bio *bbio);
* passed to btrfs_submit_bio for mapping to the physical devices.
*/
struct btrfs_bio {
/* Inode and offset into it that this I/O operates on. */
/*
* Inode and offset into it that this I/O operates on.
* Only set for data I/O.
*/
struct btrfs_inode *inode;
u64 file_offset;
@ -58,6 +61,9 @@ struct btrfs_bio {
atomic_t pending_ios;
struct work_struct end_io_work;
/* File system that this I/O operates on. */
struct btrfs_fs_info *fs_info;
/*
* This member must come last, bio_alloc_bioset will allocate enough
* bytes for entire btrfs_bio but relies on bio being last.
@ -73,11 +79,11 @@ static inline struct btrfs_bio *btrfs_bio(struct bio *bio)
int __init btrfs_bioset_init(void);
void __cold btrfs_bioset_exit(void);
void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode,
void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_fs_info *fs_info,
btrfs_bio_end_io_t end_io, void *private);
struct bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf,
struct btrfs_inode *inode,
btrfs_bio_end_io_t end_io, void *private);
struct btrfs_bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf,
struct btrfs_fs_info *fs_info,
btrfs_bio_end_io_t end_io, void *private);
static inline void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status)
{
@ -88,7 +94,11 @@ static inline void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status)
/* Bio only refers to one ordered extent. */
#define REQ_BTRFS_ONE_ORDERED REQ_DRV
void btrfs_submit_bio(struct bio *bio, int mirror_num);
/* Submit using blkcg_punt_bio_submit. */
#define REQ_BTRFS_CGROUP_PUNT REQ_FS_PRIVATE
void btrfs_submit_bio(struct btrfs_bio *bbio, int mirror_num);
void btrfs_submit_repair_write(struct btrfs_bio *bbio, int mirror_num, bool dev_replace);
int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
u64 length, u64 logical, struct page *page,
unsigned int pg_offset, int mirror_num);

40
fs/btrfs/block-group.c
View file

@ -160,15 +160,6 @@ void btrfs_put_block_group(struct btrfs_block_group *cache)
btrfs_discard_cancel_work(&cache->fs_info->discard_ctl,
cache);
/*
* If not empty, someone is still holding mutex of
* full_stripe_lock, which can only be released by caller.
* And it will definitely cause use-after-free when caller
* tries to release full stripe lock.
*
* No better way to resolve, but only to warn.
*/
WARN_ON(!RB_EMPTY_ROOT(&cache->full_stripe_locks_root.root));
kfree(cache->free_space_ctl);
kfree(cache->physical_map);
kfree(cache);
@ -1977,12 +1968,12 @@ int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start,
map = em->map_lookup;
data_stripe_length = em->orig_block_len;
io_stripe_size = map->stripe_len;
io_stripe_size = BTRFS_STRIPE_LEN;
chunk_start = em->start;
/* For RAID5/6 adjust to a full IO stripe length */
if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
io_stripe_size = map->stripe_len * nr_data_stripes(map);
io_stripe_size = nr_data_stripes(map) << BTRFS_STRIPE_LEN_SHIFT;
buf = kcalloc(map->num_stripes, sizeof(u64), GFP_NOFS);
if (!buf) {
@ -1992,28 +1983,28 @@ int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start,
for (i = 0; i < map->num_stripes; i++) {
bool already_inserted = false;
u64 stripe_nr;
u64 offset;
u32 stripe_nr;
u32 offset;
int j;
if (!in_range(physical, map->stripes[i].physical,
data_stripe_length))
continue;
stripe_nr = physical - map->stripes[i].physical;
stripe_nr = div64_u64_rem(stripe_nr, map->stripe_len, &offset);
stripe_nr = (physical - map->stripes[i].physical) >>
BTRFS_STRIPE_LEN_SHIFT;
offset = (physical - map->stripes[i].physical) &
BTRFS_STRIPE_LEN_MASK;
if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
BTRFS_BLOCK_GROUP_RAID10)) {
stripe_nr = stripe_nr * map->num_stripes + i;
stripe_nr = div_u64(stripe_nr, map->sub_stripes);
}
BTRFS_BLOCK_GROUP_RAID10))
stripe_nr = div_u64(stripe_nr * map->num_stripes + i,
map->sub_stripes);
/*
* The remaining case would be for RAID56, multiply by
* nr_data_stripes(). Alternatively, just use rmap_len below
* instead of map->stripe_len
*/
bytenr = chunk_start + stripe_nr * io_stripe_size + offset;
/* Ensure we don't add duplicate addresses */
@ -2124,8 +2115,6 @@ static struct btrfs_block_group *btrfs_create_block_group_cache(
btrfs_init_free_space_ctl(cache, cache->free_space_ctl);
atomic_set(&cache->frozen, 0);
mutex_init(&cache->free_space_lock);
cache->full_stripe_locks_root.root = RB_ROOT;
mutex_init(&cache->full_stripe_locks_root.lock);
return cache;
}
@ -2672,7 +2661,7 @@ static u64 calculate_global_root_id(struct btrfs_fs_info *fs_info, u64 offset)
}
struct btrfs_block_group *btrfs_make_block_group(struct btrfs_trans_handle *trans,
u64 bytes_used, u64 type,
u64 type,
u64 chunk_offset, u64 size)
{
struct btrfs_fs_info *fs_info = trans->fs_info;
@ -2687,7 +2676,6 @@ struct btrfs_block_group *btrfs_make_block_group(struct btrfs_trans_handle *tran
cache->length = size;
set_free_space_tree_thresholds(cache);
cache->used = bytes_used;
cache->flags = type;
cache->cached = BTRFS_CACHE_FINISHED;
cache->global_root_id = calculate_global_root_id(fs_info, cache->start);
@ -2738,9 +2726,7 @@ struct btrfs_block_group *btrfs_make_block_group(struct btrfs_trans_handle *tran
#ifdef CONFIG_BTRFS_DEBUG
if (btrfs_should_fragment_free_space(cache)) {
u64 new_bytes_used = size - bytes_used;
cache->space_info->bytes_used += new_bytes_used >> 1;
cache->space_info->bytes_used += size >> 1;
fragment_free_space(cache);
}
#endif

13
fs/btrfs/block-group.h
View file

@ -91,14 +91,6 @@ struct btrfs_caching_control {
/* Once caching_thread() finds this much free space, it will wake up waiters. */
#define CACHING_CTL_WAKE_UP SZ_2M
/*
* Tree to record all locked full stripes of a RAID5/6 block group
*/
struct btrfs_full_stripe_locks_tree {
struct rb_root root;
struct mutex lock;
};
struct btrfs_block_group {
struct btrfs_fs_info *fs_info;
struct inode *inode;
@ -229,9 +221,6 @@ struct btrfs_block_group {
*/
int swap_extents;
/* Record locked full stripes for RAID5/6 block group */
struct btrfs_full_stripe_locks_tree full_stripe_locks_root;
/*
* Allocation offset for the block group to implement sequential
* allocation. This is used only on a zoned filesystem.
@ -302,7 +291,7 @@ void btrfs_reclaim_bgs(struct btrfs_fs_info *fs_info);
void btrfs_mark_bg_to_reclaim(struct btrfs_block_group *bg);
int btrfs_read_block_groups(struct btrfs_fs_info *info);
struct btrfs_block_group *btrfs_make_block_group(struct btrfs_trans_handle *trans,
u64 bytes_used, u64 type,
u64 type,
u64 chunk_offset, u64 size);
void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans);
int btrfs_inc_block_group_ro(struct btrfs_block_group *cache,

21
fs/btrfs/block-rsv.c
View file

@ -232,9 +232,6 @@ int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_percent)
u64 num_bytes = 0;
int ret = -ENOSPC;
if (!block_rsv)
return 0;
spin_lock(&block_rsv->lock);
num_bytes = mult_perc(block_rsv->size, min_percent);
if (block_rsv->reserved >= num_bytes)
@ -245,17 +242,15 @@ int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_percent)
}
int btrfs_block_rsv_refill(struct btrfs_fs_info *fs_info,
struct btrfs_block_rsv *block_rsv, u64 min_reserved,
struct btrfs_block_rsv *block_rsv, u64 num_bytes,
enum btrfs_reserve_flush_enum flush)
{
u64 num_bytes = 0;
int ret = -ENOSPC;
if (!block_rsv)
return 0;
spin_lock(&block_rsv->lock);
num_bytes = min_reserved;
if (block_rsv->reserved >= num_bytes)
ret = 0;
else
@ -355,17 +350,19 @@ void btrfs_update_global_block_rsv(struct btrfs_fs_info *fs_info)
/*
* But we also want to reserve enough space so we can do the fallback
* global reserve for an unlink, which is an additional 5 items (see the
* comment in __unlink_start_trans for what we're modifying.)
* global reserve for an unlink, which is an additional
* BTRFS_UNLINK_METADATA_UNITS items.
*
* But we also need space for the delayed ref updates from the unlink,
* so its 10, 5 for the actual operation, and 5 for the delayed ref
* updates.
* so add BTRFS_UNLINK_METADATA_UNITS units for delayed refs, one for
* each unlink metadata item.
*/
min_items += 10;
min_items += BTRFS_UNLINK_METADATA_UNITS;
num_bytes = max_t(u64, num_bytes,
btrfs_calc_insert_metadata_size(fs_info, min_items));
btrfs_calc_insert_metadata_size(fs_info, min_items) +
btrfs_calc_delayed_ref_bytes(fs_info,
BTRFS_UNLINK_METADATA_UNITS));
spin_lock(&sinfo->lock);
spin_lock(&block_rsv->lock);

2
fs/btrfs/block-rsv.h
View file

@ -65,7 +65,7 @@ int btrfs_block_rsv_add(struct btrfs_fs_info *fs_info,
enum btrfs_reserve_flush_enum flush);
int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_percent);
int btrfs_block_rsv_refill(struct btrfs_fs_info *fs_info,
struct btrfs_block_rsv *block_rsv, u64 min_reserved,
struct btrfs_block_rsv *block_rsv, u64 num_bytes,
enum btrfs_reserve_flush_enum flush);
int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
struct btrfs_block_rsv *dst_rsv, u64 num_bytes,

35
fs/btrfs/btrfs_inode.h
View file

@ -142,11 +142,22 @@ struct btrfs_inode {
/* a local copy of root's last_log_commit */
int last_log_commit;
/*
* Total number of bytes pending delalloc, used by stat to calculate the
* real block usage of the file. This is used only for files.
*/
u64 delalloc_bytes;
union {
/*
* Total number of bytes pending delalloc, used by stat to
* calculate the real block usage of the file. This is used
* only for files.
*/
u64 delalloc_bytes;
/*
* The lowest possible index of the next dir index key which
* points to an inode that needs to be logged.
* This is used only for directories.
* Use the helpers btrfs_get_first_dir_index_to_log() and
* btrfs_set_first_dir_index_to_log() to access this field.
*/
u64 first_dir_index_to_log;
};
union {
/*
@ -247,6 +258,17 @@ struct btrfs_inode {
struct inode vfs_inode;
};
static inline u64 btrfs_get_first_dir_index_to_log(const struct btrfs_inode *inode)
{
return READ_ONCE(inode->first_dir_index_to_log);
}
static inline void btrfs_set_first_dir_index_to_log(struct btrfs_inode *inode,
u64 index)
{
WRITE_ONCE(inode->first_dir_index_to_log, index);
}
static inline struct btrfs_inode *BTRFS_I(const struct inode *inode)
{
return container_of(inode, struct btrfs_inode, vfs_inode);
@ -407,7 +429,8 @@ static inline void btrfs_inode_split_flags(u64 inode_item_flags,
int btrfs_check_sector_csum(struct btrfs_fs_info *fs_info, struct page *page,
u32 pgoff, u8 *csum, const u8 * const csum_expected);
blk_status_t btrfs_extract_ordered_extent(struct btrfs_bio *bbio);
int btrfs_extract_ordered_extent(struct btrfs_bio *bbio,
struct btrfs_ordered_extent *ordered);
bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev,
u32 bio_offset, struct bio_vec *bv);
noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,

311
fs/btrfs/compression.c
View file

@ -37,6 +37,8 @@
#include "file-item.h"
#include "super.h"
struct bio_set btrfs_compressed_bioset;
static const char* const btrfs_compress_types[] = { "", "zlib", "lzo", "zstd" };
const char* btrfs_compress_type2str(enum btrfs_compression_type type)
@ -54,6 +56,25 @@ const char* btrfs_compress_type2str(enum btrfs_compression_type type)
return NULL;
}
static inline struct compressed_bio *to_compressed_bio(struct btrfs_bio *bbio)
{
return container_of(bbio, struct compressed_bio, bbio);
}
static struct compressed_bio *alloc_compressed_bio(struct btrfs_inode *inode,
u64 start, blk_opf_t op,
btrfs_bio_end_io_t end_io)
{
struct btrfs_bio *bbio;
bbio = btrfs_bio(bio_alloc_bioset(NULL, BTRFS_MAX_COMPRESSED_PAGES, op,
GFP_NOFS, &btrfs_compressed_bioset));
btrfs_bio_init(bbio, inode->root->fs_info, end_io, NULL);
bbio->inode = inode;
bbio->file_offset = start;
return to_compressed_bio(bbio);
}
bool btrfs_compress_is_valid_type(const char *str, size_t len)
{
int i;
@ -139,32 +160,25 @@ static int compression_decompress(int type, struct list_head *ws,
}
}
static void btrfs_free_compressed_pages(struct compressed_bio *cb)
{
for (unsigned int i = 0; i < cb->nr_pages; i++)
put_page(cb->compressed_pages[i]);
kfree(cb->compressed_pages);
}
static int btrfs_decompress_bio(struct compressed_bio *cb);
static void end_compressed_bio_read(struct btrfs_bio *bbio)
{
struct compressed_bio *cb = bbio->private;
unsigned int index;
struct page *page;
struct compressed_bio *cb = to_compressed_bio(bbio);
blk_status_t status = bbio->bio.bi_status;
if (bbio->bio.bi_status)
cb->status = bbio->bio.bi_status;
else
cb->status = errno_to_blk_status(btrfs_decompress_bio(cb));
if (!status)
status = errno_to_blk_status(btrfs_decompress_bio(cb));
/* Release the compressed pages */
for (index = 0; index < cb->nr_pages; index++) {
page = cb->compressed_pages[index];
page->mapping = NULL;
put_page(page);
}
/* Do io completion on the original bio */
btrfs_bio_end_io(btrfs_bio(cb->orig_bio), cb->status);
/* Finally free the cb struct */
kfree(cb->compressed_pages);
kfree(cb);
btrfs_free_compressed_pages(cb);
btrfs_bio_end_io(cb->orig_bbio, status);
bio_put(&bbio->bio);
}
@ -172,14 +186,14 @@ static void end_compressed_bio_read(struct btrfs_bio *bbio)
* Clear the writeback bits on all of the file
* pages for a compressed write
*/
static noinline void end_compressed_writeback(struct inode *inode,
const struct compressed_bio *cb)
static noinline void end_compressed_writeback(const struct compressed_bio *cb)
{
struct inode *inode = &cb->bbio.inode->vfs_inode;
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
unsigned long index = cb->start >> PAGE_SHIFT;
unsigned long end_index = (cb->start + cb->len - 1) >> PAGE_SHIFT;
struct folio_batch fbatch;
const int errno = blk_status_to_errno(cb->status);
const int errno = blk_status_to_errno(cb->bbio.bio.bi_status);
int i;
int ret;
@ -207,45 +221,25 @@ static noinline void end_compressed_writeback(struct inode *inode,
/* the inode may be gone now */
}
static void finish_compressed_bio_write(struct compressed_bio *cb)
{
struct inode *inode = cb->inode;
unsigned int index;
/*
* Ok, we're the last bio for this extent, step one is to call back
* into the FS and do all the end_io operations.
*/
btrfs_writepage_endio_finish_ordered(BTRFS_I(inode), NULL,
cb->start, cb->start + cb->len - 1,
cb->status == BLK_STS_OK);
if (cb->writeback)
end_compressed_writeback(inode, cb);
/* Note, our inode could be gone now */
/*
* Release the compressed pages, these came from alloc_page and
* are not attached to the inode at all
*/
for (index = 0; index < cb->nr_pages; index++) {
struct page *page = cb->compressed_pages[index];
page->mapping = NULL;
put_page(page);
}
/* Finally free the cb struct */
kfree(cb->compressed_pages);
kfree(cb);
}
static void btrfs_finish_compressed_write_work(struct work_struct *work)
{
struct compressed_bio *cb =
container_of(work, struct compressed_bio, write_end_work);
finish_compressed_bio_write(cb);
/*
* Ok, we're the last bio for this extent, step one is to call back
* into the FS and do all the end_io operations.
*/
btrfs_writepage_endio_finish_ordered(cb->bbio.inode, NULL,
cb->start, cb->start + cb->len - 1,
cb->bbio.bio.bi_status == BLK_STS_OK);
if (cb->writeback)
end_compressed_writeback(cb);
/* Note, our inode could be gone now */
btrfs_free_compressed_pages(cb);
bio_put(&cb->bbio.bio);
}
/*
@ -257,13 +251,25 @@ static void btrfs_finish_compressed_write_work(struct work_struct *work)
*/
static void end_compressed_bio_write(struct btrfs_bio *bbio)
{
struct compressed_bio *cb = bbio->private;
struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb);
struct compressed_bio *cb = to_compressed_bio(bbio);
struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;
cb->status = bbio->bio.bi_status;
queue_work(fs_info->compressed_write_workers, &cb->write_end_work);
}
bio_put(&bbio->bio);
static void btrfs_add_compressed_bio_pages(struct compressed_bio *cb)
{
struct bio *bio = &cb->bbio.bio;
u32 offset = 0;
while (offset < cb->compressed_len) {
u32 len = min_t(u32, cb->compressed_len - offset, PAGE_SIZE);
/* Maximum compressed extent is smaller than bio size limit. */
__bio_add_page(bio, cb->compressed_pages[offset >> PAGE_SHIFT],
len, 0);
offset += len;
}
}
/*
@ -275,28 +281,24 @@ static void end_compressed_bio_write(struct btrfs_bio *bbio)
* This also checksums the file bytes and gets things ready for
* the end io hooks.
*/
blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start,
void btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start,
unsigned int len, u64 disk_start,
unsigned int compressed_len,
struct page **compressed_pages,
unsigned int nr_pages,
blk_opf_t write_flags,
struct cgroup_subsys_state *blkcg_css,
bool writeback)
{
struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct bio *bio = NULL;
struct compressed_bio *cb;
u64 cur_disk_bytenr = disk_start;
blk_status_t ret = BLK_STS_OK;
ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
IS_ALIGNED(len, fs_info->sectorsize));
cb = kmalloc(sizeof(struct compressed_bio), GFP_NOFS);
if (!cb)
return BLK_STS_RESOURCE;
cb->status = BLK_STS_OK;
cb->inode = &inode->vfs_inode;
write_flags |= REQ_BTRFS_ONE_ORDERED;
cb = alloc_compressed_bio(inode, start, REQ_OP_WRITE | write_flags,
end_compressed_bio_write);
cb->start = start;
cb->len = len;
cb->compressed_pages = compressed_pages;
@ -304,56 +306,10 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start,
cb->writeback = writeback;
INIT_WORK(&cb->write_end_work, btrfs_finish_compressed_write_work);
cb->nr_pages = nr_pages;
cb->bbio.bio.bi_iter.bi_sector = disk_start >> SECTOR_SHIFT;
btrfs_add_compressed_bio_pages(cb);
if (blkcg_css) {
kthread_associate_blkcg(blkcg_css);
write_flags |= REQ_CGROUP_PUNT;
}
write_flags |= REQ_BTRFS_ONE_ORDERED;
bio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_WRITE | write_flags,
BTRFS_I(cb->inode), end_compressed_bio_write, cb);
bio->bi_iter.bi_sector = cur_disk_bytenr >> SECTOR_SHIFT;
btrfs_bio(bio)->file_offset = start;
while (cur_disk_bytenr < disk_start + compressed_len) {
u64 offset = cur_disk_bytenr - disk_start;
unsigned int index = offset >> PAGE_SHIFT;
unsigned int real_size;
unsigned int added;
struct page *page = compressed_pages[index];
/*
* We have various limits on the real read size:
* - page boundary
* - compressed length boundary
*/
real_size = min_t(u64, U32_MAX, PAGE_SIZE - offset_in_page(offset));
real_size = min_t(u64, real_size, compressed_len - offset);
ASSERT(IS_ALIGNED(real_size, fs_info->sectorsize));
added = bio_add_page(bio, page, real_size, offset_in_page(offset));
/*
* Maximum compressed extent is smaller than bio size limit,
* thus bio_add_page() should always success.
*/
ASSERT(added == real_size);
cur_disk_bytenr += added;
}
/* Finished the range. */
ASSERT(bio->bi_iter.bi_size);
btrfs_submit_bio(bio, 0);
if (blkcg_css)
kthread_associate_blkcg(NULL);
return ret;
}
static u64 bio_end_offset(struct bio *bio)
{
struct bio_vec *last = bio_last_bvec_all(bio);
return page_offset(last->bv_page) + last->bv_len + last->bv_offset;
btrfs_submit_bio(&cb->bbio, 0);
}
/*
@ -374,7 +330,8 @@ static noinline int add_ra_bio_pages(struct inode *inode,
{
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
unsigned long end_index;
u64 cur = bio_end_offset(cb->orig_bio);
struct bio *orig_bio = &cb->orig_bbio->bio;
u64 cur = cb->orig_bbio->file_offset + orig_bio->bi_iter.bi_size;
u64 isize = i_size_read(inode);
int ret;
struct page *page;
@ -464,7 +421,7 @@ static noinline int add_ra_bio_pages(struct inode *inode,
*/
if (!em || cur < em->start ||
(cur + fs_info->sectorsize > extent_map_end(em)) ||
(em->block_start >> 9) != cb->orig_bio->bi_iter.bi_sector) {
(em->block_start >> 9) != orig_bio->bi_iter.bi_sector) {
free_extent_map(em);
unlock_extent(tree, cur, page_end, NULL);
unlock_page(page);
@ -484,7 +441,7 @@ static noinline int add_ra_bio_pages(struct inode *inode,
}
add_size = min(em->start + em->len, page_end + 1) - cur;
ret = bio_add_page(cb->orig_bio, page, add_size, offset_in_page(cur));
ret = bio_add_page(orig_bio, page, add_size, offset_in_page(cur));
if (ret != add_size) {
unlock_extent(tree, cur, page_end, NULL);
unlock_page(page);
@ -515,17 +472,14 @@ static noinline int add_ra_bio_pages(struct inode *inode,
* After the compressed pages are read, we copy the bytes into the
* bio we were passed and then call the bio end_io calls
*/
void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
int mirror_num)
void btrfs_submit_compressed_read(struct btrfs_bio *bbio, int mirror_num)
{
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
struct extent_map_tree *em_tree;
struct btrfs_inode *inode = bbio->inode;
struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct extent_map_tree *em_tree = &inode->extent_tree;
struct compressed_bio *cb;
unsigned int compressed_len;
struct bio *comp_bio;
const u64 disk_bytenr = bio->bi_iter.bi_sector << SECTOR_SHIFT;
u64 cur_disk_byte = disk_bytenr;
u64 file_offset;
u64 file_offset = bbio->file_offset;
u64 em_len;
u64 em_start;
struct extent_map *em;
@ -533,12 +487,6 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
int memstall = 0;
blk_status_t ret;
int ret2;
int i;
em_tree = &BTRFS_I(inode)->extent_tree;
file_offset = bio_first_bvec_all(bio)->bv_offset +
page_offset(bio_first_page_all(bio));
/* we need the actual starting offset of this extent in the file */
read_lock(&em_tree->lock);
@ -551,102 +499,54 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
ASSERT(em->compress_type != BTRFS_COMPRESS_NONE);
compressed_len = em->block_len;
cb = kmalloc(sizeof(struct compressed_bio), GFP_NOFS);
if (!cb) {
ret = BLK_STS_RESOURCE;
goto out;
}
cb->status = BLK_STS_OK;
cb->inode = inode;
cb = alloc_compressed_bio(inode, file_offset, REQ_OP_READ,
end_compressed_bio_read);
cb->start = em->orig_start;
em_len = em->len;
em_start = em->start;
cb->len = bio->bi_iter.bi_size;
cb->len = bbio->bio.bi_iter.bi_size;
cb->compressed_len = compressed_len;
cb->compress_type = em->compress_type;
cb->orig_bio = bio;
cb->orig_bbio = bbio;
free_extent_map(em);
em = NULL;
cb->nr_pages = DIV_ROUND_UP(compressed_len, PAGE_SIZE);
cb->compressed_pages = kcalloc(cb->nr_pages, sizeof(struct page *), GFP_NOFS);
if (!cb->compressed_pages) {
ret = BLK_STS_RESOURCE;
goto fail;
goto out_free_bio;
}
ret2 = btrfs_alloc_page_array(cb->nr_pages, cb->compressed_pages);
if (ret2) {
ret = BLK_STS_RESOURCE;
goto fail;
goto out_free_compressed_pages;
}
add_ra_bio_pages(inode, em_start + em_len, cb, &memstall, &pflags);
add_ra_bio_pages(&inode->vfs_inode, em_start + em_len, cb, &memstall,
&pflags);
/* include any pages we added in add_ra-bio_pages */
cb->len = bio->bi_iter.bi_size;
comp_bio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, BTRFS_I(cb->inode),
end_compressed_bio_read, cb);
comp_bio->bi_iter.bi_sector = (cur_disk_byte >> SECTOR_SHIFT);
while (cur_disk_byte < disk_bytenr + compressed_len) {
u64 offset = cur_disk_byte - disk_bytenr;
unsigned int index = offset >> PAGE_SHIFT;
unsigned int real_size;
unsigned int added;
struct page *page = cb->compressed_pages[index];
/*
* We have various limit on the real read size:
* - page boundary
* - compressed length boundary
*/
real_size = min_t(u64, U32_MAX, PAGE_SIZE - offset_in_page(offset));
real_size = min_t(u64, real_size, compressed_len - offset);
ASSERT(IS_ALIGNED(real_size, fs_info->sectorsize));
added = bio_add_page(comp_bio, page, real_size, offset_in_page(offset));
/*
* Maximum compressed extent is smaller than bio size limit,
* thus bio_add_page() should always success.
*/
ASSERT(added == real_size);
cur_disk_byte += added;
}
cb->len = bbio->bio.bi_iter.bi_size;
cb->bbio.bio.bi_iter.bi_sector = bbio->bio.bi_iter.bi_sector;
btrfs_add_compressed_bio_pages(cb);
if (memstall)
psi_memstall_leave(&pflags);
/*
* Stash the initial offset of this chunk, as there is no direct
* correlation between compressed pages and the original file offset.
* The field is only used for printing error messages anyway.
*/
btrfs_bio(comp_bio)->file_offset = file_offset;
ASSERT(comp_bio->bi_iter.bi_size);
btrfs_submit_bio(comp_bio, mirror_num);
btrfs_submit_bio(&cb->bbio, mirror_num);
return;
fail:
if (cb->compressed_pages) {
for (i = 0; i < cb->nr_pages; i++) {
if (cb->compressed_pages[i])
__free_page(cb->compressed_pages[i]);
}
}
out_free_compressed_pages:
kfree(cb->compressed_pages);
kfree(cb);
out_free_bio:
bio_put(&cb->bbio.bio);
out:
free_extent_map(em);
btrfs_bio_end_io(btrfs_bio(bio), ret);
return;
btrfs_bio_end_io(bbio, ret);
}
/*
@ -1038,6 +938,8 @@ static int btrfs_decompress_bio(struct compressed_bio *cb)
ret = compression_decompress_bio(workspace, cb);
put_workspace(type, workspace);
if (!ret)
zero_fill_bio(&cb->orig_bbio->bio);
return ret;
}
@ -1062,6 +964,10 @@ int btrfs_decompress(int type, const u8 *data_in, struct page *dest_page,
int __init btrfs_init_compress(void)
{
if (bioset_init(&btrfs_compressed_bioset, BIO_POOL_SIZE,
offsetof(struct compressed_bio, bbio.bio),
BIOSET_NEED_BVECS))
return -ENOMEM;
btrfs_init_workspace_manager(BTRFS_COMPRESS_NONE);
btrfs_init_workspace_manager(BTRFS_COMPRESS_ZLIB);
btrfs_init_workspace_manager(BTRFS_COMPRESS_LZO);
@ -1075,6 +981,7 @@ void __cold btrfs_exit_compress(void)
btrfs_cleanup_workspace_manager(BTRFS_COMPRESS_ZLIB);
btrfs_cleanup_workspace_manager(BTRFS_COMPRESS_LZO);
zstd_cleanup_workspace_manager();
bioset_exit(&btrfs_compressed_bioset);
}
/*
@ -1110,7 +1017,7 @@ void __cold btrfs_exit_compress(void)
int btrfs_decompress_buf2page(const char *buf, u32 buf_len,
struct compressed_bio *cb, u32 decompressed)
{
struct bio *orig_bio = cb->orig_bio;
struct bio *orig_bio = &cb->orig_bbio->bio;
/* Offset inside the full decompressed extent */
u32 cur_offset;

20
fs/btrfs/compression.h
View file

@ -6,8 +6,8 @@
#ifndef BTRFS_COMPRESSION_H
#define BTRFS_COMPRESSION_H
#include <linux/blk_types.h>
#include <linux/sizes.h>
#include "bio.h"
struct btrfs_inode;
@ -23,6 +23,7 @@ struct btrfs_inode;
/* Maximum length of compressed data stored on disk */
#define BTRFS_MAX_COMPRESSED (SZ_128K)
#define BTRFS_MAX_COMPRESSED_PAGES (BTRFS_MAX_COMPRESSED / PAGE_SIZE)
static_assert((BTRFS_MAX_COMPRESSED % PAGE_SIZE) == 0);
/* Maximum size of data before compression */
@ -37,9 +38,6 @@ struct compressed_bio {
/* the pages with the compressed data on them */
struct page **compressed_pages;
/* inode that owns this data */
struct inode *inode;
/* starting offset in the inode for our pages */
u64 start;
@ -55,14 +53,14 @@ struct compressed_bio {
/* Whether this is a write for writeback. */
bool writeback;
/* IO errors */
blk_status_t status;
union {
/* For reads, this is the bio we are copying the data into */
struct bio *orig_bio;
struct btrfs_bio *orig_bbio;
struct work_struct write_end_work;
};
/* Must be last. */
struct btrfs_bio bbio;
};
static inline unsigned int btrfs_compress_type(unsigned int type_level)
@ -88,16 +86,14 @@ int btrfs_decompress(int type, const u8 *data_in, struct page *dest_page,
int btrfs_decompress_buf2page(const char *buf, u32 buf_len,
struct compressed_bio *cb, u32 decompressed);
blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start,
void btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start,
unsigned int len, u64 disk_start,
unsigned int compressed_len,
struct page **compressed_pages,
unsigned int nr_pages,
blk_opf_t write_flags,
struct cgroup_subsys_state *blkcg_css,
bool writeback);
void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
int mirror_num);
void btrfs_submit_compressed_read(struct btrfs_bio *bbio, int mirror_num);
unsigned int btrfs_compress_str2level(unsigned int type, const char *str);

91
fs/btrfs/ctree.c
View file

@ -854,7 +854,8 @@ int btrfs_realloc_node(struct btrfs_trans_handle *trans,
* Search for a key in the given extent_buffer.
*
* The lower boundary for the search is specified by the slot number @first_slot.
* Use a value of 0 to search over the whole extent buffer.
* Use a value of 0 to search over the whole extent buffer. Works for both
* leaves and nodes.
*
* The slot in the extent buffer is returned via @slot. If the key exists in the
* extent buffer, then @slot will point to the slot where the key is, otherwise
@ -863,8 +864,8 @@ int btrfs_realloc_node(struct btrfs_trans_handle *trans,
* Slot may point to the total number of items (i.e. one position beyond the last
* key) if the key is bigger than the last key in the extent buffer.
*/
int btrfs_generic_bin_search(struct extent_buffer *eb, int first_slot,
const struct btrfs_key *key, int *slot)
int btrfs_bin_search(struct extent_buffer *eb, int first_slot,
const struct btrfs_key *key, int *slot)
{
unsigned long p;
int item_size;
@ -959,7 +960,7 @@ struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent,
if (slot < 0 || slot >= btrfs_header_nritems(parent))
return ERR_PTR(-ENOENT);
BUG_ON(level == 0);
ASSERT(level);
check.level = level - 1;
check.transid = btrfs_node_ptr_generation(parent, slot);
@ -1064,11 +1065,14 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4)
return 0;
left = btrfs_read_node_slot(parent, pslot - 1);
if (IS_ERR(left))
left = NULL;
if (pslot) {
left = btrfs_read_node_slot(parent, pslot - 1);
if (IS_ERR(left)) {
ret = PTR_ERR(left);
left = NULL;
goto enospc;
}
if (left) {
__btrfs_tree_lock(left, BTRFS_NESTING_LEFT);
wret = btrfs_cow_block(trans, root, left,
parent, pslot - 1, &left,
@ -1079,11 +1083,14 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
}
}
right = btrfs_read_node_slot(parent, pslot + 1);
if (IS_ERR(right))
right = NULL;
if (pslot + 1 < btrfs_header_nritems(parent)) {
right = btrfs_read_node_slot(parent, pslot + 1);
if (IS_ERR(right)) {
ret = PTR_ERR(right);
right = NULL;
goto enospc;
}
if (right) {
__btrfs_tree_lock(right, BTRFS_NESTING_RIGHT);
wret = btrfs_cow_block(trans, root, right,
parent, pslot + 1, &right,
@ -1240,14 +1247,14 @@ static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans,
if (!parent)
return 1;
left = btrfs_read_node_slot(parent, pslot - 1);
if (IS_ERR(left))
left = NULL;
/* first, try to make some room in the middle buffer */
if (left) {
if (pslot) {
u32 left_nr;
left = btrfs_read_node_slot(parent, pslot - 1);
if (IS_ERR(left))
return PTR_ERR(left);
__btrfs_tree_lock(left, BTRFS_NESTING_LEFT);
left_nr = btrfs_header_nritems(left);
@ -1292,16 +1299,17 @@ static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans,
btrfs_tree_unlock(left);
free_extent_buffer(left);
}
right = btrfs_read_node_slot(parent, pslot + 1);
if (IS_ERR(right))
right = NULL;
/*
* then try to empty the right most buffer into the middle
*/
if (right) {
if (pslot + 1 < btrfs_header_nritems(parent)) {
u32 right_nr;
right = btrfs_read_node_slot(parent, pslot + 1);
if (IS_ERR(right))
return PTR_ERR(right);
__btrfs_tree_lock(right, BTRFS_NESTING_RIGHT);
right_nr = btrfs_header_nritems(right);
@ -1864,7 +1872,7 @@ static inline int search_for_key_slot(struct extent_buffer *eb,
return 0;
}
return btrfs_generic_bin_search(eb, search_low_slot, key, slot);
return btrfs_bin_search(eb, search_low_slot, key, slot);
}
static int search_leaf(struct btrfs_trans_handle *trans,
@ -2321,7 +2329,7 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key,
*/
btrfs_unlock_up_safe(p, level + 1);
ret = btrfs_bin_search(b, key, &slot);
ret = btrfs_bin_search(b, 0, key, &slot);
if (ret < 0)
goto done;
@ -2482,26 +2490,15 @@ int btrfs_search_backwards(struct btrfs_root *root, struct btrfs_key *key,
int btrfs_get_next_valid_item(struct btrfs_root *root, struct btrfs_key *key,
struct btrfs_path *path)
{
while (1) {
if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
int ret;
const int slot = path->slots[0];
const struct extent_buffer *leaf = path->nodes[0];
/* This is where we start walking the path. */
if (slot >= btrfs_header_nritems(leaf)) {
/*
* If we've reached the last slot in this leaf we need
* to go to the next leaf and reset the path.
*/
ret = btrfs_next_leaf(root, path);
if (ret)
return ret;
continue;
}
/* Store the found, valid item in @key. */
btrfs_item_key_to_cpu(leaf, key, slot);
break;
ret = btrfs_next_leaf(root, path);
if (ret)
return ret;
}
btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]);
return 0;
}
@ -3198,12 +3195,8 @@ static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
btrfs_assert_tree_write_locked(path->nodes[1]);
right = btrfs_read_node_slot(upper, slot + 1);
/*
* slot + 1 is not valid or we fail to read the right node,
* no big deal, just return.
*/
if (IS_ERR(right))
return 1;
return PTR_ERR(right);
__btrfs_tree_lock(right, BTRFS_NESTING_RIGHT);
@ -3417,12 +3410,8 @@ static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
btrfs_assert_tree_write_locked(path->nodes[1]);
left = btrfs_read_node_slot(path->nodes[1], slot - 1);
/*
* slot - 1 is not valid or we fail to read the left node,
* no big deal, just return.
*/
if (IS_ERR(left))
return 1;
return PTR_ERR(left);
__btrfs_tree_lock(left, BTRFS_NESTING_LEFT);
@ -4576,7 +4565,7 @@ int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key,
while (1) {
nritems = btrfs_header_nritems(cur);
level = btrfs_header_level(cur);
sret = btrfs_bin_search(cur, min_key, &slot);
sret = btrfs_bin_search(cur, 0, min_key, &slot);
if (sret < 0) {
ret = sret;
goto out;

17
fs/btrfs/ctree.h
View file

@ -508,22 +508,9 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range);
int __init btrfs_ctree_init(void);
void __cold btrfs_ctree_exit(void);
int btrfs_generic_bin_search(struct extent_buffer *eb, int first_slot,
const struct btrfs_key *key, int *slot);
int btrfs_bin_search(struct extent_buffer *eb, int first_slot,
const struct btrfs_key *key, int *slot);
/*
* Simple binary search on an extent buffer. Works for both leaves and nodes, and
* always searches over the whole range of keys (slot 0 to slot 'nritems - 1').
*/
static inline int btrfs_bin_search(struct extent_buffer *eb,
const struct btrfs_key *key,
int *slot)
{
return btrfs_generic_bin_search(eb, 0, key, slot);
}
int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key,
int *slot);
int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2);
int btrfs_previous_item(struct btrfs_root *root,
struct btrfs_path *path, u64 min_objectid,

2
fs/btrfs/delalloc-space.c
View file

@ -358,8 +358,8 @@ int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes,
* racing with an ordered completion or some such that would think it
* needs to free the reservation we just made.
*/
spin_lock(&inode->lock);
nr_extents = count_max_extents(fs_info, num_bytes);
spin_lock(&inode->lock);
btrfs_mod_outstanding_extents(inode, nr_extents);
inode->csum_bytes += disk_num_bytes;
btrfs_calculate_inode_block_rsv_size(fs_info, inode);

49
fs/btrfs/delayed-ref.c
View file

@ -53,24 +53,6 @@ bool btrfs_check_space_for_delayed_refs(struct btrfs_fs_info *fs_info)
return ret;
}
int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans)
{
u64 num_entries =
atomic_read(&trans->transaction->delayed_refs.num_entries);
u64 avg_runtime;
u64 val;
smp_mb();
avg_runtime = trans->fs_info->avg_delayed_ref_runtime;
val = num_entries * avg_runtime;
if (val >= NSEC_PER_SEC)
return 1;
if (val >= NSEC_PER_SEC / 2)
return 2;
return btrfs_check_space_for_delayed_refs(trans->fs_info);
}
/*
* Release a ref head's reservation.
*
@ -83,20 +65,9 @@ int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans)
void btrfs_delayed_refs_rsv_release(struct btrfs_fs_info *fs_info, int nr)
{
struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv;
u64 num_bytes = btrfs_calc_insert_metadata_size(fs_info, nr);
const u64 num_bytes = btrfs_calc_delayed_ref_bytes(fs_info, nr);
u64 released = 0;
/*
* We have to check the mount option here because we could be enabling
* the free space tree for the first time and don't have the compat_ro
* option set yet.
*
* We need extra reservations if we have the free space tree because
* we'll have to modify that tree as well.
*/
if (btrfs_test_opt(fs_info, FREE_SPACE_TREE))
num_bytes *= 2;
released = btrfs_block_rsv_release(fs_info, block_rsv, num_bytes, NULL);
if (released)
trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv",
@ -118,18 +89,8 @@ void btrfs_update_delayed_refs_rsv(struct btrfs_trans_handle *trans)
if (!trans->delayed_ref_updates)
return;
num_bytes = btrfs_calc_insert_metadata_size(fs_info,
trans->delayed_ref_updates);
/*
* We have to check the mount option here because we could be enabling
* the free space tree for the first time and don't have the compat_ro
* option set yet.
*
* We need extra reservations if we have the free space tree because
* we'll have to modify that tree as well.
*/
if (btrfs_test_opt(fs_info, FREE_SPACE_TREE))
num_bytes *= 2;
num_bytes = btrfs_calc_delayed_ref_bytes(fs_info,
trans->delayed_ref_updates);
spin_lock(&delayed_rsv->lock);
delayed_rsv->size += num_bytes;
@ -200,7 +161,7 @@ int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info,
enum btrfs_reserve_flush_enum flush)
{
struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv;
u64 limit = btrfs_calc_insert_metadata_size(fs_info, 1);
u64 limit = btrfs_calc_delayed_ref_bytes(fs_info, 1);
u64 num_bytes = 0;
int ret = -ENOSPC;
@ -217,7 +178,7 @@ int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info,
ret = btrfs_reserve_metadata_bytes(fs_info, block_rsv, num_bytes, flush);
if (ret)
return ret;
btrfs_block_rsv_add_bytes(block_rsv, num_bytes, 0);
btrfs_block_rsv_add_bytes(block_rsv, num_bytes, false);
trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv",
0, num_bytes, 1);
return 0;

22
fs/btrfs/delayed-ref.h
View file

@ -253,6 +253,27 @@ extern struct kmem_cache *btrfs_delayed_extent_op_cachep;
int __init btrfs_delayed_ref_init(void);
void __cold btrfs_delayed_ref_exit(void);
static inline u64 btrfs_calc_delayed_ref_bytes(const struct btrfs_fs_info *fs_info,
int num_delayed_refs)
{
u64 num_bytes;
num_bytes = btrfs_calc_insert_metadata_size(fs_info, num_delayed_refs);
/*
* We have to check the mount option here because we could be enabling
* the free space tree for the first time and don't have the compat_ro
* option set yet.
*
* We need extra reservations if we have the free space tree because
* we'll have to modify that tree as well.
*/
if (btrfs_test_opt(fs_info, FREE_SPACE_TREE))
num_bytes *= 2;
return num_bytes;
}
static inline void btrfs_init_generic_ref(struct btrfs_ref *generic_ref,
int action, u64 bytenr, u64 len, u64 parent)
{
@ -385,7 +406,6 @@ int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info,
void btrfs_migrate_to_delayed_refs_rsv(struct btrfs_fs_info *fs_info,
struct btrfs_block_rsv *src,
u64 num_bytes);
int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans);
bool btrfs_check_space_for_delayed_refs(struct btrfs_fs_info *fs_info);
/*

147
fs/btrfs/disk-io.c
View file

@ -1341,17 +1341,8 @@ struct btrfs_root *btrfs_read_tree_root(struct btrfs_root *tree_root,
static int btrfs_init_fs_root(struct btrfs_root *root, dev_t anon_dev)
{
int ret;
unsigned int nofs_flag;
/*
* We might be called under a transaction (e.g. indirect backref
* resolution) which could deadlock if it triggers memory reclaim
*/
nofs_flag = memalloc_nofs_save();
ret = btrfs_drew_lock_init(&root->snapshot_lock);
memalloc_nofs_restore(nofs_flag);
if (ret)
goto fail;
btrfs_drew_lock_init(&root->snapshot_lock);
if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID &&
!btrfs_is_data_reloc_root(root)) {
@ -2065,7 +2056,6 @@ void btrfs_put_root(struct btrfs_root *root)
WARN_ON(test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state));
if (root->anon_dev)
free_anon_bdev(root->anon_dev);
btrfs_drew_lock_destroy(&root->snapshot_lock);
free_root_extent_buffers(root);
#ifdef CONFIG_BTRFS_DEBUG
spin_lock(&root->fs_info->fs_roots_radix_lock);
@ -2125,11 +2115,16 @@ static void btrfs_init_balance(struct btrfs_fs_info *fs_info)
atomic_set(&fs_info->reloc_cancel_req, 0);
}
static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info)
static int btrfs_init_btree_inode(struct super_block *sb)
{
struct inode *inode = fs_info->btree_inode;
struct btrfs_fs_info *fs_info = btrfs_sb(sb);
unsigned long hash = btrfs_inode_hash(BTRFS_BTREE_INODE_OBJECTID,
fs_info->tree_root);
struct inode *inode;
inode = new_inode(sb);
if (!inode)
return -ENOMEM;
inode->i_ino = BTRFS_BTREE_INODE_OBJECTID;
set_nlink(inode, 1);
@ -2140,6 +2135,7 @@ static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info)
*/
inode->i_size = OFFSET_MAX;
inode->i_mapping->a_ops = &btree_aops;
mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS);
RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
extent_io_tree_init(fs_info, &BTRFS_I(inode)->io_tree,
@ -2152,6 +2148,9 @@ static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info)
BTRFS_I(inode)->location.offset = 0;
set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
__insert_inode_hash(inode, hash);
fs_info->btree_inode = inode;
return 0;
}
static void btrfs_init_dev_replace_locks(struct btrfs_fs_info *fs_info)
@ -2966,7 +2965,6 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info)
atomic64_set(&fs_info->free_chunk_space, 0);
fs_info->tree_mod_log = RB_ROOT;
fs_info->commit_interval = BTRFS_DEFAULT_COMMIT_INTERVAL;
fs_info->avg_delayed_ref_runtime = NSEC_PER_SEC >> 6; /* div by 64 */
btrfs_init_ref_verify(fs_info);
fs_info->thread_pool_size = min_t(unsigned long,
@ -3344,14 +3342,11 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
struct btrfs_root *tree_root;
struct btrfs_root *chunk_root;
int ret;
int err = -EINVAL;
int level;
ret = init_mount_fs_info(fs_info, sb);
if (ret) {
err = ret;
if (ret)
goto fail;
}
/* These need to be init'ed before we start creating inodes and such. */
tree_root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID,
@ -3361,17 +3356,13 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
GFP_KERNEL);
fs_info->chunk_root = chunk_root;
if (!tree_root || !chunk_root) {
err = -ENOMEM;
ret = -ENOMEM;
goto fail;
}
fs_info->btree_inode = new_inode(sb);
if (!fs_info->btree_inode) {
err = -ENOMEM;
ret = btrfs_init_btree_inode(sb);
if (ret)
goto fail;
}
mapping_set_gfp_mask(fs_info->btree_inode->i_mapping, GFP_NOFS);
btrfs_init_btree_inode(fs_info);
invalidate_bdev(fs_devices->latest_dev->bdev);
@ -3380,7 +3371,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
*/
disk_super = btrfs_read_dev_super(fs_devices->latest_dev->bdev);
if (IS_ERR(disk_super)) {
err = PTR_ERR(disk_super);
ret = PTR_ERR(disk_super);
goto fail_alloc;
}
@ -3392,7 +3383,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
if (!btrfs_supported_super_csum(csum_type)) {
btrfs_err(fs_info, "unsupported checksum algorithm: %u",
csum_type);
err = -EINVAL;
ret = -EINVAL;
btrfs_release_disk_super(disk_super);
goto fail_alloc;
}
@ -3401,7 +3392,6 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
ret = btrfs_init_csum_hash(fs_info, csum_type);
if (ret) {
err = ret;
btrfs_release_disk_super(disk_super);
goto fail_alloc;
}
@ -3412,7 +3402,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
*/
if (btrfs_check_super_csum(fs_info, disk_super)) {
btrfs_err(fs_info, "superblock checksum mismatch");
err = -EINVAL;
ret = -EINVAL;
btrfs_release_disk_super(disk_super);
goto fail_alloc;
}
@ -3442,12 +3432,15 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
ret = btrfs_validate_mount_super(fs_info);
if (ret) {
btrfs_err(fs_info, "superblock contains fatal errors");
err = -EINVAL;
ret = -EINVAL;
goto fail_alloc;
}
if (!btrfs_super_root(disk_super))
if (!btrfs_super_root(disk_super)) {
btrfs_err(fs_info, "invalid superblock tree root bytenr");
ret = -EINVAL;
goto fail_alloc;
}
/* check FS state, whether FS is broken. */
if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_ERROR)
@ -3474,16 +3467,12 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
fs_info->stripesize = stripesize;
ret = btrfs_parse_options(fs_info, options, sb->s_flags);
if (ret) {
err = ret;
if (ret)
goto fail_alloc;
}
ret = btrfs_check_features(fs_info, !sb_rdonly(sb));
if (ret < 0) {
err = ret;
if (ret < 0)
goto fail_alloc;
}
if (sectorsize < PAGE_SIZE) {
struct btrfs_subpage_info *subpage_info;
@ -3503,17 +3492,17 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
"read-write for sector size %u with page size %lu is experimental",
sectorsize, PAGE_SIZE);
subpage_info = kzalloc(sizeof(*subpage_info), GFP_KERNEL);
if (!subpage_info)
if (!subpage_info) {
ret = -ENOMEM;
goto fail_alloc;
}
btrfs_init_subpage_info(subpage_info, sectorsize);
fs_info->subpage_info = subpage_info;
}
ret = btrfs_init_workqueues(fs_info);
if (ret) {
err = ret;
if (ret)
goto fail_sb_buffer;
}
sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super);
sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE);
@ -3559,6 +3548,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
btrfs_free_extra_devids(fs_devices);
if (!fs_devices->latest_dev->bdev) {
btrfs_err(fs_info, "failed to read devices");
ret = -EIO;
goto fail_tree_roots;
}
@ -3574,8 +3564,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
ret = btrfs_get_dev_zone_info_all_devices(fs_info);
if (ret) {
btrfs_err(fs_info,
"zoned: failed to read device zone info: %d",
ret);
"zoned: failed to read device zone info: %d", ret);
goto fail_block_groups;
}
@ -3654,19 +3643,24 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
!btrfs_check_rw_degradable(fs_info, NULL)) {
btrfs_warn(fs_info,
"writable mount is not allowed due to too many missing devices");
ret = -EINVAL;
goto fail_sysfs;
}
fs_info->cleaner_kthread = kthread_run(cleaner_kthread, fs_info,
"btrfs-cleaner");
if (IS_ERR(fs_info->cleaner_kthread))
if (IS_ERR(fs_info->cleaner_kthread)) {
ret = PTR_ERR(fs_info->cleaner_kthread);
goto fail_sysfs;
}
fs_info->transaction_kthread = kthread_run(transaction_kthread,
tree_root,
"btrfs-transaction");
if (IS_ERR(fs_info->transaction_kthread))
if (IS_ERR(fs_info->transaction_kthread)) {
ret = PTR_ERR(fs_info->transaction_kthread);
goto fail_cleaner;
}
if (!btrfs_test_opt(fs_info, NOSSD) &&
!fs_info->fs_devices->rotating) {
@ -3684,7 +3678,6 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
fs_info->fs_devices->discardable) {
btrfs_set_and_info(fs_info, DISCARD_ASYNC,
"auto enabling async discard");
btrfs_clear_opt(fs_info->mount_opt, NODISCARD);
}
#ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
@ -3711,16 +3704,14 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
!btrfs_test_opt(fs_info, NOLOGREPLAY)) {
btrfs_info(fs_info, "start tree-log replay");
ret = btrfs_replay_log(fs_info, fs_devices);
if (ret) {
err = ret;
if (ret)
goto fail_qgroup;
}
}
fs_info->fs_root = btrfs_get_fs_root(fs_info, BTRFS_FS_TREE_OBJECTID, true);
if (IS_ERR(fs_info->fs_root)) {
err = PTR_ERR(fs_info->fs_root);
btrfs_warn(fs_info, "failed to read fs tree: %d", err);
ret = PTR_ERR(fs_info->fs_root);
btrfs_warn(fs_info, "failed to read fs tree: %d", ret);
fs_info->fs_root = NULL;
goto fail_qgroup;
}
@ -3797,7 +3788,8 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
iput(fs_info->btree_inode);
fail:
btrfs_close_devices(fs_info->fs_devices);
return err;
ASSERT(ret < 0);
return ret;
}
ALLOW_ERROR_INJECTION(open_ctree, ERRNO);
@ -4094,6 +4086,8 @@ static void write_dev_flush(struct btrfs_device *device)
{
struct bio *bio = &device->flush_bio;
device->last_flush_error = BLK_STS_OK;
#ifndef CONFIG_BTRFS_FS_CHECK_INTEGRITY
/*
* When a disk has write caching disabled, we skip submission of a bio
@ -4122,25 +4116,24 @@ static void write_dev_flush(struct btrfs_device *device)
/*
* If the flush bio has been submitted by write_dev_flush, wait for it.
* Return true for any error, and false otherwise.
*/
static blk_status_t wait_dev_flush(struct btrfs_device *device)
static bool wait_dev_flush(struct btrfs_device *device)
{
struct bio *bio = &device->flush_bio;
if (!test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state))
return BLK_STS_OK;
if (!test_and_clear_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state))
return false;
clear_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
wait_for_completion_io(&device->flush_wait);
return bio->bi_status;
}
if (bio->bi_status) {
device->last_flush_error = bio->bi_status;
btrfs_dev_stat_inc_and_print(device, BTRFS_DEV_STAT_FLUSH_ERRS);
return true;
}
static int check_barrier_error(struct btrfs_fs_info *fs_info)
{
if (!btrfs_check_rw_degradable(fs_info, NULL))
return -EIO;
return 0;
return false;
}
/*
@ -4152,7 +4145,6 @@ static int barrier_all_devices(struct btrfs_fs_info *info)
struct list_head *head;
struct btrfs_device *dev;
int errors_wait = 0;
blk_status_t ret;
lockdep_assert_held(&info->fs_devices->device_list_mutex);
/* send down all the barriers */
@ -4167,7 +4159,6 @@ static int barrier_all_devices(struct btrfs_fs_info *info)
continue;
write_dev_flush(dev);
dev->last_flush_error = BLK_STS_OK;
}
/* wait for all the barriers */
@ -4182,23 +4173,17 @@ static int barrier_all_devices(struct btrfs_fs_info *info)
!test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
continue;
ret = wait_dev_flush(dev);
if (ret) {
dev->last_flush_error = ret;
btrfs_dev_stat_inc_and_print(dev,
BTRFS_DEV_STAT_FLUSH_ERRS);
if (wait_dev_flush(dev))
errors_wait++;
}
}
if (errors_wait) {
/*
* At some point we need the status of all disks
* to arrive at the volume status. So error checking
* is being pushed to a separate loop.
*/
return check_barrier_error(info);
}
/*
* Checks last_flush_error of disks in order to determine the device
* state.
*/
if (errors_wait && !btrfs_check_rw_degradable(info, NULL))
return -EIO;
return 0;
}
@ -4404,12 +4389,12 @@ int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info)
root_objectid = gang[i]->root_key.objectid;
err = btrfs_orphan_cleanup(gang[i]);
if (err)
break;
goto out;
btrfs_put_root(gang[i]);
}
root_objectid++;
}
out:
/* release the uncleaned roots due to error */
for (; i < ret; i++) {
if (gang[i])

37
fs/btrfs/extent-tree.c
View file

@ -1894,8 +1894,7 @@ static struct btrfs_delayed_ref_head *btrfs_obtain_ref_head(
}
static int btrfs_run_delayed_refs_for_head(struct btrfs_trans_handle *trans,
struct btrfs_delayed_ref_head *locked_ref,
unsigned long *run_refs)
struct btrfs_delayed_ref_head *locked_ref)
{
struct btrfs_fs_info *fs_info = trans->fs_info;
struct btrfs_delayed_ref_root *delayed_refs;
@ -1917,7 +1916,6 @@ static int btrfs_run_delayed_refs_for_head(struct btrfs_trans_handle *trans,
return -EAGAIN;
}
(*run_refs)++;
ref->in_tree = 0;
rb_erase_cached(&ref->ref_node, &locked_ref->ref_tree);
RB_CLEAR_NODE(&ref->ref_node);
@ -1981,10 +1979,8 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
struct btrfs_fs_info *fs_info = trans->fs_info;
struct btrfs_delayed_ref_root *delayed_refs;
struct btrfs_delayed_ref_head *locked_ref = NULL;
ktime_t start = ktime_get();
int ret;
unsigned long count = 0;
unsigned long actual_count = 0;
delayed_refs = &trans->transaction->delayed_refs;
do {
@ -2014,8 +2010,7 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
spin_lock(&locked_ref->lock);
btrfs_merge_delayed_refs(fs_info, delayed_refs, locked_ref);
ret = btrfs_run_delayed_refs_for_head(trans, locked_ref,
&actual_count);
ret = btrfs_run_delayed_refs_for_head(trans, locked_ref);
if (ret < 0 && ret != -EAGAIN) {
/*
* Error, btrfs_run_delayed_refs_for_head already
@ -2046,24 +2041,6 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
cond_resched();
} while ((nr != -1 && count < nr) || locked_ref);
/*
* We don't want to include ref heads since we can have empty ref heads
* and those will drastically skew our runtime down since we just do
* accounting, no actual extent tree updates.
*/
if (actual_count > 0) {
u64 runtime = ktime_to_ns(ktime_sub(ktime_get(), start));
u64 avg;
/*
* We weigh the current average higher than our current runtime
* to avoid large swings in the average.
*/
spin_lock(&delayed_refs->lock);
avg = fs_info->avg_delayed_ref_runtime * 3 + runtime;
fs_info->avg_delayed_ref_runtime = avg >> 2; /* div by 4 */
spin_unlock(&delayed_refs->lock);
}
return 0;
}
@ -5509,11 +5486,11 @@ static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
{
int level = wc->level;
int lookup_info = 1;
int ret;
int ret = 0;
while (level >= 0) {
ret = walk_down_proc(trans, root, path, wc, lookup_info);
if (ret > 0)
if (ret)
break;
if (level == 0)
@ -5528,10 +5505,10 @@ static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
path->slots[level]++;
continue;
} else if (ret < 0)
return ret;
break;
level = wc->level;
}
return 0;
return (ret == 1) ? 0 : ret;
}
static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
@ -5708,12 +5685,14 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc)
ret = walk_down_tree(trans, root, path, wc);
if (ret < 0) {
btrfs_abort_transaction(trans, ret);
err = ret;
break;
}
ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
if (ret < 0) {
btrfs_abort_transaction(trans, ret);
err = ret;
break;
}

546
fs/btrfs/extent_io.c
View file

File diff suppressed because it is too large Load diff

91
fs/btrfs/file-item.c
View file

@ -335,48 +335,6 @@ static int search_csum_tree(struct btrfs_fs_info *fs_info,
return ret;
}
/*
* Locate the file_offset of @cur_disk_bytenr of a @bio.
*
* Bio of btrfs represents read range of
* [bi_sector << 9, bi_sector << 9 + bi_size).
* Knowing this, we can iterate through each bvec to locate the page belong to
* @cur_disk_bytenr and get the file offset.
*
* @inode is used to determine if the bvec page really belongs to @inode.
*
* Return 0 if we can't find the file offset
* Return >0 if we find the file offset and restore it to @file_offset_ret
*/
static int search_file_offset_in_bio(struct bio *bio, struct inode *inode,
u64 disk_bytenr, u64 *file_offset_ret)
{
struct bvec_iter iter;
struct bio_vec bvec;
u64 cur = bio->bi_iter.bi_sector << SECTOR_SHIFT;
int ret = 0;
bio_for_each_segment(bvec, bio, iter) {
struct page *page = bvec.bv_page;
if (cur > disk_bytenr)
break;
if (cur + bvec.bv_len <= disk_bytenr) {
cur += bvec.bv_len;
continue;
}
ASSERT(in_range(disk_bytenr, cur, bvec.bv_len));
if (page->mapping && page->mapping->host &&
page->mapping->host == inode) {
ret = 1;
*file_offset_ret = page_offset(page) + bvec.bv_offset +
disk_bytenr - cur;
break;
}
}
return ret;
}
/*
* Lookup the checksum for the read bio in csum tree.
*
@ -386,17 +344,15 @@ blk_status_t btrfs_lookup_bio_sums(struct btrfs_bio *bbio)
{
struct btrfs_inode *inode = bbio->inode;
struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct extent_io_tree *io_tree = &inode->io_tree;
struct bio *bio = &bbio->bio;
struct btrfs_path *path;
const u32 sectorsize = fs_info->sectorsize;
const u32 csum_size = fs_info->csum_size;
u32 orig_len = bio->bi_iter.bi_size;
u64 orig_disk_bytenr = bio->bi_iter.bi_sector << SECTOR_SHIFT;
u64 cur_disk_bytenr;
const unsigned int nblocks = orig_len >> fs_info->sectorsize_bits;
int count = 0;
blk_status_t ret = BLK_STS_OK;
u32 bio_offset = 0;
if ((inode->flags & BTRFS_INODE_NODATASUM) ||
test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state))
@ -447,28 +403,14 @@ blk_status_t btrfs_lookup_bio_sums(struct btrfs_bio *bbio)
path->skip_locking = 1;
}
for (cur_disk_bytenr = orig_disk_bytenr;
cur_disk_bytenr < orig_disk_bytenr + orig_len;
cur_disk_bytenr += (count * sectorsize)) {
u64 search_len = orig_disk_bytenr + orig_len - cur_disk_bytenr;
unsigned int sector_offset;
u8 *csum_dst;
/*
* Although both cur_disk_bytenr and orig_disk_bytenr is u64,
* we're calculating the offset to the bio start.
*
* Bio size is limited to UINT_MAX, thus unsigned int is large
* enough to contain the raw result, not to mention the right
* shifted result.
*/
ASSERT(cur_disk_bytenr - orig_disk_bytenr < UINT_MAX);
sector_offset = (cur_disk_bytenr - orig_disk_bytenr) >>
fs_info->sectorsize_bits;
csum_dst = bbio->csum + sector_offset * csum_size;
while (bio_offset < orig_len) {
int count;
u64 cur_disk_bytenr = orig_disk_bytenr + bio_offset;
u8 *csum_dst = bbio->csum +
(bio_offset >> fs_info->sectorsize_bits) * csum_size;
count = search_csum_tree(fs_info, path, cur_disk_bytenr,
search_len, csum_dst);
orig_len - bio_offset, csum_dst);
if (count < 0) {
ret = errno_to_blk_status(count);
if (bbio->csum != bbio->csum_inline)
@ -493,14 +435,9 @@ blk_status_t btrfs_lookup_bio_sums(struct btrfs_bio *bbio)
if (inode->root->root_key.objectid ==
BTRFS_DATA_RELOC_TREE_OBJECTID) {
u64 file_offset;
int ret;
u64 file_offset = bbio->file_offset + bio_offset;
ret = search_file_offset_in_bio(bio,
&inode->vfs_inode,
cur_disk_bytenr, &file_offset);
if (ret)
set_extent_bits(io_tree, file_offset,
set_extent_bits(&inode->io_tree, file_offset,
file_offset + sectorsize - 1,
EXTENT_NODATASUM);
} else {
@ -509,6 +446,7 @@ blk_status_t btrfs_lookup_bio_sums(struct btrfs_bio *bbio)
cur_disk_bytenr, cur_disk_bytenr + sectorsize);
}
}
bio_offset += count * sectorsize;
}
btrfs_free_path(path);
@ -659,7 +597,8 @@ int btrfs_lookup_csums_list(struct btrfs_root *root, u64 start, u64 end,
* in is large enough to contain all csums.
*/
int btrfs_lookup_csums_bitmap(struct btrfs_root *root, u64 start, u64 end,
u8 *csum_buf, unsigned long *csum_bitmap)
u8 *csum_buf, unsigned long *csum_bitmap,
bool search_commit)
{
struct btrfs_fs_info *fs_info = root->fs_info;
struct btrfs_key key;
@ -676,6 +615,12 @@ int btrfs_lookup_csums_bitmap(struct btrfs_root *root, u64 start, u64 end,
if (!path)
return -ENOMEM;
if (search_commit) {
path->skip_locking = 1;
path->reada = READA_FORWARD;
path->search_commit_root = 1;
}
key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
key.type = BTRFS_EXTENT_CSUM_KEY;
key.offset = start;

3
fs/btrfs/file-item.h
View file

@ -57,7 +57,8 @@ int btrfs_lookup_csums_list(struct btrfs_root *root, u64 start, u64 end,
struct list_head *list, int search_commit,
bool nowait);
int btrfs_lookup_csums_bitmap(struct btrfs_root *root, u64 start, u64 end,
u8 *csum_buf, unsigned long *csum_bitmap);
u8 *csum_buf, unsigned long *csum_bitmap,
bool search_commit);
void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode,
const struct btrfs_path *path,
struct btrfs_file_extent_item *fi,

55
fs/btrfs/fs.h
View file

@ -24,6 +24,18 @@
#define BTRFS_SUPER_INFO_SIZE 4096
static_assert(sizeof(struct btrfs_super_block) == BTRFS_SUPER_INFO_SIZE);
/*
* Number of metadata items necessary for an unlink operation:
*
* 1 for the possible orphan item
* 1 for the dir item
* 1 for the dir index
* 1 for the inode ref
* 1 for the inode
* 1 for the parent inode
*/
#define BTRFS_UNLINK_METADATA_UNITS 6
/*
* The reserved space at the beginning of each device. It covers the primary
* super block and leaves space for potential use by other tools like
@ -193,27 +205,7 @@ enum {
#define BTRFS_FEATURE_COMPAT_RO_SAFE_SET 0ULL
#define BTRFS_FEATURE_COMPAT_RO_SAFE_CLEAR 0ULL
#ifdef CONFIG_BTRFS_DEBUG
/*
* Extent tree v2 supported only with CONFIG_BTRFS_DEBUG
*/
#define BTRFS_FEATURE_INCOMPAT_SUPP \
(BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF | \
BTRFS_FEATURE_INCOMPAT_DEFAULT_SUBVOL | \
BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \
BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \
BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \
BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD | \
BTRFS_FEATURE_INCOMPAT_RAID56 | \
BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \
BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \
BTRFS_FEATURE_INCOMPAT_NO_HOLES | \
BTRFS_FEATURE_INCOMPAT_METADATA_UUID | \
BTRFS_FEATURE_INCOMPAT_RAID1C34 | \
BTRFS_FEATURE_INCOMPAT_ZONED | \
BTRFS_FEATURE_INCOMPAT_EXTENT_TREE_V2)
#else
#define BTRFS_FEATURE_INCOMPAT_SUPP \
#define BTRFS_FEATURE_INCOMPAT_SUPP_STABLE \
(BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF | \
BTRFS_FEATURE_INCOMPAT_DEFAULT_SUBVOL | \
BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \
@ -227,6 +219,21 @@ enum {
BTRFS_FEATURE_INCOMPAT_METADATA_UUID | \
BTRFS_FEATURE_INCOMPAT_RAID1C34 | \
BTRFS_FEATURE_INCOMPAT_ZONED)
#ifdef CONFIG_BTRFS_DEBUG
/*
* Features under developmen like Extent tree v2 support is enabled
* only under CONFIG_BTRFS_DEBUG.
*/
#define BTRFS_FEATURE_INCOMPAT_SUPP \
(BTRFS_FEATURE_INCOMPAT_SUPP_STABLE | \
BTRFS_FEATURE_INCOMPAT_EXTENT_TREE_V2)
#else
#define BTRFS_FEATURE_INCOMPAT_SUPP \
(BTRFS_FEATURE_INCOMPAT_SUPP_STABLE)
#endif
#define BTRFS_FEATURE_INCOMPAT_SAFE_SET \
@ -412,7 +419,6 @@ struct btrfs_fs_info {
* Must be written and read while holding btrfs_fs_info::commit_root_sem.
*/
u64 last_reloc_trans;
u64 avg_delayed_ref_runtime;
/*
* This is updated to the current trans every time a full commit is
@ -638,7 +644,6 @@ struct btrfs_fs_info {
refcount_t scrub_workers_refcnt;
struct workqueue_struct *scrub_workers;
struct workqueue_struct *scrub_wr_completion_workers;
struct workqueue_struct *scrub_parity_workers;
struct btrfs_subpage_info *subpage_info;
struct btrfs_discard_ctl discard_ctl;
@ -828,7 +833,7 @@ static inline u64 btrfs_csum_bytes_to_leaves(
* Use this if we would be adding new items, as we could split nodes as we cow
* down the tree.
*/
static inline u64 btrfs_calc_insert_metadata_size(struct btrfs_fs_info *fs_info,
static inline u64 btrfs_calc_insert_metadata_size(const struct btrfs_fs_info *fs_info,
unsigned num_items)
{
return (u64)fs_info->nodesize * BTRFS_MAX_LEVEL * 2 * num_items;
@ -838,7 +843,7 @@ static inline u64 btrfs_calc_insert_metadata_size(struct btrfs_fs_info *fs_info,
* Doing a truncate or a modification won't result in new nodes or leaves, just
* what we need for COW.
*/
static inline u64 btrfs_calc_metadata_size(struct btrfs_fs_info *fs_info,
static inline u64 btrfs_calc_metadata_size(const struct btrfs_fs_info *fs_info,
unsigned num_items)
{
return (u64)fs_info->nodesize * BTRFS_MAX_LEVEL * num_items;

15
fs/btrfs/inode-item.c
View file

@ -527,7 +527,7 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
while (1) {
u64 clear_start = 0, clear_len = 0, extent_start = 0;
bool should_throttle = false;
bool refill_delayed_refs_rsv = false;
fi = NULL;
leaf = path->nodes[0];
@ -660,8 +660,7 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
/* No pending yet, add ourselves */
pending_del_slot = path->slots[0];
pending_del_nr = 1;
} else if (pending_del_nr &&
path->slots[0] + 1 == pending_del_slot) {
} else if (path->slots[0] + 1 == pending_del_slot) {
/* Hop on the pending chunk */
pending_del_nr++;
pending_del_slot = path->slots[0];
@ -686,10 +685,8 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
btrfs_abort_transaction(trans, ret);
break;
}
if (be_nice) {
if (btrfs_should_throttle_delayed_refs(trans))
should_throttle = true;
}
if (be_nice && btrfs_check_space_for_delayed_refs(fs_info))
refill_delayed_refs_rsv = true;
}
if (found_type == BTRFS_INODE_ITEM_KEY)
@ -697,7 +694,7 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
if (path->slots[0] == 0 ||
path->slots[0] != pending_del_slot ||
should_throttle) {
refill_delayed_refs_rsv) {
if (pending_del_nr) {
ret = btrfs_del_items(trans, root, path,
pending_del_slot,
@ -720,7 +717,7 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
* actually allocate, so just bail if we're short and
* let the normal reservation dance happen higher up.
*/
if (should_throttle) {
if (refill_delayed_refs_rsv) {
ret = btrfs_delayed_refs_rsv_refill(fs_info,
BTRFS_RESERVE_NO_FLUSH);
if (ret) {

367
fs/btrfs/inode.c
View file

@ -79,6 +79,7 @@ struct btrfs_iget_args {
struct btrfs_dio_data {
ssize_t submitted;
struct extent_changeset *data_reserved;
struct btrfs_ordered_extent *ordered;
bool data_space_reserved;
bool nocow_done;
};
@ -669,8 +670,7 @@ static noinline int compress_file_range(struct async_chunk *async_chunk)
again:
will_compress = 0;
nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
nr_pages = min_t(unsigned long, nr_pages,
BTRFS_MAX_COMPRESSED / PAGE_SIZE);
nr_pages = min_t(unsigned long, nr_pages, BTRFS_MAX_COMPRESSED_PAGES);
/*
* we don't want to send crud past the end of i_size through
@ -945,10 +945,9 @@ static int submit_uncompressed_range(struct btrfs_inode *inode,
ret = cow_file_range(inode, locked_page, start, end, &page_started,
&nr_written, 0, NULL);
/* Inline extent inserted, page gets unlocked and everything is done */
if (page_started) {
ret = 0;
goto out;
}
if (page_started)
return 0;
if (ret < 0) {
btrfs_cleanup_ordered_extents(inode, locked_page, start, end - start + 1);
if (locked_page) {
@ -962,14 +961,11 @@ static int submit_uncompressed_range(struct btrfs_inode *inode,
end_extent_writepage(locked_page, ret, page_start, page_end);
unlock_page(locked_page);
}
goto out;
return ret;
}
ret = extent_write_locked_range(&inode->vfs_inode, start, end);
/* All pages will be unlocked, including @locked_page */
out:
kfree(async_extent);
return ret;
return extent_write_locked_range(&inode->vfs_inode, start, end);
}
static int submit_one_async_extent(struct btrfs_inode *inode,
@ -987,6 +983,9 @@ static int submit_one_async_extent(struct btrfs_inode *inode,
u64 start = async_extent->start;
u64 end = async_extent->start + async_extent->ram_size - 1;
if (async_chunk->blkcg_css)
kthread_associate_blkcg(async_chunk->blkcg_css);
/*
* If async_chunk->locked_page is in the async_extent range, we need to
* handle it.
@ -1001,8 +1000,10 @@ static int submit_one_async_extent(struct btrfs_inode *inode,
lock_extent(io_tree, start, end, NULL);
/* We have fall back to uncompressed write */
if (!async_extent->pages)
return submit_uncompressed_range(inode, async_extent, locked_page);
if (!async_extent->pages) {
ret = submit_uncompressed_range(inode, async_extent, locked_page);
goto done;
}
ret = btrfs_reserve_extent(root, async_extent->ram_size,
async_extent->compressed_size,
@ -1054,24 +1055,18 @@ static int submit_one_async_extent(struct btrfs_inode *inode,
extent_clear_unlock_delalloc(inode, start, end,
NULL, EXTENT_LOCKED | EXTENT_DELALLOC,
PAGE_UNLOCK | PAGE_START_WRITEBACK);
if (btrfs_submit_compressed_write(inode, start, /* file_offset */
btrfs_submit_compressed_write(inode, start, /* file_offset */
async_extent->ram_size, /* num_bytes */
ins.objectid, /* disk_bytenr */
ins.offset, /* compressed_len */
async_extent->pages, /* compressed_pages */
async_extent->nr_pages,
async_chunk->write_flags,
async_chunk->blkcg_css, true)) {
const u64 start = async_extent->start;
const u64 end = start + async_extent->ram_size - 1;
btrfs_writepage_endio_finish_ordered(inode, NULL, start, end, 0);
extent_clear_unlock_delalloc(inode, start, end, NULL, 0,
PAGE_END_WRITEBACK | PAGE_SET_ERROR);
free_async_extent_pages(async_extent);
}
async_chunk->write_flags, true);
*alloc_hint = ins.objectid + ins.offset;
done:
if (async_chunk->blkcg_css)
kthread_associate_blkcg(NULL);
kfree(async_extent);
return ret;
@ -1086,8 +1081,7 @@ static int submit_one_async_extent(struct btrfs_inode *inode,
PAGE_UNLOCK | PAGE_START_WRITEBACK |
PAGE_END_WRITEBACK | PAGE_SET_ERROR);
free_async_extent_pages(async_extent);
kfree(async_extent);
return ret;
goto done;
}
/*
@ -1622,6 +1616,7 @@ static int cow_file_range_async(struct btrfs_inode *inode,
if (blkcg_css != blkcg_root_css) {
css_get(blkcg_css);
async_chunk[i].blkcg_css = blkcg_css;
async_chunk[i].write_flags |= REQ_BTRFS_CGROUP_PUNT;
} else {
async_chunk[i].blkcg_css = NULL;
}
@ -2521,37 +2516,31 @@ void btrfs_clear_delalloc_extent(struct btrfs_inode *inode,
}
/*
* Split an extent_map at [start, start + len]
* Split off the first pre bytes from the extent_map at [start, start + len]
*
* This function is intended to be used only for extract_ordered_extent().
*/
static int split_zoned_em(struct btrfs_inode *inode, u64 start, u64 len,
u64 pre, u64 post)
static int split_extent_map(struct btrfs_inode *inode, u64 start, u64 len, u64 pre)
{
struct extent_map_tree *em_tree = &inode->extent_tree;
struct extent_map *em;
struct extent_map *split_pre = NULL;
struct extent_map *split_mid = NULL;
struct extent_map *split_post = NULL;
int ret = 0;
unsigned long flags;
/* Sanity check */
if (pre == 0 && post == 0)
return 0;
ASSERT(pre != 0);
ASSERT(pre < len);
split_pre = alloc_extent_map();
if (pre)
split_mid = alloc_extent_map();
if (post)
split_post = alloc_extent_map();
if (!split_pre || (pre && !split_mid) || (post && !split_post)) {
if (!split_pre)
return -ENOMEM;
split_mid = alloc_extent_map();
if (!split_mid) {
ret = -ENOMEM;
goto out;
goto out_free_pre;
}
ASSERT(pre + post < len);
lock_extent(&inode->io_tree, start, start + len - 1, NULL);
write_lock(&em_tree->lock);
em = lookup_extent_mapping(em_tree, start, len);
@ -2572,7 +2561,7 @@ static int split_zoned_em(struct btrfs_inode *inode, u64 start, u64 len,
/* First, replace the em with a new extent_map starting from * em->start */
split_pre->start = em->start;
split_pre->len = (pre ? pre : em->len - post);
split_pre->len = pre;
split_pre->orig_start = split_pre->start;
split_pre->block_start = em->block_start;
split_pre->block_len = split_pre->len;
@ -2586,38 +2575,21 @@ static int split_zoned_em(struct btrfs_inode *inode, u64 start, u64 len,
/*
* Now we only have an extent_map at:
* [em->start, em->start + pre] if pre != 0
* [em->start, em->start + em->len - post] if pre == 0
* [em->start, em->start + pre]
*/
if (pre) {
/* Insert the middle extent_map */
split_mid->start = em->start + pre;
split_mid->len = em->len - pre - post;
split_mid->orig_start = split_mid->start;
split_mid->block_start = em->block_start + pre;
split_mid->block_len = split_mid->len;
split_mid->orig_block_len = split_mid->block_len;
split_mid->ram_bytes = split_mid->len;
split_mid->flags = flags;
split_mid->compress_type = em->compress_type;
split_mid->generation = em->generation;
add_extent_mapping(em_tree, split_mid, 1);
}
if (post) {
split_post->start = em->start + em->len - post;
split_post->len = post;
split_post->orig_start = split_post->start;
split_post->block_start = em->block_start + em->len - post;
split_post->block_len = split_post->len;
split_post->orig_block_len = split_post->block_len;
split_post->ram_bytes = split_post->len;
split_post->flags = flags;
split_post->compress_type = em->compress_type;
split_post->generation = em->generation;
add_extent_mapping(em_tree, split_post, 1);
}
/* Insert the middle extent_map. */
split_mid->start = em->start + pre;
split_mid->len = em->len - pre;
split_mid->orig_start = split_mid->start;
split_mid->block_start = em->block_start + pre;
split_mid->block_len = split_mid->len;
split_mid->orig_block_len = split_mid->block_len;
split_mid->ram_bytes = split_mid->len;
split_mid->flags = flags;
split_mid->compress_type = em->compress_type;
split_mid->generation = em->generation;
add_extent_mapping(em_tree, split_mid, 1);
/* Once for us */
free_extent_map(em);
@ -2627,72 +2599,41 @@ static int split_zoned_em(struct btrfs_inode *inode, u64 start, u64 len,
out_unlock:
write_unlock(&em_tree->lock);
unlock_extent(&inode->io_tree, start, start + len - 1, NULL);
out:
free_extent_map(split_pre);
free_extent_map(split_mid);
free_extent_map(split_post);
out_free_pre:
free_extent_map(split_pre);
return ret;
}
blk_status_t btrfs_extract_ordered_extent(struct btrfs_bio *bbio)
int btrfs_extract_ordered_extent(struct btrfs_bio *bbio,
struct btrfs_ordered_extent *ordered)
{
u64 start = (u64)bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT;
u64 len = bbio->bio.bi_iter.bi_size;
struct btrfs_inode *inode = bbio->inode;
struct btrfs_ordered_extent *ordered;
u64 file_len;
u64 end = start + len;
u64 ordered_end;
u64 pre, post;
u64 ordered_len = ordered->num_bytes;
int ret = 0;
ordered = btrfs_lookup_ordered_extent(inode, bbio->file_offset);
if (WARN_ON_ONCE(!ordered))
return BLK_STS_IOERR;
/* Must always be called for the beginning of an ordered extent. */
if (WARN_ON_ONCE(start != ordered->disk_bytenr))
return -EINVAL;
/* No need to split */
/* No need to split if the ordered extent covers the entire bio. */
if (ordered->disk_num_bytes == len)
goto out;
return 0;
/* We cannot split once end_bio'd ordered extent */
if (WARN_ON_ONCE(ordered->bytes_left != ordered->disk_num_bytes)) {
ret = -EINVAL;
goto out;
}
/* We cannot split a compressed ordered extent */
if (WARN_ON_ONCE(ordered->disk_num_bytes != ordered->num_bytes)) {
ret = -EINVAL;
goto out;
}
ordered_end = ordered->disk_bytenr + ordered->disk_num_bytes;
/* bio must be in one ordered extent */
if (WARN_ON_ONCE(start < ordered->disk_bytenr || end > ordered_end)) {
ret = -EINVAL;
goto out;
}
/* Checksum list should be empty */
if (WARN_ON_ONCE(!list_empty(&ordered->list))) {
ret = -EINVAL;
goto out;
}
file_len = ordered->num_bytes;
pre = start - ordered->disk_bytenr;
post = ordered_end - end;
ret = btrfs_split_ordered_extent(ordered, pre, post);
ret = btrfs_split_ordered_extent(ordered, len);
if (ret)
goto out;
ret = split_zoned_em(inode, bbio->file_offset, file_len, pre, post);
return ret;
out:
btrfs_put_ordered_extent(ordered);
/*
* Don't split the extent_map for NOCOW extents, as we're writing into
* a pre-existing one.
*/
if (test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags))
return 0;
return errno_to_blk_status(ret);
return split_extent_map(inode, bbio->file_offset, ordered_len, len);
}
/*
@ -3367,13 +3308,6 @@ int btrfs_check_sector_csum(struct btrfs_fs_info *fs_info, struct page *page,
return 0;
}
static u8 *btrfs_csum_ptr(const struct btrfs_fs_info *fs_info, u8 *csums, u64 offset)
{
u64 offset_in_sectors = offset >> fs_info->sectorsize_bits;
return csums + offset_in_sectors * fs_info->csum_size;
}
/*
* Verify the checksum of a single data sector.
*
@ -3411,7 +3345,8 @@ bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev,
return true;
}
csum_expected = btrfs_csum_ptr(fs_info, bbio->csum, bio_offset);
csum_expected = bbio->csum + (bio_offset >> fs_info->sectorsize_bits) *
fs_info->csum_size;
if (btrfs_check_sector_csum(fs_info, bv->bv_page, bv->bv_offset, csum,
csum_expected))
goto zeroit;
@ -3691,6 +3626,7 @@ int btrfs_orphan_cleanup(struct btrfs_root *root)
trans = btrfs_start_transaction(root, 1);
if (IS_ERR(trans)) {
ret = PTR_ERR(trans);
iput(inode);
goto out;
}
btrfs_debug(fs_info, "auto deleting %Lu",
@ -3698,8 +3634,10 @@ int btrfs_orphan_cleanup(struct btrfs_root *root)
ret = btrfs_del_orphan_item(trans, root,
found_key.objectid);
btrfs_end_transaction(trans);
if (ret)
if (ret) {
iput(inode);
goto out;
}
continue;
}
@ -4261,15 +4199,8 @@ static struct btrfs_trans_handle *__unlink_start_trans(struct btrfs_inode *dir)
{
struct btrfs_root *root = dir->root;
/*
* 1 for the possible orphan item
* 1 for the dir item
* 1 for the dir index
* 1 for the inode ref
* 1 for the inode
* 1 for the parent inode
*/
return btrfs_start_transaction_fallback_global_rsv(root, 6);
return btrfs_start_transaction_fallback_global_rsv(root,
BTRFS_UNLINK_METADATA_UNITS);
}
static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
@ -5243,7 +5174,7 @@ static struct btrfs_trans_handle *evict_refill_and_join(struct btrfs_root *root,
{
struct btrfs_fs_info *fs_info = root->fs_info;
struct btrfs_trans_handle *trans;
u64 delayed_refs_extra = btrfs_calc_insert_metadata_size(fs_info, 1);
u64 delayed_refs_extra = btrfs_calc_delayed_ref_bytes(fs_info, 1);
int ret;
/*
@ -5281,7 +5212,7 @@ static struct btrfs_trans_handle *evict_refill_and_join(struct btrfs_root *root,
trans->block_rsv = &fs_info->trans_block_rsv;
trans->bytes_reserved = delayed_refs_extra;
btrfs_block_rsv_migrate(rsv, trans->block_rsv,
delayed_refs_extra, 1);
delayed_refs_extra, true);
}
return trans;
}
@ -5291,7 +5222,7 @@ void btrfs_evict_inode(struct inode *inode)
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
struct btrfs_trans_handle *trans;
struct btrfs_root *root = BTRFS_I(inode)->root;
struct btrfs_block_rsv *rsv;
struct btrfs_block_rsv *rsv = NULL;
int ret;
trace_btrfs_inode_evict(inode);
@ -5308,18 +5239,18 @@ void btrfs_evict_inode(struct inode *inode)
((btrfs_root_refs(&root->root_item) != 0 &&
root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) ||
btrfs_is_free_space_inode(BTRFS_I(inode))))
goto no_delete;
goto out;
if (is_bad_inode(inode))
goto no_delete;
goto out;
if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags))
goto no_delete;
goto out;
if (inode->i_nlink > 0) {
BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID);
goto no_delete;
goto out;
}
/*
@ -5328,7 +5259,7 @@ void btrfs_evict_inode(struct inode *inode)
*/
ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));
if (ret)
goto no_delete;
goto out;
/*
* This drops any pending insert or delete operations we have for this
@ -5340,7 +5271,7 @@ void btrfs_evict_inode(struct inode *inode)
rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
if (!rsv)
goto no_delete;
goto out;
rsv->size = btrfs_calc_metadata_size(fs_info, 1);
rsv->failfast = true;
@ -5356,16 +5287,21 @@ void btrfs_evict_inode(struct inode *inode)
trans = evict_refill_and_join(root, rsv);
if (IS_ERR(trans))
goto free_rsv;
goto out;
trans->block_rsv = rsv;
ret = btrfs_truncate_inode_items(trans, root, &control);
trans->block_rsv = &fs_info->trans_block_rsv;
btrfs_end_transaction(trans);
btrfs_btree_balance_dirty(fs_info);
/*
* We have not added new delayed items for our inode after we
* have flushed its delayed items, so no need to throttle on
* delayed items. However we have modified extent buffers.
*/
btrfs_btree_balance_dirty_nodelay(fs_info);
if (ret && ret != -ENOSPC && ret != -EAGAIN)
goto free_rsv;
goto out;
else if (!ret)
break;
}
@ -5387,9 +5323,8 @@ void btrfs_evict_inode(struct inode *inode)
btrfs_end_transaction(trans);
}
free_rsv:
out:
btrfs_free_block_rsv(fs_info, rsv);
no_delete:
/*
* If we didn't successfully delete, the orphan item will still be in
* the tree and we'll retry on the next mount. Again, we might also want
@ -6981,6 +6916,7 @@ struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
}
static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode,
struct btrfs_dio_data *dio_data,
const u64 start,
const u64 len,
const u64 orig_start,
@ -6991,7 +6927,7 @@ static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode,
const int type)
{
struct extent_map *em = NULL;
int ret;
struct btrfs_ordered_extent *ordered;
if (type != BTRFS_ORDERED_NOCOW) {
em = create_io_em(inode, start, len, orig_start, block_start,
@ -7001,18 +6937,21 @@ static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode,
if (IS_ERR(em))
goto out;
}
ret = btrfs_add_ordered_extent(inode, start, len, len, block_start,
block_len, 0,
(1 << type) |
(1 << BTRFS_ORDERED_DIRECT),
BTRFS_COMPRESS_NONE);
if (ret) {
ordered = btrfs_alloc_ordered_extent(inode, start, len, len,
block_start, block_len, 0,
(1 << type) |
(1 << BTRFS_ORDERED_DIRECT),
BTRFS_COMPRESS_NONE);
if (IS_ERR(ordered)) {
if (em) {
free_extent_map(em);
btrfs_drop_extent_map_range(inode, start,
start + len - 1, false);
}
em = ERR_PTR(ret);
em = ERR_CAST(ordered);
} else {
ASSERT(!dio_data->ordered);
dio_data->ordered = ordered;
}
out:
@ -7020,6 +6959,7 @@ static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode,
}
static struct extent_map *btrfs_new_extent_direct(struct btrfs_inode *inode,
struct btrfs_dio_data *dio_data,
u64 start, u64 len)
{
struct btrfs_root *root = inode->root;
@ -7035,7 +6975,7 @@ static struct extent_map *btrfs_new_extent_direct(struct btrfs_inode *inode,
if (ret)
return ERR_PTR(ret);
em = btrfs_create_dio_extent(inode, start, ins.offset, start,
em = btrfs_create_dio_extent(inode, dio_data, start, ins.offset, start,
ins.objectid, ins.offset, ins.offset,
ins.offset, BTRFS_ORDERED_REGULAR);
btrfs_dec_block_group_reservations(fs_info, ins.objectid);
@ -7380,7 +7320,7 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map,
}
space_reserved = true;
em2 = btrfs_create_dio_extent(BTRFS_I(inode), start, len,
em2 = btrfs_create_dio_extent(BTRFS_I(inode), dio_data, start, len,
orig_start, block_start,
len, orig_block_len,
ram_bytes, type);
@ -7422,7 +7362,7 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map,
goto out;
space_reserved = true;
em = btrfs_new_extent_direct(BTRFS_I(inode), start, len);
em = btrfs_new_extent_direct(BTRFS_I(inode), dio_data, start, len);
if (IS_ERR(em)) {
ret = PTR_ERR(em);
goto out;
@ -7728,6 +7668,10 @@ static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length,
pos + length - 1, NULL);
ret = -ENOTBLK;
}
if (write) {
btrfs_put_ordered_extent(dio_data->ordered);
dio_data->ordered = NULL;
}
if (write)
extent_changeset_free(dio_data->data_reserved);
@ -7767,14 +7711,34 @@ static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio,
container_of(bbio, struct btrfs_dio_private, bbio);
struct btrfs_dio_data *dio_data = iter->private;
btrfs_bio_init(bbio, BTRFS_I(iter->inode), btrfs_dio_end_io, bio->bi_private);
btrfs_bio_init(bbio, BTRFS_I(iter->inode)->root->fs_info,
btrfs_dio_end_io, bio->bi_private);
bbio->inode = BTRFS_I(iter->inode);
bbio->file_offset = file_offset;
dip->file_offset = file_offset;
dip->bytes = bio->bi_iter.bi_size;
dio_data->submitted += bio->bi_iter.bi_size;
btrfs_submit_bio(bio, 0);
/*
* Check if we are doing a partial write. If we are, we need to split
* the ordered extent to match the submitted bio. Hang on to the
* remaining unfinishable ordered_extent in dio_data so that it can be
* cancelled in iomap_end to avoid a deadlock wherein faulting the
* remaining pages is blocked on the outstanding ordered extent.
*/
if (iter->flags & IOMAP_WRITE) {
int ret;
ret = btrfs_extract_ordered_extent(bbio, dio_data->ordered);
if (ret) {
btrfs_bio_end_io(bbio, errno_to_blk_status(ret));
return;
}
}
btrfs_submit_bio(bbio, 0);
}
static const struct iomap_ops btrfs_dio_iomap_ops = {
@ -7789,7 +7753,7 @@ static const struct iomap_dio_ops btrfs_dio_ops = {
ssize_t btrfs_dio_read(struct kiocb *iocb, struct iov_iter *iter, size_t done_before)
{
struct btrfs_dio_data data;
struct btrfs_dio_data data = { 0 };
return iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
IOMAP_DIO_PARTIAL, &data, done_before);
@ -7798,7 +7762,7 @@ ssize_t btrfs_dio_read(struct kiocb *iocb, struct iov_iter *iter, size_t done_be
struct iomap_dio *btrfs_dio_write(struct kiocb *iocb, struct iov_iter *iter,
size_t done_before)
{
struct btrfs_dio_data data;
struct btrfs_dio_data data = { 0 };
return __iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
IOMAP_DIO_PARTIAL, &data, done_before);
@ -9908,8 +9872,6 @@ static ssize_t btrfs_encoded_read_inline(
}
struct btrfs_encoded_read_private {
struct btrfs_inode *inode;
u64 file_offset;
wait_queue_head_t wait;
atomic_t pending;
blk_status_t status;
@ -9939,45 +9901,41 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,
u64 file_offset, u64 disk_bytenr,
u64 disk_io_size, struct page **pages)
{
struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct btrfs_encoded_read_private priv = {
.inode = inode,
.file_offset = file_offset,
.pending = ATOMIC_INIT(1),
};
unsigned long i = 0;
u64 cur = 0;
struct btrfs_bio *bbio;
init_waitqueue_head(&priv.wait);
/* Submit bios for the extent, splitting due to bio limits as necessary. */
while (cur < disk_io_size) {
struct bio *bio = NULL;
u64 remaining = disk_io_size - cur;
while (bio || remaining) {
size_t bytes = min_t(u64, remaining, PAGE_SIZE);
bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, fs_info,
btrfs_encoded_read_endio, &priv);
bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
bbio->inode = inode;
if (!bio) {
bio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ,
inode,
btrfs_encoded_read_endio,
&priv);
bio->bi_iter.bi_sector =
(disk_bytenr + cur) >> SECTOR_SHIFT;
}
do {
size_t bytes = min_t(u64, disk_io_size, PAGE_SIZE);
if (!bytes ||
bio_add_page(bio, pages[i], bytes, 0) < bytes) {
atomic_inc(&priv.pending);
btrfs_submit_bio(bio, 0);
bio = NULL;
continue;
}
if (bio_add_page(&bbio->bio, pages[i], bytes, 0) < bytes) {
atomic_inc(&priv.pending);
btrfs_submit_bio(bbio, 0);
i++;
cur += bytes;
remaining -= bytes;
bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, fs_info,
btrfs_encoded_read_endio, &priv);
bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
bbio->inode = inode;
continue;
}
}
i++;
disk_bytenr += bytes;
disk_io_size -= bytes;
} while (disk_io_size);
atomic_inc(&priv.pending);
btrfs_submit_bio(bbio, 0);
if (atomic_dec_return(&priv.pending))
io_wait_event(priv.wait, !atomic_read(&priv.pending));
@ -10398,13 +10356,8 @@ ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from,
btrfs_delalloc_release_extents(inode, num_bytes);
if (btrfs_submit_compressed_write(inode, start, num_bytes, ins.objectid,
ins.offset, pages, nr_pages, 0, NULL,
false)) {
btrfs_writepage_endio_finish_ordered(inode, pages[0], start, end, 0);
ret = -EIO;
goto out_pages;
}
btrfs_submit_compressed_write(inode, start, num_bytes, ins.objectid,
ins.offset, pages, nr_pages, 0, false);
ret = orig_count;
goto out;

5
fs/btrfs/ioctl.c
View file

@ -3161,6 +3161,11 @@ static long btrfs_ioctl_scrub(struct file *file, void __user *arg)
if (IS_ERR(sa))
return PTR_ERR(sa);
if (sa->flags & ~BTRFS_SCRUB_SUPPORTED_FLAGS) {
ret = -EOPNOTSUPP;
goto out;
}
if (!(sa->flags & BTRFS_SCRUB_READONLY)) {
ret = mnt_want_write_file(file);
if (ret)

25
fs/btrfs/locking.c
View file

@ -325,24 +325,12 @@ struct extent_buffer *btrfs_try_read_lock_root_node(struct btrfs_root *root)
* acquire the lock.
*/
int btrfs_drew_lock_init(struct btrfs_drew_lock *lock)
void btrfs_drew_lock_init(struct btrfs_drew_lock *lock)
{
int ret;
ret = percpu_counter_init(&lock->writers, 0, GFP_KERNEL);
if (ret)
return ret;
atomic_set(&lock->readers, 0);
atomic_set(&lock->writers, 0);
init_waitqueue_head(&lock->pending_readers);
init_waitqueue_head(&lock->pending_writers);
return 0;
}
void btrfs_drew_lock_destroy(struct btrfs_drew_lock *lock)
{
percpu_counter_destroy(&lock->writers);
}
/* Return true if acquisition is successful, false otherwise */
@ -351,10 +339,10 @@ bool btrfs_drew_try_write_lock(struct btrfs_drew_lock *lock)
if (atomic_read(&lock->readers))
return false;
percpu_counter_inc(&lock->writers);
atomic_inc(&lock->writers);
/* Ensure writers count is updated before we check for pending readers */
smp_mb();
smp_mb__after_atomic();
if (atomic_read(&lock->readers)) {
btrfs_drew_write_unlock(lock);
return false;
@ -374,7 +362,7 @@ void btrfs_drew_write_lock(struct btrfs_drew_lock *lock)
void btrfs_drew_write_unlock(struct btrfs_drew_lock *lock)
{
percpu_counter_dec(&lock->writers);
atomic_dec(&lock->writers);
cond_wake_up(&lock->pending_readers);
}
@ -390,8 +378,7 @@ void btrfs_drew_read_lock(struct btrfs_drew_lock *lock)
*/
smp_mb__after_atomic();
wait_event(lock->pending_readers,
percpu_counter_sum(&lock->writers) == 0);
wait_event(lock->pending_readers, atomic_read(&lock->writers) == 0);
}
void btrfs_drew_read_unlock(struct btrfs_drew_lock *lock)

5
fs/btrfs/locking.h
View file

@ -195,13 +195,12 @@ static inline void btrfs_tree_unlock_rw(struct extent_buffer *eb, int rw)
struct btrfs_drew_lock {
atomic_t readers;
struct percpu_counter writers;
atomic_t writers;
wait_queue_head_t pending_writers;
wait_queue_head_t pending_readers;
};
int btrfs_drew_lock_init(struct btrfs_drew_lock *lock);
void btrfs_drew_lock_destroy(struct btrfs_drew_lock *lock);
void btrfs_drew_lock_init(struct btrfs_drew_lock *lock);
void btrfs_drew_write_lock(struct btrfs_drew_lock *lock);
bool btrfs_drew_try_write_lock(struct btrfs_drew_lock *lock);
void btrfs_drew_write_unlock(struct btrfs_drew_lock *lock);

5
fs/btrfs/lru_cache.h
View file

@ -55,11 +55,6 @@ static inline unsigned int btrfs_lru_cache_size(const struct btrfs_lru_cache *ca
return cache->size;
}
static inline bool btrfs_lru_cache_is_full(const struct btrfs_lru_cache *cache)
{
return cache->size >= cache->max_size;
}
static inline struct btrfs_lru_cache_entry *btrfs_lru_cache_lru_entry(
struct btrfs_lru_cache *cache)
{

17
fs/btrfs/lzo.c
View file

@ -17,6 +17,7 @@
#include "compression.h"
#include "ctree.h"
#include "super.h"
#include "btrfs_inode.h"
#define LZO_LEN 4
@ -329,7 +330,7 @@ static void copy_compressed_segment(struct compressed_bio *cb,
int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
{
struct workspace *workspace = list_entry(ws, struct workspace, list);
const struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb);
const struct btrfs_fs_info *fs_info = cb->bbio.inode->root->fs_info;
const u32 sectorsize = fs_info->sectorsize;
char *kaddr;
int ret;
@ -388,8 +389,7 @@ int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
*/
btrfs_err(fs_info, "unexpectedly large lzo segment len %u",
seg_len);
ret = -EIO;
goto out;
return -EIO;
}
/* Copy the compressed segment payload into workspace */
@ -400,8 +400,7 @@ int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
workspace->buf, &out_len);
if (ret != LZO_E_OK) {
btrfs_err(fs_info, "failed to decompress");
ret = -EIO;
goto out;
return -EIO;
}
/* Copy the data into inode pages */
@ -410,7 +409,7 @@ int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
/* All data read, exit */
if (ret == 0)
goto out;
return 0;
ret = 0;
/* Check if the sector has enough space for a segment header */
@ -421,10 +420,8 @@ int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
/* Skip the padding zeros */
cur_in += sector_bytes_left;
}
out:
if (!ret)
zero_fill_bio(cb->orig_bio);
return ret;
return 0;
}
int lzo_decompress(struct list_head *ws, const u8 *data_in,

2
fs/btrfs/messages.c
View file

@ -253,7 +253,7 @@ void __cold _btrfs_printk(const struct btrfs_fs_info *fs_info, const char *fmt,
#endif
#ifdef CONFIG_BTRFS_ASSERT
void __cold btrfs_assertfail(const char *expr, const char *file, int line)
void __cold __noreturn btrfs_assertfail(const char *expr, const char *file, int line)
{
pr_err("assertion failed: %s, in %s:%d\n", expr, file, line);
BUG();

2
fs/btrfs/messages.h
View file

@ -160,7 +160,7 @@ do { \
} while (0)
#ifdef CONFIG_BTRFS_ASSERT
void __cold btrfs_assertfail(const char *expr, const char *file, int line);
void __cold __noreturn btrfs_assertfail(const char *expr, const char *file, int line);
#define ASSERT(expr) \
(likely(expr) ? (void)0 : btrfs_assertfail(#expr, __FILE__, __LINE__))

120
fs/btrfs/ordered-data.c
View file

@ -160,14 +160,16 @@ static inline struct rb_node *tree_search(struct btrfs_ordered_inode_tree *tree,
* @compress_type: Compression algorithm used for data.
*
* Most of these parameters correspond to &struct btrfs_file_extent_item. The
* tree is given a single reference on the ordered extent that was inserted.
* tree is given a single reference on the ordered extent that was inserted, and
* the returned pointer is given a second reference.
*
* Return: 0 or -ENOMEM.
* Return: the new ordered extent or error pointer.
*/
int btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset,
u64 num_bytes, u64 ram_bytes, u64 disk_bytenr,
u64 disk_num_bytes, u64 offset, unsigned flags,
int compress_type)
struct btrfs_ordered_extent *btrfs_alloc_ordered_extent(
struct btrfs_inode *inode, u64 file_offset,
u64 num_bytes, u64 ram_bytes, u64 disk_bytenr,
u64 disk_num_bytes, u64 offset, unsigned long flags,
int compress_type)
{
struct btrfs_root *root = inode->root;
struct btrfs_fs_info *fs_info = root->fs_info;
@ -181,7 +183,7 @@ int btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset,
/* For nocow write, we can release the qgroup rsv right now */
ret = btrfs_qgroup_free_data(inode, NULL, file_offset, num_bytes);
if (ret < 0)
return ret;
return ERR_PTR(ret);
ret = 0;
} else {
/*
@ -190,11 +192,11 @@ int btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset,
*/
ret = btrfs_qgroup_release_data(inode, file_offset, num_bytes);
if (ret < 0)
return ret;
return ERR_PTR(ret);
}
entry = kmem_cache_zalloc(btrfs_ordered_extent_cache, GFP_NOFS);
if (!entry)
return -ENOMEM;
return ERR_PTR(-ENOMEM);
entry->file_offset = file_offset;
entry->num_bytes = num_bytes;
@ -256,6 +258,32 @@ int btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset,
btrfs_mod_outstanding_extents(inode, 1);
spin_unlock(&inode->lock);
/* One ref for the returned entry to match semantics of lookup. */
refcount_inc(&entry->refs);
return entry;
}
/*
* Add a new btrfs_ordered_extent for the range, but drop the reference instead
* of returning it to the caller.
*/
int btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset,
u64 num_bytes, u64 ram_bytes, u64 disk_bytenr,
u64 disk_num_bytes, u64 offset, unsigned long flags,
int compress_type)
{
struct btrfs_ordered_extent *ordered;
ordered = btrfs_alloc_ordered_extent(inode, file_offset, num_bytes,
ram_bytes, disk_bytenr,
disk_num_bytes, offset, flags,
compress_type);
if (IS_ERR(ordered))
return PTR_ERR(ordered);
btrfs_put_ordered_extent(ordered);
return 0;
}
@ -1088,39 +1116,37 @@ bool btrfs_try_lock_ordered_range(struct btrfs_inode *inode, u64 start, u64 end,
return false;
}
static int clone_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pos,
u64 len)
{
struct inode *inode = ordered->inode;
struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
u64 file_offset = ordered->file_offset + pos;
u64 disk_bytenr = ordered->disk_bytenr + pos;
unsigned long flags = ordered->flags & BTRFS_ORDERED_TYPE_FLAGS;
/*
* The splitting extent is already counted and will be added again in
* btrfs_add_ordered_extent_*(). Subtract len to avoid double counting.
*/
percpu_counter_add_batch(&fs_info->ordered_bytes, -len,
fs_info->delalloc_batch);
WARN_ON_ONCE(flags & (1 << BTRFS_ORDERED_COMPRESSED));
return btrfs_add_ordered_extent(BTRFS_I(inode), file_offset, len, len,
disk_bytenr, len, 0, flags,
ordered->compress_type);
}
int btrfs_split_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pre,
u64 post)
/* Split out a new ordered extent for this first @len bytes of @ordered. */
int btrfs_split_ordered_extent(struct btrfs_ordered_extent *ordered, u64 len)
{
struct inode *inode = ordered->inode;
struct btrfs_ordered_inode_tree *tree = &BTRFS_I(inode)->ordered_tree;
struct rb_node *node;
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
int ret = 0;
u64 file_offset = ordered->file_offset;
u64 disk_bytenr = ordered->disk_bytenr;
unsigned long flags = ordered->flags & BTRFS_ORDERED_TYPE_FLAGS;
struct rb_node *node;
trace_btrfs_ordered_extent_split(BTRFS_I(inode), ordered);
ASSERT(!(flags & (1U << BTRFS_ORDERED_COMPRESSED)));
/*
* The entire bio must be covered by the ordered extent, but we can't
* reduce the original extent to a zero length either.
*/
if (WARN_ON_ONCE(len >= ordered->num_bytes))
return -EINVAL;
/* We cannot split once ordered extent is past end_bio. */
if (WARN_ON_ONCE(ordered->bytes_left != ordered->disk_num_bytes))
return -EINVAL;
/* We cannot split a compressed ordered extent. */
if (WARN_ON_ONCE(ordered->disk_num_bytes != ordered->num_bytes))
return -EINVAL;
/* Checksum list should be empty. */
if (WARN_ON_ONCE(!list_empty(&ordered->list)))
return -EINVAL;
spin_lock_irq(&tree->lock);
/* Remove from tree once */
node = &ordered->rb_node;
@ -1129,11 +1155,11 @@ int btrfs_split_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pre,
if (tree->last == node)
tree->last = NULL;
ordered->file_offset += pre;
ordered->disk_bytenr += pre;
ordered->num_bytes -= (pre + post);
ordered->disk_num_bytes -= (pre + post);
ordered->bytes_left -= (pre + post);
ordered->file_offset += len;
ordered->disk_bytenr += len;
ordered->num_bytes -= len;
ordered->disk_num_bytes -= len;
ordered->bytes_left -= len;
/* Re-insert the node */
node = tree_insert(&tree->tree, ordered->file_offset, &ordered->rb_node);
@ -1144,13 +1170,15 @@ int btrfs_split_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pre,
spin_unlock_irq(&tree->lock);
if (pre)
ret = clone_ordered_extent(ordered, 0, pre);
if (ret == 0 && post)
ret = clone_ordered_extent(ordered, pre + ordered->disk_num_bytes,
post);
/*
* The splitting extent is already counted and will be added again in
* btrfs_add_ordered_extent(). Subtract len to avoid double counting.
*/
percpu_counter_add_batch(&fs_info->ordered_bytes, -len, fs_info->delalloc_batch);
return ret;
return btrfs_add_ordered_extent(BTRFS_I(inode), file_offset, len, len,
disk_bytenr, len, 0, flags,
ordered->compress_type);
}
int __init ordered_data_init(void)

10
fs/btrfs/ordered-data.h
View file

@ -178,9 +178,14 @@ void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode,
bool btrfs_dec_test_ordered_pending(struct btrfs_inode *inode,
struct btrfs_ordered_extent **cached,
u64 file_offset, u64 io_size);
struct btrfs_ordered_extent *btrfs_alloc_ordered_extent(
struct btrfs_inode *inode, u64 file_offset,
u64 num_bytes, u64 ram_bytes, u64 disk_bytenr,
u64 disk_num_bytes, u64 offset, unsigned long flags,
int compress_type);
int btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset,
u64 num_bytes, u64 ram_bytes, u64 disk_bytenr,
u64 disk_num_bytes, u64 offset, unsigned flags,
u64 disk_num_bytes, u64 offset, unsigned long flags,
int compress_type);
void btrfs_add_ordered_sum(struct btrfs_ordered_extent *entry,
struct btrfs_ordered_sum *sum);
@ -207,8 +212,7 @@ void btrfs_lock_and_flush_ordered_range(struct btrfs_inode *inode, u64 start,
struct extent_state **cached_state);
bool btrfs_try_lock_ordered_range(struct btrfs_inode *inode, u64 start, u64 end,
struct extent_state **cached_state);
int btrfs_split_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pre,
u64 post);
int btrfs_split_ordered_extent(struct btrfs_ordered_extent *ordered, u64 len);
int __init ordered_data_init(void);
void __cold ordered_data_exit(void);

162
fs/btrfs/raid56.c
View file

@ -202,7 +202,7 @@ static void cache_rbio_pages(struct btrfs_raid_bio *rbio)
*/
static int rbio_bucket(struct btrfs_raid_bio *rbio)
{
u64 num = rbio->bioc->raid_map[0];
u64 num = rbio->bioc->full_stripe_logical;
/*
* we shift down quite a bit. We're using byte
@ -407,16 +407,15 @@ static void __remove_rbio_from_cache(struct btrfs_raid_bio *rbio)
static void remove_rbio_from_cache(struct btrfs_raid_bio *rbio)
{
struct btrfs_stripe_hash_table *table;
unsigned long flags;
if (!test_bit(RBIO_CACHE_BIT, &rbio->flags))
return;
table = rbio->bioc->fs_info->stripe_hash_table;
spin_lock_irqsave(&table->cache_lock, flags);
spin_lock(&table->cache_lock);
__remove_rbio_from_cache(rbio);
spin_unlock_irqrestore(&table->cache_lock, flags);
spin_unlock(&table->cache_lock);
}
/*
@ -425,19 +424,18 @@ static void remove_rbio_from_cache(struct btrfs_raid_bio *rbio)
static void btrfs_clear_rbio_cache(struct btrfs_fs_info *info)
{
struct btrfs_stripe_hash_table *table;
unsigned long flags;
struct btrfs_raid_bio *rbio;
table = info->stripe_hash_table;
spin_lock_irqsave(&table->cache_lock, flags);
spin_lock(&table->cache_lock);
while (!list_empty(&table->stripe_cache)) {
rbio = list_entry(table->stripe_cache.next,
struct btrfs_raid_bio,
stripe_cache);
__remove_rbio_from_cache(rbio);
}
spin_unlock_irqrestore(&table->cache_lock, flags);
spin_unlock(&table->cache_lock);
}
/*
@ -467,14 +465,13 @@ void btrfs_free_stripe_hash_table(struct btrfs_fs_info *info)
static void cache_rbio(struct btrfs_raid_bio *rbio)
{
struct btrfs_stripe_hash_table *table;
unsigned long flags;
if (!test_bit(RBIO_CACHE_READY_BIT, &rbio->flags))
return;
table = rbio->bioc->fs_info->stripe_hash_table;
spin_lock_irqsave(&table->cache_lock, flags);
spin_lock(&table->cache_lock);
spin_lock(&rbio->bio_list_lock);
/* bump our ref if we were not in the list before */
@ -501,7 +498,7 @@ static void cache_rbio(struct btrfs_raid_bio *rbio)
__remove_rbio_from_cache(found);
}
spin_unlock_irqrestore(&table->cache_lock, flags);
spin_unlock(&table->cache_lock);
}
/*
@ -530,15 +527,14 @@ static void run_xor(void **pages, int src_cnt, ssize_t len)
*/
static int rbio_is_full(struct btrfs_raid_bio *rbio)
{
unsigned long flags;
unsigned long size = rbio->bio_list_bytes;
int ret = 1;
spin_lock_irqsave(&rbio->bio_list_lock, flags);
spin_lock(&rbio->bio_list_lock);
if (size != rbio->nr_data * BTRFS_STRIPE_LEN)
ret = 0;
BUG_ON(size > rbio->nr_data * BTRFS_STRIPE_LEN);
spin_unlock_irqrestore(&rbio->bio_list_lock, flags);
spin_unlock(&rbio->bio_list_lock);
return ret;
}
@ -571,7 +567,7 @@ static int rbio_can_merge(struct btrfs_raid_bio *last,
test_bit(RBIO_CACHE_BIT, &cur->flags))
return 0;
if (last->bioc->raid_map[0] != cur->bioc->raid_map[0])
if (last->bioc->full_stripe_logical != cur->bioc->full_stripe_logical)
return 0;
/* we can't merge with different operations */
@ -657,16 +653,15 @@ static noinline int lock_stripe_add(struct btrfs_raid_bio *rbio)
struct btrfs_stripe_hash *h;
struct btrfs_raid_bio *cur;
struct btrfs_raid_bio *pending;
unsigned long flags;
struct btrfs_raid_bio *freeit = NULL;
struct btrfs_raid_bio *cache_drop = NULL;
int ret = 0;
h = rbio->bioc->fs_info->stripe_hash_table->table + rbio_bucket(rbio);
spin_lock_irqsave(&h->lock, flags);
spin_lock(&h->lock);
list_for_each_entry(cur, &h->hash_list, hash_list) {
if (cur->bioc->raid_map[0] != rbio->bioc->raid_map[0])
if (cur->bioc->full_stripe_logical != rbio->bioc->full_stripe_logical)
continue;
spin_lock(&cur->bio_list_lock);
@ -724,7 +719,7 @@ static noinline int lock_stripe_add(struct btrfs_raid_bio *rbio)
refcount_inc(&rbio->refs);
list_add(&rbio->hash_list, &h->hash_list);
out:
spin_unlock_irqrestore(&h->lock, flags);
spin_unlock(&h->lock);
if (cache_drop)
remove_rbio_from_cache(cache_drop);
if (freeit)
@ -742,7 +737,6 @@ static noinline void unlock_stripe(struct btrfs_raid_bio *rbio)
{
int bucket;
struct btrfs_stripe_hash *h;
unsigned long flags;
int keep_cache = 0;
bucket = rbio_bucket(rbio);
@ -751,7 +745,7 @@ static noinline void unlock_stripe(struct btrfs_raid_bio *rbio)
if (list_empty(&rbio->plug_list))
cache_rbio(rbio);
spin_lock_irqsave(&h->lock, flags);
spin_lock(&h->lock);
spin_lock(&rbio->bio_list_lock);
if (!list_empty(&rbio->hash_list)) {
@ -788,7 +782,7 @@ static noinline void unlock_stripe(struct btrfs_raid_bio *rbio)
list_add(&next->hash_list, &h->hash_list);
refcount_inc(&next->refs);
spin_unlock(&rbio->bio_list_lock);
spin_unlock_irqrestore(&h->lock, flags);
spin_unlock(&h->lock);
if (next->operation == BTRFS_RBIO_READ_REBUILD)
start_async_work(next, recover_rbio_work_locked);
@ -808,7 +802,7 @@ static noinline void unlock_stripe(struct btrfs_raid_bio *rbio)
}
done:
spin_unlock(&rbio->bio_list_lock);
spin_unlock_irqrestore(&h->lock, flags);
spin_unlock(&h->lock);
done_nolock:
if (!keep_cache)
@ -891,16 +885,16 @@ static struct sector_ptr *sector_in_rbio(struct btrfs_raid_bio *rbio,
index = stripe_nr * rbio->stripe_nsectors + sector_nr;
ASSERT(index >= 0 && index < rbio->nr_sectors);
spin_lock_irq(&rbio->bio_list_lock);
spin_lock(&rbio->bio_list_lock);
sector = &rbio->bio_sectors[index];
if (sector->page || bio_list_only) {
/* Don't return sector without a valid page pointer */
if (!sector->page)
sector = NULL;
spin_unlock_irq(&rbio->bio_list_lock);
spin_unlock(&rbio->bio_list_lock);
return sector;
}
spin_unlock_irq(&rbio->bio_list_lock);
spin_unlock(&rbio->bio_list_lock);
return &rbio->stripe_sectors[index];
}
@ -912,7 +906,7 @@ static struct sector_ptr *sector_in_rbio(struct btrfs_raid_bio *rbio,
static struct btrfs_raid_bio *alloc_rbio(struct btrfs_fs_info *fs_info,
struct btrfs_io_context *bioc)
{
const unsigned int real_stripes = bioc->num_stripes - bioc->num_tgtdevs;
const unsigned int real_stripes = bioc->num_stripes - bioc->replace_nr_stripes;
const unsigned int stripe_npages = BTRFS_STRIPE_LEN >> PAGE_SHIFT;
const unsigned int num_pages = stripe_npages * real_stripes;
const unsigned int stripe_nsectors =
@ -1108,7 +1102,7 @@ static int rbio_add_io_sector(struct btrfs_raid_bio *rbio,
bio->bi_iter.bi_sector = disk_start >> 9;
bio->bi_private = rbio;
bio_add_page(bio, sector->page, sectorsize, sector->pgoff);
__bio_add_page(bio, sector->page, sectorsize, sector->pgoff);
bio_list_add(bio_list, bio);
return 0;
}
@ -1119,7 +1113,7 @@ static void index_one_bio(struct btrfs_raid_bio *rbio, struct bio *bio)
struct bio_vec bvec;
struct bvec_iter iter;
u32 offset = (bio->bi_iter.bi_sector << SECTOR_SHIFT) -
rbio->bioc->raid_map[0];
rbio->bioc->full_stripe_logical;
bio_for_each_segment(bvec, bio, iter) {
u32 bvec_offset;
@ -1148,11 +1142,11 @@ static void index_rbio_pages(struct btrfs_raid_bio *rbio)
{
struct bio *bio;
spin_lock_irq(&rbio->bio_list_lock);
spin_lock(&rbio->bio_list_lock);
bio_list_for_each(bio, &rbio->bio_list)
index_one_bio(rbio, bio);
spin_unlock_irq(&rbio->bio_list_lock);
spin_unlock(&rbio->bio_list_lock);
}
static void bio_get_trace_info(struct btrfs_raid_bio *rbio, struct bio *bio,
@ -1282,10 +1276,16 @@ static int rmw_assemble_write_bios(struct btrfs_raid_bio *rbio,
goto error;
}
if (likely(!rbio->bioc->num_tgtdevs))
if (likely(!rbio->bioc->replace_nr_stripes))
return 0;
/* Make a copy for the replace target device. */
/*
* Make a copy for the replace target device.
*
* Thus the source stripe number (in replace_stripe_src) should be valid.
*/
ASSERT(rbio->bioc->replace_stripe_src >= 0);
for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors;
total_sector_nr++) {
struct sector_ptr *sector;
@ -1293,7 +1293,12 @@ static int rmw_assemble_write_bios(struct btrfs_raid_bio *rbio,
stripe = total_sector_nr / rbio->stripe_nsectors;
sectornr = total_sector_nr % rbio->stripe_nsectors;
if (!rbio->bioc->tgtdev_map[stripe]) {
/*
* For RAID56, there is only one device that can be replaced,
* and replace_stripe_src[0] indicates the stripe number we
* need to copy from.
*/
if (stripe != rbio->bioc->replace_stripe_src) {
/*
* We can skip the whole stripe completely, note
* total_sector_nr will be increased by one anyway.
@ -1316,7 +1321,7 @@ static int rmw_assemble_write_bios(struct btrfs_raid_bio *rbio,
}
ret = rbio_add_io_sector(rbio, bio_list, sector,
rbio->bioc->tgtdev_map[stripe],
rbio->real_stripes,
sectornr, REQ_OP_WRITE);
if (ret)
goto error;
@ -1332,7 +1337,7 @@ static void set_rbio_range_error(struct btrfs_raid_bio *rbio, struct bio *bio)
{
struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
u32 offset = (bio->bi_iter.bi_sector << SECTOR_SHIFT) -
rbio->bioc->raid_map[0];
rbio->bioc->full_stripe_logical;
int total_nr_sector = offset >> fs_info->sectorsize_bits;
ASSERT(total_nr_sector < rbio->nr_data * rbio->stripe_nsectors);
@ -1609,7 +1614,7 @@ static void rbio_add_bio(struct btrfs_raid_bio *rbio, struct bio *orig_bio)
{
const struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
const u64 orig_logical = orig_bio->bi_iter.bi_sector << SECTOR_SHIFT;
const u64 full_stripe_start = rbio->bioc->raid_map[0];
const u64 full_stripe_start = rbio->bioc->full_stripe_logical;
const u32 orig_len = orig_bio->bi_iter.bi_size;
const u32 sectorsize = fs_info->sectorsize;
u64 cur_logical;
@ -1796,9 +1801,8 @@ static int recover_vertical(struct btrfs_raid_bio *rbio, int sector_nr,
* here due to a crc mismatch and we can't give them the
* data they want.
*/
if (rbio->bioc->raid_map[failb] == RAID6_Q_STRIPE) {
if (rbio->bioc->raid_map[faila] ==
RAID5_P_STRIPE)
if (failb == rbio->real_stripes - 1) {
if (faila == rbio->real_stripes - 2)
/*
* Only P and Q are corrupted.
* We only care about data stripes recovery,
@ -1812,7 +1816,7 @@ static int recover_vertical(struct btrfs_raid_bio *rbio, int sector_nr,
goto pstripe;
}
if (rbio->bioc->raid_map[failb] == RAID5_P_STRIPE) {
if (failb == rbio->real_stripes - 2) {
raid6_datap_recov(rbio->real_stripes, sectorsize,
faila, pointers);
} else {
@ -1895,9 +1899,9 @@ static int recover_sectors(struct btrfs_raid_bio *rbio)
if (rbio->operation == BTRFS_RBIO_READ_REBUILD ||
rbio->operation == BTRFS_RBIO_REBUILD_MISSING) {
spin_lock_irq(&rbio->bio_list_lock);
spin_lock(&rbio->bio_list_lock);
set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
spin_unlock_irq(&rbio->bio_list_lock);
spin_unlock(&rbio->bio_list_lock);
}
index_rbio_pages(rbio);
@ -2075,8 +2079,8 @@ static void fill_data_csums(struct btrfs_raid_bio *rbio)
{
struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
struct btrfs_root *csum_root = btrfs_csum_root(fs_info,
rbio->bioc->raid_map[0]);
const u64 start = rbio->bioc->raid_map[0];
rbio->bioc->full_stripe_logical);
const u64 start = rbio->bioc->full_stripe_logical;
const u32 len = (rbio->nr_data * rbio->stripe_nsectors) <<
fs_info->sectorsize_bits;
int ret;
@ -2109,7 +2113,7 @@ static void fill_data_csums(struct btrfs_raid_bio *rbio)
}
ret = btrfs_lookup_csums_bitmap(csum_root, start, start + len - 1,
rbio->csum_buf, rbio->csum_bitmap);
rbio->csum_buf, rbio->csum_bitmap, false);
if (ret < 0)
goto error;
if (bitmap_empty(rbio->csum_bitmap, len >> fs_info->sectorsize_bits))
@ -2124,7 +2128,7 @@ static void fill_data_csums(struct btrfs_raid_bio *rbio)
*/
btrfs_warn_rl(fs_info,
"sub-stripe write for full stripe %llu is not safe, failed to get csum: %d",
rbio->bioc->raid_map[0], ret);
rbio->bioc->full_stripe_logical, ret);
no_csum:
kfree(rbio->csum_buf);
bitmap_free(rbio->csum_bitmap);
@ -2265,9 +2269,9 @@ static void rmw_rbio(struct btrfs_raid_bio *rbio)
* bio list any more, anyone else that wants to change this stripe
* needs to do their own rmw.
*/
spin_lock_irq(&rbio->bio_list_lock);
spin_lock(&rbio->bio_list_lock);
set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
spin_unlock_irq(&rbio->bio_list_lock);
spin_unlock(&rbio->bio_list_lock);
bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors);
@ -2372,23 +2376,6 @@ struct btrfs_raid_bio *raid56_parity_alloc_scrub_rbio(struct bio *bio,
return rbio;
}
/* Used for both parity scrub and missing. */
void raid56_add_scrub_pages(struct btrfs_raid_bio *rbio, struct page *page,
unsigned int pgoff, u64 logical)
{
const u32 sectorsize = rbio->bioc->fs_info->sectorsize;
int stripe_offset;
int index;
ASSERT(logical >= rbio->bioc->raid_map[0]);
ASSERT(logical + sectorsize <= rbio->bioc->raid_map[0] +
BTRFS_STRIPE_LEN * rbio->nr_data);
stripe_offset = (int)(logical - rbio->bioc->raid_map[0]);
index = stripe_offset / sectorsize;
rbio->bio_sectors[index].page = page;
rbio->bio_sectors[index].pgoff = pgoff;
}
/*
* We just scrub the parity that we have correct data on the same horizontal,
* so we needn't allocate all pages for all the stripes.
@ -2442,7 +2429,11 @@ static int finish_parity_scrub(struct btrfs_raid_bio *rbio, int need_check)
else
BUG();
if (bioc->num_tgtdevs && bioc->tgtdev_map[rbio->scrubp]) {
/*
* Replace is running and our P/Q stripe is being replaced, then we
* need to duplicate the final write to replace target.
*/
if (bioc->replace_nr_stripes && bioc->replace_stripe_src == rbio->scrubp) {
is_replace = 1;
bitmap_copy(pbitmap, &rbio->dbitmap, rbio->stripe_nsectors);
}
@ -2544,13 +2535,18 @@ static int finish_parity_scrub(struct btrfs_raid_bio *rbio, int need_check)
if (!is_replace)
goto submit_write;
/*
* Replace is running and our parity stripe needs to be duplicated to
* the target device. Check we have a valid source stripe number.
*/
ASSERT(rbio->bioc->replace_stripe_src >= 0);
for_each_set_bit(sectornr, pbitmap, rbio->stripe_nsectors) {
struct sector_ptr *sector;
sector = rbio_stripe_sector(rbio, rbio->scrubp, sectornr);
ret = rbio_add_io_sector(rbio, &bio_list, sector,
bioc->tgtdev_map[rbio->scrubp],
sectornr, REQ_OP_WRITE);
rbio->real_stripes,
sectornr, REQ_OP_WRITE);
if (ret)
goto cleanup;
}
@ -2751,33 +2747,3 @@ void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio)
if (!lock_stripe_add(rbio))
start_async_work(rbio, scrub_rbio_work_locked);
}
/* The following code is used for dev replace of a missing RAID 5/6 device. */
struct btrfs_raid_bio *
raid56_alloc_missing_rbio(struct bio *bio, struct btrfs_io_context *bioc)
{
struct btrfs_fs_info *fs_info = bioc->fs_info;
struct btrfs_raid_bio *rbio;
rbio = alloc_rbio(fs_info, bioc);
if (IS_ERR(rbio))
return NULL;
rbio->operation = BTRFS_RBIO_REBUILD_MISSING;
bio_list_add(&rbio->bio_list, bio);
/*
* This is a special bio which is used to hold the completion handler
* and make the scrub rbio is similar to the other types
*/
ASSERT(!bio->bi_iter.bi_size);
set_rbio_range_error(rbio, bio);
return rbio;
}
void raid56_submit_missing_rbio(struct btrfs_raid_bio *rbio)
{
start_async_work(rbio, recover_rbio_work);
}

12
fs/btrfs/raid56.h
View file

@ -170,6 +170,11 @@ static inline int nr_data_stripes(const struct map_lookup *map)
return map->num_stripes - btrfs_nr_parity_stripes(map->type);
}
static inline int nr_bioc_data_stripes(const struct btrfs_io_context *bioc)
{
return bioc->num_stripes - btrfs_nr_parity_stripes(bioc->map_type);
}
#define RAID5_P_STRIPE ((u64)-2)
#define RAID6_Q_STRIPE ((u64)-1)
@ -182,19 +187,12 @@ void raid56_parity_recover(struct bio *bio, struct btrfs_io_context *bioc,
int mirror_num);
void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc);
void raid56_add_scrub_pages(struct btrfs_raid_bio *rbio, struct page *page,
unsigned int pgoff, u64 logical);
struct btrfs_raid_bio *raid56_parity_alloc_scrub_rbio(struct bio *bio,
struct btrfs_io_context *bioc,
struct btrfs_device *scrub_dev,
unsigned long *dbitmap, int stripe_nsectors);
void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio);
struct btrfs_raid_bio *
raid56_alloc_missing_rbio(struct bio *bio, struct btrfs_io_context *bioc);
void raid56_submit_missing_rbio(struct btrfs_raid_bio *rbio);
int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info);
void btrfs_free_stripe_hash_table(struct btrfs_fs_info *info);

6
fs/btrfs/relocation.c
View file

@ -1266,7 +1266,7 @@ int replace_path(struct btrfs_trans_handle *trans, struct reloc_control *rc,
level = btrfs_header_level(parent);
ASSERT(level >= lowest_level);
ret = btrfs_bin_search(parent, &key, &slot);
ret = btrfs_bin_search(parent, 0, &key, &slot);
if (ret < 0)
break;
if (ret && slot > 0)
@ -2407,7 +2407,7 @@ static int do_relocation(struct btrfs_trans_handle *trans,
if (upper->eb && !upper->locked) {
if (!lowest) {
ret = btrfs_bin_search(upper->eb, key, &slot);
ret = btrfs_bin_search(upper->eb, 0, key, &slot);
if (ret < 0)
goto next;
BUG_ON(ret);
@ -2441,7 +2441,7 @@ static int do_relocation(struct btrfs_trans_handle *trans,
slot = path->slots[upper->level];
btrfs_release_path(path);
} else {
ret = btrfs_bin_search(upper->eb, key, &slot);
ret = btrfs_bin_search(upper->eb, 0, key, &slot);
if (ret < 0)
goto next;
BUG_ON(ret);

4228
fs/btrfs/scrub.c
View file

File diff suppressed because it is too large Load diff

2
fs/btrfs/send.c
View file

@ -1875,7 +1875,7 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen,
int left_ret;
int right_ret;
u64 left_gen;
u64 right_gen;
u64 right_gen = 0;
struct btrfs_inode_info info;
ret = get_inode_info(sctx->send_root, ino, &info);

32
fs/btrfs/space-info.c
View file

@ -537,7 +537,7 @@ void btrfs_dump_space_info(struct btrfs_fs_info *fs_info,
up_read(&info->groups_sem);
}
static inline u64 calc_reclaim_items_nr(struct btrfs_fs_info *fs_info,
static inline u64 calc_reclaim_items_nr(const struct btrfs_fs_info *fs_info,
u64 to_reclaim)
{
u64 bytes;
@ -550,6 +550,18 @@ static inline u64 calc_reclaim_items_nr(struct btrfs_fs_info *fs_info,
return nr;
}
static inline u64 calc_delayed_refs_nr(const struct btrfs_fs_info *fs_info,
u64 to_reclaim)
{
const u64 bytes = btrfs_calc_delayed_ref_bytes(fs_info, 1);
u64 nr;
nr = div64_u64(to_reclaim, bytes);
if (!nr)
nr = 1;
return nr;
}
#define EXTENT_SIZE_PER_ITEM SZ_256K
/*
@ -727,7 +739,7 @@ static void flush_space(struct btrfs_fs_info *fs_info,
break;
}
if (state == FLUSH_DELAYED_REFS_NR)
nr = calc_reclaim_items_nr(fs_info, num_bytes);
nr = calc_delayed_refs_nr(fs_info, num_bytes);
else
nr = 0;
btrfs_run_delayed_refs(trans, nr);
@ -1599,11 +1611,22 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info,
struct reserve_ticket ticket;
u64 start_ns = 0;
u64 used;
int ret = 0;
int ret = -ENOSPC;
bool pending_tickets;
ASSERT(orig_bytes);
ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_ALL);
/*
* If have a transaction handle (current->journal_info != NULL), then
* the flush method can not be neither BTRFS_RESERVE_FLUSH_ALL* nor
* BTRFS_RESERVE_FLUSH_EVICT, as we could deadlock because those
* flushing methods can trigger transaction commits.
*/
if (current->journal_info) {
/* One assert per line for easier debugging. */
ASSERT(flush != BTRFS_RESERVE_FLUSH_ALL);
ASSERT(flush != BTRFS_RESERVE_FLUSH_ALL_STEAL);
ASSERT(flush != BTRFS_RESERVE_FLUSH_EVICT);
}
if (flush == BTRFS_RESERVE_FLUSH_DATA)
async_work = &fs_info->async_data_reclaim_work;
@ -1611,7 +1634,6 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info,
async_work = &fs_info->async_reclaim_work;
spin_lock(&space_info->lock);
ret = -ENOSPC;
used = btrfs_space_info_used(space_info, true);
/*

1
fs/btrfs/space-info.h
View file

@ -27,6 +27,7 @@ enum btrfs_reserve_flush_enum {
* - Running delayed refs
* - Running delalloc and waiting for ordered extents
* - Allocating a new chunk
* - Committing transaction
*/
BTRFS_RESERVE_FLUSH_EVICT,

3
fs/btrfs/super.c
View file

@ -1158,6 +1158,7 @@ static int btrfs_fill_super(struct super_block *sb,
inode = btrfs_iget(sb, BTRFS_FIRST_FREE_OBJECTID, fs_info->fs_root);
if (IS_ERR(inode)) {
err = PTR_ERR(inode);
btrfs_handle_fs_error(fs_info, err, NULL);
goto fail_close;
}
@ -2412,7 +2413,7 @@ static int __init btrfs_print_mod_info(void)
", fsverity=no"
#endif
;
pr_info("Btrfs loaded, crc32c=%s%s\n", crc32c_impl(), options);
pr_info("Btrfs loaded%s\n", options);
return 0;
}

5
fs/btrfs/sysfs.c
View file

@ -1262,8 +1262,13 @@ static ssize_t btrfs_bg_reclaim_threshold_store(struct kobject *kobj,
if (ret)
return ret;
#ifdef CONFIG_BTRFS_DEBUG
if (thresh != 0 && (thresh > 100))
return -EINVAL;
#else
if (thresh != 0 && (thresh <= 50 || thresh > 100))
return -EINVAL;
#endif
WRITE_ONCE(fs_info->bg_reclaim_threshold, thresh);

1
fs/btrfs/tests/extent-map-tests.c
View file

@ -486,7 +486,6 @@ static int test_rmap_block(struct btrfs_fs_info *fs_info,
em->map_lookup = map;
map->num_stripes = test->num_stripes;
map->stripe_len = BTRFS_STRIPE_LEN;
map->type = test->raid_type;
for (i = 0; i < map->num_stripes; i++) {

28
fs/btrfs/transaction.c
View file

@ -601,15 +601,16 @@ start_transaction(struct btrfs_root *root, unsigned int num_items,
/*
* We want to reserve all the bytes we may need all at once, so
* we only do 1 enospc flushing cycle per transaction start. We
* accomplish this by simply assuming we'll do 2 x num_items
* worth of delayed refs updates in this trans handle, and
* refill that amount for whatever is missing in the reserve.
* accomplish this by simply assuming we'll do num_items worth
* of delayed refs updates in this trans handle, and refill that
* amount for whatever is missing in the reserve.
*/
num_bytes = btrfs_calc_insert_metadata_size(fs_info, num_items);
if (flush == BTRFS_RESERVE_FLUSH_ALL &&
btrfs_block_rsv_full(delayed_refs_rsv) == 0) {
delayed_refs_bytes = num_bytes;
num_bytes <<= 1;
!btrfs_block_rsv_full(delayed_refs_rsv)) {
delayed_refs_bytes = btrfs_calc_delayed_ref_bytes(fs_info,
num_items);
num_bytes += delayed_refs_bytes;
}
/*
@ -942,16 +943,6 @@ void btrfs_throttle(struct btrfs_fs_info *fs_info)
wait_current_trans(fs_info);
}
static bool should_end_transaction(struct btrfs_trans_handle *trans)
{
struct btrfs_fs_info *fs_info = trans->fs_info;
if (btrfs_check_space_for_delayed_refs(fs_info))
return true;
return !!btrfs_block_rsv_check(&fs_info->global_block_rsv, 50);
}
bool btrfs_should_end_transaction(struct btrfs_trans_handle *trans)
{
struct btrfs_transaction *cur_trans = trans->transaction;
@ -960,7 +951,10 @@ bool btrfs_should_end_transaction(struct btrfs_trans_handle *trans)
test_bit(BTRFS_DELAYED_REFS_FLUSHING, &cur_trans->delayed_refs.flags))
return true;
return should_end_transaction(trans);
if (btrfs_check_space_for_delayed_refs(trans->fs_info))
return true;
return !!btrfs_block_rsv_check(&trans->fs_info->global_block_rsv, 50);
}
static void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans)

14
fs/btrfs/tree-checker.c
View file

@ -849,6 +849,20 @@ int btrfs_check_chunk_valid(struct extent_buffer *leaf,
stripe_len);
return -EUCLEAN;
}
/*
* We artificially limit the chunk size, so that the number of stripes
* inside a chunk can be fit into a U32. The current limit (256G) is
* way too large for real world usage anyway, and it's also much larger
* than our existing limit (10G).
*
* Thus it should be a good way to catch obvious bitflips.
*/
if (unlikely(length >= ((u64)U32_MAX << BTRFS_STRIPE_LEN_SHIFT))) {
chunk_err(leaf, chunk, logical,
"chunk length too large: have %llu limit %llu",
length, (u64)U32_MAX << BTRFS_STRIPE_LEN_SHIFT);
return -EUCLEAN;
}
if (unlikely(type & ~(BTRFS_BLOCK_GROUP_TYPE_MASK |
BTRFS_BLOCK_GROUP_PROFILE_MASK))) {
chunk_err(leaf, chunk, logical,

171
fs/btrfs/tree-log.c
View file

@ -2563,6 +2563,28 @@ static void unaccount_log_buffer(struct btrfs_fs_info *fs_info, u64 start)
btrfs_put_block_group(cache);
}
static int clean_log_buffer(struct btrfs_trans_handle *trans,
struct extent_buffer *eb)
{
int ret;
btrfs_tree_lock(eb);
btrfs_clear_buffer_dirty(trans, eb);
wait_on_extent_buffer_writeback(eb);
btrfs_tree_unlock(eb);
if (trans) {
ret = btrfs_pin_reserved_extent(trans, eb->start, eb->len);
if (ret)
return ret;
btrfs_redirty_list_add(trans->transaction, eb);
} else {
unaccount_log_buffer(eb->fs_info, eb->start);
}
return 0;
}
static noinline int walk_down_log_tree(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
struct btrfs_path *path, int *level,
@ -2573,7 +2595,6 @@ static noinline int walk_down_log_tree(struct btrfs_trans_handle *trans,
u64 ptr_gen;
struct extent_buffer *next;
struct extent_buffer *cur;
u32 blocksize;
int ret = 0;
while (*level > 0) {
@ -2593,7 +2614,6 @@ static noinline int walk_down_log_tree(struct btrfs_trans_handle *trans,
check.level = *level - 1;
check.has_first_key = true;
btrfs_node_key_to_cpu(cur, &check.first_key, path->slots[*level]);
blocksize = fs_info->nodesize;
next = btrfs_find_create_tree_block(fs_info, bytenr,
btrfs_header_owner(cur),
@ -2617,22 +2637,10 @@ static noinline int walk_down_log_tree(struct btrfs_trans_handle *trans,
return ret;
}
btrfs_tree_lock(next);
btrfs_clear_buffer_dirty(trans, next);
wait_on_extent_buffer_writeback(next);
btrfs_tree_unlock(next);
if (trans) {
ret = btrfs_pin_reserved_extent(trans,
bytenr, blocksize);
if (ret) {
free_extent_buffer(next);
return ret;
}
btrfs_redirty_list_add(
trans->transaction, next);
} else {
unaccount_log_buffer(fs_info, bytenr);
ret = clean_log_buffer(trans, next);
if (ret) {
free_extent_buffer(next);
return ret;
}
}
free_extent_buffer(next);
@ -2662,7 +2670,6 @@ static noinline int walk_up_log_tree(struct btrfs_trans_handle *trans,
struct btrfs_path *path, int *level,
struct walk_control *wc)
{
struct btrfs_fs_info *fs_info = root->fs_info;
int i;
int slot;
int ret;
@ -2682,27 +2689,9 @@ static noinline int walk_up_log_tree(struct btrfs_trans_handle *trans,
return ret;
if (wc->free) {
struct extent_buffer *next;
next = path->nodes[*level];
btrfs_tree_lock(next);
btrfs_clear_buffer_dirty(trans, next);
wait_on_extent_buffer_writeback(next);
btrfs_tree_unlock(next);
if (trans) {
ret = btrfs_pin_reserved_extent(trans,
path->nodes[*level]->start,
path->nodes[*level]->len);
if (ret)
return ret;
btrfs_redirty_list_add(trans->transaction,
next);
} else {
unaccount_log_buffer(fs_info,
path->nodes[*level]->start);
}
ret = clean_log_buffer(trans, path->nodes[*level]);
if (ret)
return ret;
}
free_extent_buffer(path->nodes[*level]);
path->nodes[*level] = NULL;
@ -2720,7 +2709,6 @@ static noinline int walk_up_log_tree(struct btrfs_trans_handle *trans,
static int walk_log_tree(struct btrfs_trans_handle *trans,
struct btrfs_root *log, struct walk_control *wc)
{
struct btrfs_fs_info *fs_info = log->fs_info;
int ret = 0;
int wret;
int level;
@ -2762,26 +2750,8 @@ static int walk_log_tree(struct btrfs_trans_handle *trans,
orig_level);
if (ret)
goto out;
if (wc->free) {
struct extent_buffer *next;
next = path->nodes[orig_level];
btrfs_tree_lock(next);
btrfs_clear_buffer_dirty(trans, next);
wait_on_extent_buffer_writeback(next);
btrfs_tree_unlock(next);
if (trans) {
ret = btrfs_pin_reserved_extent(trans,
next->start, next->len);
if (ret)
goto out;
btrfs_redirty_list_add(trans->transaction, next);
} else {
unaccount_log_buffer(fs_info, next->start);
}
}
if (wc->free)
ret = clean_log_buffer(trans, path->nodes[orig_level]);
}
out:
@ -3648,6 +3618,9 @@ static int flush_dir_items_batch(struct btrfs_trans_handle *trans,
ret = BTRFS_LOG_FORCE_COMMIT;
else
inode->last_dir_index_offset = last_index;
if (btrfs_get_first_dir_index_to_log(inode) == 0)
btrfs_set_first_dir_index_to_log(inode, batch.keys[0].offset);
out:
kfree(ins_data);
@ -4099,7 +4072,7 @@ static int drop_inode_items(struct btrfs_trans_handle *trans,
found_key.offset = 0;
found_key.type = 0;
ret = btrfs_bin_search(path->nodes[0], &found_key, &start_slot);
ret = btrfs_bin_search(path->nodes[0], 0, &found_key, &start_slot);
if (ret < 0)
break;
@ -5406,6 +5379,7 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans,
LIST_HEAD(dir_list);
struct btrfs_dir_list *dir_elem;
u64 ino = btrfs_ino(start_inode);
struct btrfs_inode *curr_inode = start_inode;
int ret = 0;
/*
@ -5420,43 +5394,39 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans,
if (!path)
return -ENOMEM;
/* Pairs with btrfs_add_delayed_iput below. */
ihold(&curr_inode->vfs_inode);
while (true) {
struct extent_buffer *leaf;
struct btrfs_key min_key;
struct inode *vfs_inode;
struct btrfs_key key;
struct btrfs_key found_key;
u64 next_index;
bool continue_curr_inode = true;
int nritems;
int i;
int iter_ret;
min_key.objectid = ino;
min_key.type = BTRFS_DIR_INDEX_KEY;
min_key.offset = 0;
key.objectid = ino;
key.type = BTRFS_DIR_INDEX_KEY;
key.offset = btrfs_get_first_dir_index_to_log(curr_inode);
next_index = key.offset;
again:
btrfs_release_path(path);
ret = btrfs_search_forward(root, &min_key, path, trans->transid);
if (ret < 0) {
break;
} else if (ret > 0) {
ret = 0;
goto next;
}
leaf = path->nodes[0];
nritems = btrfs_header_nritems(leaf);
for (i = path->slots[0]; i < nritems; i++) {
btrfs_for_each_slot(root->log_root, &key, &found_key, path, iter_ret) {
struct extent_buffer *leaf = path->nodes[0];
struct btrfs_dir_item *di;
struct btrfs_key di_key;
struct inode *di_inode;
int log_mode = LOG_INODE_EXISTS;
int type;
btrfs_item_key_to_cpu(leaf, &min_key, i);
if (min_key.objectid != ino ||
min_key.type != BTRFS_DIR_INDEX_KEY) {
if (found_key.objectid != ino ||
found_key.type != BTRFS_DIR_INDEX_KEY) {
continue_curr_inode = false;
break;
}
di = btrfs_item_ptr(leaf, i, struct btrfs_dir_item);
next_index = found_key.offset + 1;
di = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dir_item);
type = btrfs_dir_ftype(leaf, di);
if (btrfs_dir_transid(leaf, di) < trans->transid)
continue;
@ -5496,12 +5466,24 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans,
break;
}
if (continue_curr_inode && min_key.offset < (u64)-1) {
min_key.offset++;
btrfs_release_path(path);
if (iter_ret < 0) {
ret = iter_ret;
goto out;
} else if (iter_ret > 0) {
continue_curr_inode = false;
} else {
key = found_key;
}
if (continue_curr_inode && key.offset < (u64)-1) {
key.offset++;
goto again;
}
next:
btrfs_set_first_dir_index_to_log(curr_inode, next_index);
if (list_empty(&dir_list))
break;
@ -5509,9 +5491,22 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans,
ino = dir_elem->ino;
list_del(&dir_elem->list);
kfree(dir_elem);
btrfs_add_delayed_iput(curr_inode);
curr_inode = NULL;
vfs_inode = btrfs_iget(fs_info->sb, ino, root);
if (IS_ERR(vfs_inode)) {
ret = PTR_ERR(vfs_inode);
break;
}
curr_inode = BTRFS_I(vfs_inode);
}
out:
btrfs_free_path(path);
if (curr_inode)
btrfs_add_delayed_iput(curr_inode);
if (ret) {
struct btrfs_dir_list *next;

597
fs/btrfs/volumes.c
View file

@ -395,7 +395,6 @@ void btrfs_free_device(struct btrfs_device *device)
{
WARN_ON(!list_empty(&device->post_commit_list));
rcu_string_free(device->name);
extent_io_tree_release(&device->alloc_state);
btrfs_destroy_dev_zone_info(device);
kfree(device);
}
@ -1150,10 +1149,10 @@ static void btrfs_close_one_device(struct btrfs_device *device)
device->last_flush_error = 0;
/* Verify the device is back in a pristine state */
ASSERT(!test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state));
ASSERT(!test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
ASSERT(list_empty(&device->dev_alloc_list));
ASSERT(list_empty(&device->post_commit_list));
WARN_ON(test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state));
WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
WARN_ON(!list_empty(&device->dev_alloc_list));
WARN_ON(!list_empty(&device->post_commit_list));
}
static void close_fs_devices(struct btrfs_fs_devices *fs_devices)
@ -2618,7 +2617,7 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
struct block_device *bdev;
struct super_block *sb = fs_info->sb;
struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
struct btrfs_fs_devices *seed_devices;
struct btrfs_fs_devices *seed_devices = NULL;
u64 orig_super_total_bytes;
u64 orig_super_num_devices;
int ret = 0;
@ -5125,7 +5124,7 @@ static void init_alloc_chunk_ctl_policy_regular(
/* We don't want a chunk larger than 10% of writable space */
ctl->max_chunk_size = min(mult_perc(fs_devices->total_rw_bytes, 10),
ctl->max_chunk_size);
ctl->dev_extent_min = BTRFS_STRIPE_LEN * ctl->dev_stripes;
ctl->dev_extent_min = ctl->dev_stripes << BTRFS_STRIPE_LEN_SHIFT;
}
static void init_alloc_chunk_ctl_policy_zoned(
@ -5407,7 +5406,6 @@ static struct btrfs_block_group *create_chunk(struct btrfs_trans_handle *trans,
j * ctl->stripe_size;
}
}
map->stripe_len = BTRFS_STRIPE_LEN;
map->io_align = BTRFS_STRIPE_LEN;
map->io_width = BTRFS_STRIPE_LEN;
map->type = type;
@ -5438,7 +5436,7 @@ static struct btrfs_block_group *create_chunk(struct btrfs_trans_handle *trans,
}
write_unlock(&em_tree->lock);
block_group = btrfs_make_block_group(trans, 0, type, start, ctl->chunk_size);
block_group = btrfs_make_block_group(trans, type, start, ctl->chunk_size);
if (IS_ERR(block_group))
goto error_del_extent;
@ -5615,11 +5613,11 @@ int btrfs_chunk_alloc_add_chunk_item(struct btrfs_trans_handle *trans,
btrfs_set_stack_chunk_length(chunk, bg->length);
btrfs_set_stack_chunk_owner(chunk, BTRFS_EXTENT_TREE_OBJECTID);
btrfs_set_stack_chunk_stripe_len(chunk, map->stripe_len);
btrfs_set_stack_chunk_stripe_len(chunk, BTRFS_STRIPE_LEN);
btrfs_set_stack_chunk_type(chunk, map->type);
btrfs_set_stack_chunk_num_stripes(chunk, map->num_stripes);
btrfs_set_stack_chunk_io_align(chunk, map->stripe_len);
btrfs_set_stack_chunk_io_width(chunk, map->stripe_len);
btrfs_set_stack_chunk_io_align(chunk, BTRFS_STRIPE_LEN);
btrfs_set_stack_chunk_io_width(chunk, BTRFS_STRIPE_LEN);
btrfs_set_stack_chunk_sector_size(chunk, fs_info->sectorsize);
btrfs_set_stack_chunk_sub_stripes(chunk, map->sub_stripes);
@ -5784,13 +5782,6 @@ int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
*/
ret = map->num_stripes;
free_extent_map(em);
down_read(&fs_info->dev_replace.rwsem);
if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace) &&
fs_info->dev_replace.tgtdev)
ret++;
up_read(&fs_info->dev_replace.rwsem);
return ret;
}
@ -5809,7 +5800,7 @@ unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info,
if (!WARN_ON(IS_ERR(em))) {
map = em->map_lookup;
if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
len = map->stripe_len * nr_data_stripes(map);
len = nr_data_stripes(map) << BTRFS_STRIPE_LEN_SHIFT;
free_extent_map(em);
}
return len;
@ -5895,41 +5886,16 @@ static int find_live_mirror(struct btrfs_fs_info *fs_info,
return preferred_mirror;
}
/* Bubble-sort the stripe set to put the parity/syndrome stripes last */
static void sort_parity_stripes(struct btrfs_io_context *bioc, int num_stripes)
{
int i;
int again = 1;
while (again) {
again = 0;
for (i = 0; i < num_stripes - 1; i++) {
/* Swap if parity is on a smaller index */
if (bioc->raid_map[i] > bioc->raid_map[i + 1]) {
swap(bioc->stripes[i], bioc->stripes[i + 1]);
swap(bioc->raid_map[i], bioc->raid_map[i + 1]);
again = 1;
}
}
}
}
static struct btrfs_io_context *alloc_btrfs_io_context(struct btrfs_fs_info *fs_info,
int total_stripes,
int real_stripes)
u16 total_stripes)
{
struct btrfs_io_context *bioc = kzalloc(
struct btrfs_io_context *bioc;
bioc = kzalloc(
/* The size of btrfs_io_context */
sizeof(struct btrfs_io_context) +
/* Plus the variable array for the stripes */
sizeof(struct btrfs_io_stripe) * (total_stripes) +
/* Plus the variable array for the tgt dev */
sizeof(int) * (real_stripes) +
/*
* Plus the raid_map, which includes both the tgt dev
* and the stripes.
*/
sizeof(u64) * (total_stripes),
sizeof(struct btrfs_io_stripe) * (total_stripes),
GFP_NOFS);
if (!bioc)
@ -5938,8 +5904,8 @@ static struct btrfs_io_context *alloc_btrfs_io_context(struct btrfs_fs_info *fs_
refcount_set(&bioc->refs, 1);
bioc->fs_info = fs_info;
bioc->tgtdev_map = (int *)(bioc->stripes + total_stripes);
bioc->raid_map = (u64 *)(bioc->tgtdev_map + real_stripes);
bioc->replace_stripe_src = -1;
bioc->full_stripe_logical = (u64)-1;
return bioc;
}
@ -5971,16 +5937,15 @@ struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info,
struct btrfs_discard_stripe *stripes;
u64 length = *length_ret;
u64 offset;
u64 stripe_nr;
u64 stripe_nr_end;
u32 stripe_nr;
u32 stripe_nr_end;
u32 stripe_cnt;
u64 stripe_end_offset;
u64 stripe_cnt;
u64 stripe_len;
u64 stripe_offset;
u32 stripe_index;
u32 factor = 0;
u32 sub_stripes = 0;
u64 stripes_per_dev = 0;
u32 stripes_per_dev = 0;
u32 remaining_stripes = 0;
u32 last_stripe = 0;
int ret;
@ -5996,26 +5961,25 @@ struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info,
if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
ret = -EOPNOTSUPP;
goto out_free_map;
}
}
offset = logical - em->start;
length = min_t(u64, em->start + em->len - logical, length);
*length_ret = length;
stripe_len = map->stripe_len;
/*
* stripe_nr counts the total number of stripes we have to stride
* to get to this block
*/
stripe_nr = div64_u64(offset, stripe_len);
stripe_nr = offset >> BTRFS_STRIPE_LEN_SHIFT;
/* stripe_offset is the offset of this block in its stripe */
stripe_offset = offset - stripe_nr * stripe_len;
stripe_offset = offset - (stripe_nr << BTRFS_STRIPE_LEN_SHIFT);
stripe_nr_end = round_up(offset + length, map->stripe_len);
stripe_nr_end = div64_u64(stripe_nr_end, map->stripe_len);
stripe_nr_end = round_up(offset + length, BTRFS_STRIPE_LEN) >>
BTRFS_STRIPE_LEN_SHIFT;
stripe_cnt = stripe_nr_end - stripe_nr;
stripe_end_offset = stripe_nr_end * map->stripe_len -
stripe_end_offset = (stripe_nr_end << BTRFS_STRIPE_LEN_SHIFT) -
(offset + length);
/*
* after this, stripe_nr is the number of stripes on this
@ -6034,18 +5998,19 @@ struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info,
factor = map->num_stripes / sub_stripes;
*num_stripes = min_t(u64, map->num_stripes,
sub_stripes * stripe_cnt);
stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index);
stripe_index = stripe_nr % factor;
stripe_nr /= factor;
stripe_index *= sub_stripes;
stripes_per_dev = div_u64_rem(stripe_cnt, factor,
&remaining_stripes);
div_u64_rem(stripe_nr_end - 1, factor, &last_stripe);
last_stripe *= sub_stripes;
remaining_stripes = stripe_cnt % factor;
stripes_per_dev = stripe_cnt / factor;
last_stripe = ((stripe_nr_end - 1) % factor) * sub_stripes;
} else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK |
BTRFS_BLOCK_GROUP_DUP)) {
*num_stripes = map->num_stripes;
} else {
stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
&stripe_index);
stripe_index = stripe_nr % map->num_stripes;
stripe_nr /= map->num_stripes;
}
stripes = kcalloc(*num_stripes, sizeof(*stripes), GFP_NOFS);
@ -6057,15 +6022,15 @@ struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info,
for (i = 0; i < *num_stripes; i++) {
stripes[i].physical =
map->stripes[stripe_index].physical +
stripe_offset + stripe_nr * map->stripe_len;
stripe_offset + (stripe_nr << BTRFS_STRIPE_LEN_SHIFT);
stripes[i].dev = map->stripes[stripe_index].dev;
if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
BTRFS_BLOCK_GROUP_RAID10)) {
stripes[i].length = stripes_per_dev * map->stripe_len;
stripes[i].length = stripes_per_dev << BTRFS_STRIPE_LEN_SHIFT;
if (i / sub_stripes < remaining_stripes)
stripes[i].length += map->stripe_len;
stripes[i].length += BTRFS_STRIPE_LEN;
/*
* Special for the first stripe and
@ -6103,83 +6068,6 @@ struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info,
return ERR_PTR(ret);
}
/*
* In dev-replace case, for repair case (that's the only case where the mirror
* is selected explicitly when calling btrfs_map_block), blocks left of the
* left cursor can also be read from the target drive.
*
* For REQ_GET_READ_MIRRORS, the target drive is added as the last one to the
* array of stripes.
* For READ, it also needs to be supported using the same mirror number.
*
* If the requested block is not left of the left cursor, EIO is returned. This
* can happen because btrfs_num_copies() returns one more in the dev-replace
* case.
*/
static int get_extra_mirror_from_replace(struct btrfs_fs_info *fs_info,
u64 logical, u64 length,
u64 srcdev_devid, int *mirror_num,
u64 *physical)
{
struct btrfs_io_context *bioc = NULL;
int num_stripes;
int index_srcdev = 0;
int found = 0;
u64 physical_of_found = 0;
int i;
int ret = 0;
ret = __btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS,
logical, &length, &bioc, NULL, NULL, 0);
if (ret) {
ASSERT(bioc == NULL);
return ret;
}
num_stripes = bioc->num_stripes;
if (*mirror_num > num_stripes) {
/*
* BTRFS_MAP_GET_READ_MIRRORS does not contain this mirror,
* that means that the requested area is not left of the left
* cursor
*/
btrfs_put_bioc(bioc);
return -EIO;
}
/*
* process the rest of the function using the mirror_num of the source
* drive. Therefore look it up first. At the end, patch the device
* pointer to the one of the target drive.
*/
for (i = 0; i < num_stripes; i++) {
if (bioc->stripes[i].dev->devid != srcdev_devid)
continue;
/*
* In case of DUP, in order to keep it simple, only add the
* mirror with the lowest physical address
*/
if (found &&
physical_of_found <= bioc->stripes[i].physical)
continue;
index_srcdev = i;
found = 1;
physical_of_found = bioc->stripes[i].physical;
}
btrfs_put_bioc(bioc);
ASSERT(found);
if (!found)
return -EIO;
*mirror_num = index_srcdev + 1;
*physical = physical_of_found;
return ret;
}
static bool is_block_group_to_copy(struct btrfs_fs_info *fs_info, u64 logical)
{
struct btrfs_block_group *cache;
@ -6198,101 +6086,80 @@ static bool is_block_group_to_copy(struct btrfs_fs_info *fs_info, u64 logical)
}
static void handle_ops_on_dev_replace(enum btrfs_map_op op,
struct btrfs_io_context **bioc_ret,
struct btrfs_io_context *bioc,
struct btrfs_dev_replace *dev_replace,
u64 logical,
int *num_stripes_ret, int *max_errors_ret)
{
struct btrfs_io_context *bioc = *bioc_ret;
u64 srcdev_devid = dev_replace->srcdev->devid;
int tgtdev_indexes = 0;
/*
* At this stage, num_stripes is still the real number of stripes,
* excluding the duplicated stripes.
*/
int num_stripes = *num_stripes_ret;
int nr_extra_stripes = 0;
int max_errors = *max_errors_ret;
int i;
if (op == BTRFS_MAP_WRITE) {
int index_where_to_add;
/*
* A block group which has "to_copy" set will eventually be copied by
* the dev-replace process. We can avoid cloning IO here.
*/
if (is_block_group_to_copy(dev_replace->srcdev->fs_info, logical))
return;
/*
* Duplicate the write operations while the dev-replace procedure is
* running. Since the copying of the old disk to the new disk takes
* place at run time while the filesystem is mounted writable, the
* regular write operations to the old disk have to be duplicated to go
* to the new disk as well.
*
* Note that device->missing is handled by the caller, and that the
* write to the old disk is already set up in the stripes array.
*/
for (i = 0; i < num_stripes; i++) {
struct btrfs_io_stripe *old = &bioc->stripes[i];
struct btrfs_io_stripe *new = &bioc->stripes[num_stripes + nr_extra_stripes];
if (old->dev->devid != srcdev_devid)
continue;
new->physical = old->physical;
new->dev = dev_replace->tgtdev;
if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK)
bioc->replace_stripe_src = i;
nr_extra_stripes++;
}
/* We can only have at most 2 extra nr_stripes (for DUP). */
ASSERT(nr_extra_stripes <= 2);
/*
* For GET_READ_MIRRORS, we can only return at most 1 extra stripe for
* replace.
* If we have 2 extra stripes, only choose the one with smaller physical.
*/
if (op == BTRFS_MAP_GET_READ_MIRRORS && nr_extra_stripes == 2) {
struct btrfs_io_stripe *first = &bioc->stripes[num_stripes];
struct btrfs_io_stripe *second = &bioc->stripes[num_stripes + 1];
/* Only DUP can have two extra stripes. */
ASSERT(bioc->map_type & BTRFS_BLOCK_GROUP_DUP);
/*
* A block group which have "to_copy" set will eventually
* copied by dev-replace process. We can avoid cloning IO here.
* Swap the last stripe stripes and reduce @nr_extra_stripes.
* The extra stripe would still be there, but won't be accessed.
*/
if (is_block_group_to_copy(dev_replace->srcdev->fs_info, logical))
return;
/*
* duplicate the write operations while the dev replace
* procedure is running. Since the copying of the old disk to
* the new disk takes place at run time while the filesystem is
* mounted writable, the regular write operations to the old
* disk have to be duplicated to go to the new disk as well.
*
* Note that device->missing is handled by the caller, and that
* the write to the old disk is already set up in the stripes
* array.
*/
index_where_to_add = num_stripes;
for (i = 0; i < num_stripes; i++) {
if (bioc->stripes[i].dev->devid == srcdev_devid) {
/* write to new disk, too */
struct btrfs_io_stripe *new =
bioc->stripes + index_where_to_add;
struct btrfs_io_stripe *old =
bioc->stripes + i;
new->physical = old->physical;
new->dev = dev_replace->tgtdev;
bioc->tgtdev_map[i] = index_where_to_add;
index_where_to_add++;
max_errors++;
tgtdev_indexes++;
}
}
num_stripes = index_where_to_add;
} else if (op == BTRFS_MAP_GET_READ_MIRRORS) {
int index_srcdev = 0;
int found = 0;
u64 physical_of_found = 0;
/*
* During the dev-replace procedure, the target drive can also
* be used to read data in case it is needed to repair a corrupt
* block elsewhere. This is possible if the requested area is
* left of the left cursor. In this area, the target drive is a
* full copy of the source drive.
*/
for (i = 0; i < num_stripes; i++) {
if (bioc->stripes[i].dev->devid == srcdev_devid) {
/*
* In case of DUP, in order to keep it simple,
* only add the mirror with the lowest physical
* address
*/
if (found &&
physical_of_found <= bioc->stripes[i].physical)
continue;
index_srcdev = i;
found = 1;
physical_of_found = bioc->stripes[i].physical;
}
}
if (found) {
struct btrfs_io_stripe *tgtdev_stripe =
bioc->stripes + num_stripes;
tgtdev_stripe->physical = physical_of_found;
tgtdev_stripe->dev = dev_replace->tgtdev;
bioc->tgtdev_map[index_srcdev] = num_stripes;
tgtdev_indexes++;
num_stripes++;
if (first->physical > second->physical) {
swap(second->physical, first->physical);
swap(second->dev, first->dev);
nr_extra_stripes--;
}
}
*num_stripes_ret = num_stripes;
*max_errors_ret = max_errors;
bioc->num_tgtdevs = tgtdev_indexes;
*bioc_ret = bioc;
*num_stripes_ret = num_stripes + nr_extra_stripes;
*max_errors_ret = max_errors + nr_extra_stripes;
bioc->replace_nr_stripes = nr_extra_stripes;
}
static bool need_full_stripe(enum btrfs_map_op op)
@ -6301,25 +6168,35 @@ static bool need_full_stripe(enum btrfs_map_op op)
}
static u64 btrfs_max_io_len(struct map_lookup *map, enum btrfs_map_op op,
u64 offset, u64 *stripe_nr, u64 *stripe_offset,
u64 offset, u32 *stripe_nr, u64 *stripe_offset,
u64 *full_stripe_start)
{
u32 stripe_len = map->stripe_len;
ASSERT(op != BTRFS_MAP_DISCARD);
/*
* Stripe_nr is the stripe where this block falls. stripe_offset is
* the offset of this block in its stripe.
*/
*stripe_nr = div64_u64_rem(offset, stripe_len, stripe_offset);
*stripe_offset = offset & BTRFS_STRIPE_LEN_MASK;
*stripe_nr = offset >> BTRFS_STRIPE_LEN_SHIFT;
ASSERT(*stripe_offset < U32_MAX);
if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
unsigned long full_stripe_len = stripe_len * nr_data_stripes(map);
unsigned long full_stripe_len = nr_data_stripes(map) <<
BTRFS_STRIPE_LEN_SHIFT;
/*
* For full stripe start, we use previously calculated
* @stripe_nr. Align it to nr_data_stripes, then multiply with
* STRIPE_LEN.
*
* By this we can avoid u64 division completely. And we have
* to go rounddown(), not round_down(), as nr_data_stripes is
* not ensured to be power of 2.
*/
*full_stripe_start =
div64_u64(offset, full_stripe_len) * full_stripe_len;
rounddown(*stripe_nr, nr_data_stripes(map)) <<
BTRFS_STRIPE_LEN_SHIFT;
/*
* For writes to RAID56, allow to write a full stripe set, but
@ -6334,16 +6211,16 @@ static u64 btrfs_max_io_len(struct map_lookup *map, enum btrfs_map_op op,
* a single disk).
*/
if (map->type & BTRFS_BLOCK_GROUP_STRIPE_MASK)
return stripe_len - *stripe_offset;
return BTRFS_STRIPE_LEN - *stripe_offset;
return U64_MAX;
}
static void set_io_stripe(struct btrfs_io_stripe *dst, const struct map_lookup *map,
u32 stripe_index, u64 stripe_offset, u64 stripe_nr)
u32 stripe_index, u64 stripe_offset, u32 stripe_nr)
{
dst->dev = map->stripes[stripe_index].dev;
dst->physical = map->stripes[stripe_index].physical +
stripe_offset + stripe_nr * map->stripe_len;
stripe_offset + (stripe_nr << BTRFS_STRIPE_LEN_SHIFT);
}
int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
@ -6356,35 +6233,35 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
struct map_lookup *map;
u64 map_offset;
u64 stripe_offset;
u64 stripe_nr;
u64 stripe_len;
u32 stripe_nr;
u32 stripe_index;
int data_stripes;
int i;
int ret = 0;
int mirror_num = (mirror_num_ret ? *mirror_num_ret : 0);
int num_stripes;
int num_copies;
int max_errors = 0;
int tgtdev_indexes = 0;
struct btrfs_io_context *bioc = NULL;
struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
int dev_replace_is_ongoing = 0;
int num_alloc_stripes;
int patch_the_first_stripe_for_dev_replace = 0;
u64 physical_to_patch_in_first_stripe = 0;
u16 num_alloc_stripes;
u64 raid56_full_stripe_start = (u64)-1;
u64 max_len;
ASSERT(bioc_ret);
ASSERT(op != BTRFS_MAP_DISCARD);
num_copies = btrfs_num_copies(fs_info, logical, fs_info->sectorsize);
if (mirror_num > num_copies)
return -EINVAL;
em = btrfs_get_chunk_map(fs_info, logical, *length);
if (IS_ERR(em))
return PTR_ERR(em);
map = em->map_lookup;
data_stripes = nr_data_stripes(map);
stripe_len = map->stripe_len;
map_offset = logical - em->start;
max_len = btrfs_max_io_len(map, op, map_offset, &stripe_nr,
@ -6400,25 +6277,11 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
if (!dev_replace_is_ongoing)
up_read(&dev_replace->rwsem);
if (dev_replace_is_ongoing && mirror_num == map->num_stripes + 1 &&
!need_full_stripe(op) && dev_replace->tgtdev != NULL) {
ret = get_extra_mirror_from_replace(fs_info, logical, *length,
dev_replace->srcdev->devid,
&mirror_num,
&physical_to_patch_in_first_stripe);
if (ret)
goto out;
else
patch_the_first_stripe_for_dev_replace = 1;
} else if (mirror_num > map->num_stripes) {
mirror_num = 0;
}
num_stripes = 1;
stripe_index = 0;
if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
&stripe_index);
stripe_index = stripe_nr % map->num_stripes;
stripe_nr /= map->num_stripes;
if (!need_full_stripe(op))
mirror_num = 1;
} else if (map->type & BTRFS_BLOCK_GROUP_RAID1_MASK) {
@ -6444,8 +6307,8 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
} else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
u32 factor = map->num_stripes / map->sub_stripes;
stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index);
stripe_index *= map->sub_stripes;
stripe_index = (stripe_nr % factor) * map->sub_stripes;
stripe_nr /= factor;
if (need_full_stripe(op))
num_stripes = map->sub_stripes;
@ -6460,11 +6323,17 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
}
} else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
ASSERT(map->stripe_len == BTRFS_STRIPE_LEN);
if (need_raid_map && (need_full_stripe(op) || mirror_num > 1)) {
/* push stripe_nr back to the start of the full stripe */
stripe_nr = div64_u64(raid56_full_stripe_start,
stripe_len * data_stripes);
/*
* Push stripe_nr back to the start of the full stripe
* For those cases needing a full stripe, @stripe_nr
* is the full stripe number.
*
* Originally we go raid56_full_stripe_start / full_stripe_len,
* but that can be expensive. Here we just divide
* @stripe_nr with @data_stripes.
*/
stripe_nr /= data_stripes;
/* RAID[56] write or recovery. Return all stripes */
num_stripes = map->num_stripes;
@ -6473,7 +6342,7 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
/* Return the length to the full stripe end */
*length = min(logical + *length,
raid56_full_stripe_start + em->start +
data_stripes * stripe_len) - logical;
(data_stripes << BTRFS_STRIPE_LEN_SHIFT)) - logical;
stripe_index = 0;
stripe_offset = 0;
} else {
@ -6482,25 +6351,24 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
* Mirror #2 is RAID5 parity block.
* Mirror #3 is RAID6 Q block.
*/
stripe_nr = div_u64_rem(stripe_nr,
data_stripes, &stripe_index);
stripe_index = stripe_nr % data_stripes;
stripe_nr /= data_stripes;
if (mirror_num > 1)
stripe_index = data_stripes + mirror_num - 2;
/* We distribute the parity blocks across stripes */
div_u64_rem(stripe_nr + stripe_index, map->num_stripes,
&stripe_index);
stripe_index = (stripe_nr + stripe_index) % map->num_stripes;
if (!need_full_stripe(op) && mirror_num <= 1)
mirror_num = 1;
}
} else {
/*
* after this, stripe_nr is the number of stripes on this
* After this, stripe_nr is the number of stripes on this
* device we have to walk to find the data, and stripe_index is
* the number of our device in the stripe array
*/
stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
&stripe_index);
stripe_index = stripe_nr % map->num_stripes;
stripe_nr /= map->num_stripes;
mirror_num = stripe_index + 1;
}
if (stripe_index >= map->num_stripes) {
@ -6512,13 +6380,16 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
}
num_alloc_stripes = num_stripes;
if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL) {
if (op == BTRFS_MAP_WRITE)
num_alloc_stripes <<= 1;
if (op == BTRFS_MAP_GET_READ_MIRRORS)
num_alloc_stripes++;
tgtdev_indexes = num_stripes;
}
if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL &&
op != BTRFS_MAP_READ)
/*
* For replace case, we need to add extra stripes for extra
* duplicated stripes.
*
* For both WRITE and GET_READ_MIRRORS, we may have at most
* 2 more stripes (DUP types, otherwise 1).
*/
num_alloc_stripes += 2;
/*
* If this I/O maps to a single device, try to return the device and
@ -6529,53 +6400,53 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
!((map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) && mirror_num > 1) &&
(!need_full_stripe(op) || !dev_replace_is_ongoing ||
!dev_replace->tgtdev)) {
if (patch_the_first_stripe_for_dev_replace) {
smap->dev = dev_replace->tgtdev;
smap->physical = physical_to_patch_in_first_stripe;
*mirror_num_ret = map->num_stripes + 1;
} else {
set_io_stripe(smap, map, stripe_index, stripe_offset,
stripe_nr);
*mirror_num_ret = mirror_num;
}
set_io_stripe(smap, map, stripe_index, stripe_offset, stripe_nr);
*mirror_num_ret = mirror_num;
*bioc_ret = NULL;
ret = 0;
goto out;
}
bioc = alloc_btrfs_io_context(fs_info, num_alloc_stripes, tgtdev_indexes);
bioc = alloc_btrfs_io_context(fs_info, num_alloc_stripes);
if (!bioc) {
ret = -ENOMEM;
goto out;
}
bioc->map_type = map->type;
for (i = 0; i < num_stripes; i++) {
set_io_stripe(&bioc->stripes[i], map, stripe_index, stripe_offset,
stripe_nr);
stripe_index++;
}
/* Build raid_map */
/*
* For RAID56 full map, we need to make sure the stripes[] follows the
* rule that data stripes are all ordered, then followed with P and Q
* (if we have).
*
* It's still mostly the same as other profiles, just with extra rotation.
*/
if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK && need_raid_map &&
(need_full_stripe(op) || mirror_num > 1)) {
u64 tmp;
unsigned rot;
/* Work out the disk rotation on this stripe-set */
div_u64_rem(stripe_nr, num_stripes, &rot);
/* Fill in the logical address of each stripe */
tmp = stripe_nr * data_stripes;
for (i = 0; i < data_stripes; i++)
bioc->raid_map[(i + rot) % num_stripes] =
em->start + (tmp + i) * map->stripe_len;
bioc->raid_map[(i + rot) % map->num_stripes] = RAID5_P_STRIPE;
if (map->type & BTRFS_BLOCK_GROUP_RAID6)
bioc->raid_map[(i + rot + 1) % num_stripes] =
RAID6_Q_STRIPE;
sort_parity_stripes(bioc, num_stripes);
/*
* For RAID56 @stripe_nr is already the number of full stripes
* before us, which is also the rotation value (needs to modulo
* with num_stripes).
*
* In this case, we just add @stripe_nr with @i, then do the
* modulo, to reduce one modulo call.
*/
bioc->full_stripe_logical = em->start +
((stripe_nr * data_stripes) << BTRFS_STRIPE_LEN_SHIFT);
for (i = 0; i < num_stripes; i++)
set_io_stripe(&bioc->stripes[i], map,
(i + stripe_nr) % num_stripes,
stripe_offset, stripe_nr);
} else {
/*
* For all other non-RAID56 profiles, just copy the target
* stripe into the bioc.
*/
for (i = 0; i < num_stripes; i++) {
set_io_stripe(&bioc->stripes[i], map, stripe_index,
stripe_offset, stripe_nr);
stripe_index++;
}
}
if (need_full_stripe(op))
@ -6583,27 +6454,15 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL &&
need_full_stripe(op)) {
handle_ops_on_dev_replace(op, &bioc, dev_replace, logical,
handle_ops_on_dev_replace(op, bioc, dev_replace, logical,
&num_stripes, &max_errors);
}
*bioc_ret = bioc;
bioc->map_type = map->type;
bioc->num_stripes = num_stripes;
bioc->max_errors = max_errors;
bioc->mirror_num = mirror_num;
/*
* this is the case that REQ_READ && dev_replace_is_ongoing &&
* mirror_num == num_stripes + 1 && dev_replace target drive is
* available as a mirror
*/
if (patch_the_first_stripe_for_dev_replace && num_stripes > 0) {
WARN_ON(num_stripes > 1);
bioc->stripes[0].dev = dev_replace->tgtdev;
bioc->stripes[0].physical = physical_to_patch_in_first_stripe;
bioc->mirror_num = map->num_stripes + 1;
}
out:
if (dev_replace_is_ongoing) {
lockdep_assert_held(&dev_replace->rwsem);
@ -6941,7 +6800,6 @@ static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf,
map->num_stripes = num_stripes;
map->io_width = btrfs_chunk_io_width(leaf, chunk);
map->io_align = btrfs_chunk_io_align(leaf, chunk);
map->stripe_len = btrfs_chunk_stripe_len(leaf, chunk);
map->type = type;
/*
* We can't use the sub_stripes value, as for profiles other than
@ -8161,3 +8019,76 @@ bool btrfs_repair_one_zone(struct btrfs_fs_info *fs_info, u64 logical)
return true;
}
static void map_raid56_repair_block(struct btrfs_io_context *bioc,
struct btrfs_io_stripe *smap,
u64 logical)
{
int data_stripes = nr_bioc_data_stripes(bioc);
int i;
for (i = 0; i < data_stripes; i++) {
u64 stripe_start = bioc->full_stripe_logical +
(i << BTRFS_STRIPE_LEN_SHIFT);
if (logical >= stripe_start &&
logical < stripe_start + BTRFS_STRIPE_LEN)
break;
}
ASSERT(i < data_stripes);
smap->dev = bioc->stripes[i].dev;
smap->physical = bioc->stripes[i].physical +
((logical - bioc->full_stripe_logical) &
BTRFS_STRIPE_LEN_MASK);
}
/*
* Map a repair write into a single device.
*
* A repair write is triggered by read time repair or scrub, which would only
* update the contents of a single device.
* Not update any other mirrors nor go through RMW path.
*
* Callers should ensure:
*
* - Call btrfs_bio_counter_inc_blocked() first
* - The range does not cross stripe boundary
* - Has a valid @mirror_num passed in.
*/
int btrfs_map_repair_block(struct btrfs_fs_info *fs_info,
struct btrfs_io_stripe *smap, u64 logical,
u32 length, int mirror_num)
{
struct btrfs_io_context *bioc = NULL;
u64 map_length = length;
int mirror_ret = mirror_num;
int ret;
ASSERT(mirror_num > 0);
ret = __btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical, &map_length,
&bioc, smap, &mirror_ret, true);
if (ret < 0)
return ret;
/* The map range should not cross stripe boundary. */
ASSERT(map_length >= length);
/* Already mapped to single stripe. */
if (!bioc)
goto out;
/* Map the RAID56 multi-stripe writes to a single one. */
if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
map_raid56_repair_block(bioc, smap, logical);
goto out;
}
ASSERT(mirror_num <= bioc->num_stripes);
smap->dev = bioc->stripes[mirror_num - 1].dev;
smap->physical = bioc->stripes[mirror_num - 1].physical;
out:
btrfs_put_bioc(bioc);
ASSERT(smap->dev);
return 0;
}

85
fs/btrfs/volumes.h
View file

@ -17,7 +17,11 @@
extern struct mutex uuid_mutex;
#define BTRFS_STRIPE_LEN SZ_64K
#define BTRFS_STRIPE_LEN SZ_64K
#define BTRFS_STRIPE_LEN_SHIFT (16)
#define BTRFS_STRIPE_LEN_MASK (BTRFS_STRIPE_LEN - 1)
static_assert(const_ilog2(BTRFS_STRIPE_LEN) == BTRFS_STRIPE_LEN_SHIFT);
/* Used by sanity check for btrfs_raid_types. */
#define const_ffs(n) (__builtin_ctzll(n) + 1)
@ -404,17 +408,74 @@ struct btrfs_io_context {
u64 map_type; /* get from map_lookup->type */
struct bio *orig_bio;
atomic_t error;
int max_errors;
int num_stripes;
int mirror_num;
int num_tgtdevs;
int *tgtdev_map;
u16 max_errors;
/*
* logical block numbers for the start of each stripe
* The last one or two are p/q. These are sorted,
* so raid_map[0] is the start of our full stripe
* The total number of stripes, including the extra duplicated
* stripe for replace.
*/
u64 *raid_map;
u16 num_stripes;
/*
* The mirror_num of this bioc.
*
* This is for reads which use 0 as mirror_num, thus we should return a
* valid mirror_num (>0) for the reader.
*/
u16 mirror_num;
/*
* The following two members are for dev-replace case only.
*
* @replace_nr_stripes: Number of duplicated stripes which need to be
* written to replace target.
* Should be <= 2 (2 for DUP, otherwise <= 1).
* @replace_stripe_src: The array indicates where the duplicated stripes
* are from.
*
* The @replace_stripe_src[] array is mostly for RAID56 cases.
* As non-RAID56 stripes share the same contents of the mapped range,
* thus no need to bother where the duplicated ones are from.
*
* But for RAID56 case, all stripes contain different contents, thus
* we need a way to know the mapping.
*
* There is an example for the two members, using a RAID5 write:
*
* num_stripes: 4 (3 + 1 duplicated write)
* stripes[0]: dev = devid 1, physical = X
* stripes[1]: dev = devid 2, physical = Y
* stripes[2]: dev = devid 3, physical = Z
* stripes[3]: dev = devid 0, physical = Y
*
* replace_nr_stripes = 1
* replace_stripe_src = 1 <- Means stripes[1] is involved in replace.
* The duplicated stripe index would be
* (@num_stripes - 1).
*
* Note, that we can still have cases replace_nr_stripes = 2 for DUP.
* In that case, all stripes share the same content, thus we don't
* need to bother @replace_stripe_src value at all.
*/
u16 replace_nr_stripes;
s16 replace_stripe_src;
/*
* Logical bytenr of the full stripe start, only for RAID56 cases.
*
* When this value is set to other than (u64)-1, the stripes[] should
* follow this pattern:
*
* (real_stripes = num_stripes - replace_nr_stripes)
* (data_stripes = (is_raid6) ? (real_stripes - 2) : (real_stripes - 1))
*
* stripes[0]: The first data stripe
* stripes[1]: The second data stripe
* ...
* stripes[data_stripes - 1]: The last data stripe
* stripes[data_stripes]: The P stripe
* stripes[data_stripes + 1]: The Q stripe (only for RAID6).
*/
u64 full_stripe_logical;
struct btrfs_io_stripe stripes[];
};
@ -446,7 +507,6 @@ struct map_lookup {
u64 type;
int io_align;
int io_width;
u32 stripe_len;
int num_stripes;
int sub_stripes;
int verified_stripes; /* For mount time dev extent verification */
@ -527,6 +587,9 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
struct btrfs_io_context **bioc_ret,
struct btrfs_io_stripe *smap, int *mirror_num_ret,
int need_raid_map);
int btrfs_map_repair_block(struct btrfs_fs_info *fs_info,
struct btrfs_io_stripe *smap, u64 logical,
u32 length, int mirror_num);
struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info,
u64 logical, u64 *length_ret,
u32 *num_stripes);

2
fs/btrfs/zlib.c
View file

@ -350,8 +350,6 @@ int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
zlib_inflateEnd(&workspace->strm);
if (data_in)
kunmap_local(data_in);
if (!ret)
zero_fill_bio(cb->orig_bio);
return ret;
}

4
fs/btrfs/zoned.c
View file

@ -1640,14 +1640,14 @@ bool btrfs_use_zone_append(struct btrfs_bio *bbio)
{
u64 start = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT);
struct btrfs_inode *inode = bbio->inode;
struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct btrfs_fs_info *fs_info = bbio->fs_info;
struct btrfs_block_group *cache;
bool ret = false;
if (!btrfs_is_zoned(fs_info))
return false;
if (!is_data_inode(&inode->vfs_inode))
if (!inode || !is_data_inode(&inode->vfs_inode))
return false;
if (btrfs_op(&bbio->bio) != BTRFS_MAP_WRITE)

1
fs/btrfs/zstd.c
View file

@ -609,7 +609,6 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
}
}
ret = 0;
zero_fill_bio(cb->orig_bio);
done:
if (workspace->in_buf.src)
kunmap_local(workspace->in_buf.src);

5
include/linux/bio.h
View file

@ -500,6 +500,7 @@ void bio_associate_blkg(struct bio *bio);
void bio_associate_blkg_from_css(struct bio *bio,
struct cgroup_subsys_state *css);
void bio_clone_blkg_association(struct bio *dst, struct bio *src);
void blkcg_punt_bio_submit(struct bio *bio);
#else /* CONFIG_BLK_CGROUP */
static inline void bio_associate_blkg(struct bio *bio) { }
static inline void bio_associate_blkg_from_css(struct bio *bio,
@ -507,6 +508,10 @@ static inline void bio_associate_blkg_from_css(struct bio *bio,
{ }
static inline void bio_clone_blkg_association(struct bio *dst,
struct bio *src) { }
static inline void blkcg_punt_bio_submit(struct bio *bio)
{
submit_bio(bio);
}
#endif /* CONFIG_BLK_CGROUP */
static inline void bio_set_dev(struct bio *bio, struct block_device *bdev)

18
include/linux/blk_types.h
View file

@ -404,18 +404,11 @@ enum req_flag_bits {
__REQ_RAHEAD, /* read ahead, can fail anytime */
__REQ_BACKGROUND, /* background IO */
__REQ_NOWAIT, /* Don't wait if request will block */
/*
* When a shared kthread needs to issue a bio for a cgroup, doing
* so synchronously can lead to priority inversions as the kthread
* can be trapped waiting for that cgroup. CGROUP_PUNT flag makes
* submit_bio() punt the actual issuing to a dedicated per-blkcg
* work item to avoid such priority inversions.
*/
__REQ_CGROUP_PUNT,
__REQ_POLLED, /* caller polls for completion using bio_poll */
__REQ_ALLOC_CACHE, /* allocate IO from cache if available */
__REQ_SWAP, /* swap I/O */
__REQ_DRV, /* for driver use */
__REQ_FS_PRIVATE, /* for file system (submitter) use */
/*
* Command specific flags, keep last:
@ -443,14 +436,13 @@ enum req_flag_bits {
#define REQ_RAHEAD (__force blk_opf_t)(1ULL << __REQ_RAHEAD)
#define REQ_BACKGROUND (__force blk_opf_t)(1ULL << __REQ_BACKGROUND)
#define REQ_NOWAIT (__force blk_opf_t)(1ULL << __REQ_NOWAIT)
#define REQ_CGROUP_PUNT (__force blk_opf_t)(1ULL << __REQ_CGROUP_PUNT)
#define REQ_NOUNMAP (__force blk_opf_t)(1ULL << __REQ_NOUNMAP)
#define REQ_POLLED (__force blk_opf_t)(1ULL << __REQ_POLLED)
#define REQ_ALLOC_CACHE (__force blk_opf_t)(1ULL << __REQ_ALLOC_CACHE)
#define REQ_DRV (__force blk_opf_t)(1ULL << __REQ_DRV)
#define REQ_SWAP (__force blk_opf_t)(1ULL << __REQ_SWAP)
#define REQ_DRV (__force blk_opf_t)(1ULL << __REQ_DRV)
#define REQ_FS_PRIVATE (__force blk_opf_t)(1ULL << __REQ_FS_PRIVATE)
#define REQ_NOUNMAP (__force blk_opf_t)(1ULL << __REQ_NOUNMAP)
#define REQ_FAILFAST_MASK \
(REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER)

1
include/linux/crc32c.h
View file

@ -5,7 +5,6 @@
#include <linux/types.h>
extern u32 crc32c(u32 crc, const void *address, unsigned int length);
extern const char *crc32c_impl(void);
/* This macro exists for backwards-compatibility. */
#define crc32c_le crc32c

5
include/linux/writeback.h
View file

@ -70,8 +70,6 @@ struct writeback_control {
*/
unsigned no_cgroup_owner:1;
unsigned punt_to_cgroup:1; /* cgrp punting, see __REQ_CGROUP_PUNT */
/* To enable batching of swap writes to non-block-device backends,
* "plug" can be set point to a 'struct swap_iocb *'. When all swap
* writes have been submitted, if with swap_iocb is not NULL,
@ -97,9 +95,6 @@ static inline blk_opf_t wbc_to_write_flags(struct writeback_control *wbc)
{
blk_opf_t flags = 0;
if (wbc->punt_to_cgroup)
flags = REQ_CGROUP_PUNT;
if (wbc->sync_mode == WB_SYNC_ALL)
flags |= REQ_SYNC;
else if (wbc->for_kupdate || wbc->for_background)

2
include/trace/events/btrfs.h
View file

@ -2422,7 +2422,7 @@ DECLARE_EVENT_CLASS(btrfs_raid56_bio,
),
TP_fast_assign_btrfs(rbio->bioc->fs_info,
__entry->full_stripe = rbio->bioc->raid_map[0];
__entry->full_stripe = rbio->bioc->full_stripe_logical;
__entry->physical = bio->bi_iter.bi_sector << SECTOR_SHIFT;
__entry->len = bio->bi_iter.bi_size;
__entry->opf = bio_op(bio);

1
include/uapi/linux/btrfs.h
View file

@ -187,6 +187,7 @@ struct btrfs_scrub_progress {
};
#define BTRFS_SCRUB_READONLY 1
#define BTRFS_SCRUB_SUPPORTED_FLAGS (BTRFS_SCRUB_READONLY)
struct btrfs_ioctl_scrub_args {
__u64 devid; /* in */
__u64 start; /* in */

6
lib/libcrc32c.c
View file

@ -65,12 +65,6 @@ static void __exit libcrc32c_mod_fini(void)
crypto_free_shash(tfm);
}
const char *crc32c_impl(void)
{
return crypto_shash_driver_name(tfm);
}
EXPORT_SYMBOL(crc32c_impl);
module_init(libcrc32c_mod_init);
module_exit(libcrc32c_mod_fini);

1
tools/objtool/check.c
View file

@ -202,6 +202,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
"__reiserfs_panic",
"__stack_chk_fail",
"__ubsan_handle_builtin_unreachable",
"btrfs_assertfail",
"cpu_bringup_and_idle",
"cpu_startup_entry",
"do_exit",