2017-03-17 06:18:50 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
/*
|
|
|
|
* bcachefs journalling code, for btree insertions
|
|
|
|
*
|
|
|
|
* Copyright 2012 Google, Inc.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "bcachefs.h"
|
2018-10-06 04:46:55 +00:00
|
|
|
#include "alloc_foreground.h"
|
2017-03-17 06:18:50 +00:00
|
|
|
#include "bkey_methods.h"
|
|
|
|
#include "btree_gc.h"
|
|
|
|
#include "buckets.h"
|
|
|
|
#include "journal.h"
|
|
|
|
#include "journal_io.h"
|
|
|
|
#include "journal_reclaim.h"
|
|
|
|
#include "journal_seq_blacklist.h"
|
|
|
|
#include "super-io.h"
|
|
|
|
#include "trace.h"
|
|
|
|
|
2020-11-14 21:04:30 +00:00
|
|
|
static u64 last_unwritten_seq(struct journal *j)
|
|
|
|
{
|
|
|
|
union journal_res_state s = READ_ONCE(j->reservations);
|
|
|
|
|
|
|
|
lockdep_assert_held(&j->lock);
|
|
|
|
|
|
|
|
return journal_cur_seq(j) - s.prev_buf_unwritten;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool journal_seq_unwritten(struct journal *j, u64 seq)
|
|
|
|
{
|
|
|
|
return seq >= last_unwritten_seq(j);
|
|
|
|
}
|
2020-10-07 02:18:21 +00:00
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
static bool __journal_entry_is_open(union journal_res_state state)
|
2017-03-17 06:18:50 +00:00
|
|
|
{
|
2019-02-18 22:39:42 +00:00
|
|
|
return state.cur_entry_offset < JOURNAL_ENTRY_CLOSED_VAL;
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
static bool journal_entry_is_open(struct journal *j)
|
2017-03-17 06:18:50 +00:00
|
|
|
{
|
2019-02-18 22:39:42 +00:00
|
|
|
return __journal_entry_is_open(j->reservations);
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
2020-11-14 21:04:30 +00:00
|
|
|
static inline struct journal_buf *
|
|
|
|
journal_seq_to_buf(struct journal *j, u64 seq)
|
|
|
|
{
|
|
|
|
struct journal_buf *buf = NULL;
|
|
|
|
|
|
|
|
EBUG_ON(seq > journal_cur_seq(j));
|
|
|
|
EBUG_ON(seq == journal_cur_seq(j) &&
|
|
|
|
j->reservations.cur_entry_offset == JOURNAL_ENTRY_CLOSED_VAL);
|
|
|
|
|
|
|
|
if (journal_seq_unwritten(j, seq)) {
|
|
|
|
buf = j->buf + (seq & 1);
|
|
|
|
EBUG_ON(le64_to_cpu(buf->data->seq) != seq);
|
|
|
|
}
|
|
|
|
return buf;
|
|
|
|
}
|
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
static void journal_pin_new_entry(struct journal *j, int count)
|
|
|
|
{
|
|
|
|
struct journal_entry_pin_list *p;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The fifo_push() needs to happen at the same time as j->seq is
|
|
|
|
* incremented for journal_last_seq() to be calculated correctly
|
|
|
|
*/
|
|
|
|
atomic64_inc(&j->seq);
|
|
|
|
p = fifo_push_ref(&j->pin);
|
|
|
|
|
|
|
|
INIT_LIST_HEAD(&p->list);
|
|
|
|
INIT_LIST_HEAD(&p->flushed);
|
|
|
|
atomic_set(&p->count, count);
|
|
|
|
p->devs.nr = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void bch2_journal_buf_init(struct journal *j)
|
|
|
|
{
|
|
|
|
struct journal_buf *buf = journal_cur_buf(j);
|
|
|
|
|
2020-11-14 21:04:30 +00:00
|
|
|
bkey_extent_init(&buf->key);
|
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
memset(buf->has_inode, 0, sizeof(buf->has_inode));
|
|
|
|
|
|
|
|
memset(buf->data, 0, sizeof(*buf->data));
|
|
|
|
buf->data->seq = cpu_to_le64(journal_cur_seq(j));
|
|
|
|
buf->data->u64s = 0;
|
|
|
|
}
|
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
void bch2_journal_halt(struct journal *j)
|
|
|
|
{
|
|
|
|
union journal_res_state old, new;
|
|
|
|
u64 v = atomic64_read(&j->reservations.counter);
|
|
|
|
|
|
|
|
do {
|
|
|
|
old.v = new.v = v;
|
|
|
|
if (old.cur_entry_offset == JOURNAL_ENTRY_ERROR_VAL)
|
|
|
|
return;
|
|
|
|
|
|
|
|
new.cur_entry_offset = JOURNAL_ENTRY_ERROR_VAL;
|
|
|
|
} while ((v = atomic64_cmpxchg(&j->reservations.counter,
|
|
|
|
old.v, new.v)) != old.v);
|
|
|
|
|
2020-11-14 21:04:30 +00:00
|
|
|
j->err_seq = journal_cur_seq(j);
|
2019-02-18 22:39:42 +00:00
|
|
|
journal_wake(j);
|
|
|
|
closure_wake_up(&journal_cur_buf(j)->wait);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* journal entry close/open: */
|
|
|
|
|
|
|
|
void __bch2_journal_buf_put(struct journal *j, bool need_write_just_set)
|
|
|
|
{
|
|
|
|
if (!need_write_just_set &&
|
|
|
|
test_bit(JOURNAL_NEED_WRITE, &j->flags))
|
|
|
|
bch2_time_stats_update(j->delay_time,
|
|
|
|
j->need_write_time);
|
|
|
|
|
|
|
|
clear_bit(JOURNAL_NEED_WRITE, &j->flags);
|
|
|
|
|
|
|
|
closure_call(&j->io, bch2_journal_write, system_highpri_wq, NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Returns true if journal entry is now closed:
|
|
|
|
*/
|
|
|
|
static bool __journal_entry_close(struct journal *j)
|
2017-03-17 06:18:50 +00:00
|
|
|
{
|
|
|
|
struct bch_fs *c = container_of(j, struct bch_fs, journal);
|
2018-07-23 11:52:00 +00:00
|
|
|
struct journal_buf *buf = journal_cur_buf(j);
|
2017-03-17 06:18:50 +00:00
|
|
|
union journal_res_state old, new;
|
|
|
|
u64 v = atomic64_read(&j->reservations.counter);
|
2019-02-18 22:39:42 +00:00
|
|
|
bool set_need_write = false;
|
|
|
|
unsigned sectors;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
lockdep_assert_held(&j->lock);
|
|
|
|
|
|
|
|
do {
|
|
|
|
old.v = new.v = v;
|
|
|
|
if (old.cur_entry_offset == JOURNAL_ENTRY_CLOSED_VAL)
|
2019-02-18 22:39:42 +00:00
|
|
|
return true;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2018-07-23 11:52:00 +00:00
|
|
|
if (old.cur_entry_offset == JOURNAL_ENTRY_ERROR_VAL) {
|
|
|
|
/* this entry will never be written: */
|
|
|
|
closure_wake_up(&buf->wait);
|
2019-02-18 22:39:42 +00:00
|
|
|
return true;
|
2018-07-23 11:52:00 +00:00
|
|
|
}
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
if (!test_bit(JOURNAL_NEED_WRITE, &j->flags)) {
|
|
|
|
set_bit(JOURNAL_NEED_WRITE, &j->flags);
|
|
|
|
j->need_write_time = local_clock();
|
|
|
|
set_need_write = true;
|
|
|
|
}
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
if (new.prev_buf_unwritten)
|
|
|
|
return false;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
new.cur_entry_offset = JOURNAL_ENTRY_CLOSED_VAL;
|
|
|
|
new.idx++;
|
|
|
|
new.prev_buf_unwritten = 1;
|
|
|
|
|
|
|
|
BUG_ON(journal_state_count(new, new.idx));
|
|
|
|
} while ((v = atomic64_cmpxchg(&j->reservations.counter,
|
|
|
|
old.v, new.v)) != old.v);
|
|
|
|
|
|
|
|
buf->data->u64s = cpu_to_le32(old.cur_entry_offset);
|
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
sectors = vstruct_blocks_plus(buf->data, c->block_bits,
|
|
|
|
buf->u64s_reserved) << c->block_bits;
|
|
|
|
BUG_ON(sectors > buf->sectors);
|
|
|
|
buf->sectors = sectors;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2018-07-17 16:19:14 +00:00
|
|
|
/*
|
|
|
|
* We have to set last_seq here, _before_ opening a new journal entry:
|
|
|
|
*
|
|
|
|
* A threads may replace an old pin with a new pin on their current
|
|
|
|
* journal reservation - the expectation being that the journal will
|
|
|
|
* contain either what the old pin protected or what the new pin
|
|
|
|
* protects.
|
|
|
|
*
|
|
|
|
* After the old pin is dropped journal_last_seq() won't include the old
|
|
|
|
* pin, so we can only write the updated last_seq on the entry that
|
|
|
|
* contains whatever the new pin protects.
|
|
|
|
*
|
|
|
|
* Restated, we can _not_ update last_seq for a given entry if there
|
|
|
|
* could be a newer entry open with reservations/pins that have been
|
|
|
|
* taken against it.
|
|
|
|
*
|
|
|
|
* Hence, we want update/set last_seq on the current journal entry right
|
|
|
|
* before we open a new one:
|
|
|
|
*/
|
2017-03-17 06:18:50 +00:00
|
|
|
buf->data->last_seq = cpu_to_le64(journal_last_seq(j));
|
|
|
|
|
|
|
|
journal_pin_new_entry(j, 1);
|
|
|
|
|
|
|
|
bch2_journal_buf_init(j);
|
|
|
|
|
|
|
|
cancel_delayed_work(&j->write_work);
|
|
|
|
|
2019-02-21 18:33:21 +00:00
|
|
|
bch2_journal_space_available(j);
|
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
bch2_journal_buf_put(j, old.idx, set_need_write);
|
|
|
|
return true;
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
static bool journal_entry_close(struct journal *j)
|
2017-03-17 06:18:50 +00:00
|
|
|
{
|
2019-02-18 22:39:42 +00:00
|
|
|
bool ret;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
spin_lock(&j->lock);
|
|
|
|
ret = __journal_entry_close(j);
|
|
|
|
spin_unlock(&j->lock);
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
return ret;
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* should _only_ called from journal_res_get() - when we actually want a
|
|
|
|
* journal reservation - journal entry is open means journal is dirty:
|
|
|
|
*
|
|
|
|
* returns:
|
2019-02-18 22:39:42 +00:00
|
|
|
* 0: success
|
|
|
|
* -ENOSPC: journal currently full, must invoke reclaim
|
|
|
|
* -EAGAIN: journal blocked, must wait
|
|
|
|
* -EROFS: insufficient rw devices or journal error
|
2017-03-17 06:18:50 +00:00
|
|
|
*/
|
|
|
|
static int journal_entry_open(struct journal *j)
|
|
|
|
{
|
2020-11-20 01:55:33 +00:00
|
|
|
struct bch_fs *c = container_of(j, struct bch_fs, journal);
|
2017-03-17 06:18:50 +00:00
|
|
|
struct journal_buf *buf = journal_cur_buf(j);
|
|
|
|
union journal_res_state old, new;
|
2019-02-21 18:33:21 +00:00
|
|
|
int u64s;
|
2017-03-17 06:18:50 +00:00
|
|
|
u64 v;
|
|
|
|
|
2020-11-20 01:55:33 +00:00
|
|
|
BUG_ON(BCH_SB_CLEAN(c->disk_sb.sb));
|
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
lockdep_assert_held(&j->lock);
|
|
|
|
BUG_ON(journal_entry_is_open(j));
|
|
|
|
|
2019-02-14 23:38:52 +00:00
|
|
|
if (j->blocked)
|
2020-11-18 18:21:59 +00:00
|
|
|
return cur_entry_blocked;
|
2019-02-14 23:38:52 +00:00
|
|
|
|
2019-02-21 18:33:21 +00:00
|
|
|
if (j->cur_entry_error)
|
|
|
|
return j->cur_entry_error;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-02-21 18:33:21 +00:00
|
|
|
BUG_ON(!j->cur_entry_sectors);
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-01-24 21:50:48 +00:00
|
|
|
buf->u64s_reserved = j->entry_u64s_reserved;
|
2019-02-18 22:39:42 +00:00
|
|
|
buf->disk_sectors = j->cur_entry_sectors;
|
|
|
|
buf->sectors = min(buf->disk_sectors, buf->buf_size >> 9);
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
u64s = (int) (buf->sectors << 9) / sizeof(u64) -
|
|
|
|
journal_entry_overhead(j);
|
|
|
|
u64s = clamp_t(int, u64s, 0, JOURNAL_ENTRY_CLOSED_VAL - 1);
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
if (u64s <= le32_to_cpu(buf->data->u64s))
|
2020-11-18 18:21:59 +00:00
|
|
|
return cur_entry_journal_full;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Must be set before marking the journal entry as open:
|
|
|
|
*/
|
|
|
|
j->cur_entry_u64s = u64s;
|
|
|
|
|
|
|
|
v = atomic64_read(&j->reservations.counter);
|
|
|
|
do {
|
|
|
|
old.v = new.v = v;
|
|
|
|
|
|
|
|
if (old.cur_entry_offset == JOURNAL_ENTRY_ERROR_VAL)
|
2020-11-18 18:21:59 +00:00
|
|
|
return cur_entry_insufficient_devices;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
/* Handle any already added entries */
|
|
|
|
new.cur_entry_offset = le32_to_cpu(buf->data->u64s);
|
2019-03-08 04:14:35 +00:00
|
|
|
|
|
|
|
EBUG_ON(journal_state_count(new, new.idx));
|
2019-02-18 22:39:42 +00:00
|
|
|
journal_state_inc(&new);
|
2017-03-17 06:18:50 +00:00
|
|
|
} while ((v = atomic64_cmpxchg(&j->reservations.counter,
|
|
|
|
old.v, new.v)) != old.v);
|
|
|
|
|
|
|
|
if (j->res_get_blocked_start)
|
|
|
|
bch2_time_stats_update(j->blocked_time,
|
|
|
|
j->res_get_blocked_start);
|
|
|
|
j->res_get_blocked_start = 0;
|
|
|
|
|
|
|
|
mod_delayed_work(system_freezable_wq,
|
|
|
|
&j->write_work,
|
|
|
|
msecs_to_jiffies(j->write_delay_ms));
|
|
|
|
journal_wake(j);
|
2019-02-18 22:39:42 +00:00
|
|
|
return 0;
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
2019-02-14 23:38:52 +00:00
|
|
|
static bool journal_quiesced(struct journal *j)
|
|
|
|
{
|
2019-02-18 22:39:42 +00:00
|
|
|
union journal_res_state state = READ_ONCE(j->reservations);
|
|
|
|
bool ret = !state.prev_buf_unwritten && !__journal_entry_is_open(state);
|
2019-02-14 23:38:52 +00:00
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
if (!ret)
|
|
|
|
journal_entry_close(j);
|
2019-02-14 23:38:52 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void journal_quiesce(struct journal *j)
|
|
|
|
{
|
|
|
|
wait_event(j->wait, journal_quiesced(j));
|
|
|
|
}
|
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
static void journal_write_work(struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct journal *j = container_of(work, struct journal, write_work.work);
|
|
|
|
|
2018-07-23 11:52:00 +00:00
|
|
|
journal_entry_close(j);
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Given an inode number, if that inode number has data in the journal that
|
|
|
|
* hasn't yet been flushed, return the journal sequence number that needs to be
|
|
|
|
* flushed:
|
|
|
|
*/
|
|
|
|
u64 bch2_inode_journal_seq(struct journal *j, u64 inode)
|
|
|
|
{
|
|
|
|
size_t h = hash_64(inode, ilog2(sizeof(j->buf[0].has_inode) * 8));
|
|
|
|
u64 seq = 0;
|
|
|
|
|
|
|
|
if (!test_bit(h, j->buf[0].has_inode) &&
|
|
|
|
!test_bit(h, j->buf[1].has_inode))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
spin_lock(&j->lock);
|
|
|
|
if (test_bit(h, journal_cur_buf(j)->has_inode))
|
|
|
|
seq = journal_cur_seq(j);
|
|
|
|
else if (test_bit(h, journal_prev_buf(j)->has_inode))
|
|
|
|
seq = journal_cur_seq(j) - 1;
|
|
|
|
spin_unlock(&j->lock);
|
|
|
|
|
|
|
|
return seq;
|
|
|
|
}
|
|
|
|
|
2020-10-07 02:18:21 +00:00
|
|
|
void bch2_journal_set_has_inum(struct journal *j, u64 inode, u64 seq)
|
|
|
|
{
|
|
|
|
size_t h = hash_64(inode, ilog2(sizeof(j->buf[0].has_inode) * 8));
|
|
|
|
struct journal_buf *buf;
|
|
|
|
|
|
|
|
spin_lock(&j->lock);
|
|
|
|
|
|
|
|
if ((buf = journal_seq_to_buf(j, seq)))
|
|
|
|
set_bit(h, buf->has_inode);
|
|
|
|
|
|
|
|
spin_unlock(&j->lock);
|
|
|
|
}
|
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
static int __journal_res_get(struct journal *j, struct journal_res *res,
|
2018-11-19 02:35:59 +00:00
|
|
|
unsigned flags)
|
2017-03-17 06:18:50 +00:00
|
|
|
{
|
|
|
|
struct bch_fs *c = container_of(j, struct bch_fs, journal);
|
|
|
|
struct journal_buf *buf;
|
2019-03-03 23:39:07 +00:00
|
|
|
bool can_discard;
|
2017-03-17 06:18:50 +00:00
|
|
|
int ret;
|
|
|
|
retry:
|
2018-11-19 02:35:59 +00:00
|
|
|
if (journal_res_get_fast(j, res, flags))
|
|
|
|
return 0;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
if (bch2_journal_error(j))
|
|
|
|
return -EROFS;
|
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
spin_lock(&j->lock);
|
2019-02-18 22:39:42 +00:00
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
/*
|
|
|
|
* Recheck after taking the lock, so we don't race with another thread
|
|
|
|
* that just did journal_entry_open() and call journal_entry_close()
|
|
|
|
* unnecessarily
|
|
|
|
*/
|
2018-11-19 02:35:59 +00:00
|
|
|
if (journal_res_get_fast(j, res, flags)) {
|
2017-03-17 06:18:50 +00:00
|
|
|
spin_unlock(&j->lock);
|
2018-11-19 02:35:59 +00:00
|
|
|
return 0;
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
2019-02-19 18:41:36 +00:00
|
|
|
if (!(flags & JOURNAL_RES_GET_RESERVED) &&
|
|
|
|
!test_bit(JOURNAL_MAY_GET_UNRESERVED, &j->flags)) {
|
|
|
|
/*
|
|
|
|
* Don't want to close current journal entry, just need to
|
|
|
|
* invoke reclaim:
|
|
|
|
*/
|
2020-11-18 18:21:59 +00:00
|
|
|
ret = cur_entry_journal_full;
|
2019-02-19 18:41:36 +00:00
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
/*
|
|
|
|
* If we couldn't get a reservation because the current buf filled up,
|
|
|
|
* and we had room for a bigger entry on disk, signal that we want to
|
|
|
|
* realloc the journal bufs:
|
|
|
|
*/
|
|
|
|
buf = journal_cur_buf(j);
|
|
|
|
if (journal_entry_is_open(j) &&
|
2019-02-18 22:39:42 +00:00
|
|
|
buf->buf_size >> 9 < buf->disk_sectors &&
|
|
|
|
buf->buf_size < JOURNAL_ENTRY_SIZE_MAX)
|
|
|
|
j->buf_size_want = max(j->buf_size_want, buf->buf_size << 1);
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
if (journal_entry_is_open(j) &&
|
|
|
|
!__journal_entry_close(j)) {
|
2018-11-19 02:35:59 +00:00
|
|
|
/*
|
2019-02-18 22:39:42 +00:00
|
|
|
* We failed to get a reservation on the current open journal
|
|
|
|
* entry because it's full, and we can't close it because
|
|
|
|
* there's still a previous one in flight:
|
2018-11-19 02:35:59 +00:00
|
|
|
*/
|
2017-03-17 06:18:50 +00:00
|
|
|
trace_journal_entry_full(c);
|
2020-11-18 18:21:59 +00:00
|
|
|
ret = cur_entry_blocked;
|
2019-02-18 22:39:42 +00:00
|
|
|
} else {
|
|
|
|
ret = journal_entry_open(j);
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
2019-02-19 18:41:36 +00:00
|
|
|
unlock:
|
2020-11-18 18:21:59 +00:00
|
|
|
if ((ret && ret != cur_entry_insufficient_devices) &&
|
|
|
|
!j->res_get_blocked_start) {
|
2019-02-18 22:39:42 +00:00
|
|
|
j->res_get_blocked_start = local_clock() ?: 1;
|
2020-11-18 18:21:59 +00:00
|
|
|
trace_journal_full(c);
|
|
|
|
}
|
2019-02-18 22:39:42 +00:00
|
|
|
|
2019-03-03 23:39:07 +00:00
|
|
|
can_discard = j->can_discard;
|
2017-03-17 06:18:50 +00:00
|
|
|
spin_unlock(&j->lock);
|
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
if (!ret)
|
2017-03-17 06:18:50 +00:00
|
|
|
goto retry;
|
2019-03-03 23:39:07 +00:00
|
|
|
|
2020-11-18 18:21:59 +00:00
|
|
|
if (WARN_ONCE(ret == cur_entry_journal_full &&
|
|
|
|
!can_discard &&
|
|
|
|
(flags & JOURNAL_RES_GET_RESERVED),
|
|
|
|
"JOURNAL_RES_GET_RESERVED set but journal full")) {
|
|
|
|
char *buf;
|
|
|
|
|
|
|
|
buf = kmalloc(4096, GFP_NOFS);
|
|
|
|
if (buf) {
|
|
|
|
bch2_journal_debug_to_text(&_PBUF(buf, 4096), j);
|
|
|
|
pr_err("\n%s", buf);
|
|
|
|
kfree(buf);
|
2020-11-16 23:21:55 +00:00
|
|
|
}
|
2020-11-18 18:21:59 +00:00
|
|
|
}
|
2019-02-19 18:41:36 +00:00
|
|
|
|
2020-11-18 18:21:59 +00:00
|
|
|
/*
|
|
|
|
* Journal is full - can't rely on reclaim from work item due to
|
|
|
|
* freezing:
|
|
|
|
*/
|
|
|
|
if ((ret == cur_entry_journal_full ||
|
|
|
|
ret == cur_entry_journal_pin_full) &&
|
|
|
|
!(flags & JOURNAL_RES_GET_NONBLOCK)) {
|
|
|
|
if (can_discard) {
|
|
|
|
bch2_journal_do_discards(j);
|
|
|
|
goto retry;
|
2019-03-03 23:39:07 +00:00
|
|
|
}
|
|
|
|
|
2020-11-18 18:21:59 +00:00
|
|
|
if (mutex_trylock(&j->reclaim_lock)) {
|
|
|
|
bch2_journal_reclaim(j);
|
|
|
|
mutex_unlock(&j->reclaim_lock);
|
|
|
|
}
|
2019-02-18 22:39:42 +00:00
|
|
|
}
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2020-11-18 18:21:59 +00:00
|
|
|
return ret == cur_entry_insufficient_devices ? -EROFS : -EAGAIN;
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Essentially the entry function to the journaling code. When bcachefs is doing
|
|
|
|
* a btree insert, it calls this function to get the current journal write.
|
|
|
|
* Journal write is the structure used set up journal writes. The calling
|
|
|
|
* function will then add its keys to the structure, queuing them for the next
|
|
|
|
* write.
|
|
|
|
*
|
|
|
|
* To ensure forward progress, the current task must not be holding any
|
|
|
|
* btree node write locks.
|
|
|
|
*/
|
|
|
|
int bch2_journal_res_get_slowpath(struct journal *j, struct journal_res *res,
|
2018-11-19 02:35:59 +00:00
|
|
|
unsigned flags)
|
2017-03-17 06:18:50 +00:00
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
2019-02-21 18:33:21 +00:00
|
|
|
closure_wait_event(&j->async_wait,
|
2018-11-19 02:35:59 +00:00
|
|
|
(ret = __journal_res_get(j, res, flags)) != -EAGAIN ||
|
|
|
|
(flags & JOURNAL_RES_GET_NONBLOCK));
|
|
|
|
return ret;
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
2019-02-19 18:41:36 +00:00
|
|
|
/* journal_preres: */
|
|
|
|
|
|
|
|
static bool journal_preres_available(struct journal *j,
|
|
|
|
struct journal_preres *res,
|
2020-06-10 00:54:36 +00:00
|
|
|
unsigned new_u64s,
|
|
|
|
unsigned flags)
|
2019-02-19 18:41:36 +00:00
|
|
|
{
|
2020-06-10 00:54:36 +00:00
|
|
|
bool ret = bch2_journal_preres_get_fast(j, res, new_u64s, flags);
|
2019-02-19 18:41:36 +00:00
|
|
|
|
2020-11-20 01:55:33 +00:00
|
|
|
if (!ret && mutex_trylock(&j->reclaim_lock)) {
|
|
|
|
bch2_journal_reclaim(j);
|
|
|
|
mutex_unlock(&j->reclaim_lock);
|
|
|
|
}
|
2019-02-19 18:41:36 +00:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
int __bch2_journal_preres_get(struct journal *j,
|
|
|
|
struct journal_preres *res,
|
2020-06-10 00:54:36 +00:00
|
|
|
unsigned new_u64s,
|
|
|
|
unsigned flags)
|
2019-02-19 18:41:36 +00:00
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
closure_wait_event(&j->preres_wait,
|
|
|
|
(ret = bch2_journal_error(j)) ||
|
2020-06-10 00:54:36 +00:00
|
|
|
journal_preres_available(j, res, new_u64s, flags));
|
2019-02-19 18:41:36 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2019-01-24 21:50:48 +00:00
|
|
|
/* journal_entry_res: */
|
|
|
|
|
|
|
|
void bch2_journal_entry_res_resize(struct journal *j,
|
|
|
|
struct journal_entry_res *res,
|
|
|
|
unsigned new_u64s)
|
|
|
|
{
|
|
|
|
union journal_res_state state;
|
|
|
|
int d = new_u64s - res->u64s;
|
|
|
|
|
|
|
|
spin_lock(&j->lock);
|
|
|
|
|
|
|
|
j->entry_u64s_reserved += d;
|
|
|
|
if (d <= 0)
|
2019-02-18 22:39:42 +00:00
|
|
|
goto out;
|
2019-01-24 21:50:48 +00:00
|
|
|
|
2019-02-28 00:14:23 +00:00
|
|
|
j->cur_entry_u64s = max_t(int, 0, j->cur_entry_u64s - d);
|
2019-01-24 21:50:48 +00:00
|
|
|
smp_mb();
|
|
|
|
state = READ_ONCE(j->reservations);
|
|
|
|
|
|
|
|
if (state.cur_entry_offset < JOURNAL_ENTRY_CLOSED_VAL &&
|
|
|
|
state.cur_entry_offset > j->cur_entry_u64s) {
|
|
|
|
j->cur_entry_u64s += d;
|
|
|
|
/*
|
|
|
|
* Not enough room in current journal entry, have to flush it:
|
|
|
|
*/
|
|
|
|
__journal_entry_close(j);
|
2019-02-18 22:39:42 +00:00
|
|
|
} else {
|
|
|
|
journal_cur_buf(j)->u64s_reserved += d;
|
2019-01-24 21:50:48 +00:00
|
|
|
}
|
|
|
|
out:
|
2019-02-18 22:39:42 +00:00
|
|
|
spin_unlock(&j->lock);
|
2019-01-24 21:50:48 +00:00
|
|
|
res->u64s += d;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* journal flushing: */
|
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
/**
|
|
|
|
* bch2_journal_flush_seq_async - wait for a journal entry to be written
|
|
|
|
*
|
|
|
|
* like bch2_journal_wait_on_seq, except that it triggers a write immediately if
|
|
|
|
* necessary
|
|
|
|
*/
|
2020-11-14 21:04:30 +00:00
|
|
|
int bch2_journal_flush_seq_async(struct journal *j, u64 seq,
|
2020-11-21 00:27:57 +00:00
|
|
|
struct closure *parent)
|
2017-03-17 06:18:50 +00:00
|
|
|
{
|
|
|
|
struct journal_buf *buf;
|
2020-11-14 21:04:30 +00:00
|
|
|
int ret = 0;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2020-11-21 00:27:57 +00:00
|
|
|
if (seq <= j->err_seq)
|
|
|
|
return -EIO;
|
|
|
|
|
|
|
|
if (seq <= j->seq_ondisk)
|
|
|
|
return 1;
|
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
spin_lock(&j->lock);
|
2020-11-21 00:27:57 +00:00
|
|
|
|
|
|
|
/* Recheck under lock: */
|
2020-11-14 21:04:30 +00:00
|
|
|
if (seq <= j->err_seq) {
|
|
|
|
ret = -EIO;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (seq <= j->seq_ondisk) {
|
|
|
|
ret = 1;
|
|
|
|
goto out;
|
|
|
|
}
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2018-07-23 11:52:00 +00:00
|
|
|
if (parent &&
|
|
|
|
(buf = journal_seq_to_buf(j, seq)))
|
2017-03-17 06:18:50 +00:00
|
|
|
if (!closure_wait(&buf->wait, parent))
|
|
|
|
BUG();
|
|
|
|
|
2018-07-23 11:52:00 +00:00
|
|
|
if (seq == journal_cur_seq(j))
|
|
|
|
__journal_entry_close(j);
|
2020-11-14 21:04:30 +00:00
|
|
|
out:
|
2019-02-18 22:39:42 +00:00
|
|
|
spin_unlock(&j->lock);
|
2017-03-17 06:18:50 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
int bch2_journal_flush_seq(struct journal *j, u64 seq)
|
|
|
|
{
|
|
|
|
u64 start_time = local_clock();
|
|
|
|
int ret, ret2;
|
|
|
|
|
2020-11-14 21:04:30 +00:00
|
|
|
ret = wait_event_killable(j->wait, (ret2 = bch2_journal_flush_seq_async(j, seq, NULL)));
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
bch2_time_stats_update(j->flush_seq_time, start_time);
|
|
|
|
|
|
|
|
return ret ?: ret2 < 0 ? ret2 : 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int bch2_journal_meta(struct journal *j)
|
|
|
|
{
|
|
|
|
struct journal_res res;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
memset(&res, 0, sizeof(res));
|
|
|
|
|
2018-11-19 02:35:59 +00:00
|
|
|
ret = bch2_journal_res_get(j, &res, jset_u64s(0), 0);
|
2017-03-17 06:18:50 +00:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
bch2_journal_res_put(j, &res);
|
|
|
|
|
|
|
|
return bch2_journal_flush_seq(j, res.seq);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* bch2_journal_flush_async - if there is an open journal entry, or a journal
|
|
|
|
* still being written, write it and wait for the write to complete
|
|
|
|
*/
|
|
|
|
void bch2_journal_flush_async(struct journal *j, struct closure *parent)
|
|
|
|
{
|
|
|
|
u64 seq, journal_seq;
|
|
|
|
|
|
|
|
spin_lock(&j->lock);
|
|
|
|
journal_seq = journal_cur_seq(j);
|
|
|
|
|
|
|
|
if (journal_entry_is_open(j)) {
|
|
|
|
seq = journal_seq;
|
|
|
|
} else if (journal_seq) {
|
|
|
|
seq = journal_seq - 1;
|
|
|
|
} else {
|
|
|
|
spin_unlock(&j->lock);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
spin_unlock(&j->lock);
|
|
|
|
|
|
|
|
bch2_journal_flush_seq_async(j, seq, parent);
|
|
|
|
}
|
|
|
|
|
|
|
|
int bch2_journal_flush(struct journal *j)
|
|
|
|
{
|
|
|
|
u64 seq, journal_seq;
|
|
|
|
|
|
|
|
spin_lock(&j->lock);
|
|
|
|
journal_seq = journal_cur_seq(j);
|
|
|
|
|
|
|
|
if (journal_entry_is_open(j)) {
|
|
|
|
seq = journal_seq;
|
|
|
|
} else if (journal_seq) {
|
|
|
|
seq = journal_seq - 1;
|
|
|
|
} else {
|
|
|
|
spin_unlock(&j->lock);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
spin_unlock(&j->lock);
|
|
|
|
|
|
|
|
return bch2_journal_flush_seq(j, seq);
|
|
|
|
}
|
|
|
|
|
2019-02-14 23:38:52 +00:00
|
|
|
/* block/unlock the journal: */
|
|
|
|
|
|
|
|
void bch2_journal_unblock(struct journal *j)
|
|
|
|
{
|
|
|
|
spin_lock(&j->lock);
|
|
|
|
j->blocked--;
|
|
|
|
spin_unlock(&j->lock);
|
|
|
|
|
|
|
|
journal_wake(j);
|
|
|
|
}
|
|
|
|
|
|
|
|
void bch2_journal_block(struct journal *j)
|
|
|
|
{
|
|
|
|
spin_lock(&j->lock);
|
|
|
|
j->blocked++;
|
|
|
|
spin_unlock(&j->lock);
|
|
|
|
|
|
|
|
journal_quiesce(j);
|
|
|
|
}
|
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
/* allocate journal on a device: */
|
|
|
|
|
|
|
|
static int __bch2_set_nr_journal_buckets(struct bch_dev *ca, unsigned nr,
|
|
|
|
bool new_fs, struct closure *cl)
|
|
|
|
{
|
|
|
|
struct bch_fs *c = ca->fs;
|
|
|
|
struct journal_device *ja = &ca->journal;
|
|
|
|
struct bch_sb_field_journal *journal_buckets;
|
|
|
|
u64 *new_bucket_seq = NULL, *new_buckets = NULL;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
/* don't handle reducing nr of buckets yet: */
|
|
|
|
if (nr <= ja->nr)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
new_buckets = kzalloc(nr * sizeof(u64), GFP_KERNEL);
|
|
|
|
new_bucket_seq = kzalloc(nr * sizeof(u64), GFP_KERNEL);
|
2020-11-20 21:12:39 +00:00
|
|
|
if (!new_buckets || !new_bucket_seq) {
|
|
|
|
ret = -ENOMEM;
|
2017-03-17 06:18:50 +00:00
|
|
|
goto err;
|
2020-11-20 21:12:39 +00:00
|
|
|
}
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
journal_buckets = bch2_sb_resize_journal(&ca->disk_sb,
|
2020-11-16 19:23:06 +00:00
|
|
|
nr + sizeof(*journal_buckets) / sizeof(u64));
|
2020-11-20 21:12:39 +00:00
|
|
|
if (!journal_buckets) {
|
|
|
|
ret = -ENOSPC;
|
2017-03-17 06:18:50 +00:00
|
|
|
goto err;
|
2020-11-20 21:12:39 +00:00
|
|
|
}
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2018-07-21 02:08:17 +00:00
|
|
|
/*
|
|
|
|
* We may be called from the device add path, before the new device has
|
|
|
|
* actually been added to the running filesystem:
|
|
|
|
*/
|
2017-03-17 06:18:50 +00:00
|
|
|
if (c)
|
|
|
|
spin_lock(&c->journal.lock);
|
|
|
|
|
|
|
|
memcpy(new_buckets, ja->buckets, ja->nr * sizeof(u64));
|
|
|
|
memcpy(new_bucket_seq, ja->bucket_seq, ja->nr * sizeof(u64));
|
|
|
|
swap(new_buckets, ja->buckets);
|
|
|
|
swap(new_bucket_seq, ja->bucket_seq);
|
|
|
|
|
|
|
|
if (c)
|
|
|
|
spin_unlock(&c->journal.lock);
|
|
|
|
|
|
|
|
while (ja->nr < nr) {
|
|
|
|
struct open_bucket *ob = NULL;
|
2019-03-03 20:15:55 +00:00
|
|
|
unsigned pos;
|
2017-03-17 06:18:50 +00:00
|
|
|
long bucket;
|
|
|
|
|
|
|
|
if (new_fs) {
|
|
|
|
bucket = bch2_bucket_alloc_new_fs(ca);
|
|
|
|
if (bucket < 0) {
|
|
|
|
ret = -ENOSPC;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
} else {
|
2020-11-20 21:12:39 +00:00
|
|
|
rcu_read_lock();
|
2018-10-06 08:12:42 +00:00
|
|
|
ob = bch2_bucket_alloc(c, ca, RESERVE_ALLOC,
|
|
|
|
false, cl);
|
2020-11-20 21:12:39 +00:00
|
|
|
rcu_read_unlock();
|
2018-10-06 08:12:42 +00:00
|
|
|
if (IS_ERR(ob)) {
|
2017-03-17 06:18:50 +00:00
|
|
|
ret = cl ? -EAGAIN : -ENOSPC;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
|
|
|
|
bucket = sector_to_bucket(ca, ob->ptr.offset);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (c) {
|
2018-11-26 05:13:33 +00:00
|
|
|
percpu_down_read(&c->mark_lock);
|
2017-03-17 06:18:50 +00:00
|
|
|
spin_lock(&c->journal.lock);
|
|
|
|
}
|
|
|
|
|
2020-11-16 19:23:06 +00:00
|
|
|
/*
|
|
|
|
* XXX
|
|
|
|
* For resize at runtime, we should be writing the new
|
|
|
|
* superblock before inserting into the journal array
|
|
|
|
*/
|
|
|
|
|
2019-03-03 20:15:55 +00:00
|
|
|
pos = ja->nr ? (ja->cur_idx + 1) % ja->nr : 0;
|
|
|
|
__array_insert_item(ja->buckets, ja->nr, pos);
|
|
|
|
__array_insert_item(ja->bucket_seq, ja->nr, pos);
|
|
|
|
__array_insert_item(journal_buckets->buckets, ja->nr, pos);
|
|
|
|
ja->nr++;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-03-03 20:15:55 +00:00
|
|
|
ja->buckets[pos] = bucket;
|
|
|
|
ja->bucket_seq[pos] = 0;
|
|
|
|
journal_buckets->buckets[pos] = cpu_to_le64(bucket);
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-03-03 20:15:55 +00:00
|
|
|
if (pos <= ja->discard_idx)
|
|
|
|
ja->discard_idx = (ja->discard_idx + 1) % ja->nr;
|
|
|
|
if (pos <= ja->dirty_idx_ondisk)
|
|
|
|
ja->dirty_idx_ondisk = (ja->dirty_idx_ondisk + 1) % ja->nr;
|
|
|
|
if (pos <= ja->dirty_idx)
|
|
|
|
ja->dirty_idx = (ja->dirty_idx + 1) % ja->nr;
|
|
|
|
if (pos <= ja->cur_idx)
|
|
|
|
ja->cur_idx = (ja->cur_idx + 1) % ja->nr;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2020-07-09 22:28:11 +00:00
|
|
|
bch2_mark_metadata_bucket(c, ca, bucket, BCH_DATA_journal,
|
2019-02-18 22:39:42 +00:00
|
|
|
ca->mi.bucket_size,
|
|
|
|
gc_phase(GC_PHASE_SB),
|
|
|
|
0);
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
if (c) {
|
|
|
|
spin_unlock(&c->journal.lock);
|
2018-11-26 05:13:33 +00:00
|
|
|
percpu_up_read(&c->mark_lock);
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!new_fs)
|
|
|
|
bch2_open_bucket_put(c, ob);
|
|
|
|
}
|
|
|
|
err:
|
2020-11-16 19:23:06 +00:00
|
|
|
bch2_sb_resize_journal(&ca->disk_sb,
|
|
|
|
ja->nr + sizeof(*journal_buckets) / sizeof(u64));
|
2017-03-17 06:18:50 +00:00
|
|
|
kfree(new_bucket_seq);
|
|
|
|
kfree(new_buckets);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Allocate more journal space at runtime - not currently making use if it, but
|
|
|
|
* the code works:
|
|
|
|
*/
|
|
|
|
int bch2_set_nr_journal_buckets(struct bch_fs *c, struct bch_dev *ca,
|
|
|
|
unsigned nr)
|
|
|
|
{
|
|
|
|
struct journal_device *ja = &ca->journal;
|
|
|
|
struct closure cl;
|
|
|
|
unsigned current_nr;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
closure_init_stack(&cl);
|
|
|
|
|
|
|
|
do {
|
|
|
|
struct disk_reservation disk_res = { 0, 0 };
|
|
|
|
|
|
|
|
closure_sync(&cl);
|
|
|
|
|
|
|
|
mutex_lock(&c->sb_lock);
|
|
|
|
current_nr = ja->nr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* note: journal buckets aren't really counted as _sectors_ used yet, so
|
|
|
|
* we don't need the disk reservation to avoid the BUG_ON() in buckets.c
|
|
|
|
* when space used goes up without a reservation - but we do need the
|
|
|
|
* reservation to ensure we'll actually be able to allocate:
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (bch2_disk_reservation_get(c, &disk_res,
|
2019-02-18 22:39:42 +00:00
|
|
|
bucket_to_sector(ca, nr - ja->nr), 1, 0)) {
|
2017-03-17 06:18:50 +00:00
|
|
|
mutex_unlock(&c->sb_lock);
|
|
|
|
return -ENOSPC;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = __bch2_set_nr_journal_buckets(ca, nr, false, &cl);
|
|
|
|
|
|
|
|
bch2_disk_reservation_put(c, &disk_res);
|
|
|
|
|
|
|
|
if (ja->nr != current_nr)
|
|
|
|
bch2_write_super(c);
|
|
|
|
mutex_unlock(&c->sb_lock);
|
|
|
|
} while (ret == -EAGAIN);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
int bch2_dev_journal_alloc(struct bch_dev *ca)
|
|
|
|
{
|
|
|
|
unsigned nr;
|
|
|
|
|
|
|
|
if (dynamic_fault("bcachefs:add:journal_alloc"))
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* clamp journal size to 1024 buckets or 512MB (in sectors), whichever
|
|
|
|
* is smaller:
|
|
|
|
*/
|
|
|
|
nr = clamp_t(unsigned, ca->mi.nbuckets >> 8,
|
|
|
|
BCH_JOURNAL_BUCKETS_MIN,
|
|
|
|
min(1 << 10,
|
|
|
|
(1 << 20) / ca->mi.bucket_size));
|
|
|
|
|
|
|
|
return __bch2_set_nr_journal_buckets(ca, nr, true, NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* startup/shutdown: */
|
|
|
|
|
|
|
|
static bool bch2_journal_writing_to_device(struct journal *j, unsigned dev_idx)
|
|
|
|
{
|
|
|
|
union journal_res_state state;
|
|
|
|
struct journal_buf *w;
|
|
|
|
bool ret;
|
|
|
|
|
|
|
|
spin_lock(&j->lock);
|
|
|
|
state = READ_ONCE(j->reservations);
|
|
|
|
w = j->buf + !state.idx;
|
|
|
|
|
|
|
|
ret = state.prev_buf_unwritten &&
|
2019-11-16 01:40:15 +00:00
|
|
|
bch2_bkey_has_device(bkey_i_to_s_c(&w->key), dev_idx);
|
2017-03-17 06:18:50 +00:00
|
|
|
spin_unlock(&j->lock);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
void bch2_dev_journal_stop(struct journal *j, struct bch_dev *ca)
|
|
|
|
{
|
|
|
|
wait_event(j->wait, !bch2_journal_writing_to_device(j, ca->dev_idx));
|
|
|
|
}
|
|
|
|
|
|
|
|
void bch2_fs_journal_stop(struct journal *j)
|
|
|
|
{
|
2019-05-10 15:58:00 +00:00
|
|
|
bch2_journal_flush_all_pins(j);
|
|
|
|
|
2018-07-23 11:52:00 +00:00
|
|
|
wait_event(j->wait, journal_entry_close(j));
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2020-10-25 05:08:28 +00:00
|
|
|
/*
|
|
|
|
* Always write a new journal entry, to make sure the clock hands are up
|
|
|
|
* to date (and match the superblock)
|
|
|
|
*/
|
|
|
|
bch2_journal_meta(j);
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-02-14 23:38:52 +00:00
|
|
|
journal_quiesce(j);
|
2018-07-23 11:52:00 +00:00
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
BUG_ON(!bch2_journal_error(j) &&
|
2020-11-14 21:04:30 +00:00
|
|
|
(journal_entry_is_open(j) ||
|
|
|
|
j->last_empty_seq + 1 != journal_cur_seq(j)));
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
cancel_delayed_work_sync(&j->write_work);
|
2020-11-20 01:55:33 +00:00
|
|
|
bch2_journal_reclaim_stop(j);
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
2019-04-05 01:53:12 +00:00
|
|
|
int bch2_fs_journal_start(struct journal *j, u64 cur_seq,
|
|
|
|
struct list_head *journal_entries)
|
2017-03-17 06:18:50 +00:00
|
|
|
{
|
2018-07-22 02:57:20 +00:00
|
|
|
struct bch_fs *c = container_of(j, struct bch_fs, journal);
|
2019-04-05 01:53:12 +00:00
|
|
|
struct journal_entry_pin_list *p;
|
|
|
|
struct journal_replay *i;
|
|
|
|
u64 last_seq = cur_seq, nr, seq;
|
|
|
|
|
|
|
|
if (!list_empty(journal_entries))
|
2020-06-13 22:43:14 +00:00
|
|
|
last_seq = le64_to_cpu(list_last_entry(journal_entries,
|
|
|
|
struct journal_replay, list)->j.last_seq);
|
2019-04-05 01:53:12 +00:00
|
|
|
|
|
|
|
nr = cur_seq - last_seq;
|
|
|
|
|
|
|
|
if (nr + 1 > j->pin.size) {
|
|
|
|
free_fifo(&j->pin);
|
|
|
|
init_fifo(&j->pin, roundup_pow_of_two(nr + 1), GFP_KERNEL);
|
|
|
|
if (!j->pin.data) {
|
|
|
|
bch_err(c, "error reallocating journal fifo (%llu open entries)", nr);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-12 02:39:39 +00:00
|
|
|
j->replay_journal_seq = last_seq;
|
|
|
|
j->replay_journal_seq_end = cur_seq;
|
2019-04-05 01:53:12 +00:00
|
|
|
j->last_seq_ondisk = last_seq;
|
|
|
|
j->pin.front = last_seq;
|
|
|
|
j->pin.back = cur_seq;
|
|
|
|
atomic64_set(&j->seq, cur_seq - 1);
|
|
|
|
|
|
|
|
fifo_for_each_entry_ptr(p, &j->pin, seq) {
|
|
|
|
INIT_LIST_HEAD(&p->list);
|
|
|
|
INIT_LIST_HEAD(&p->flushed);
|
2019-04-12 02:39:39 +00:00
|
|
|
atomic_set(&p->count, 1);
|
2019-04-05 01:53:12 +00:00
|
|
|
p->devs.nr = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
list_for_each_entry(i, journal_entries, list) {
|
|
|
|
seq = le64_to_cpu(i->j.seq);
|
2020-06-13 22:43:14 +00:00
|
|
|
BUG_ON(seq >= cur_seq);
|
2019-04-05 01:53:12 +00:00
|
|
|
|
2020-06-13 22:43:14 +00:00
|
|
|
if (seq < last_seq)
|
|
|
|
continue;
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2019-04-12 02:39:39 +00:00
|
|
|
journal_seq_pin(j, seq)->devs = i->devs;
|
2019-04-05 01:53:12 +00:00
|
|
|
}
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
spin_lock(&j->lock);
|
|
|
|
|
|
|
|
set_bit(JOURNAL_STARTED, &j->flags);
|
|
|
|
|
|
|
|
journal_pin_new_entry(j, 1);
|
2020-11-14 21:04:30 +00:00
|
|
|
|
|
|
|
j->reservations.idx = journal_cur_seq(j);
|
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
bch2_journal_buf_init(j);
|
|
|
|
|
2018-07-22 02:57:20 +00:00
|
|
|
c->last_bucket_seq_cleanup = journal_cur_seq(j);
|
|
|
|
|
2019-02-21 18:33:21 +00:00
|
|
|
bch2_journal_space_available(j);
|
2017-03-17 06:18:50 +00:00
|
|
|
spin_unlock(&j->lock);
|
|
|
|
|
2019-04-05 01:53:12 +00:00
|
|
|
return 0;
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* init/exit: */
|
|
|
|
|
|
|
|
void bch2_dev_journal_exit(struct bch_dev *ca)
|
|
|
|
{
|
|
|
|
kfree(ca->journal.bio);
|
|
|
|
kfree(ca->journal.buckets);
|
|
|
|
kfree(ca->journal.bucket_seq);
|
|
|
|
|
|
|
|
ca->journal.bio = NULL;
|
|
|
|
ca->journal.buckets = NULL;
|
|
|
|
ca->journal.bucket_seq = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
int bch2_dev_journal_init(struct bch_dev *ca, struct bch_sb *sb)
|
|
|
|
{
|
|
|
|
struct journal_device *ja = &ca->journal;
|
|
|
|
struct bch_sb_field_journal *journal_buckets =
|
|
|
|
bch2_sb_get_journal(sb);
|
|
|
|
unsigned i, nr_bvecs;
|
|
|
|
|
|
|
|
ja->nr = bch2_nr_journal_buckets(journal_buckets);
|
|
|
|
|
|
|
|
ja->bucket_seq = kcalloc(ja->nr, sizeof(u64), GFP_KERNEL);
|
|
|
|
if (!ja->bucket_seq)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
nr_bvecs = DIV_ROUND_UP(JOURNAL_ENTRY_SIZE_MAX, PAGE_SIZE);
|
|
|
|
|
|
|
|
ca->journal.bio = bio_kmalloc(nr_bvecs, GFP_KERNEL);
|
|
|
|
if (!ca->journal.bio)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
bio_init(ca->journal.bio, NULL, ca->journal.bio->bi_inline_vecs, nr_bvecs, 0);
|
|
|
|
|
|
|
|
ja->buckets = kcalloc(ja->nr, sizeof(u64), GFP_KERNEL);
|
|
|
|
if (!ja->buckets)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
for (i = 0; i < ja->nr; i++)
|
|
|
|
ja->buckets[i] = le64_to_cpu(journal_buckets->buckets[i]);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void bch2_fs_journal_exit(struct journal *j)
|
|
|
|
{
|
2019-02-18 22:39:42 +00:00
|
|
|
kvpfree(j->buf[1].data, j->buf[1].buf_size);
|
|
|
|
kvpfree(j->buf[0].data, j->buf[0].buf_size);
|
2017-03-17 06:18:50 +00:00
|
|
|
free_fifo(&j->pin);
|
|
|
|
}
|
|
|
|
|
|
|
|
int bch2_fs_journal_init(struct journal *j)
|
|
|
|
{
|
|
|
|
struct bch_fs *c = container_of(j, struct bch_fs, journal);
|
|
|
|
static struct lock_class_key res_key;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
pr_verbose_init(c->opts, "");
|
|
|
|
|
|
|
|
spin_lock_init(&j->lock);
|
|
|
|
spin_lock_init(&j->err_lock);
|
|
|
|
init_waitqueue_head(&j->wait);
|
|
|
|
INIT_DELAYED_WORK(&j->write_work, journal_write_work);
|
2018-07-17 16:19:14 +00:00
|
|
|
init_waitqueue_head(&j->pin_flush_wait);
|
2017-03-17 06:18:50 +00:00
|
|
|
mutex_init(&j->reclaim_lock);
|
2019-03-03 20:15:55 +00:00
|
|
|
mutex_init(&j->discard_lock);
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
lockdep_init_map(&j->res_map, "journal res", &res_key, 0);
|
|
|
|
|
2019-02-18 22:39:42 +00:00
|
|
|
j->buf[0].buf_size = JOURNAL_ENTRY_SIZE_MIN;
|
|
|
|
j->buf[1].buf_size = JOURNAL_ENTRY_SIZE_MIN;
|
2017-03-17 06:18:50 +00:00
|
|
|
j->write_delay_ms = 1000;
|
|
|
|
j->reclaim_delay_ms = 100;
|
|
|
|
|
2019-01-24 21:50:48 +00:00
|
|
|
/* Btree roots: */
|
|
|
|
j->entry_u64s_reserved +=
|
|
|
|
BTREE_ID_NR * (JSET_KEYS_U64s + BKEY_EXTENT_U64s_MAX);
|
|
|
|
|
2017-03-17 06:18:50 +00:00
|
|
|
atomic64_set(&j->reservations.counter,
|
|
|
|
((union journal_res_state)
|
|
|
|
{ .cur_entry_offset = JOURNAL_ENTRY_CLOSED_VAL }).v);
|
|
|
|
|
|
|
|
if (!(init_fifo(&j->pin, JOURNAL_PIN, GFP_KERNEL)) ||
|
2019-02-18 22:39:42 +00:00
|
|
|
!(j->buf[0].data = kvpmalloc(j->buf[0].buf_size, GFP_KERNEL)) ||
|
|
|
|
!(j->buf[1].data = kvpmalloc(j->buf[1].buf_size, GFP_KERNEL))) {
|
2017-03-17 06:18:50 +00:00
|
|
|
ret = -ENOMEM;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
j->pin.front = j->pin.back = 1;
|
|
|
|
out:
|
|
|
|
pr_verbose_init(c->opts, "ret %i", ret);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* debug: */
|
|
|
|
|
2020-07-25 21:06:11 +00:00
|
|
|
void bch2_journal_debug_to_text(struct printbuf *out, struct journal *j)
|
2017-03-17 06:18:50 +00:00
|
|
|
{
|
|
|
|
struct bch_fs *c = container_of(j, struct bch_fs, journal);
|
2019-02-26 19:28:08 +00:00
|
|
|
union journal_res_state s;
|
2017-03-17 06:18:50 +00:00
|
|
|
struct bch_dev *ca;
|
|
|
|
unsigned iter;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
spin_lock(&j->lock);
|
2019-02-26 19:28:08 +00:00
|
|
|
s = READ_ONCE(j->reservations);
|
2017-03-17 06:18:50 +00:00
|
|
|
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out,
|
2018-11-09 06:24:07 +00:00
|
|
|
"active journal entries:\t%llu\n"
|
|
|
|
"seq:\t\t\t%llu\n"
|
|
|
|
"last_seq:\t\t%llu\n"
|
|
|
|
"last_seq_ondisk:\t%llu\n"
|
2019-02-19 18:41:36 +00:00
|
|
|
"prereserved:\t\t%u/%u\n"
|
2020-11-20 01:55:33 +00:00
|
|
|
"nr direct reclaim:\t%llu\n"
|
|
|
|
"nr background reclaim:\t%llu\n"
|
2019-02-19 18:41:36 +00:00
|
|
|
"current entry sectors:\t%u\n"
|
2020-11-18 18:21:59 +00:00
|
|
|
"current entry error:\t%u\n"
|
2019-02-26 19:28:08 +00:00
|
|
|
"current entry:\t\t",
|
2018-11-09 06:24:07 +00:00
|
|
|
fifo_used(&j->pin),
|
|
|
|
journal_cur_seq(j),
|
|
|
|
journal_last_seq(j),
|
2019-02-19 18:41:36 +00:00
|
|
|
j->last_seq_ondisk,
|
|
|
|
j->prereserved.reserved,
|
|
|
|
j->prereserved.remaining,
|
2020-11-20 01:55:33 +00:00
|
|
|
j->nr_direct_reclaim,
|
|
|
|
j->nr_background_reclaim,
|
2020-11-18 18:21:59 +00:00
|
|
|
j->cur_entry_sectors,
|
|
|
|
j->cur_entry_error);
|
2019-02-26 19:28:08 +00:00
|
|
|
|
|
|
|
switch (s.cur_entry_offset) {
|
|
|
|
case JOURNAL_ENTRY_ERROR_VAL:
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out, "error\n");
|
2019-02-26 19:28:08 +00:00
|
|
|
break;
|
|
|
|
case JOURNAL_ENTRY_CLOSED_VAL:
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out, "closed\n");
|
2019-02-26 19:28:08 +00:00
|
|
|
break;
|
|
|
|
default:
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out, "%u/%u\n",
|
2019-02-26 19:28:08 +00:00
|
|
|
s.cur_entry_offset,
|
|
|
|
j->cur_entry_u64s);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out,
|
2019-02-26 19:28:08 +00:00
|
|
|
"current entry refs:\t%u\n"
|
|
|
|
"prev entry unwritten:\t",
|
|
|
|
journal_state_count(s, s.idx));
|
|
|
|
|
|
|
|
if (s.prev_buf_unwritten)
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out, "yes, ref %u sectors %u\n",
|
2019-02-19 18:41:36 +00:00
|
|
|
journal_state_count(s, !s.idx),
|
|
|
|
journal_prev_buf(j)->sectors);
|
2019-02-26 19:28:08 +00:00
|
|
|
else
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out, "no\n");
|
2019-02-26 19:28:08 +00:00
|
|
|
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out,
|
2019-02-26 19:28:08 +00:00
|
|
|
"need write:\t\t%i\n"
|
|
|
|
"replay done:\t\t%i\n",
|
2018-11-09 06:24:07 +00:00
|
|
|
test_bit(JOURNAL_NEED_WRITE, &j->flags),
|
|
|
|
test_bit(JOURNAL_REPLAY_DONE, &j->flags));
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
for_each_member_device_rcu(ca, c, iter,
|
2020-07-09 22:28:11 +00:00
|
|
|
&c->rw_devs[BCH_DATA_journal]) {
|
2017-03-17 06:18:50 +00:00
|
|
|
struct journal_device *ja = &ca->journal;
|
|
|
|
|
|
|
|
if (!ja->nr)
|
|
|
|
continue;
|
|
|
|
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out,
|
2018-11-09 06:24:07 +00:00
|
|
|
"dev %u:\n"
|
|
|
|
"\tnr\t\t%u\n"
|
2019-02-21 18:33:21 +00:00
|
|
|
"\tavailable\t%u:%u\n"
|
2019-03-03 20:15:55 +00:00
|
|
|
"\tdiscard_idx\t\t%u\n"
|
|
|
|
"\tdirty_idx_ondisk\t%u (seq %llu)\n"
|
|
|
|
"\tdirty_idx\t\t%u (seq %llu)\n"
|
|
|
|
"\tcur_idx\t\t%u (seq %llu)\n",
|
2018-11-09 06:24:07 +00:00
|
|
|
iter, ja->nr,
|
2019-03-03 21:50:40 +00:00
|
|
|
bch2_journal_dev_buckets_available(j, ja, journal_space_discarded),
|
2019-02-21 18:33:21 +00:00
|
|
|
ja->sectors_free,
|
2019-03-03 20:15:55 +00:00
|
|
|
ja->discard_idx,
|
|
|
|
ja->dirty_idx_ondisk, ja->bucket_seq[ja->dirty_idx_ondisk],
|
|
|
|
ja->dirty_idx, ja->bucket_seq[ja->dirty_idx],
|
|
|
|
ja->cur_idx, ja->bucket_seq[ja->cur_idx]);
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock(&j->lock);
|
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
|
2020-07-25 21:06:11 +00:00
|
|
|
void bch2_journal_pins_to_text(struct printbuf *out, struct journal *j)
|
2017-03-17 06:18:50 +00:00
|
|
|
{
|
|
|
|
struct journal_entry_pin_list *pin_list;
|
|
|
|
struct journal_entry_pin *pin;
|
|
|
|
u64 i;
|
|
|
|
|
|
|
|
spin_lock(&j->lock);
|
|
|
|
fifo_for_each_entry_ptr(pin_list, &j->pin, i) {
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out, "%llu: count %u\n",
|
2018-11-09 06:24:07 +00:00
|
|
|
i, atomic_read(&pin_list->count));
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
list_for_each_entry(pin, &pin_list->list, list)
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out, "\t%px %ps\n",
|
2018-11-09 06:24:07 +00:00
|
|
|
pin, pin->flush);
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
if (!list_empty(&pin_list->flushed))
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out, "flushed:\n");
|
2017-03-17 06:18:50 +00:00
|
|
|
|
|
|
|
list_for_each_entry(pin, &pin_list->flushed, list)
|
2020-07-25 21:06:11 +00:00
|
|
|
pr_buf(out, "\t%px %ps\n",
|
2018-11-09 06:24:07 +00:00
|
|
|
pin, pin->flush);
|
2017-03-17 06:18:50 +00:00
|
|
|
}
|
|
|
|
spin_unlock(&j->lock);
|
|
|
|
}
|