mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-27 11:19:37 +00:00
![Jingbo Xu](/assets/img/avatar_default.png)
As the first step of converting hardcoded blocksize to that specified in on-disk superblock, convert all call sites of hardcoded blocksize to sb->s_blocksize except for: 1) use sbi->blkszbits instead of sb->s_blocksize in erofs_superblock_csum_verify() since sb->s_blocksize has not been updated with the on-disk blocksize yet when the function is called. 2) use inode->i_blkbits instead of sb->s_blocksize in erofs_bread(), since the inode operated on may be an anonymous inode in fscache mode. Currently the anonymous inode is allocated from an anonymous mount maintained in erofs, while in the near future we may allocate anonymous inodes from a generic API directly and thus have no access to the anonymous inode's i_sb. Thus we keep the block size in i_blkbits for anonymous inodes in fscache mode. Be noted that this patch only gets rid of the hardcoded blocksize, in preparation for actually setting the on-disk block size in the following patch. The hard limit of constraining the block size to PAGE_SIZE still exists until the next patch. Signed-off-by: Jingbo Xu <jefflexu@linux.alibaba.com> Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com> Reviewed-by: Yue Hu <huyue2@coolpad.com> Reviewed-by: Chao Yu <chao@kernel.org> Link: https://lore.kernel.org/r/20230313135309.75269-2-jefflexu@linux.alibaba.com [ Gao Xiang: fold a patch to fix incorrect truncated offsets. ] Link: https://lore.kernel.org/r/20230413035734.15457-1-zhujia.zj@bytedance.com Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
291 lines
7.6 KiB
C
291 lines
7.6 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
#include <linux/xz.h>
|
|
#include <linux/module.h>
|
|
#include "compress.h"
|
|
|
|
struct z_erofs_lzma {
|
|
struct z_erofs_lzma *next;
|
|
struct xz_dec_microlzma *state;
|
|
struct xz_buf buf;
|
|
u8 bounce[PAGE_SIZE];
|
|
};
|
|
|
|
/* considering the LZMA performance, no need to use a lockless list for now */
|
|
static DEFINE_SPINLOCK(z_erofs_lzma_lock);
|
|
static unsigned int z_erofs_lzma_max_dictsize;
|
|
static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
|
|
static struct z_erofs_lzma *z_erofs_lzma_head;
|
|
static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
|
|
|
|
module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
|
|
|
|
void z_erofs_lzma_exit(void)
|
|
{
|
|
/* there should be no running fs instance */
|
|
while (z_erofs_lzma_avail_strms) {
|
|
struct z_erofs_lzma *strm;
|
|
|
|
spin_lock(&z_erofs_lzma_lock);
|
|
strm = z_erofs_lzma_head;
|
|
if (!strm) {
|
|
spin_unlock(&z_erofs_lzma_lock);
|
|
DBG_BUGON(1);
|
|
return;
|
|
}
|
|
z_erofs_lzma_head = NULL;
|
|
spin_unlock(&z_erofs_lzma_lock);
|
|
|
|
while (strm) {
|
|
struct z_erofs_lzma *n = strm->next;
|
|
|
|
if (strm->state)
|
|
xz_dec_microlzma_end(strm->state);
|
|
kfree(strm);
|
|
--z_erofs_lzma_avail_strms;
|
|
strm = n;
|
|
}
|
|
}
|
|
}
|
|
|
|
int __init z_erofs_lzma_init(void)
|
|
{
|
|
unsigned int i;
|
|
|
|
/* by default, use # of possible CPUs instead */
|
|
if (!z_erofs_lzma_nstrms)
|
|
z_erofs_lzma_nstrms = num_possible_cpus();
|
|
|
|
for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
|
|
struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
|
|
|
|
if (!strm) {
|
|
z_erofs_lzma_exit();
|
|
return -ENOMEM;
|
|
}
|
|
spin_lock(&z_erofs_lzma_lock);
|
|
strm->next = z_erofs_lzma_head;
|
|
z_erofs_lzma_head = strm;
|
|
spin_unlock(&z_erofs_lzma_lock);
|
|
++z_erofs_lzma_avail_strms;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int z_erofs_load_lzma_config(struct super_block *sb,
|
|
struct erofs_super_block *dsb,
|
|
struct z_erofs_lzma_cfgs *lzma, int size)
|
|
{
|
|
static DEFINE_MUTEX(lzma_resize_mutex);
|
|
unsigned int dict_size, i;
|
|
struct z_erofs_lzma *strm, *head = NULL;
|
|
int err;
|
|
|
|
if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
|
|
erofs_err(sb, "invalid lzma cfgs, size=%u", size);
|
|
return -EINVAL;
|
|
}
|
|
if (lzma->format) {
|
|
erofs_err(sb, "unidentified lzma format %x, please check kernel version",
|
|
le16_to_cpu(lzma->format));
|
|
return -EINVAL;
|
|
}
|
|
dict_size = le32_to_cpu(lzma->dict_size);
|
|
if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
|
|
erofs_err(sb, "unsupported lzma dictionary size %u",
|
|
dict_size);
|
|
return -EINVAL;
|
|
}
|
|
|
|
erofs_info(sb, "EXPERIMENTAL MicroLZMA in use. Use at your own risk!");
|
|
|
|
/* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
|
|
mutex_lock(&lzma_resize_mutex);
|
|
|
|
if (z_erofs_lzma_max_dictsize >= dict_size) {
|
|
mutex_unlock(&lzma_resize_mutex);
|
|
return 0;
|
|
}
|
|
|
|
/* 1. collect/isolate all streams for the following check */
|
|
for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
|
|
struct z_erofs_lzma *last;
|
|
|
|
again:
|
|
spin_lock(&z_erofs_lzma_lock);
|
|
strm = z_erofs_lzma_head;
|
|
if (!strm) {
|
|
spin_unlock(&z_erofs_lzma_lock);
|
|
wait_event(z_erofs_lzma_wq,
|
|
READ_ONCE(z_erofs_lzma_head));
|
|
goto again;
|
|
}
|
|
z_erofs_lzma_head = NULL;
|
|
spin_unlock(&z_erofs_lzma_lock);
|
|
|
|
for (last = strm; last->next; last = last->next)
|
|
++i;
|
|
last->next = head;
|
|
head = strm;
|
|
}
|
|
|
|
err = 0;
|
|
/* 2. walk each isolated stream and grow max dict_size if needed */
|
|
for (strm = head; strm; strm = strm->next) {
|
|
if (strm->state)
|
|
xz_dec_microlzma_end(strm->state);
|
|
strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
|
|
if (!strm->state)
|
|
err = -ENOMEM;
|
|
}
|
|
|
|
/* 3. push back all to the global list and update max dict_size */
|
|
spin_lock(&z_erofs_lzma_lock);
|
|
DBG_BUGON(z_erofs_lzma_head);
|
|
z_erofs_lzma_head = head;
|
|
spin_unlock(&z_erofs_lzma_lock);
|
|
wake_up_all(&z_erofs_lzma_wq);
|
|
|
|
z_erofs_lzma_max_dictsize = dict_size;
|
|
mutex_unlock(&lzma_resize_mutex);
|
|
return err;
|
|
}
|
|
|
|
int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
|
|
struct page **pagepool)
|
|
{
|
|
const unsigned int nrpages_out =
|
|
PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
|
|
const unsigned int nrpages_in =
|
|
PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
|
|
unsigned int inlen, outlen, pageofs;
|
|
struct z_erofs_lzma *strm;
|
|
u8 *kin;
|
|
bool bounced = false;
|
|
int no, ni, j, err = 0;
|
|
|
|
/* 1. get the exact LZMA compressed size */
|
|
kin = kmap(*rq->in);
|
|
err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in,
|
|
min_t(unsigned int, rq->inputsize,
|
|
rq->sb->s_blocksize - rq->pageofs_in));
|
|
if (err) {
|
|
kunmap(*rq->in);
|
|
return err;
|
|
}
|
|
|
|
/* 2. get an available lzma context */
|
|
again:
|
|
spin_lock(&z_erofs_lzma_lock);
|
|
strm = z_erofs_lzma_head;
|
|
if (!strm) {
|
|
spin_unlock(&z_erofs_lzma_lock);
|
|
wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
|
|
goto again;
|
|
}
|
|
z_erofs_lzma_head = strm->next;
|
|
spin_unlock(&z_erofs_lzma_lock);
|
|
|
|
/* 3. multi-call decompress */
|
|
inlen = rq->inputsize;
|
|
outlen = rq->outputsize;
|
|
xz_dec_microlzma_reset(strm->state, inlen, outlen,
|
|
!rq->partial_decoding);
|
|
pageofs = rq->pageofs_out;
|
|
strm->buf.in = kin + rq->pageofs_in;
|
|
strm->buf.in_pos = 0;
|
|
strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in);
|
|
inlen -= strm->buf.in_size;
|
|
strm->buf.out = NULL;
|
|
strm->buf.out_pos = 0;
|
|
strm->buf.out_size = 0;
|
|
|
|
for (ni = 0, no = -1;;) {
|
|
enum xz_ret xz_err;
|
|
|
|
if (strm->buf.out_pos == strm->buf.out_size) {
|
|
if (strm->buf.out) {
|
|
kunmap(rq->out[no]);
|
|
strm->buf.out = NULL;
|
|
}
|
|
|
|
if (++no >= nrpages_out || !outlen) {
|
|
erofs_err(rq->sb, "decompressed buf out of bound");
|
|
err = -EFSCORRUPTED;
|
|
break;
|
|
}
|
|
strm->buf.out_pos = 0;
|
|
strm->buf.out_size = min_t(u32, outlen,
|
|
PAGE_SIZE - pageofs);
|
|
outlen -= strm->buf.out_size;
|
|
if (!rq->out[no] && rq->fillgaps) /* deduped */
|
|
rq->out[no] = erofs_allocpage(pagepool,
|
|
GFP_KERNEL | __GFP_NOFAIL);
|
|
if (rq->out[no])
|
|
strm->buf.out = kmap(rq->out[no]) + pageofs;
|
|
pageofs = 0;
|
|
} else if (strm->buf.in_pos == strm->buf.in_size) {
|
|
kunmap(rq->in[ni]);
|
|
|
|
if (++ni >= nrpages_in || !inlen) {
|
|
erofs_err(rq->sb, "compressed buf out of bound");
|
|
err = -EFSCORRUPTED;
|
|
break;
|
|
}
|
|
strm->buf.in_pos = 0;
|
|
strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE);
|
|
inlen -= strm->buf.in_size;
|
|
kin = kmap(rq->in[ni]);
|
|
strm->buf.in = kin;
|
|
bounced = false;
|
|
}
|
|
|
|
/*
|
|
* Handle overlapping: Use bounced buffer if the compressed
|
|
* data is under processing; Otherwise, Use short-lived pages
|
|
* from the on-stack pagepool where pages share with the same
|
|
* request.
|
|
*/
|
|
if (!bounced && rq->out[no] == rq->in[ni]) {
|
|
memcpy(strm->bounce, strm->buf.in, strm->buf.in_size);
|
|
strm->buf.in = strm->bounce;
|
|
bounced = true;
|
|
}
|
|
for (j = ni + 1; j < nrpages_in; ++j) {
|
|
struct page *tmppage;
|
|
|
|
if (rq->out[no] != rq->in[j])
|
|
continue;
|
|
|
|
DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb),
|
|
rq->in[j]));
|
|
tmppage = erofs_allocpage(pagepool,
|
|
GFP_KERNEL | __GFP_NOFAIL);
|
|
set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE);
|
|
copy_highpage(tmppage, rq->in[j]);
|
|
rq->in[j] = tmppage;
|
|
}
|
|
xz_err = xz_dec_microlzma_run(strm->state, &strm->buf);
|
|
DBG_BUGON(strm->buf.out_pos > strm->buf.out_size);
|
|
DBG_BUGON(strm->buf.in_pos > strm->buf.in_size);
|
|
|
|
if (xz_err != XZ_OK) {
|
|
if (xz_err == XZ_STREAM_END && !outlen)
|
|
break;
|
|
erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]",
|
|
xz_err, rq->inputsize, rq->outputsize);
|
|
err = -EFSCORRUPTED;
|
|
break;
|
|
}
|
|
}
|
|
if (no < nrpages_out && strm->buf.out)
|
|
kunmap(rq->out[no]);
|
|
if (ni < nrpages_in)
|
|
kunmap(rq->in[ni]);
|
|
/* 4. push back LZMA stream context to the global list */
|
|
spin_lock(&z_erofs_lzma_lock);
|
|
strm->next = z_erofs_lzma_head;
|
|
z_erofs_lzma_head = strm;
|
|
spin_unlock(&z_erofs_lzma_lock);
|
|
wake_up(&z_erofs_lzma_wq);
|
|
return err;
|
|
}
|