kernel-brax3-ubuntu-touch/drivers/scsi/ufs/vendor/ufsshpb.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Universal Flash Storage Host Performance Booster
 *
 * Copyright (C) 2017-2018 Samsung Electronics Co., Ltd.
 *
 * Authors:
 *	Yongmyung Lee <ymhungry.lee@samsung.com>
 *	Jinyoung Choi <j-young.choi@samsung.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * See the COPYING file in the top-level directory or visit
 * <http://www.gnu.org/licenses/gpl-2.0.html>
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * This program is provided "AS IS" and "WITH ALL FAULTS" and
 * without warranty of any kind. You are solely responsible for
 * determining the appropriateness of using and distributing
 * the program and assume all risks associated with your exercise
 * of rights with respect to the program, including but not limited
 * to infringement of third party rights, the risks and costs of
 * program errors, damage to or loss of data, programs or equipment,
 * and unavailability or interruption of operations. Under no
 * circumstances will the contributor of this Program be liable for
 * any damages of any kind arising from your use or distribution of
 * this program.
 *
 * The Linux Foundation chooses to take subject only to the GPLv2
 * license terms, and distributes only under these terms.
 */

#include "ufshcd.h"
#include "ufsfeature.h"
#include "ufsshpb.h"

#define UFSHCD_REQ_SENSE_SIZE	18

static int ufsshpb_create_sysfs(struct ufsf_feature *ufsf,
			       struct ufsshpb_lu *hpb);
static void ufsshpb_remove_sysfs(struct ufsshpb_lu *hpb);


static inline bool ufsshpb_is_triggered_by_fs(struct ufsshpb_region *rgn)
{
	return atomic_read(&rgn->reason) == HPB_UPDATE_FROM_FS;
}

static inline void
ufsshpb_get_pos_from_lpn(struct ufsshpb_lu *hpb, unsigned long lpn, int *rgn_idx,
			int *srgn_idx, int *offset)
{
	int rgn_offset;

	*rgn_idx = lpn >> hpb->entries_per_rgn_shift;
	rgn_offset = lpn & hpb->entries_per_rgn_mask;
	*srgn_idx = rgn_offset >> hpb->entries_per_srgn_shift;
	*offset = rgn_offset & hpb->entries_per_srgn_mask;
}

inline int ufsshpb_valid_srgn(struct ufsshpb_region *rgn,
			     struct ufsshpb_subregion *srgn)
{
	return rgn->rgn_state != HPB_RGN_INACTIVE &&
		srgn->srgn_state == HPB_SRGN_CLEAN;
}

/* Must be held hpb_lock  */
static bool ufsshpb_ppn_dirty_check(struct ufsshpb_lu *hpb, unsigned long lpn,
				   int transfer_len)
{
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;
	unsigned long cur_lpn = lpn;
	int rgn_idx, srgn_idx, srgn_offset, find_size;
	int scan_cnt = transfer_len;

	do {
		ufsshpb_get_pos_from_lpn(hpb, cur_lpn, &rgn_idx, &srgn_idx,
					&srgn_offset);
		rgn = hpb->rgn_tbl + rgn_idx;
		srgn = rgn->srgn_tbl + srgn_idx;

		if (!ufsshpb_valid_srgn(rgn, srgn))
			return true;

		if (unlikely(!srgn->mctx || !srgn->mctx->ppn_dirty))
			return true;

		if (hpb->entries_per_srgn < srgn_offset + scan_cnt) {
			int cnt = hpb->entries_per_srgn - srgn_offset;

			find_size = hpb->entries_per_srgn;
			scan_cnt -= cnt;
			cur_lpn += cnt;
		} else {
			find_size = srgn_offset + scan_cnt;
			scan_cnt = 0;
		}

		srgn_offset =
			find_next_bit((unsigned long *)srgn->mctx->ppn_dirty,
				      hpb->entries_per_srgn, srgn_offset);

		if (srgn_offset < hpb->entries_per_srgn)
			return srgn_offset < find_size;

	} while (scan_cnt);

	return false;
}

static void ufsshpb_set_read16_cmd(struct ufsshpb_lu *hpb, struct scsi_cmnd *cmd,
				  u64 ppn, unsigned int transfer_len,
				  int hpb_ctx_id)
{
	unsigned char *cdb = cmd->cmnd;

	cdb[0] = READ_16;
	cdb[2] = cmd->cmnd[2];
	cdb[3] = cmd->cmnd[3];
	cdb[4] = cmd->cmnd[4];
	cdb[5] = cmd->cmnd[5];
	cdb[6] = GET_BYTE_7(ppn);
	cdb[7] = GET_BYTE_6(ppn);
	cdb[8] = GET_BYTE_5(ppn);
	cdb[9] = GET_BYTE_4(ppn);
	cdb[10] = GET_BYTE_3(ppn);
	cdb[11] = GET_BYTE_2(ppn);
	cdb[12] = GET_BYTE_1(ppn);
	cdb[13] = GET_BYTE_0(ppn);

	if (hpb_ctx_id < MAX_HPB_CONTEXT_ID)
		cdb[14] = (1 << 7) | hpb_ctx_id;
	else
		cdb[14] = UFSSHPB_GROUP_NUMBER;

	cdb[15] = transfer_len;

	cmd->cmd_len = UFS_CDB_SIZE;
}

/* called with hpb_lock (irq) */
static inline void
ufsshpb_set_dirty_bits(struct ufsshpb_lu *hpb, struct ufsshpb_region *rgn,
		      struct ufsshpb_subregion *srgn, int dword, int offset,
		      unsigned int cnt)
{
	const unsigned long mask = ((1UL << cnt) - 1) & 0xffffffff;

	if (rgn->rgn_state == HPB_RGN_INACTIVE)
		return;

	BUG_ON(!srgn->mctx);
	srgn->mctx->ppn_dirty[dword] |= (mask << offset);
}

static inline void ufsshpb_get_bit_offset(struct ufsshpb_lu *hpb, int srgn_offset,
					 int *dword, int *offset)
{
	*dword = srgn_offset >> bits_per_dword_shift;
	*offset = srgn_offset & bits_per_dword_mask;
}

static void ufsshpb_set_dirty(struct ufsshpb_lu *hpb, struct ufshcd_lrb *lrbp,
			     int rgn_idx, int srgn_idx, int srgn_offset)
{
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;
	int cnt, bit_cnt, bit_dword, bit_offset;

	cnt = blk_rq_sectors(lrbp->cmd->request) >> sects_per_blk_shift;
	ufsshpb_get_bit_offset(hpb, srgn_offset, &bit_dword, &bit_offset);

	do {
		bit_cnt = min(cnt, BITS_PER_DWORD - bit_offset);

		rgn = hpb->rgn_tbl + rgn_idx;
		srgn = rgn->srgn_tbl + srgn_idx;

		ufsshpb_set_dirty_bits(hpb, rgn, srgn, bit_dword, bit_offset,
				      bit_cnt);

		bit_offset = 0;
		bit_dword++;

		if (bit_dword == hpb->dwords_per_srgn) {
			bit_dword = 0;
			srgn_idx++;

			if (srgn_idx == hpb->srgns_per_rgn) {
				srgn_idx = 0;
				rgn_idx++;
			}
		}
		cnt -= bit_cnt;
	} while (cnt);

	BUG_ON(cnt < 0);
}

static inline bool ufsshpb_is_read_cmd(struct scsi_cmnd *cmd)
{
	return (cmd->cmnd[0] == READ_10 || cmd->cmnd[0] == READ_16);
}

static inline bool ufsshpb_is_write_cmd(struct scsi_cmnd *cmd)
{
	return (cmd->cmnd[0] == WRITE_10 || cmd->cmnd[0] == WRITE_16);
}

static inline bool ufsshpb_is_discard_cmd(struct scsi_cmnd *cmd)
{
	return cmd->cmnd[0] == UNMAP;
}

static u64 ufsshpb_get_ppn(struct ufsshpb_map_ctx *mctx, int pos, int *error)
{
	u64 *ppn_table;
	struct page *page = NULL;
	int index, offset;

	index = pos / HPB_ENTREIS_PER_OS_PAGE;
	offset = pos % HPB_ENTREIS_PER_OS_PAGE;

	page = mctx->m_page[index];
	if (unlikely(!page)) {
		*error = -ENOMEM;
		ERR_MSG("mctx %p cannot get m_page", mctx);
		return 0;
	}

	ppn_table = page_address(page);
	if (unlikely(!ppn_table)) {
		*error = -ENOMEM;
		ERR_MSG("mctx %p cannot get ppn_table vm", mctx);
		return 0;
	}

	return ppn_table[offset];
}

#if defined(CONFIG_HPB_ERR_INJECTION)
static u64 ufsshpb_get_bitflip_ppn(struct ufsshpb_lu *hpb, u64 ppn)
{
	return (ppn ^ hpb->err_injection_bitflip);
}

static u64 ufsshpb_get_offset_ppn(struct ufsshpb_lu *hpb, u64 ppn)
{
	return (ppn + hpb->err_injection_offset);
}

static u64 ufsshpb_get_random_ppn(struct ufsshpb_lu *hpb, u64 ppn)
{
	u64 random_ppn = 0;

	if (!hpb->err_injection_random)
		return ppn;

	get_random_bytes(&random_ppn, sizeof(random_ppn));

	return random_ppn;
}

static u64 ufsshpb_get_err_injection_ppn(struct ufsshpb_lu *hpb, u64 ppn)
{
	u64 new_ppn = 0;

	if (hpb->err_injection_select == HPB_ERR_INJECTION_BITFLIP)
		new_ppn = ufsshpb_get_bitflip_ppn(hpb, ppn);
	else if (hpb->err_injection_select == HPB_ERR_INJECTION_OFFSET)
		new_ppn = ufsshpb_get_offset_ppn(hpb, ppn);
	else if (hpb->err_injection_select == HPB_ERR_INJECTION_RANDOM)
		new_ppn = ufsshpb_get_random_ppn(hpb, ppn);

	return new_ppn;
}
#endif

inline int ufsshpb_get_state(struct ufsf_feature *ufsf)
{
	return atomic_read(&ufsf->hpb_state);
}

inline void ufsshpb_set_state(struct ufsf_feature *ufsf, int state)
{
	atomic_set(&ufsf->hpb_state, state);
}

static inline int ufsshpb_lu_get(struct ufsshpb_lu *hpb)
{
	if (!hpb || ufsshpb_get_state(hpb->ufsf) != HPB_PRESENT)
		return -ENODEV;

	kref_get(&hpb->ufsf->hpb_kref);
	return 0;
}

static inline void ufsshpb_schedule_error_handler(struct kref *kref)
{
	struct ufsf_feature *ufsf;

	ufsf = container_of(kref, struct ufsf_feature, hpb_kref);
	schedule_work(&ufsf->hpb_eh_work);
}

static inline void ufsshpb_lu_put(struct ufsshpb_lu *hpb)
{
	kref_put(&hpb->ufsf->hpb_kref, ufsshpb_schedule_error_handler);
}

static void ufsshpb_failed(struct ufsshpb_lu *hpb, const char *f)
{
	ERR_MSG("ufsshpb_driver failed. function (%s)", f);
	ufsshpb_set_state(hpb->ufsf, HPB_FAILED);
	ufsshpb_lu_put(hpb);
}

/*
 * end_io() is called with held the queue lock.
 * so, blk_put_request is not available directly.
 */
static void ufsshpb_put_blk_request(struct request *req)
{
	struct request_queue *q = req->q;

	if (q->mq_ops) {
		blk_mq_free_request(req);
	} else {
		lockdep_assert_held(&q->queue_lock);
		blk_put_request(req); // modify
	}
}

static inline void ufsshpb_put_pre_req(struct ufsshpb_lu *hpb,
				      struct ufsshpb_req *pre_req)
{
	pre_req->req = NULL;
	pre_req->bio = NULL;
	list_add_tail(&pre_req->list_req, &hpb->lh_pre_req_free);
	hpb->num_inflight_pre_req--;
}

static struct ufsshpb_req *ufsshpb_get_pre_req(struct ufsshpb_lu *hpb)
{
	struct ufsshpb_req *pre_req;

	if (hpb->num_inflight_pre_req >= hpb->throttle_pre_req) {
#if defined(CONFIG_HPB_DEBUG)
		HPB_DEBUG(hpb, "pre_req throttle. inflight %d throttle %d",
			  hpb->num_inflight_pre_req, hpb->throttle_pre_req);
#endif
		return NULL;
	}

	pre_req = list_first_entry_or_null(&hpb->lh_pre_req_free,
					   struct ufsshpb_req, list_req);
	if (!pre_req) {
#if defined(CONFIG_HPB_DEBUG)
		HPB_DEBUG(hpb, "There is no pre_req");
#endif
		return NULL;
	}

	list_del_init(&pre_req->list_req);
	hpb->num_inflight_pre_req++;

	return pre_req;
}

static void ufsshpb_pre_req_compl_fn(struct request *req, blk_status_t error)
{
	struct ufsshpb_req *pre_req = (struct ufsshpb_req *)req->end_io_data;
	struct ufsshpb_lu *hpb = pre_req->hpb;
	unsigned long flags;
	struct scsi_sense_hdr sshdr;

	if (error) {
		ERR_MSG("block status %d", error);
		scsi_normalize_sense(pre_req->sense, SCSI_SENSE_BUFFERSIZE,
				     &sshdr);
		ERR_MSG("code %x sense_key %x asc %x ascq %x",
			sshdr.response_code,
			sshdr.sense_key, sshdr.asc, sshdr.ascq);
		ERR_MSG("byte4 %x byte5 %x byte6 %x additional_len %x",
			sshdr.byte4, sshdr.byte5,
			sshdr.byte6, sshdr.additional_length);
	}

	bio_put(pre_req->bio);
	ufsshpb_put_blk_request(pre_req->req);
	spin_lock_irqsave(&hpb->hpb_lock, flags);
	ufsshpb_put_pre_req(pre_req->hpb, pre_req);
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);

	ufsshpb_lu_put(pre_req->hpb);
}

static int ufsshpb_prep_entry(struct ufsshpb_req *pre_req, struct page *page)
{
	struct ufsshpb_lu *hpb = pre_req->hpb;
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;
	u64 *addr;
	u64 entry_ppn = 0;
	unsigned long lpn = pre_req->wb.lpn;
	int rgn_idx, srgn_idx, srgn_offset;
	int i, error = 0;
	unsigned long flags;

	addr = page_address(page);

	spin_lock_irqsave(&hpb->hpb_lock, flags);
	for (i = 0; i < pre_req->wb.len; i++, lpn++) {
		ufsshpb_get_pos_from_lpn(hpb, lpn, &rgn_idx, &srgn_idx,
					&srgn_offset);

		rgn = hpb->rgn_tbl + rgn_idx;
		srgn = rgn->srgn_tbl + srgn_idx;

		if (!ufsshpb_valid_srgn(rgn, srgn))
			goto mctx_error;

		BUG_ON(!srgn->mctx);

		entry_ppn = ufsshpb_get_ppn(srgn->mctx, srgn_offset, &error);
		if (error)
			goto mctx_error;

#if defined(CONFIG_HPB_ERR_INJECTION)
		if (hpb->err_injection_select != HPB_ERR_INJECTION_DISABLE)
			entry_ppn =
				ufsshpb_get_err_injection_ppn(hpb, entry_ppn);
#endif

		addr[i] = entry_ppn;
	}
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	return 0;
mctx_error:
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	return -ENOMEM;
}

static int ufsshpb_pre_req_add_bio_page(struct request_queue *q,
				       struct ufsshpb_req *pre_req)
{
	struct page *page = pre_req->wb.m_page;
	struct bio *bio = pre_req->bio;
	int entries_bytes, ret;

	BUG_ON(!page);

	ret = ufsshpb_prep_entry(pre_req, page);
	if (ret)
		return ret;

	entries_bytes = pre_req->wb.len * sizeof(u64);

	ret = bio_add_pc_page(q, bio, page, entries_bytes, 0);
	if (ret != entries_bytes) {
		ERR_MSG("bio_add_pc_page fail: %d", ret);
		return -ENOMEM;
	}

	return 0;
}

static inline unsigned long ufsshpb_get_lpn(struct request *rq)
{
	return blk_rq_pos(rq) / SECTORS_PER_BLOCK;
}

static inline unsigned int ufsshpb_get_len(struct request *rq)
{
	return blk_rq_sectors(rq) / SECTORS_PER_BLOCK;
}

static inline unsigned int ufsshpb_is_unaligned(struct request *rq)
{
	return blk_rq_sectors(rq) % SECTORS_PER_BLOCK;
}

static inline int ufsshpb_issue_ctx_id_ticket(struct ufsshpb_lu *hpb)
{
	int hpb_ctx_id;

	hpb->ctx_id_ticket++;
	if (hpb->ctx_id_ticket >= MAX_HPB_CONTEXT_ID)
		hpb->ctx_id_ticket = 0;
	hpb_ctx_id = hpb->ctx_id_ticket;

	return hpb_ctx_id;
}

static inline void ufsshpb_set_prefetch_cmd(unsigned char *cdb,
					   unsigned long lpn, unsigned int len,
					   int hpb_ctx_id)
{
	int len_byte = len * HPB_ENTRY_SIZE;

	cdb[0] = UFSSHPB_WRITE_BUFFER;
	cdb[1] = UFSSHPB_WB_ID_PREFETCH;
	cdb[2] = GET_BYTE_3(lpn);
	cdb[3] = GET_BYTE_2(lpn);
	cdb[4] = GET_BYTE_1(lpn);
	cdb[5] = GET_BYTE_0(lpn);
	cdb[6] = (1 << 7) | hpb_ctx_id;
	cdb[7] = GET_BYTE_1(len_byte);
	cdb[8] = GET_BYTE_0(len_byte);
	cdb[9] = 0x00;	/* Control = 0x00 */
}

static int ufsshpb_execute_pre_req(struct ufsshpb_lu *hpb, struct scsi_cmnd *cmd,
				  struct ufsshpb_req *pre_req, int hpb_ctx_id)
{
	struct scsi_device *sdev = cmd->device;
	struct request_queue *q = sdev->request_queue;
	struct request *req;
	struct blk_mq_hw_ctx *hctx;
	struct scsi_request *rq;
	struct bio *bio = pre_req->bio;
	int ret = 0;

	pre_req->hpb = hpb;
	pre_req->wb.lpn = ufsshpb_get_lpn(cmd->request);
	pre_req->wb.len = ufsshpb_get_len(cmd->request);
	ret = ufsshpb_pre_req_add_bio_page(q, pre_req);
	if (ret)
		return ret;

	req = pre_req->req;
	hctx = blk_mq_map_queue(req->q, REQ_OP_WRITE, req->mq_ctx);

	/* 1. request setup */
	blk_rq_append_bio(req, &bio);
	req->rq_flags |= RQF_QUIET;
	req->timeout = 30 * HZ;
	req->end_io_data = (void *)pre_req;

	/* 2. scsi_request setup */
	rq = scsi_req(req);
	rq->retries = 1;
	ufsshpb_set_prefetch_cmd(rq->cmd, pre_req->wb.lpn, pre_req->wb.len,
				hpb_ctx_id);
	rq->cmd_len = scsi_command_size(rq->cmd);

	ret = ufsf_blk_execute_rq_nowait_mimic(req, hctx, ufsshpb_pre_req_compl_fn);

	if (!ret)
		atomic64_inc(&hpb->pre_req_cnt);

	return ret;
}

static int ufsshpb_issue_pre_req(struct ufsshpb_lu *hpb, struct scsi_cmnd *cmd,
				int *hpb_ctx_id)
{
	struct ufsshpb_req *pre_req;
	struct request *req = NULL;
	struct bio *bio = NULL;
	unsigned long flags;
	int ctx_id;
	int ret = 0;

	req = blk_get_request(cmd->device->request_queue,
			      REQ_OP_SCSI_OUT | REQ_SYNC,
			      BLK_MQ_REQ_PM | BLK_MQ_REQ_NOWAIT);
	if (IS_ERR(req))
		return -EAGAIN;

	bio = bio_alloc(GFP_ATOMIC, 1);
	if (!bio) {
		blk_put_request(req);
		return -EAGAIN;
	}

	spin_lock_irqsave(&hpb->hpb_lock, flags);
	pre_req = ufsshpb_get_pre_req(hpb);
	if (!pre_req) {
		ret = -EAGAIN;
		goto unlock_out;
	}
	ctx_id = ufsshpb_issue_ctx_id_ticket(hpb);
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);

	pre_req->req = req;
	pre_req->bio = bio;

#if defined(CONFIG_HPB_DEBUG)
	{
	unsigned int start_lpn, end_lpn;
	int s_rgn_idx, s_srgn_idx, s_offset;
	int e_rgn_idx, e_srgn_idx, e_offset;
		start_lpn = ufsshpb_get_lpn(cmd->request);
		end_lpn = start_lpn + ufsshpb_get_len(cmd->request) - 1;

		ufsshpb_get_pos_from_lpn(hpb, start_lpn, &s_rgn_idx, &s_srgn_idx,
					&s_offset);
		ufsshpb_get_pos_from_lpn(hpb, end_lpn, &e_rgn_idx, &e_srgn_idx,
					&e_offset);
		trace_printk("[wb.set]\t rgn %04d (%d) - rgn %04d (%d)\n",
			     s_rgn_idx, hpb->rgn_tbl[s_rgn_idx].rgn_state,
			     e_rgn_idx, hpb->rgn_tbl[e_rgn_idx].rgn_state);
	}
#endif

	ret = ufsshpb_lu_get(hpb);
	if (unlikely(ret)) {
		ERR_MSG("ufsshpb_lu_get failed. (%d)", ret);
		goto free_pre_req;
	}

	ret = ufsshpb_execute_pre_req(hpb, cmd, pre_req, ctx_id);
	if (ret) {
		ufsshpb_lu_put(hpb);
		goto free_pre_req;
	}

	*hpb_ctx_id = ctx_id;

	return ret;
free_pre_req:
	spin_lock_irqsave(&hpb->hpb_lock, flags);
	ufsshpb_put_pre_req(hpb, pre_req);
unlock_out:
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	bio_put(bio);
	blk_put_request(req);
	return ret;
}

static inline bool ufsshpb_is_support_chunk(int transfer_len)
{
	return transfer_len <= HPB_MULTI_CHUNK_HIGH;
}

static inline void ufsshpb_put_map_req(struct ufsshpb_lu *hpb,
				      struct ufsshpb_req *map_req)
{
	map_req->req = NULL;
	map_req->bio = NULL;
	list_add_tail(&map_req->list_req, &hpb->lh_map_req_free);
	hpb->num_inflight_map_req--;
	hpb->num_inflight_hcm_req--;
}

static struct ufsshpb_req *ufsshpb_get_map_req(struct ufsshpb_lu *hpb)
{
	struct ufsshpb_req *map_req;

	if (hpb->num_inflight_map_req >= hpb->throttle_map_req ||
	    hpb->num_inflight_hcm_req >= hpb->throttle_hcm_req) {
#if defined(CONFIG_HPB_DEBUG)
		HPB_DEBUG(hpb, "map_req throttle. inflight %d throttle %d",
			  hpb->num_inflight_map_req, hpb->throttle_map_req);
		HPB_DEBUG(hpb, "hcm_req throttle. hcm inflight %d hcm throttle %d",
			  hpb->num_inflight_hcm_req, hpb->throttle_hcm_req);
#endif
		return NULL;
	}

	map_req = list_first_entry_or_null(&hpb->lh_map_req_free,
					   struct ufsshpb_req, list_req);
	if (!map_req) {
#if defined(CONFIG_HPB_DEBUG)
		HPB_DEBUG(hpb, "There is no map_req");
#endif
		return NULL;
	}

	list_del_init(&map_req->list_req);
	hpb->num_inflight_map_req++;
	hpb->num_inflight_hcm_req++;

	return map_req;
}

static int ufsshpb_clean_dirty_bitmap(struct ufsshpb_lu *hpb,
				     struct ufsshpb_subregion *srgn)
{
	struct ufsshpb_region *rgn;

	BUG_ON(!srgn->mctx);

	rgn = hpb->rgn_tbl + srgn->rgn_idx;

	if (rgn->rgn_state == HPB_RGN_INACTIVE) {
		HPB_DEBUG(hpb, "%d - %d evicted", srgn->rgn_idx,
			  srgn->srgn_idx);
		return -EINVAL;
	}

	memset(srgn->mctx->ppn_dirty, 0x00,
	       hpb->entries_per_srgn >> bits_per_byte_shift);

	return 0;
}

static void ufsshpb_clean_active_subregion(struct ufsshpb_lu *hpb,
					  struct ufsshpb_subregion *srgn)
{
	struct ufsshpb_region *rgn;

	BUG_ON(!srgn->mctx);

	rgn = hpb->rgn_tbl + srgn->rgn_idx;

	if (rgn->rgn_state == HPB_RGN_INACTIVE) {
		HPB_DEBUG(hpb, "%d - %d evicted", srgn->rgn_idx,
			  srgn->srgn_idx);
		return;
	}
	srgn->srgn_state = HPB_SRGN_CLEAN;
}

static void ufsshpb_error_active_subregion(struct ufsshpb_lu *hpb,
					  struct ufsshpb_subregion *srgn)
{
	struct ufsshpb_region *rgn;

	BUG_ON(!srgn->mctx);

	rgn = hpb->rgn_tbl + srgn->rgn_idx;

	if (rgn->rgn_state == HPB_RGN_INACTIVE) {
		ERR_MSG("%d - %d evicted", srgn->rgn_idx, srgn->srgn_idx);
		return;
	}
	srgn->srgn_state = HPB_SRGN_DIRTY;
}

#if defined(CONFIG_HPB_DEBUG)
static void ufsshpb_check_ppn(struct ufsshpb_lu *hpb, int rgn_idx, int srgn_idx,
			     struct ufsshpb_map_ctx *mctx, const char *str)
{
	int error = 0;
	u64 val[2];

	BUG_ON(!mctx);

	val[0] = ufsshpb_get_ppn(mctx, 0, &error);
	if (!error)
		val[1] = ufsshpb_get_ppn(mctx, hpb->entries_per_srgn - 1,
					&error);
	if (error)
		val[0] = val[1] = 0;

	HPB_DEBUG(hpb, "%s READ BUFFER %d - %d ( %llx ~ %llx )", str, rgn_idx,
		  srgn_idx, val[0], val[1]);
}
#endif

static bool is_clear_reason(struct ufsshpb_lu *hpb, struct ufsshpb_region *rgn)
{
	struct ufsshpb_subregion *srgn;
	unsigned long flags;
	bool ret = true;
	int srgn_idx;

	spin_lock_irqsave(&hpb->rsp_list_lock, flags);
	if (!list_empty(&rgn->list_inact_rgn))
		ret = false;
	else {
		for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) {
			srgn = rgn->srgn_tbl + srgn_idx;

			if (!list_empty(&srgn->list_act_srgn)) {
				ret = false;
				break;
			}
		}
	}
	spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);

	return ret;
}

static void ufsshpb_map_compl_process(struct ufsshpb_req *map_req)
{
	struct ufsshpb_lu *hpb = map_req->hpb;
	struct ufsshpb_region *rgn = hpb->rgn_tbl + map_req->rb.rgn_idx;
	struct ufsshpb_subregion *srgn = rgn->srgn_tbl + map_req->rb.srgn_idx;
	unsigned long flags;

	atomic64_inc(&hpb->map_compl_cnt);

#if defined(CONFIG_HPB_DEBUG)
	if (hpb->debug)
		ufsshpb_check_ppn(hpb, srgn->rgn_idx, srgn->srgn_idx, srgn->mctx,
				 "COMPL");

	TMSG(hpb->ufsf, hpb->lun, "Noti: C RB %d - %d", map_req->rb.rgn_idx,
	     map_req->rb.srgn_idx);
	trace_printk("[rb.cpl]\t rgn %04d (%d)\n", map_req->rb.rgn_idx, rgn->rgn_state);
#endif

	spin_lock_irqsave(&hpb->hpb_lock, flags);
	ufsshpb_clean_active_subregion(hpb, srgn);
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);

	if (is_clear_reason(hpb, rgn))
		atomic_set(&rgn->reason, HPB_UPDATE_NONE);
}

static inline bool ufsshpb_is_hpb_flag(struct request *rq)
{
	return false;
}

static inline void ufsshpb_act_rsp_list_add(struct ufsshpb_lu *hpb,
					   struct ufsshpb_subregion *srgn)
{
#if defined(CONFIG_HPB_DEBUG_SYSFS)
	atomic64_inc(&hpb->act_rsp_list_cnt);
#endif
	list_add_tail(&srgn->list_act_srgn, &hpb->lh_act_srgn);
#if defined(CONFIG_HPB_DEBUG)
	trace_printk("[al.add]\t rgn %04d (%d) srgn %d\n",
		     srgn->rgn_idx, hpb->rgn_tbl[srgn->rgn_idx].rgn_state,
		     srgn->srgn_idx);
#endif
}

static inline void ufsshpb_act_rsp_list_del(struct ufsshpb_lu *hpb,
					   struct ufsshpb_subregion *srgn)
{
#if defined(CONFIG_HPB_DEBUG_SYSFS)
	atomic64_dec(&hpb->act_rsp_list_cnt);
#endif
	list_del_init(&srgn->list_act_srgn);
#if defined(CONFIG_HPB_DEBUG)
	trace_printk("[al.del]\t rgn %04d (%d)\n",
		     srgn->rgn_idx, hpb->rgn_tbl[srgn->rgn_idx].rgn_state);
#endif
}

static inline void ufsshpb_inact_rsp_list_add(struct ufsshpb_lu *hpb,
					     struct ufsshpb_region *rgn)
{
#if defined(CONFIG_HPB_DEBUG_SYSFS)
	atomic64_inc(&hpb->inact_rsp_list_cnt);
#endif
	list_add_tail(&rgn->list_inact_rgn, &hpb->lh_inact_rgn);
#if defined(CONFIG_HPB_DEBUG)
	trace_printk("[il.add]\t rgn %04d (%d)\n",
		     rgn->rgn_idx, hpb->rgn_tbl[rgn->rgn_idx].rgn_state);
#endif
}

static inline void ufsshpb_inact_rsp_list_del(struct ufsshpb_lu *hpb,
					     struct ufsshpb_region *rgn)
{
#if defined(CONFIG_HPB_DEBUG_SYSFS)
	atomic64_dec(&hpb->inact_rsp_list_cnt);
#endif
	list_del_init(&rgn->list_inact_rgn);
#if defined(CONFIG_HPB_DEBUG)
	trace_printk("[il.del]\t rgn %04d (%d)\n",
		     rgn->rgn_idx, hpb->rgn_tbl[rgn->rgn_idx].rgn_state);
#endif
}

static void ufsshpb_update_active_info(struct ufsshpb_lu *hpb, int rgn_idx,
				      int srgn_idx, enum HPB_UPDATE_INFO info)
{
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;

	rgn = hpb->rgn_tbl + rgn_idx;
	srgn = rgn->srgn_tbl + srgn_idx;

	if (!list_empty(&rgn->list_inact_rgn))
		ufsshpb_inact_rsp_list_del(hpb, rgn);

	if (list_empty(&srgn->list_act_srgn)) {
		ufsshpb_act_rsp_list_add(hpb, srgn);
#if defined(CONFIG_HPB_DEBUG)
		if (info == HPB_UPDATE_FROM_FS) {
			HPB_DEBUG(hpb, "*** Noti [HOST]: HCM SET %d - %d",
				  rgn_idx, srgn_idx);
			TMSG(hpb->ufsf, hpb->lun, "*** Noti [HOST]: HCM SET %d - %d",
			     rgn_idx, srgn_idx);
			trace_printk("[act.fs]\t HCM SET rgn %04d (%d)\n",
				     rgn_idx, hpb->rgn_tbl[rgn_idx].rgn_state);
		} else if (info == HPB_UPDATE_FROM_DEV) {
			HPB_DEBUG(hpb, "*** Noti [DEV]: ACT %d - %d",
				  rgn_idx, srgn_idx);
			TMSG(hpb->ufsf, hpb->lun, "*** Noti [DEV]: ACT %d - %d",
			     rgn_idx, srgn_idx);
			trace_printk("[act.dv]\t ACT SET rgn %04d (%d)\n",
				     rgn_idx, hpb->rgn_tbl[rgn_idx].rgn_state);
		}
#endif
	}
}

static void ufsshpb_update_inactive_info(struct ufsshpb_lu *hpb, int rgn_idx,
					enum HPB_UPDATE_INFO info)
{
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;
	int srgn_idx;

	rgn = hpb->rgn_tbl + rgn_idx;

	for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) {
		srgn = rgn->srgn_tbl + srgn_idx;

		if (!list_empty(&srgn->list_act_srgn))
			ufsshpb_act_rsp_list_del(hpb, srgn);
	}

	if (list_empty(&rgn->list_inact_rgn)) {
		ufsshpb_inact_rsp_list_add(hpb, rgn);
#if defined(CONFIG_HPB_DEBUG)
		if (info == HPB_UPDATE_FROM_FS) {
			HPB_DEBUG(hpb, "*** Noti [HOST]: HCM UNSET %d", rgn_idx);
			TMSG(hpb->ufsf, hpb->lun, "*** Noti [HOST]: HCM UNSET %d",
			     rgn_idx);
			trace_printk("[ina.fs]\t HCM UNSET rgn %04d (%d)\n",
				     rgn_idx, hpb->rgn_tbl[rgn_idx].rgn_state);
		} else if (info == HPB_UPDATE_FROM_DEV) {
			HPB_DEBUG(hpb, "*** Noti [DEV]: INACT %d", rgn_idx);
			TMSG(hpb->ufsf, hpb->lun, "*** Noti [DEV]: INACT %d",
			     rgn_idx);
			trace_printk("[ina.dv]\t INACT rgn %04d (%d)\n",
				     rgn_idx, hpb->rgn_tbl[rgn_idx].rgn_state);
		} else {
			HPB_DEBUG(hpb, "*** ERR : INACT %d", rgn_idx);
			TMSG(hpb->ufsf, hpb->lun, "*** ERR [DEV]: INACT %d",
			     rgn_idx);
			trace_printk("[ina.--]\t ERR INACT rgn %04d (%d)\n",
				     rgn_idx, hpb->rgn_tbl[rgn_idx].rgn_state);
		}
#endif
	}
}

static void ufsshpb_update_active_info_by_flag(struct ufsshpb_lu *hpb,
					      struct scsi_cmnd *cmd)
{
	struct ufsshpb_region *rgn;
	struct request *rq = cmd->request;
	int s_rgn_idx, e_rgn_idx, rgn_idx, srgn_idx;
	bool is_update = false;
	unsigned int start_lpn, end_lpn;
	unsigned long flags;

	start_lpn = ufsshpb_get_lpn(rq);
	end_lpn = start_lpn + ufsshpb_get_len(rq) - 1;

	s_rgn_idx = start_lpn >> hpb->entries_per_rgn_shift;
	e_rgn_idx = end_lpn >> hpb->entries_per_rgn_shift;

	for (rgn_idx = s_rgn_idx; rgn_idx <= e_rgn_idx; rgn_idx++) {
		is_update = false;
		rgn = hpb->rgn_tbl + rgn_idx;

		spin_lock_irqsave(&hpb->hpb_lock, flags);
		if (rgn->rgn_state == HPB_RGN_INACTIVE ||
		    rgn->rgn_state == HPB_RGN_ACTIVE) {
			is_update = true;
			atomic_set(&rgn->reason, HPB_UPDATE_FROM_FS);
		} else if (rgn->rgn_state == HPB_RGN_HCM) {
			if (ufsshpb_is_write_cmd(cmd))
				is_update = true;
		}
		spin_unlock_irqrestore(&hpb->hpb_lock, flags);

		if (is_update) {
#if defined(CONFIG_HPB_DEBUG_SYSFS)
			atomic64_inc(&hpb->fs_set_hcm_cnt);
#endif
#if defined(CONFIG_HPB_DEBUG)
			trace_printk("[fs.add]\t rgn %04d (%d)\n",
				     rgn_idx, rgn->rgn_state);

			TMSG(hpb->ufsf, hpb->lun, "%llu + %u set hcm %d",
			     (unsigned long long) blk_rq_pos(rq),
			     (unsigned int) blk_rq_sectors(rq), rgn_idx);
#endif
			spin_lock_irqsave(&hpb->rsp_list_lock, flags);
			for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++)
				ufsshpb_update_active_info(hpb, rgn_idx,
							  srgn_idx,
							  HPB_UPDATE_FROM_FS);
			spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
		}
	}
}

static void ufsshpb_update_inactive_info_by_flag(struct ufsshpb_lu *hpb,
						struct scsi_cmnd *cmd)
{
	struct ufsshpb_region *rgn;
	struct request *rq = cmd->request;
	int s_rgn_idx, s_srgn_idx, s_offset;
	int e_rgn_idx, e_srgn_idx, e_offset;
	int rgn_idx;
	bool is_update = false;
	unsigned int start_lpn, end_lpn;
	unsigned long flags;

	start_lpn = ufsshpb_get_lpn(rq);
	end_lpn = start_lpn + ufsshpb_get_len(rq) - 1;

	ufsshpb_get_pos_from_lpn(hpb, start_lpn, &s_rgn_idx, &s_srgn_idx,
				&s_offset);
	ufsshpb_get_pos_from_lpn(hpb, end_lpn, &e_rgn_idx, &e_srgn_idx,
				&e_offset);

	if (ufsshpb_is_read_cmd(cmd) && !ufsshpb_is_hpb_flag(rq)) {
		if (s_srgn_idx != 0 || s_offset != 0)
			s_rgn_idx++;

		if (e_srgn_idx != (hpb->srgns_per_rgn - 1) ||
		    e_offset != (hpb->entries_per_srgn - 1))
			e_rgn_idx--;
	}

	for (rgn_idx = s_rgn_idx; rgn_idx <= e_rgn_idx; rgn_idx++) {
		is_update = false;
		rgn = hpb->rgn_tbl + rgn_idx;

		spin_lock_irqsave(&hpb->hpb_lock, flags);
		if (rgn->rgn_state == HPB_RGN_HCM) {
			is_update = true;
			atomic_set(&rgn->reason, HPB_UPDATE_FROM_FS);
		}
		spin_unlock_irqrestore(&hpb->hpb_lock, flags);

		if (is_update) {
#if defined(CONFIG_HPB_DEBUG_SYSFS)
			atomic64_inc(&hpb->fs_unset_hcm_cnt);
#endif
#if defined(CONFIG_HPB_DEBUG)
			trace_printk("[fs.del]\t rgn %04d (%d)\n",
				     rgn_idx, rgn->rgn_state);
			TMSG(hpb->ufsf, hpb->lun, "%llu + %u unset hcm %d",
			     (unsigned long long) blk_rq_pos(rq),
			     (unsigned int) blk_rq_sectors(rq), rgn_idx);
#endif
			spin_lock_irqsave(&hpb->rsp_list_lock, flags);
			ufsshpb_update_inactive_info(hpb, rgn_idx,
						    HPB_UPDATE_FROM_FS);
			spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
		}
	}
}

static void ufsshpb_hit_lru_info_by_read(struct victim_select_info *lru_info,
					struct ufsshpb_region *rgn);

#if defined(CONFIG_HPB_DEBUG_SYSFS)
static void ufsshpb_increase_hit_count(struct ufsshpb_lu *hpb,
				      struct ufsshpb_region *rgn,
				      int transfer_len)
{
	if (rgn->rgn_state == HPB_RGN_PINNED ||
	    rgn->rgn_state == HPB_RGN_HCM) {
		if (transfer_len < HPB_MULTI_CHUNK_LOW) {
			atomic64_inc(&hpb->pinned_low_hit);
			atomic64_inc(&rgn->rgn_pinned_low_hit);
		} else {
			atomic64_inc(&hpb->pinned_high_hit);
			atomic64_inc(&rgn->rgn_pinned_high_hit);
		}
	} else if (rgn->rgn_state == HPB_RGN_ACTIVE) {
		if (transfer_len < HPB_MULTI_CHUNK_LOW) {
			atomic64_inc(&hpb->active_low_hit);
			atomic64_inc(&rgn->rgn_active_low_hit);
		} else {
			atomic64_inc(&hpb->active_high_hit);
			atomic64_inc(&rgn->rgn_active_high_hit);
		}
	}
}

static void ufsshpb_increase_miss_count(struct ufsshpb_lu *hpb,
				       struct ufsshpb_region *rgn,
				       int transfer_len)
{
	if (rgn->rgn_state == HPB_RGN_PINNED ||
	    rgn->rgn_state == HPB_RGN_HCM) {
		if (transfer_len < HPB_MULTI_CHUNK_LOW) {
			atomic64_inc(&hpb->pinned_low_miss);
			atomic64_inc(&rgn->rgn_pinned_low_miss);
		} else {
			atomic64_inc(&hpb->pinned_high_miss);
			atomic64_inc(&rgn->rgn_pinned_high_miss);
		}
	} else if (rgn->rgn_state == HPB_RGN_ACTIVE) {
		if (transfer_len < HPB_MULTI_CHUNK_LOW) {
			atomic64_inc(&hpb->active_low_miss);
			atomic64_inc(&rgn->rgn_active_low_miss);
		} else {
			atomic64_inc(&hpb->active_high_miss);
			atomic64_inc(&rgn->rgn_active_high_miss);
		}
	} else if (rgn->rgn_state == HPB_RGN_INACTIVE) {
		if (transfer_len < HPB_MULTI_CHUNK_LOW) {
			atomic64_inc(&hpb->inactive_low_read);
			atomic64_inc(&rgn->rgn_inactive_low_read);
		} else {
			atomic64_inc(&hpb->inactive_high_read);
			atomic64_inc(&rgn->rgn_inactive_high_read);
		}
	}
}

static void ufsshpb_increase_rb_count(struct ufsshpb_lu *hpb,
				     struct ufsshpb_region *rgn)
{
	if (rgn->rgn_state == HPB_RGN_PINNED ||
	    rgn->rgn_state == HPB_RGN_HCM) {
		atomic64_inc(&hpb->pinned_rb_cnt);
		atomic64_inc(&rgn->rgn_pinned_rb_cnt);
	} else if (rgn->rgn_state == HPB_RGN_ACTIVE) {
		atomic64_inc(&hpb->active_rb_cnt);
		atomic64_inc(&rgn->rgn_active_rb_cnt);
	}
}

static void ufsshpb_debug_sys_init(struct ufsshpb_lu *hpb)
{
	struct ufsshpb_region *rgn;
	int rgn_idx;

	atomic64_set(&hpb->pinned_low_hit, 0);
	atomic64_set(&hpb->pinned_high_hit, 0);
	atomic64_set(&hpb->pinned_low_miss, 0);
	atomic64_set(&hpb->pinned_high_miss, 0);
	atomic64_set(&hpb->active_low_hit, 0);
	atomic64_set(&hpb->active_high_hit, 0);
	atomic64_set(&hpb->active_low_miss, 0);
	atomic64_set(&hpb->active_high_miss, 0);
	atomic64_set(&hpb->inactive_low_read, 0);
	atomic64_set(&hpb->inactive_high_read, 0);
	atomic64_set(&hpb->pinned_rb_cnt, 0);
	atomic64_set(&hpb->active_rb_cnt, 0);
	atomic64_set(&hpb->fs_set_hcm_cnt, 0);
	atomic64_set(&hpb->fs_unset_hcm_cnt, 0);
	atomic64_set(&hpb->unset_hcm_victim, 0);

	for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) {
		rgn = hpb->rgn_tbl + rgn_idx;
		atomic64_set(&rgn->rgn_pinned_low_hit, 0);
		atomic64_set(&rgn->rgn_pinned_high_hit, 0);
		atomic64_set(&rgn->rgn_pinned_low_miss, 0);
		atomic64_set(&rgn->rgn_pinned_high_miss, 0);
		atomic64_set(&rgn->rgn_active_low_hit, 0);
		atomic64_set(&rgn->rgn_active_high_hit, 0);
		atomic64_set(&rgn->rgn_active_low_miss, 0);
		atomic64_set(&rgn->rgn_active_high_miss, 0);
		atomic64_set(&rgn->rgn_inactive_low_read, 0);
		atomic64_set(&rgn->rgn_inactive_high_read, 0);
		atomic64_set(&rgn->rgn_pinned_rb_cnt, 0);
		atomic64_set(&rgn->rgn_active_rb_cnt, 0);
	}
}
#endif

#if defined(CONFIG_HPB_DEBUG)
static void ufsshpb_blk_fill_rwbs(char *rwbs, unsigned int op)
{
	int i = 0;

	if (op & REQ_PREFLUSH)
		rwbs[i++] = 'F';

	switch (op & REQ_OP_MASK) {
	case REQ_OP_WRITE:
	case REQ_OP_WRITE_SAME:
		rwbs[i++] = 'W';
		break;
	case REQ_OP_DISCARD:
		rwbs[i++] = 'D';
		break;
	case REQ_OP_SECURE_ERASE:
		rwbs[i++] = 'D';
		rwbs[i++] = 'E';
		break;
	case REQ_OP_FLUSH:
		rwbs[i++] = 'F';
		break;
	case REQ_OP_READ:
		rwbs[i++] = 'R';
		break;
	default:
		rwbs[i++] = 'N';
	}

	if (op & REQ_FUA)
		rwbs[i++] = 'F';
	if (op & REQ_RAHEAD)
		rwbs[i++] = 'A';
	if (op & REQ_SYNC)
		rwbs[i++] = 'S';
	if (op & REQ_META)
		rwbs[i++] = 'M';

	rwbs[i] = '\0';
}
#endif

static int ufsshpb_fill_passthrough_rq(struct scsi_cmnd *cmd)
{
	struct request *rq = cmd->request;
	struct scsi_device *sdp = cmd->device;
	struct bio *bio = rq->bio;
	sector_t sector = 0;
	unsigned int data_len = 0;
	u8 *p = NULL;

	/*
	 * UFS is 4K block device.
	 */
	if (unlikely(sdp->sector_size != BLOCK)) {
		ERR_MSG("this device is not 4K block device");
		return -EACCES;
	}

	switch (cmd->cmnd[0]) {
	case UNMAP:
		/*
		 * UNMAP command has just a block descriptor(=a phys_segment).
		 */
		if (unlikely(!bio) ||
		    unlikely(bio->bi_vcnt != 1)) {
		    //unlikely(bio->bi_phys_segments != 1)) {
			ERR_MSG("UNMAP bio info is incorrect");
			return -EINVAL;
		}

		p = page_address(bio->bi_io_vec->bv_page);
		if (unlikely(!p)) {
			ERR_MSG("can't get bio page address");
			return -ENOMEM;
		}

		sector = LI_EN_64(p + 8);
		data_len = LI_EN_32(p + 16);
		break;
	case READ_10:
	case WRITE_10:
		sector = LI_EN_32(cmd->cmnd + 2);
		data_len = LI_EN_16(cmd->cmnd + 7);
		break;
	case READ_16:
	case WRITE_16:
		sector = LI_EN_64(cmd->cmnd + 2);
		data_len = LI_EN_32(cmd->cmnd + 10);
		break;
	default:
		return 0;
	};

	rq->__sector = sector << sects_per_blk_shift;
	rq->__data_len = data_len << sects_per_blk_shift << SECTOR_SHIFT;

	return 0;
}

/* routine : READ10 -> HPB_READ  */
int ufsshpb_prep_fn(struct ufsf_feature *ufsf, struct ufshcd_lrb *lrbp)
{
	struct ufsshpb_lu *hpb;
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;
	struct request *rq;
	struct scsi_cmnd *cmd = lrbp->cmd;
	u64 ppn = 0;
	unsigned long lpn, flags;
	int hpb_ctx_id = MAX_HPB_CONTEXT_ID;
	int transfer_len = TRANSFER_LEN;
	int rgn_idx, srgn_idx, srgn_offset, ret, error = 0;
#if defined(CONFIG_HPB_DEBUG)
	char hint;
	char rwbs[8];
#endif

	/* WKLU could not be HPB-LU */
	if (!lrbp || !ufsf_is_valid_lun(lrbp->lun))
		return 0;

	if (!ufsshpb_is_read_cmd(cmd) && !ufsshpb_is_write_cmd(cmd) &&
	    !ufsshpb_is_discard_cmd(cmd))
		return 0;

	rq = cmd->request;
	if (unlikely(blk_rq_is_passthrough(rq) &&
		     rq->__sector == (sector_t)-1)) {
		ret = ufsshpb_fill_passthrough_rq(cmd);
		if (ret)
			return 0;
	}

	hpb = ufsf->hpb_lup[lrbp->lun];
	ret = ufsshpb_lu_get(hpb);
	if (unlikely(ret))
		return 0;

	lpn = ufsshpb_get_lpn(rq);
	ufsshpb_get_pos_from_lpn(hpb, lpn, &rgn_idx, &srgn_idx, &srgn_offset);
	rgn = hpb->rgn_tbl + rgn_idx;
	srgn = rgn->srgn_tbl + srgn_idx;

	if (hpb->force_disable) {
#if defined(CONFIG_HPB_DEBUG)
		if (ufsshpb_is_read_cmd(cmd)) {
			TMSG(ufsf, hpb->lun, "%llu + %u READ_10",
			     (unsigned long long) blk_rq_pos(rq),
			     (unsigned int) blk_rq_sectors(rq));
			trace_printk("[rd.nor]\t %llu + %u READ_10 (force_disable)\n",
			     (unsigned long long) blk_rq_pos(rq),
			     (unsigned int) blk_rq_sectors(rq));
		}
#endif
		goto put_hpb;
	}

#if defined(CONFIG_HPB_DEBUG)
	if (cmd->request->bio == NULL)
		hint = '-';
	else if (cmd->request->bio->bi_write_hint == WRITE_LIFE_EXTREME)
		hint = 'E';
	else if (cmd->request->bio->bi_write_hint == WRITE_LIFE_MEDIUM)
		hint = 'M';
	else if (cmd->request->bio->bi_write_hint == WRITE_LIFE_LONG)
		hint = 'L';
	else if (cmd->request->bio->bi_write_hint == WRITE_LIFE_SHORT)
		hint = 'S';
	else if (cmd->request->bio->bi_write_hint == WRITE_LIFE_NONE)
		hint = 'N';
	else if (cmd->request->bio->bi_write_hint == WRITE_LIFE_NOT_SET)
		hint = '-';
	ufsshpb_blk_fill_rwbs(rwbs, cmd->request->cmd_flags);
	if (rwbs[0] != 'N')
		trace_printk("[%s]\t%llu\t+%u\t<%c> rgn %05d (%d)\n",
			     rwbs, (u64)blk_rq_pos(cmd->request)>>3,
			     blk_rq_sectors(cmd->request)>>3, hint,
			     rgn_idx, rgn->rgn_state);
#endif

	/*
	 * If cmd type is WRITE, bitmap set to dirty.
	 */
	if (ufsshpb_is_write_cmd(cmd) || ufsshpb_is_discard_cmd(cmd)) {
		spin_lock_irqsave(&hpb->hpb_lock, flags);
		if (rgn->rgn_state != HPB_RGN_INACTIVE) {
			ufsshpb_set_dirty(hpb, lrbp, rgn_idx, srgn_idx,
					 srgn_offset);
		}
		spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	}

	if (ufsshpb_is_read_cmd(cmd)) {
		spin_lock_irqsave(&hpb->hpb_lock, flags);
		if (!list_empty(&rgn->list_lru_rgn))
			ufsshpb_hit_lru_info_by_read(&hpb->lru_info, rgn);
		spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	}

	if (hpb->hcm_disable != true) {
		if (ufsshpb_is_read_cmd(cmd) && ufsshpb_is_hpb_flag(rq))
			ufsshpb_update_active_info_by_flag(hpb, cmd);
		else if (ufsshpb_is_discard_cmd(cmd) ||
			 (ufsshpb_is_read_cmd(cmd) && !ufsshpb_is_hpb_flag(rq)))
			ufsshpb_update_inactive_info_by_flag(hpb, cmd);
	}

	if (!ufsshpb_is_read_cmd(cmd))
		goto put_hpb;

	if (unlikely(ufsshpb_is_unaligned(rq))) {
#if defined(CONFIG_HPB_DEBUG)
		TMSG_CMD(hpb, "READ_10 not aligned 4KB", rq, rgn_idx, srgn_idx);
		trace_printk("[rd.nor]\t %llu + %u READ_10 rgn %04d (%d) (not aligned)\n",
		     (unsigned long long) blk_rq_pos(rq),
		     (unsigned int) blk_rq_sectors(rq),
		     rgn_idx, rgn->rgn_state);
#endif
		goto put_hpb;
	}

	transfer_len = ufsshpb_get_len(rq);
	if (unlikely(!transfer_len))
		goto put_hpb;

	if (!ufsshpb_is_support_chunk(transfer_len)) {
#if defined(CONFIG_HPB_DEBUG)
		TMSG_CMD(hpb, "READ_10 doesn't support chunk size",
			 rq, rgn_idx, srgn_idx);
		trace_printk("[rd.nor]\t %llu + %u READ_10 rgn %04d (%d) (not support chunk)\n",
		     (unsigned long long) blk_rq_pos(rq),
		     (unsigned int) blk_rq_sectors(rq),
		     rgn_idx, rgn->rgn_state);
#endif
		goto put_hpb;
	}

	spin_lock_irqsave(&hpb->hpb_lock, flags);
	if (ufsshpb_ppn_dirty_check(hpb, lpn, transfer_len)) {
		atomic64_inc(&hpb->miss);
#if defined(CONFIG_HPB_DEBUG_SYSFS)
		ufsshpb_increase_miss_count(hpb, rgn, transfer_len);
#endif
		spin_unlock_irqrestore(&hpb->hpb_lock, flags);
#if defined(CONFIG_HPB_DEBUG)
		TMSG_CMD(hpb, "READ_10 E_D", rq, rgn_idx, srgn_idx);
		trace_printk("[rd.nor]\t %llu + %u READ_10 rgn %04d (%d) (hpb map invalid)\n",
		     (unsigned long long) blk_rq_pos(rq),
		     (unsigned int) blk_rq_sectors(rq),
		     rgn_idx, rgn->rgn_state);
#endif
		goto put_hpb;
	}

	ppn = ufsshpb_get_ppn(srgn->mctx, srgn_offset, &error);
#if defined(CONFIG_HPB_ERR_INJECTION)
	if (hpb->err_injection_select != HPB_ERR_INJECTION_DISABLE)
		ppn = ufsshpb_get_err_injection_ppn(hpb, ppn);
#endif
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	if (unlikely(error)) {
		ERR_MSG("get_ppn failed.. err %d region %d subregion %d",
			error, rgn_idx, srgn_idx);
		ufsshpb_lu_put(hpb);
		goto wakeup_ee_worker;
	}

	/*
	 * WRITE_BUFFER CMD support 36K (len=9) ~ 512K (len=128) default.
	 * it is possible to change range of transfer_len through sysfs.
	 */
	if (transfer_len >= hpb->pre_req_min_tr_len &&
	    transfer_len <= hpb->pre_req_max_tr_len) {
		ret = ufsshpb_issue_pre_req(hpb, cmd, &hpb_ctx_id);
		if (ret) {
			if (ret == -EAGAIN) {
				unsigned long timeout;

				timeout = cmd->jiffies_at_alloc +
				     msecs_to_jiffies(hpb->requeue_timeout_ms);

				if (time_before(jiffies, timeout))
					goto requeue;
			}

			atomic64_inc(&hpb->miss);
			goto put_hpb;
		}
	}

	ufsshpb_set_read16_cmd(hpb, cmd, ppn, transfer_len, hpb_ctx_id);
#if defined(CONFIG_HPB_DEBUG)
	trace_printk("[rd.hpb]\t %llu + %u READ_HPB rgn %04d (%d)- context_id %d\n",
	     (unsigned long long) blk_rq_pos(rq),
	     (unsigned int) blk_rq_sectors(rq),
	     rgn_idx, rgn->rgn_state, hpb_ctx_id);
	TMSG(ufsf, hpb->lun, "%llu + %u HPB_READ %d - %d context_id %d",
	     (unsigned long long) blk_rq_pos(rq),
	     (unsigned int) blk_rq_sectors(rq), rgn_idx,
	     srgn_idx, hpb_ctx_id);
#endif

	atomic64_inc(&hpb->hit);
#if defined(CONFIG_HPB_DEBUG_SYSFS)
	ufsshpb_increase_hit_count(hpb, rgn, transfer_len);
#endif
put_hpb:
	ufsshpb_lu_put(hpb);
	return 0;
wakeup_ee_worker:
	ufsshpb_failed(hpb, __func__);
	return 0;
requeue:
	ufsshpb_lu_put(hpb);
	return ret;
}

static int ufsshpb_map_req_error(struct ufsshpb_req *map_req)
{
	struct ufsshpb_lu *hpb = map_req->hpb;
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;
	struct scsi_sense_hdr sshdr;
	unsigned long flags;

	rgn = hpb->rgn_tbl + map_req->rb.rgn_idx;
	srgn = rgn->srgn_tbl + map_req->rb.srgn_idx;

	scsi_normalize_sense(map_req->sense, SCSI_SENSE_BUFFERSIZE, &sshdr);

	ERR_MSG("code %x sense_key %x asc %x ascq %x", sshdr.response_code,
		sshdr.sense_key, sshdr.asc, sshdr.ascq);
	ERR_MSG("byte4 %x byte5 %x byte6 %x additional_len %x", sshdr.byte4,
		sshdr.byte5, sshdr.byte6, sshdr.additional_length);

	if (sshdr.sense_key != ILLEGAL_REQUEST)
		return 0;

	spin_lock_irqsave(&hpb->hpb_lock, flags);
	if (rgn->rgn_state == HPB_RGN_PINNED) {
		if (sshdr.asc == 0x06 && sshdr.ascq == 0x01) {
#if defined(CONFIG_HPB_DEBUG)
			HPB_DEBUG(hpb, "retry pinned rb %d - %d",
				  map_req->rb.rgn_idx, map_req->rb.srgn_idx);
#endif
			spin_unlock_irqrestore(&hpb->hpb_lock, flags);
			spin_lock(&hpb->retry_list_lock);
			list_add_tail(&map_req->list_req,
				      &hpb->lh_map_req_retry);
			spin_unlock(&hpb->retry_list_lock);

			schedule_delayed_work(&hpb->retry_work,
					      msecs_to_jiffies(RETRY_DELAY_MS));
			return -EAGAIN;
		}
#if defined(CONFIG_HPB_DEBUG)
		HPB_DEBUG(hpb, "pinned rb %d - %d(dirty)",
			  map_req->rb.rgn_idx, map_req->rb.srgn_idx);
#endif
		ufsshpb_error_active_subregion(hpb, srgn);
		spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	} else {
		ufsshpb_error_active_subregion(hpb, srgn);

		spin_unlock_irqrestore(&hpb->hpb_lock, flags);

		spin_lock_irqsave(&hpb->rsp_list_lock, flags);
		ufsshpb_update_inactive_info(hpb, map_req->rb.rgn_idx,
					    HPB_UPDATE_NONE);
		spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
#if defined(CONFIG_HPB_DEBUG)
		HPB_DEBUG(hpb, "Non-pinned rb %d will be inactive",
			  map_req->rb.rgn_idx);
#endif
		schedule_work(&hpb->task_work);
	}

	return 0;
}

/* routine : map_req compl */
static void ufsshpb_map_req_compl_fn(struct request *req, blk_status_t error)
{
	struct ufsshpb_req *map_req = (struct ufsshpb_req *) req->end_io_data;
	struct ufsshpb_lu *hpb = map_req->hpb;
	unsigned long flags;
	int ret;

	if (ufsshpb_get_state(hpb->ufsf) == HPB_FAILED)
		goto free_map_req;

	if (error) {
		ERR_MSG("COMP_NOTI: RB number %d ( %d - %d )", error,
			map_req->rb.rgn_idx, map_req->rb.srgn_idx);

		ret = ufsshpb_map_req_error(map_req);
		if (ret)
			goto retry_map_req;
	} else
		ufsshpb_map_compl_process(map_req);

free_map_req:
	bio_put(map_req->bio);
	ufsshpb_put_blk_request(map_req->req);
	spin_lock_irqsave(&hpb->hpb_lock, flags);
	ufsshpb_put_map_req(map_req->hpb, map_req);
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
retry_map_req:
	scsi_device_put(hpb->ufsf->sdev_ufs_lu[hpb->lun]);
	ufsshpb_lu_put(hpb);
}

static inline void ufsshpb_set_read_buf_cmd(unsigned char *cdb, int rgn_idx,
					   int srgn_idx, int srgn_mem_size)
{
	cdb[0] = UFSSHPB_READ_BUFFER;
	cdb[1] = UFSSHPB_RB_ID_READ;
	cdb[2] = GET_BYTE_1(rgn_idx);
	cdb[3] = GET_BYTE_0(rgn_idx);
	cdb[4] = GET_BYTE_1(srgn_idx);
	cdb[5] = GET_BYTE_0(srgn_idx);
	cdb[6] = GET_BYTE_2(srgn_mem_size);
	cdb[7] = GET_BYTE_1(srgn_mem_size);
	cdb[8] = GET_BYTE_0(srgn_mem_size);
	cdb[9] = 0x00;
}

static inline void ufsshpb_set_hcm_cmd(unsigned char *cdb, int rgn_idx,
				      int srgn_idx, int srgn_mem_size)
{
	cdb[0] = UFSSHPB_READ_BUFFER;
	cdb[1] = UFSSHPB_RB_ID_SET_HCM;
	cdb[2] = GET_BYTE_1(rgn_idx);
	cdb[3] = GET_BYTE_0(rgn_idx);
	cdb[4] = GET_BYTE_1(srgn_idx);
	cdb[5] = GET_BYTE_0(srgn_idx);
	cdb[6] = GET_BYTE_2(srgn_mem_size);
	cdb[7] = GET_BYTE_1(srgn_mem_size);
	cdb[8] = GET_BYTE_0(srgn_mem_size);
	cdb[9] = 0x00;
}

static int ufsshpb_map_req_add_bio_page(struct ufsshpb_lu *hpb,
				       struct request_queue *q, struct bio *bio,
				       struct ufsshpb_map_ctx *mctx)
{
	struct page *page = NULL;
	int i, ret = 0;

	bio_reset(bio);

	for (i = 0; i < hpb->mpages_per_srgn; i++) {
		/* virt_to_page(p + (OS_PAGE_SIZE * i)); */
		page = mctx->m_page[i];
		if (!page)
			return -ENOMEM;

		ret = bio_add_pc_page(q, bio, page, hpb->mpage_bytes, 0);

		if (ret != hpb->mpage_bytes) {
			ERR_MSG("bio_add_pc_page fail: %d", ret);
			return -ENOMEM;
		}
	}

	return 0;
}

static int ufsshpb_execute_map_req(struct ufsshpb_lu *hpb,
				  struct ufsshpb_req *map_req)
{
	struct scsi_device *sdev;
	struct request_queue *q;
	struct request *req;
	struct bio *bio = map_req->bio;
	struct scsi_request *rq;
	struct ufsshpb_region *rgn = hpb->rgn_tbl + map_req->rb.rgn_idx;
	int ret = 0;

	sdev = hpb->ufsf->sdev_ufs_lu[hpb->lun];
	if (!sdev) {
		ERR_MSG("cannot find scsi_device");
		return -ENODEV;
	}

	ret = ufsf_get_scsi_device(hpb->ufsf->hba, sdev);
	if (ret)
		return ret;

	q = sdev->request_queue;

	ret = ufsshpb_map_req_add_bio_page(hpb, q, bio, map_req->rb.mctx);
	if (ret) {
		scsi_device_put(sdev);
		return ret;
	}

	req = map_req->req;

	/* 1. request setup */
	blk_rq_append_bio(req, &bio); /* req->__data_len is setted */
	req->rq_flags |= RQF_QUIET;
	req->timeout = 30 * HZ;
	req->end_io_data = (void *)map_req;

	/* 2. scsi_request setup */
	rq = scsi_req(req);
	rq->retries = 3;

	if (ufsshpb_is_triggered_by_fs(rgn))
		ufsshpb_set_hcm_cmd(rq->cmd, map_req->rb.rgn_idx,
				   map_req->rb.srgn_idx, hpb->srgn_mem_size);
	else
		ufsshpb_set_read_buf_cmd(rq->cmd, map_req->rb.rgn_idx,
					map_req->rb.srgn_idx, hpb->srgn_mem_size);

	rq->cmd_len = scsi_command_size(rq->cmd);

#if defined(CONFIG_HPB_DEBUG)
	if (hpb->debug)
		ufsshpb_check_ppn(hpb, map_req->rb.rgn_idx, map_req->rb.srgn_idx,
				 map_req->rb.mctx, "ISSUE");

	TMSG(hpb->ufsf, hpb->lun, "Noti: I RB %d - %d", map_req->rb.rgn_idx,
	     map_req->rb.srgn_idx);
	trace_printk("[rb.iss]\t rgn %04d (%d) (reason %d) act_rsp_list_cnt %lld\n",
		     rgn->rgn_idx, rgn->rgn_state, atomic_read(&rgn->reason),
		     atomic64_read(&hpb->act_rsp_list_cnt));
#endif

	blk_execute_rq_nowait(q, NULL, req, 1, ufsshpb_map_req_compl_fn);

	if (ufsshpb_is_triggered_by_fs(rgn))
		atomic64_inc(&hpb->set_hcm_req_cnt);
	else
		atomic64_inc(&hpb->map_req_cnt);

#if defined(CONFIG_HPB_DEBUG_SYSFS)
	ufsshpb_increase_rb_count(hpb, rgn);
#endif
	return 0;
}

static struct ufsshpb_map_ctx *ufsshpb_get_map_ctx(struct ufsshpb_lu *hpb,
						 int *err)
{
	struct ufsshpb_map_ctx *mctx;

	mctx = list_first_entry_or_null(&hpb->lh_map_ctx_free,
					struct ufsshpb_map_ctx, list_table);
	if (mctx) {
		list_del_init(&mctx->list_table);
		hpb->debug_free_table--;
		return mctx;
	}
	*err = -ENOMEM;
	return NULL;
}

static inline void ufsshpb_add_lru_info(struct victim_select_info *lru_info,
				       struct ufsshpb_region *rgn)
{
	if (ufsshpb_is_triggered_by_fs(rgn)) {
		rgn->rgn_state = HPB_RGN_HCM;
		list_add_tail(&rgn->list_lru_rgn,
			      &lru_info->lh_lru_hcm_region);
	} else {
		rgn->rgn_state = HPB_RGN_ACTIVE;
		list_add_tail(&rgn->list_lru_rgn, &lru_info->lh_lru_rgn);
	}

	atomic64_inc(&lru_info->active_cnt);
}

static inline int ufsshpb_add_region(struct ufsshpb_lu *hpb,
				    struct ufsshpb_region *rgn)
{
	struct victim_select_info *lru_info;
	int srgn_idx;
	int err = 0;

	lru_info = &hpb->lru_info;

	for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) {
		struct ufsshpb_subregion *srgn;

		srgn = rgn->srgn_tbl + srgn_idx;

		srgn->mctx = ufsshpb_get_map_ctx(hpb, &err);
		if (!srgn->mctx) {
			HPB_DEBUG(hpb, "get mctx err %d srgn %d free_table %d",
				  err, srgn_idx, hpb->debug_free_table);
			goto out;
		}

		srgn->srgn_state = HPB_SRGN_DIRTY;
	}

#if defined(CONFIG_HPB_DEBUG)
	HPB_DEBUG(hpb, "\x1b[44m\x1b[32m E->active region: %d \x1b[0m",
		  rgn->rgn_idx);
	TMSG(hpb->ufsf, hpb->lun, "Noti: ACT RG: %d", rgn->rgn_idx);
	trace_printk("[rg.add]\t rgn %04d (%d)\n", rgn->rgn_idx, rgn->rgn_state);
#endif

	ufsshpb_add_lru_info(lru_info, rgn);
out:
	return err;
}

static inline void ufsshpb_put_map_ctx(struct ufsshpb_lu *hpb,
				      struct ufsshpb_map_ctx *mctx)
{
	list_add(&mctx->list_table, &hpb->lh_map_ctx_free);
	hpb->debug_free_table++;
}

static inline void ufsshpb_purge_active_subregion(struct ufsshpb_lu *hpb,
						 struct ufsshpb_subregion *srgn,
						 int state)
{
	if (state == HPB_SRGN_UNUSED) {
		ufsshpb_put_map_ctx(hpb, srgn->mctx);
		srgn->mctx = NULL;
	}

	srgn->srgn_state = state;
}

static inline void ufsshpb_cleanup_lru_info(struct victim_select_info *lru_info,
					   struct ufsshpb_region *rgn)
{
	list_del_init(&rgn->list_lru_rgn);
	rgn->rgn_state = HPB_RGN_INACTIVE;
	atomic64_dec(&lru_info->active_cnt);
}

static void __ufsshpb_evict_region(struct ufsshpb_lu *hpb,
				  struct ufsshpb_region *rgn)
{
	struct victim_select_info *lru_info;
	struct ufsshpb_subregion *srgn;
	int srgn_idx;

	lru_info = &hpb->lru_info;

#if defined(CONFIG_HPB_DEBUG)
	HPB_DEBUG(hpb, "\x1b[41m\x1b[33m C->EVICT region: %d \x1b[0m",
		  rgn->rgn_idx);
	TMSG(hpb->ufsf, hpb->lun, "Noti: EVIC RG: %d", rgn->rgn_idx);
	trace_printk("[rg.evt]\t rgn %04d (%d)\n", rgn->rgn_idx, rgn->rgn_state);
#endif

	ufsshpb_cleanup_lru_info(lru_info, rgn);

	for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) {
		srgn = rgn->srgn_tbl + srgn_idx;

		ufsshpb_purge_active_subregion(hpb, srgn, HPB_SRGN_UNUSED);
	}
}

static inline void ufsshpb_update_lru_list(enum selection_type selection_type,
					  struct list_head *target_lh,
					  struct ufsshpb_region *rgn)
{
	switch (selection_type) {
	case LRU:
		list_move_tail(&rgn->list_lru_rgn, target_lh);
		break;
	default:
		break;
	}
}

static void ufsshpb_promote_lru_info(struct victim_select_info *lru_info,
				    struct ufsshpb_region *rgn)
{
	struct list_head *target_lh = NULL;

	if (rgn->rgn_state == HPB_RGN_ACTIVE) {
		if (ufsshpb_is_triggered_by_fs(rgn)) {
			rgn->rgn_state = HPB_RGN_HCM;
			target_lh = &lru_info->lh_lru_hcm_region;
#if defined(CONFIG_HPB_DEBUG)
			trace_printk("[fs.prm]\t rgn %04d (%d)\n",
				     rgn->rgn_idx, rgn->rgn_state);
#endif
		}
	}

	if (!target_lh)
		return;

	ufsshpb_update_lru_list(lru_info->selection_type, target_lh, rgn);
}

static void ufsshpb_hit_lru_info_by_read(struct victim_select_info *lru_info,
					struct ufsshpb_region *rgn)
{
	struct list_head *target_lh = NULL;

	if (rgn->rgn_state == HPB_RGN_ACTIVE)
		target_lh = &lru_info->lh_lru_rgn;
	else if (rgn->rgn_state == HPB_RGN_HCM)
		target_lh = &lru_info->lh_lru_hcm_region;

	if (!target_lh)
		return;

	ufsshpb_update_lru_list(lru_info->selection_type, target_lh, rgn);
}

/*
 *  Must be held hpb_lock before call this func.
 */
static int ufsshpb_check_issue_state_srgns(struct ufsshpb_lu *hpb,
					  struct ufsshpb_region *rgn)
{
	struct ufsshpb_subregion *srgn;
	int srgn_idx;

	for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) {
		srgn  = rgn->srgn_tbl + srgn_idx;

		if (srgn->srgn_state == HPB_SRGN_ISSUED) {
#if defined(CONFIG_HPB_DEBUG)
			trace_printk("[check]\t rgn %04d (%d)\n",
				     rgn->rgn_idx, rgn->rgn_state);
#endif
			return -EPERM;
		}
	}
	return 0;
}

static struct ufsshpb_region *ufsshpb_victim_lru_info(struct ufsshpb_lu *hpb,
						    struct ufsshpb_region *load_rgn)
{
	struct victim_select_info *lru_info = &hpb->lru_info;
	struct ufsshpb_region *rgn;
	struct ufsshpb_region *victim_rgn = NULL;

	switch (lru_info->selection_type) {
	case LRU:
		list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) {
			if (!rgn)
				break;

			if (ufsshpb_check_issue_state_srgns(hpb, rgn))
				continue;

			victim_rgn = rgn;
			break;
		}

		if (victim_rgn || !ufsshpb_is_triggered_by_fs(load_rgn))
			break;

		list_for_each_entry(rgn, &lru_info->lh_lru_hcm_region,
				    list_lru_rgn) {
			if (!rgn)
				break;

			if (ufsshpb_check_issue_state_srgns(hpb, rgn))
				continue;

			victim_rgn = rgn;
			break;

		}
		break;
	default:
		break;
	}

	return victim_rgn;
}

/* routine : unset_hcm compl */
static void ufsshpb_unset_hcm_req_compl_fn(struct request *req,
					  blk_status_t error)
{
	struct ufsshpb_req *pre_req = (struct ufsshpb_req *) req->end_io_data;
	struct ufsshpb_lu *hpb = pre_req->hpb;
	struct ufsshpb_region *rgn = hpb->rgn_tbl + pre_req->rb.rgn_idx;
	unsigned long flags;

	if (ufsshpb_get_state(hpb->ufsf) != HPB_PRESENT) {
		spin_lock_irqsave(&hpb->hpb_lock, flags);
		goto free_pre_req;
	}

	if (error) {
		ERR_MSG("COMPL UNSET_HCM err %d rgn %d", error, pre_req->rb.rgn_idx);
		spin_lock_irqsave(&hpb->hpb_lock, flags);
		goto free_pre_req;
	}

#if defined(CONFIG_HPB_DEBUG)
	HPB_DEBUG(hpb, "COMPL UNSET_HCM %d", pre_req->rb.rgn_idx);
	TMSG(hpb->ufsf, hpb->lun, "COMPL UNSET_HCM %d", pre_req->rb.rgn_idx);
	trace_printk("[us.cpl]\t COMPL UNSET_HCM rgn %04d (%d)\n",
		     pre_req->rb.rgn_idx, rgn->rgn_state);
#endif

	spin_lock_irqsave(&hpb->hpb_lock, flags);
	if (rgn->rgn_state != HPB_RGN_INACTIVE)
		rgn->rgn_state = HPB_RGN_ACTIVE;
free_pre_req:
	ufsshpb_put_pre_req(pre_req->hpb, pre_req);
	hpb->num_inflight_hcm_req--;
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	ufsshpb_put_blk_request(req);

	if (is_clear_reason(hpb, rgn))
		atomic_set(&rgn->reason, HPB_UPDATE_NONE);

	scsi_device_put(hpb->ufsf->sdev_ufs_lu[hpb->lun]);
	ufsshpb_lu_put(hpb);
}

static inline void ufsshpb_set_unset_hcm_cmd(unsigned char *cdb, u16 rgn_idx)
{
	cdb[0] = UFSSHPB_WRITE_BUFFER;
	cdb[1] = UFSSHPB_WB_ID_UNSET_HCM;
	cdb[2] = GET_BYTE_1(rgn_idx);
	cdb[3] = GET_BYTE_0(rgn_idx);
	cdb[9] = 0x00;	/* Control = 0x00 */
}

static int ufsshpb_execute_unset_hcm_req(struct ufsshpb_lu *hpb,
					struct scsi_device *sdev,
					struct ufsshpb_req *pre_req)
{
	struct request_queue *q = sdev->request_queue;
	struct request *req;
	struct scsi_request *rq;
	int ret;

	ret = ufsf_get_scsi_device(hpb->ufsf->hba, sdev);
	if (ret) {
		ERR_MSG("issue unset_hcm_req failed. rgn %d err %d",
			pre_req->rb.rgn_idx, ret);
		return ret;
	}

	req = pre_req->req;

	/* 1. request setup */
	req->rq_flags |= RQF_QUIET;
	req->timeout = 30 * HZ;
	req->end_io_data = (void *)pre_req;

	/* 2. scsi_request setup */
	rq = scsi_req(req);
	ufsshpb_set_unset_hcm_cmd(rq->cmd, pre_req->rb.rgn_idx);
	rq->cmd_len = scsi_command_size(rq->cmd);

#if defined(CONFIG_HPB_DEBUG)
	HPB_DEBUG(hpb, "ISSUE UNSET_HCM %d", pre_req->rb.rgn_idx);
	TMSG(hpb->ufsf, hpb->lun, "ISSUE UNSET_HCM %d", pre_req->rb.rgn_idx);
	trace_printk("[us.iss]\t rgn %04d (%d) inact_rsp_list_cnt %lld\n",
		     pre_req->rb.rgn_idx, hpb->rgn_tbl[pre_req->rb.rgn_idx].rgn_state,
		     atomic64_read(&hpb->inact_rsp_list_cnt));
#endif

	blk_execute_rq_nowait(q, NULL, req, 1, ufsshpb_unset_hcm_req_compl_fn);

	atomic64_inc(&hpb->unset_hcm_req_cnt);

	return 0;
}

static int ufsshpb_issue_unset_hcm_req(struct ufsshpb_lu *hpb,
				      struct ufsshpb_region *rgn)
{
	struct ufsshpb_req *pre_req;
	struct request *req = NULL;
	unsigned long flags;
	struct scsi_device *sdev;
	struct ufsf_feature *ufsf = hpb->ufsf;
	int ret = 0;

	sdev = ufsf->sdev_ufs_lu[hpb->lun];
	if (!sdev) {
		ERR_MSG("cannot find scsi_device");
		goto wakeup_ee_worker;
	}

	req = blk_get_request(sdev->request_queue,
			      REQ_OP_SCSI_OUT, BLK_MQ_REQ_PM);
	if (IS_ERR(req))
		return -EAGAIN;

	spin_lock_irqsave(&hpb->hpb_lock, flags);
	if (hpb->num_inflight_hcm_req >= hpb->throttle_hcm_req) {
#if defined(CONFIG_HPB_DEBUG)
		HPB_DEBUG(hpb, "hcm_req throttle. hcm inflight %d hcm throttle %d rgn (%04d)",
			  hpb->num_inflight_hcm_req, hpb->throttle_hcm_req,
			  rgn->rgn_idx);
#endif
		ret = -ENOMEM;
		goto unlock_out;
	}
	pre_req = ufsshpb_get_pre_req(hpb);
	if (!pre_req) {
		ret = -ENOMEM;
		goto unlock_out;
	}
	hpb->num_inflight_hcm_req++;
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);

	pre_req->hpb = hpb;
	pre_req->rb.rgn_idx = rgn->rgn_idx;
	pre_req->req = req;

	ret = ufsshpb_lu_get(hpb);
	if (ret) {
		WARN_MSG("warning: ufsshpb_lu_get failed.. %d", ret);
		spin_lock_irqsave(&hpb->hpb_lock, flags);
		ufsshpb_put_pre_req(hpb, pre_req);
		hpb->num_inflight_hcm_req--;
		goto unlock_out;
	}

	ret = ufsshpb_execute_unset_hcm_req(hpb, sdev, pre_req);
	if (ret) {
		ERR_MSG("issue unset_hcm_req failed. rgn %d err %d",
			rgn->rgn_idx, ret);
		ufsshpb_lu_put(hpb);
		goto wakeup_ee_worker;
	}

	return ret;
unlock_out:
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	blk_put_request(req);
	return ret;
wakeup_ee_worker:
	ufsshpb_failed(hpb, __func__);
	return ret;
}

static int ufsshpb_evict_region(struct ufsshpb_lu *hpb, struct ufsshpb_region *rgn)
{
	unsigned long flags, flags2;
	int ret;
	int srgn_idx;

	spin_lock_irqsave(&hpb->hpb_lock, flags);
	if (rgn->rgn_state == HPB_RGN_PINNED) {
		/*
		 * Pinned region should not drop-out.
		 * But if so, it would treat error as critical,
		 * and it will run hpb_eh_work
		 */
		ERR_MSG("pinned region drop-out error");
		goto out;
	} else if (rgn->rgn_state == HPB_RGN_HCM) {
		if (atomic_read(&rgn->reason) == HPB_UPDATE_FROM_DEV) {
			atomic64_inc(&hpb->dev_inact_hcm_cnt);
			/*
			 * Device may issue inactive noti for the HCM active
			 * region. We issues RB again to keep valid HPB entries
			 * for HCM active region.
			 */
			spin_lock_irqsave(&hpb->rsp_list_lock, flags2);
			for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++)
				ufsshpb_update_active_info(hpb, rgn->rgn_idx,
							  srgn_idx,
							  HPB_UPDATE_FROM_DEV);
			spin_unlock_irqrestore(&hpb->rsp_list_lock, flags2);
			goto out;
		} else if (ufsshpb_is_triggered_by_fs(rgn)) {
			if (ufsshpb_check_issue_state_srgns(hpb, rgn))
				goto evict_fail;

			spin_unlock_irqrestore(&hpb->hpb_lock, flags);
			ret = ufsshpb_issue_unset_hcm_req(hpb, rgn);
			spin_lock_irqsave(&hpb->hpb_lock, flags);
			if (ret)
				goto evict_fail;
		}
	}

	if (!list_empty(&rgn->list_lru_rgn)) {
		if (ufsshpb_check_issue_state_srgns(hpb, rgn))
			goto evict_fail;

		__ufsshpb_evict_region(hpb, rgn);
	}

out:
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	return 0;
evict_fail:
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	return -EPERM;
}

static inline struct
ufsshpb_rsp_field *ufsshpb_get_hpb_rsp(struct ufshcd_lrb *lrbp)
{
	return (struct ufsshpb_rsp_field *)&lrbp->ucd_rsp_ptr->sr.sense_data_len;
}

static int ufsshpb_issue_map_req(struct ufsshpb_lu *hpb,
				struct ufsshpb_subregion *srgn)
{
	struct scsi_device *sdev = hpb->ufsf->sdev_ufs_lu[hpb->lun];
	struct request *req = NULL;
	struct bio *bio = NULL;
	struct ufsshpb_req *map_req = NULL;
	unsigned long flags;
	int ret = 0;

	req = blk_get_request(sdev->request_queue, REQ_OP_SCSI_IN,
			      BLK_MQ_REQ_PM);
	if (IS_ERR(req))
		return -EAGAIN;

	bio = bio_alloc(GFP_KERNEL, hpb->mpages_per_srgn);
	if (!bio) {
		blk_put_request(req);
		return -EAGAIN;
	}

	spin_lock_irqsave(&hpb->hpb_lock, flags);

	if (srgn->srgn_state == HPB_SRGN_ISSUED) {
		ret = -EAGAIN;
		goto unlock_out;
	}

	ret = ufsshpb_clean_dirty_bitmap(hpb, srgn);
	if (ret) {
		ret = -EAGAIN;
		goto unlock_out;
	}

	map_req = ufsshpb_get_map_req(hpb);
	if (!map_req) {
		ret = -ENOMEM;
		goto unlock_out;
	}

	srgn->srgn_state = HPB_SRGN_ISSUED;

	spin_unlock_irqrestore(&hpb->hpb_lock, flags);

	map_req->req = req;
	map_req->bio = bio;
	map_req->hpb = hpb;
	map_req->rb.rgn_idx = srgn->rgn_idx;
	map_req->rb.srgn_idx = srgn->srgn_idx;
	map_req->rb.mctx = srgn->mctx;
	map_req->rb.lun = hpb->lun;

	ret = ufsshpb_lu_get(hpb);
	if (unlikely(ret)) {
		ERR_MSG("ufsshpb_lu_get failed. (%d)", ret);
		goto free_map_req;
	}

	ret = ufsshpb_execute_map_req(hpb, map_req);
	if (ret) {
		ERR_MSG("issue map_req failed. [%d-%d] err %d",
			srgn->rgn_idx, srgn->srgn_idx, ret);
		ufsshpb_lu_put(hpb);
		goto wakeup_ee_worker;
	}
	return ret;
free_map_req:
	spin_lock_irqsave(&hpb->hpb_lock, flags);
	srgn->srgn_state = HPB_SRGN_DIRTY;
	ufsshpb_put_map_req(hpb, map_req);
unlock_out:
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	bio_put(bio);
	blk_put_request(req);
	return ret;
wakeup_ee_worker:
	ufsshpb_failed(hpb, __func__);
	return ret;
}

static int ufsshpb_load_region(struct ufsshpb_lu *hpb, struct ufsshpb_region *rgn)
{
	struct ufsshpb_region *victim_rgn;
	struct victim_select_info *lru_info = &hpb->lru_info;
	unsigned long flags;
	int ret = 0;

	/*
	 * if already region is added to lru_list,
	 * just initiate the information of lru.
	 * because the region already has the map ctx.
	 * (!list_empty(&rgn->list_region) == region->state=active...)
	 */
	spin_lock_irqsave(&hpb->hpb_lock, flags);
	if (!list_empty(&rgn->list_lru_rgn)) {
		ufsshpb_promote_lru_info(lru_info, rgn);
		goto out;
	}

	if (rgn->rgn_state == HPB_RGN_INACTIVE) {
		if (atomic64_read(&lru_info->active_cnt)
		    == lru_info->max_lru_active_cnt) {
			victim_rgn = ufsshpb_victim_lru_info(hpb, rgn);
			if (!victim_rgn) {
#if defined(CONFIG_HPB_DEBUG)
				HPB_DEBUG(hpb, "UFSSHPB victim_rgn is NULL");
#endif
				if (list_empty(&lru_info->lh_lru_rgn) &&
				    !ufsshpb_is_triggered_by_fs(rgn))
					ret = -EACCES;
				else
					ret = -ENOMEM;
				goto out;
			}

			if (victim_rgn->rgn_state == HPB_RGN_HCM) {
				spin_unlock_irqrestore(&hpb->hpb_lock, flags);
				ret = ufsshpb_issue_unset_hcm_req(hpb, victim_rgn);
#if defined(CONFIG_HPB_DEBUG_SYSFS)
				atomic64_inc(&hpb->unset_hcm_victim);
#endif
				spin_lock_irqsave(&hpb->hpb_lock, flags);
				if (ret) {
					ERR_MSG("issue unset_hcm failed. [%d] err %d",
						rgn->rgn_idx, ret);
					goto out;
				}
			}

#if defined(CONFIG_HPB_DEBUG)
			TMSG(hpb->ufsf, hpb->lun, "Noti: VT RG %d state %d",
			     victim_rgn->rgn_idx, victim_rgn->rgn_state);
			HPB_DEBUG(hpb,
				  "LRU MAX(=%lld). victim choose %d state %d",
				  (long long)atomic64_read(&lru_info->active_cnt),
				  victim_rgn->rgn_idx, victim_rgn->rgn_state);
			trace_printk("[victim]\t rgn %04d (%d)\n",
				     victim_rgn->rgn_idx, victim_rgn->rgn_state);
#endif

			__ufsshpb_evict_region(hpb, victim_rgn);
		}

		ret = ufsshpb_add_region(hpb, rgn);
		if (ret) {
			ERR_MSG("UFSSHPB memory allocation failed (%d)", ret);
			spin_unlock_irqrestore(&hpb->hpb_lock, flags);
			goto wakeup_ee_worker;
		}
	}
out:
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	return ret;
wakeup_ee_worker:
	ufsshpb_failed(hpb, __func__);
	return ret;
}

static void ufsshpb_rsp_req_region_update(struct ufsshpb_lu *hpb,
					 struct ufsshpb_rsp_field *rsp_field)
{
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;
	int num, rgn_idx, srgn_idx;

	/*
	 * If active rgn = inactive rgn, choose inactive rgn.
	 * So, active process -> inactive process
	 */
	for (num = 0; num < rsp_field->active_rgn_cnt; num++) {
		rgn_idx = be16_to_cpu(rsp_field->hpb_active_field[num].active_rgn);
		srgn_idx = be16_to_cpu(rsp_field->hpb_active_field[num].active_srgn);

		if (unlikely(rgn_idx >= hpb->rgns_per_lu)) {
			ERR_MSG("rgn_idx %d is wrong. (Max %d)",
				rgn_idx, hpb->rgns_per_lu);
			continue;
		}

		rgn = hpb->rgn_tbl + rgn_idx;

		if (unlikely(srgn_idx >= rgn->srgn_cnt)) {
			ERR_MSG("srgn_idx %d is wrong. (Max %d)",
				srgn_idx, rgn->srgn_cnt);
			continue;
		}

		srgn = rgn->srgn_tbl + srgn_idx;

		if (ufsshpb_is_triggered_by_fs(rgn))
			continue;

		HPB_DEBUG(hpb, "act num: %d, region: %d, subregion: %d",
			  num + 1, rgn_idx, srgn_idx);
		spin_lock(&hpb->rsp_list_lock);
		ufsshpb_update_active_info(hpb, rgn_idx, srgn_idx,
					  HPB_UPDATE_FROM_DEV);
		spin_unlock(&hpb->rsp_list_lock);

		/* It is just blocking HPB_READ */
		spin_lock(&hpb->hpb_lock);
		if (srgn->srgn_state == HPB_SRGN_CLEAN)
			srgn->srgn_state = HPB_SRGN_DIRTY;
		spin_unlock(&hpb->hpb_lock);

		atomic64_inc(&hpb->rb_active_cnt);
	}

	for (num = 0; num < rsp_field->inactive_rgn_cnt; num++) {
		rgn_idx = be16_to_cpu(rsp_field->hpb_inactive_field[num]);

		if (unlikely(rgn_idx >= hpb->rgns_per_lu)) {
			ERR_MSG("rgn_idx %d is wrong. (Max %d)",
				rgn_idx, hpb->rgns_per_lu);
			continue;
		}

		rgn = hpb->rgn_tbl + rgn_idx;

		if (ufsshpb_is_triggered_by_fs(rgn))
			continue;


		/*
		 * to block HPB_READ
		 */
		spin_lock(&hpb->hpb_lock);
		if (rgn->rgn_state != HPB_RGN_INACTIVE) {
			for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) {
				srgn = rgn->srgn_tbl + srgn_idx;
				if (srgn->srgn_state == HPB_SRGN_CLEAN)
					srgn->srgn_state = HPB_SRGN_DIRTY;
			}
		}
		spin_unlock(&hpb->hpb_lock);

		HPB_DEBUG(hpb, "inact num: %d, region: %d", num + 1, rgn_idx);
		atomic_set(&rgn->reason, HPB_UPDATE_FROM_DEV);
		spin_lock(&hpb->rsp_list_lock);
		ufsshpb_update_inactive_info(hpb, rgn_idx,
					    HPB_UPDATE_FROM_DEV);
		spin_unlock(&hpb->rsp_list_lock);

		atomic64_inc(&hpb->rb_inactive_cnt);
	}

	TMSG(hpb->ufsf, hpb->lun, "Noti: #ACT %u, #INACT %u",
	     rsp_field->active_rgn_cnt, rsp_field->inactive_rgn_cnt);
	trace_printk("[rspnoti]\t noti ACT %d INACT %d\n",
		     rsp_field->active_rgn_cnt, rsp_field->inactive_rgn_cnt);
	schedule_work(&hpb->task_work);
}

static inline int ufsshpb_may_field_valid(struct ufshcd_lrb *lrbp,
					 struct ufsshpb_rsp_field *rsp_field)
{
	if (be16_to_cpu(rsp_field->sense_data_len) != DEV_SENSE_SEG_LEN ||
	    rsp_field->desc_type != DEV_DES_TYPE ||
	    rsp_field->additional_len != DEV_ADDITIONAL_LEN ||
	    rsp_field->hpb_type == HPB_RSP_NONE ||
	    rsp_field->active_rgn_cnt > MAX_ACTIVE_NUM ||
	    rsp_field->inactive_rgn_cnt > MAX_INACTIVE_NUM ||
	    (!rsp_field->active_rgn_cnt && !rsp_field->inactive_rgn_cnt))
		return -EINVAL;

	if (!ufsf_is_valid_lun(lrbp->lun)) {
		ERR_MSG("LU(%d) is not supported", lrbp->lun);
		return -EINVAL;
	}

	return 0;
}

static bool ufsshpb_is_empty_rsp_lists(struct ufsshpb_lu *hpb)
{
	bool ret = true;
	unsigned long flags;

	spin_lock_irqsave(&hpb->rsp_list_lock, flags);
	if (!list_empty(&hpb->lh_inact_rgn) || !list_empty(&hpb->lh_act_srgn))
		ret = false;
	spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);

	return ret;
}

/* routine : isr (ufs) */
void ufsshpb_rsp_upiu(struct ufsf_feature *ufsf, struct ufshcd_lrb *lrbp)
{
	struct ufsshpb_lu *hpb;
	struct ufsshpb_rsp_field *rsp_field;
	int data_seg_len, ret;

	data_seg_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2)
		& MASK_RSP_UPIU_DATA_SEG_LEN;

	if (!data_seg_len) {
		bool do_task_work = false;

		if (!ufsf_is_valid_lun(lrbp->lun))
			return;

		hpb = ufsf->hpb_lup[lrbp->lun];
		ret = ufsshpb_lu_get(hpb);
		if (unlikely(ret))
			return;

		do_task_work = !ufsshpb_is_empty_rsp_lists(hpb);
		if (do_task_work)
			schedule_work(&hpb->task_work);

		goto put_hpb;
	}

	rsp_field = ufsshpb_get_hpb_rsp(lrbp);

	if (ufsshpb_may_field_valid(lrbp, rsp_field)) {
		WARN_ON(rsp_field->additional_len != DEV_ADDITIONAL_LEN);
		return;
	}

	hpb = ufsf->hpb_lup[lrbp->lun];
	ret = ufsshpb_lu_get(hpb);
	if (unlikely(ret)) {
		ERR_MSG("ufsshpb_lu_get failed. (%d)", ret);
		return;
	}

	if (hpb->force_map_req_disable)
		goto put_hpb;

	HPB_DEBUG(hpb, "**** HPB Noti %u LUN %u Seg-Len %u, #ACT %u, #INACT %u",
		  rsp_field->hpb_type, lrbp->lun,
		  be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
		  MASK_RSP_UPIU_DATA_SEG_LEN, rsp_field->active_rgn_cnt,
		  rsp_field->inactive_rgn_cnt);
	atomic64_inc(&hpb->rb_noti_cnt);

	switch (rsp_field->hpb_type) {
	case HPB_RSP_REQ_REGION_UPDATE:
		WARN_ON(data_seg_len != DEV_DATA_SEG_LEN);
		ufsshpb_rsp_req_region_update(hpb, rsp_field);
		goto put_hpb;
	default:
		HPB_DEBUG(hpb, "hpb_type is not available : %d",
			  rsp_field->hpb_type);
		goto put_hpb;
	}

put_hpb:
	ufsshpb_lu_put(hpb);
}

static int ufsshpb_execute_map_req_wait(struct ufsshpb_lu *hpb,
				       unsigned char *cmd,
				       struct ufsshpb_subregion *srgn)
{
	struct ufsf_feature *ufsf = hpb->ufsf;
	struct scsi_device *sdev;
	struct request_queue *q;
	struct request *req;
	struct scsi_request *rq;
	struct bio *bio;
	struct scsi_sense_hdr sshdr;
	unsigned long flags;
	int ret = 0;

	sdev = ufsf->sdev_ufs_lu[hpb->lun];
	if (!sdev) {
		ERR_MSG("cannot find scsi_device");
		return -ENODEV;
	}

	q = sdev->request_queue;

	ret = ufsf_get_scsi_device(ufsf->hba, sdev);
	if (ret)
		return ret;

	req = blk_get_request(q, REQ_OP_SCSI_IN, BLK_MQ_REQ_PM);
	if (IS_ERR(req)) {
		ERR_MSG("cannot get request");
		ret = -EIO;
		goto sdev_put_out;
	}

	bio = bio_alloc(GFP_KERNEL, hpb->mpages_per_srgn);
	if (!bio) {
		ret = -ENOMEM;
		goto req_put_out;
	}

	ret = ufsshpb_map_req_add_bio_page(hpb, q, bio, srgn->mctx);
	if (ret)
		goto mem_free_out;

	/* 1. request setup*/
	blk_rq_append_bio(req, &bio); /* req->__data_len */
	req->rq_flags |= RQF_QUIET;
	req->timeout = 30 * HZ;

	/* 2. scsi_request setup */
	rq = scsi_req(req);
	rq->retries = 3;
	rq->cmd_len = scsi_command_size(cmd);
	memcpy(rq->cmd, cmd, rq->cmd_len);

	blk_execute_rq(q, NULL, req, 1);
	if (rq->result) {
		ret = -EIO;
		scsi_normalize_sense(rq->sense, SCSI_SENSE_BUFFERSIZE, &sshdr);
		ERR_MSG("code %x sense_key %x asc %x ascq %x",
			sshdr.response_code, sshdr.sense_key, sshdr.asc,
			sshdr.ascq);
		ERR_MSG("byte4 %x byte5 %x byte6 %x additional_len %x",
			sshdr.byte4, sshdr.byte5, sshdr.byte6,
			sshdr.additional_length);
		spin_lock_irqsave(&hpb->hpb_lock, flags);
		ufsshpb_error_active_subregion(hpb, srgn);
		spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	}
mem_free_out:
	bio_put(bio);
req_put_out:
	blk_put_request(req);
sdev_put_out:
	scsi_device_put(sdev);
	return ret;
}

static inline void ufsshpb_set_unset_hcm_all_cmd(unsigned char *cdb)
{
	cdb[0] = UFSSHPB_WRITE_BUFFER;
	cdb[1] = UFSSHPB_WB_ID_UNSET_HCM_ALL;
	cdb[9] = 0x00;	/* Control = 0x00 */
}

static int ufsshpb_issue_map_req_from_list(struct ufsshpb_lu *hpb)
{
	struct ufsshpb_subregion *srgn;
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&hpb->rsp_list_lock, flags);

	while ((srgn = list_first_entry_or_null(&hpb->lh_pinned_srgn,
						struct ufsshpb_subregion,
						list_act_srgn))) {
		unsigned char cmd[10] = { 0 };

		list_del_init(&srgn->list_act_srgn);
		spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
		ufsshpb_set_read_buf_cmd(cmd, srgn->rgn_idx, srgn->srgn_idx,
					hpb->srgn_mem_size);

#if defined(CONFIG_HPB_DEBUG)
		if (hpb->debug)
			ufsshpb_check_ppn(hpb, srgn->rgn_idx, srgn->srgn_idx,
					 srgn->mctx, "ISSUE");

		TMSG(hpb->ufsf, hpb->lun, "Pinned: I RB %d - %d",
		     srgn->rgn_idx, srgn->srgn_idx);
		trace_printk("[rb.iss]\t PINNED rgn %04d (%d)\n",
			     srgn->rgn_idx, hpb->rgn_tbl[srgn->rgn_idx].rgn_state);
#endif
		ret = ufsshpb_execute_map_req_wait(hpb, cmd, srgn);
		if (ret < 0) {
			ERR_MSG("region %d sub %d failed with err %d",
				srgn->rgn_idx, srgn->srgn_idx, ret);
			spin_lock_irqsave(&hpb->rsp_list_lock, flags);
			if (list_empty(&srgn->list_act_srgn))
				list_add(&srgn->list_act_srgn,
					 &hpb->lh_pinned_srgn);
			continue;
		}
#if defined(CONFIG_HPB_DEBUG)
		if (hpb->debug)
			ufsshpb_check_ppn(hpb, srgn->rgn_idx, srgn->srgn_idx,
					 srgn->mctx, "COMPL");
		TMSG(hpb->ufsf, hpb->lun, "Noti: C RB %d - %d",
		     srgn->rgn_idx, srgn->srgn_idx);
		trace_printk("[rb.cpl]\t PINNED rgn %04d (%d)\n",
			     srgn->rgn_idx, hpb->rgn_tbl[srgn->rgn_idx].rgn_state);
#endif
		spin_lock_irqsave(&hpb->hpb_lock, flags);
		ufsshpb_clean_active_subregion(hpb, srgn);
		spin_unlock_irqrestore(&hpb->hpb_lock, flags);

		spin_lock_irqsave(&hpb->rsp_list_lock, flags);
	}

	spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);

	return 0;
}

static void ufsshpb_pinned_work_handler(struct work_struct *work)
{
	struct ufsshpb_lu *hpb;
	int ret;

	hpb = container_of(work, struct ufsshpb_lu, pinned_work);
#if defined(CONFIG_HPB_DEBUG)
	HPB_DEBUG(hpb, "worker start for pinned region");
#endif
	pm_runtime_get_sync(hpb->ufsf->hba->dev);
	ufsshpb_lu_get(hpb);

	if (!list_empty(&hpb->lh_pinned_srgn)) {
		ret = ufsshpb_issue_map_req_from_list(hpb);
		/*
		 * if its function failed at init time,
		 * ufsshpb-device will request map-req,
		 * so it is not critical-error, and just finish work-handler
		 */
		if (ret)
			HPB_DEBUG(hpb, "failed map-issue. ret %d", ret);
	}
	ufsshpb_lu_put(hpb);
	pm_runtime_put_sync(hpb->ufsf->hba->dev);
	HPB_DEBUG(hpb, "worker end");
}

static int ufsshpb_check_pm(struct ufsshpb_lu *hpb)
{
	struct ufs_hba *hba = hpb->ufsf->hba;

	if (hba->pm_op_in_progress ||
	    hba->curr_dev_pwr_mode != UFS_ACTIVE_PWR_MODE) {
		INFO_MSG("hba current power state %d pm_progress %d",
			 hba->curr_dev_pwr_mode,
			 hba->pm_op_in_progress);
		return -ENODEV;
	}
	return 0;
}

static void ufsshpb_retry_work_handler(struct work_struct *work)
{
	struct ufsshpb_lu *hpb;
	struct delayed_work *dwork = to_delayed_work(work);
	struct ufsshpb_req *map_req, *next;
	unsigned long flags;
	bool do_retry = false;
	int ret = 0;

	LIST_HEAD(retry_list);

	hpb = container_of(dwork, struct ufsshpb_lu, retry_work);

	if (ufsshpb_check_pm(hpb))
		return;

	ret = ufsshpb_lu_get(hpb);
	if (unlikely(ret)) {
		ERR_MSG("ufsshpb_lu_get failed. (%d)", ret);
		return;
	}
#if defined(CONFIG_HPB_DEBUG)
	HPB_DEBUG(hpb, "retry worker start");
#endif
	spin_lock_bh(&hpb->retry_list_lock);
	list_splice_init(&hpb->lh_map_req_retry, &retry_list);
	spin_unlock_bh(&hpb->retry_list_lock);

	list_for_each_entry_safe(map_req, next, &retry_list, list_req) {
		if (ufsshpb_get_state(hpb->ufsf) == HPB_SUSPEND) {
			INFO_MSG("suspend state. Issue READ_BUFFER in the next turn");
			spin_lock_bh(&hpb->retry_list_lock);
			list_splice_init(&retry_list, &hpb->lh_map_req_retry);
			spin_unlock_bh(&hpb->retry_list_lock);
			do_retry = true;
			break;
		}

		list_del_init(&map_req->list_req);

		ret = ufsshpb_lu_get(hpb);
		if (unlikely(ret)) {
			WARN_MSG("ufsshpb_lu_get failed (%d)", ret);
			bio_put(map_req->bio);
			blk_put_request(map_req->req);
			spin_lock_irqsave(&hpb->hpb_lock, flags);
			ufsshpb_put_map_req(hpb, map_req);
			spin_unlock_irqrestore(&hpb->hpb_lock, flags);
			continue;
		}

		ret = ufsshpb_execute_map_req(hpb, map_req);
		if (ret) {
			ERR_MSG("issue map_req failed. [%d-%d] err %d",
				map_req->rb.rgn_idx, map_req->rb.srgn_idx, ret);
			ufsshpb_lu_put(hpb);
			goto wakeup_ee_worker;
		}
	}
	HPB_DEBUG(hpb, "worker end");
	ufsshpb_lu_put(hpb);

	if (do_retry)
		schedule_delayed_work(&hpb->retry_work,
				      msecs_to_jiffies(RETRY_DELAY_MS));
	return;
wakeup_ee_worker:
	ufsshpb_lu_put(hpb);
	ufsshpb_failed(hpb, __func__);
}

static void ufsshpb_add_starved_list(struct ufsshpb_lu *hpb,
				    struct ufsshpb_region *rgn,
				    struct list_head *starved_list)
{
	struct ufsshpb_subregion *srgn;
	int srgn_idx;

	if (!list_empty(&rgn->list_inact_rgn))
		return;

	for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) {
		srgn = rgn->srgn_tbl + srgn_idx;

		if (!list_empty(&srgn->list_act_srgn))
			return;
	}

	/* It will be added to hpb->lh_inact_rgn */
#if defined(CONFIG_HPB_DEBUG_SYSFS)
	atomic64_inc(&hpb->inact_rsp_list_cnt);
#endif
	list_add_tail(&rgn->list_inact_rgn, starved_list);
#if defined(CONFIG_HPB_DEBUG)
	trace_printk("[sl.add]\t rgn %04d (%d)\n",
		     rgn->rgn_idx, rgn->rgn_state);
#endif
}

static void ufsshpb_run_inactive_region_list(struct ufsshpb_lu *hpb)
{
	struct ufsshpb_region *rgn;
	unsigned long flags;
	int ret;
	LIST_HEAD(starved_list);

	spin_lock_irqsave(&hpb->rsp_list_lock, flags);
	while ((rgn = list_first_entry_or_null(&hpb->lh_inact_rgn,
					       struct ufsshpb_region,
					       list_inact_rgn))) {
		if (ufsshpb_get_state(hpb->ufsf) == HPB_SUSPEND) {
			INFO_MSG("suspend state. Inactivate rgn the next turn");
			break;
		}

#if defined(CONFIG_HPB_DEBUG)
		trace_printk("[tw.ina]\t inact_rgn rgn %04d (%d)\n",
			     rgn->rgn_idx, rgn->rgn_state);
#endif
		ufsshpb_inact_rsp_list_del(hpb, rgn);
		spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);

		ret = ufsshpb_evict_region(hpb, rgn);
		if (ret) {
			spin_lock_irqsave(&hpb->rsp_list_lock, flags);
			ufsshpb_add_starved_list(hpb, rgn, &starved_list);
			spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
		}

		spin_lock_irqsave(&hpb->rsp_list_lock, flags);
	}

	list_splice(&starved_list, &hpb->lh_inact_rgn);
	spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
}

static void ufsshpb_add_active_list(struct ufsshpb_lu *hpb,
				   struct ufsshpb_region *rgn,
				   struct ufsshpb_subregion *srgn)
{
	if (!list_empty(&rgn->list_inact_rgn))
		return;

	if (!list_empty(&srgn->list_act_srgn)) {
		list_move(&srgn->list_act_srgn, &hpb->lh_act_srgn);
		return;
	}
#if defined(CONFIG_HPB_DEBUG)
	trace_printk("[al.add]\trgn: %04d (%d)\n", rgn->rgn_idx, rgn->rgn_state);
#endif

	ufsshpb_act_rsp_list_add(hpb, srgn);
}

static void ufsshpb_run_active_subregion_list(struct ufsshpb_lu *hpb)
{
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;
	unsigned long flags;
	int ret = 0;

	spin_lock_irqsave(&hpb->rsp_list_lock, flags);
	while ((srgn = list_first_entry_or_null(&hpb->lh_act_srgn,
						struct ufsshpb_subregion,
						list_act_srgn))) {
		if (ufsshpb_get_state(hpb->ufsf) == HPB_SUSPEND) {
			INFO_MSG("suspend state. Issuing READ_BUFFER the next turn");
			break;
		}

		ufsshpb_act_rsp_list_del(hpb, srgn);

		if (hpb->force_map_req_disable) {
#if defined(CONFIG_HPB_DEBUG)
			HPB_DEBUG(hpb, "map_req disabled");
#endif
			continue;
		}

		spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);

		rgn = hpb->rgn_tbl + srgn->rgn_idx;
#if defined(CONFIG_HPB_DEBUG)
		trace_printk("[tw.act]\t run_act %04d (%d)\n",
			     rgn->rgn_idx, rgn->rgn_state);
#endif

		ret = ufsshpb_load_region(hpb, rgn);
		if (ret) {
			spin_lock_irqsave(&hpb->rsp_list_lock, flags);
			if (ret == -EACCES) {
				continue;
			} else {
				ufsshpb_add_active_list(hpb, rgn, srgn);
				break;
			}
		}

		ret = ufsshpb_issue_map_req(hpb, srgn);
		if (ret) {
			spin_lock_irqsave(&hpb->rsp_list_lock, flags);
			ufsshpb_add_active_list(hpb, rgn, srgn);
			break;
		}

		spin_lock_irqsave(&hpb->rsp_list_lock, flags);
	}
	spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
}

static void ufsshpb_task_work_handler(struct work_struct *work)
{
	struct ufsshpb_lu *hpb;
	int ret;

	hpb = container_of(work, struct ufsshpb_lu, task_work);
	ret = ufsshpb_lu_get(hpb);
	if (unlikely(ret)) {
		if (hpb)
			WARN_MSG("lu_get failed.. hpb_state %d",
				 ufsshpb_get_state(hpb->ufsf));
		WARN_MSG("warning: ufsshpb_lu_get failed %d..", ret);
		return;
	}

	ufsshpb_run_inactive_region_list(hpb);
	ufsshpb_run_active_subregion_list(hpb);

	ufsshpb_lu_put(hpb);
}

static inline void ufsshpb_map_req_mempool_remove(struct ufsshpb_lu *hpb)
{
	struct ufsshpb_req *map_req;
	int i;

	for (i = 0; i < hpb->qd; i++) {
		map_req = hpb->map_req + i;
		if (map_req->req)
			blk_put_request(map_req->req);
		if (map_req->bio)
			bio_put(map_req->bio);
	}

	kfree(hpb->map_req);
}

static inline void ufsshpb_pre_req_mempool_remove(struct ufsshpb_lu *hpb)
{
	struct ufsshpb_req *pre_req;
	int i;

	for (i = 0; i < hpb->qd; i++) {
		pre_req = hpb->pre_req + i;
		if (pre_req->req)
			blk_put_request(pre_req->req);
		if (pre_req->bio)
			bio_put(pre_req->bio);
		__free_page(pre_req->wb.m_page);
	}

	kfree(hpb->pre_req);
}

static void ufsshpb_table_mempool_remove(struct ufsshpb_lu *hpb)
{
	struct ufsshpb_map_ctx *mctx, *next;
	int i;

	/*
	 * the mctx in the lh_map_ctx_free has been allocated completely.
	 */
	list_for_each_entry_safe(mctx, next, &hpb->lh_map_ctx_free,
				 list_table) {
		for (i = 0; i < hpb->mpages_per_srgn; i++)
			__free_page(mctx->m_page[i]);

		vfree(mctx->ppn_dirty);
		kfree(mctx->m_page);
		kfree(mctx);
		hpb->alloc_mctx--;
	}
}

/*
 * this function doesn't need to hold lock due to be called in init.
 * (hpb_lock, rsp_list_lock, etc..)
 */
static int ufsshpb_init_pinned_active_region(struct ufsshpb_lu *hpb,
					    struct ufsshpb_region *rgn)
{
	struct ufsshpb_subregion *srgn;
	int srgn_idx, j;
	int err = 0;

	for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) {
		srgn = rgn->srgn_tbl + srgn_idx;

		srgn->mctx = ufsshpb_get_map_ctx(hpb, &err);
		if (err) {
			ERR_MSG("get mctx err %d srgn %d free_table %d",
				err, srgn_idx, hpb->debug_free_table);
			goto release;
		}

		srgn->srgn_state = HPB_SRGN_ISSUED;
		/*
		 * no need to clean ppn_dirty bitmap
		 * because it is vzalloc-ed @ufsshpb_table_mempool_init()
		 */
		list_add_tail(&srgn->list_act_srgn, &hpb->lh_pinned_srgn);
	}

	rgn->rgn_state = HPB_RGN_PINNED;

	return 0;

release:
	for (j = 0; j < srgn_idx; j++) {
		srgn = rgn->srgn_tbl + j;
		ufsshpb_put_map_ctx(hpb, srgn->mctx);
	}

	return err;
}

static inline bool ufsshpb_is_pinned_region(struct ufsshpb_lu *hpb, int rgn_idx)
{
	if (hpb->lu_pinned_end_offset != -1 &&
	    rgn_idx >= hpb->lu_pinned_rgn_startidx &&
	    rgn_idx <= hpb->lu_pinned_end_offset)
		return true;

	return false;
}

static inline void ufsshpb_init_jobs(struct ufsshpb_lu *hpb)
{
	INIT_WORK(&hpb->pinned_work, ufsshpb_pinned_work_handler);
	INIT_DELAYED_WORK(&hpb->retry_work, ufsshpb_retry_work_handler);
	INIT_WORK(&hpb->task_work, ufsshpb_task_work_handler);
}

static inline void ufsshpb_cancel_jobs(struct ufsshpb_lu *hpb)
{
	cancel_work_sync(&hpb->pinned_work);
	cancel_delayed_work_sync(&hpb->retry_work);
	cancel_work_sync(&hpb->task_work);
}

static void ufsshpb_init_subregion_tbl(struct ufsshpb_lu *hpb,
				      struct ufsshpb_region *rgn)
{
	int srgn_idx;

	for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) {
		struct ufsshpb_subregion *srgn = rgn->srgn_tbl + srgn_idx;

		INIT_LIST_HEAD(&srgn->list_act_srgn);

		srgn->rgn_idx = rgn->rgn_idx;
		srgn->srgn_idx = srgn_idx;
		srgn->srgn_state = HPB_SRGN_UNUSED;
	}
}

static inline int ufsshpb_alloc_subregion_tbl(struct ufsshpb_lu *hpb,
					     struct ufsshpb_region *rgn,
					     int srgn_cnt)
{
	rgn->srgn_tbl =
		kzalloc(sizeof(struct ufsshpb_subregion) * srgn_cnt, GFP_KERNEL);
	if (!rgn->srgn_tbl)
		return -ENOMEM;

	rgn->srgn_cnt = srgn_cnt;
	return 0;
}

static int ufsshpb_table_mempool_init(struct ufsshpb_lu *hpb)
{
	struct ufsshpb_map_ctx *mctx = NULL;
	int i, j, k;

	INIT_LIST_HEAD(&hpb->lh_map_ctx_free);

	hpb->alloc_mctx = hpb->lu_max_active_rgns * hpb->srgns_per_rgn;

	for (i = 0; i < hpb->alloc_mctx; i++) {
		mctx = kmalloc(sizeof(struct ufsshpb_map_ctx), GFP_KERNEL);
		if (!mctx)
			goto release_mem;

		mctx->m_page =
			kzalloc(sizeof(struct page *) * hpb->mpages_per_srgn,
				GFP_KERNEL);
		if (!mctx->m_page)
			goto release_mem;

		mctx->ppn_dirty = vzalloc(hpb->entries_per_srgn >>
					  bits_per_byte_shift);
		if (!mctx->ppn_dirty)
			goto release_mem;

		for (j = 0; j < hpb->mpages_per_srgn; j++) {
			mctx->m_page[j] = alloc_page(GFP_KERNEL | __GFP_ZERO);
			if (!mctx->m_page[j]) {
				for (k = 0; k < j; k++)
					__free_page(mctx->m_page[k]);
				goto release_mem;
			}
		}

		INIT_LIST_HEAD(&mctx->list_table);
		list_add(&mctx->list_table, &hpb->lh_map_ctx_free);

		hpb->debug_free_table++;
	}

	INFO_MSG("The number of mctx = (%d). debug_free_table (%d)",
		 hpb->alloc_mctx, hpb->debug_free_table);
	return 0;
release_mem:
	/*
	 * mctxs already added in lh_map_ctx_free will be removed
	 * in the caller function.
	 */
	if (mctx) {
		kfree(mctx->m_page);
		vfree(mctx->ppn_dirty);
		kfree(mctx);
	}
	return -ENOMEM;
}

static int ufsshpb_map_req_mempool_init(struct ufsshpb_lu *hpb)
{
	struct scsi_device *sdev;
	struct request_queue *q;
	struct ufsshpb_req *map_req = NULL;
	int qd = hpb->qd;
	int i;

	sdev = hpb->ufsf->sdev_ufs_lu[hpb->lun];
	q = sdev->request_queue;

	INIT_LIST_HEAD(&hpb->lh_map_req_free);
	INIT_LIST_HEAD(&hpb->lh_map_req_retry);

	hpb->map_req = kzalloc(sizeof(struct ufsshpb_req) * qd, GFP_KERNEL);
	if (!hpb->map_req)
		goto release_mem;

	for (i = 0; i < qd; i++) {
		map_req = hpb->map_req + i;
		INIT_LIST_HEAD(&map_req->list_req);
		map_req->req = NULL;
		map_req->bio = NULL;

		list_add_tail(&map_req->list_req, &hpb->lh_map_req_free);
	}

	return 0;
release_mem:
	kfree(hpb->map_req);
	return -ENOMEM;
}

static int ufsshpb_pre_req_mempool_init(struct ufsshpb_lu *hpb)
{
	struct scsi_device *sdev;
	struct request_queue *q;
	struct ufsshpb_req *pre_req = NULL;
	int qd = hpb->qd;
	int i, j;

	INIT_LIST_HEAD(&hpb->lh_pre_req_free);

	sdev = hpb->ufsf->sdev_ufs_lu[hpb->lun];
	q = sdev->request_queue;

	hpb->pre_req = kzalloc(sizeof(struct ufsshpb_req) * qd, GFP_KERNEL);
	if (!hpb->pre_req)
		goto release_mem;

	for (i = 0; i < qd; i++) {
		pre_req = hpb->pre_req + i;
		INIT_LIST_HEAD(&pre_req->list_req);
		pre_req->req = NULL;
		pre_req->bio = NULL;

		pre_req->wb.m_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
		if (!pre_req->wb.m_page) {
			for (j = 0; j < i; j++)
				__free_page(hpb->pre_req[j].wb.m_page);

			goto release_mem;
		}
		list_add_tail(&pre_req->list_req, &hpb->lh_pre_req_free);
	}

	return 0;
release_mem:
	kfree(hpb->pre_req);
	return -ENOMEM;
}

static void ufsshpb_find_lu_qd(struct ufsshpb_lu *hpb)
{
	struct scsi_device *sdev;
	struct ufs_hba *hba;

	sdev = hpb->ufsf->sdev_ufs_lu[hpb->lun];
	hba = hpb->ufsf->hba;

	/*
	 * ufshcd_slave_alloc(sdev) -> ufshcd_set_queue_depth(sdev)
	 * a lu-queue-depth compared with lu_info and hba->nutrs
	 * is selected in ufshcd_set_queue_depth()
	 */
	hpb->qd = sdev->queue_depth;
	INFO_MSG("lu (%d) queue_depth (%d)", hpb->lun, hpb->qd);
	if (!hpb->qd) {
		hpb->qd = hba->nutrs;
		INFO_MSG("lu_queue_depth is 0. we use device's queue info.");
		INFO_MSG("hba->nutrs = (%d)", hba->nutrs);
	}

	hpb->throttle_map_req = hpb->qd;
	hpb->throttle_pre_req = hpb->qd;
	hpb->throttle_hcm_req = hpb->qd;
	hpb->num_inflight_map_req = 0;
	hpb->num_inflight_pre_req = 0;
	hpb->num_inflight_hcm_req = 0;
}

static void ufsshpb_init_lu_constant(struct ufsshpb_dev_info *hpb_dev_info,
				    struct ufsshpb_lu *hpb)
{
	unsigned long long rgn_unit_size, rgn_mem_size;
	int entries_per_rgn;

	hpb->debug = false;

	ufsshpb_find_lu_qd(hpb);

	/* for pre_req */
	hpb->pre_req_min_tr_len = HPB_MULTI_CHUNK_LOW;
	hpb->pre_req_max_tr_len = HPB_MULTI_CHUNK_HIGH;
	hpb->ctx_id_ticket = 0;
	hpb->requeue_timeout_ms = DEFAULT_WB_REQUEUE_TIME_MS;

	/*	From descriptors	*/
	rgn_unit_size = (unsigned long long)
		SECTOR * (0x01 << hpb_dev_info->rgn_size);
	rgn_mem_size = rgn_unit_size / BLOCK * HPB_ENTRY_SIZE;

	hpb->srgn_unit_size = (unsigned long long)
		SECTOR * (0x01 << hpb_dev_info->srgn_size);
	hpb->srgn_mem_size = hpb->srgn_unit_size / BLOCK * HPB_ENTRY_SIZE;

	/* relation : lu <-> region <-> sub region <-> entry */
	entries_per_rgn = rgn_mem_size / HPB_ENTRY_SIZE;
	hpb->entries_per_srgn = hpb->srgn_mem_size / HPB_ENTRY_SIZE;
	hpb->srgns_per_rgn = rgn_mem_size / hpb->srgn_mem_size;

	/*
	 * regions_per_lu = (lu_num_blocks * 4096) / region_unit_size
	 *	          = (lu_num_blocks * HPB_ENTRY_SIZE) / region_mem_size
	 */
	hpb->rgns_per_lu =
		((unsigned long long)hpb->lu_num_blocks
		 + (rgn_mem_size / HPB_ENTRY_SIZE) - 1)
		/ (rgn_mem_size / HPB_ENTRY_SIZE);
	hpb->srgns_per_lu =
		((unsigned long long)hpb->lu_num_blocks
		 + (hpb->srgn_mem_size / HPB_ENTRY_SIZE) - 1)
		/ (hpb->srgn_mem_size / HPB_ENTRY_SIZE);

	/* mempool info */
	hpb->mpage_bytes = OS_PAGE_SIZE;
	hpb->mpages_per_srgn = hpb->srgn_mem_size / hpb->mpage_bytes;

	/* Bitmask Info. */
	hpb->dwords_per_srgn = hpb->entries_per_srgn / BITS_PER_DWORD;
	hpb->entries_per_rgn_shift = ffs(entries_per_rgn) - 1;
	hpb->entries_per_rgn_mask = entries_per_rgn - 1;
	hpb->entries_per_srgn_shift = ffs(hpb->entries_per_srgn) - 1;
	hpb->entries_per_srgn_mask = hpb->entries_per_srgn - 1;

	INFO_MSG("===== From Device Descriptor! =====");
	INFO_MSG("hpb_region_size = (%d), hpb_subregion_size = (%d)",
		 hpb_dev_info->rgn_size, hpb_dev_info->srgn_size);
	INFO_MSG("=====   Constant Values(LU)   =====");
	INFO_MSG("region_unit_size = (%lld), region_mem_size (%lld)",
		 rgn_unit_size, rgn_mem_size);
	INFO_MSG("subregion_unit_size = (%lld), subregion_mem_size (%d)",
		 hpb->srgn_unit_size, hpb->srgn_mem_size);

	INFO_MSG("lu_num_blocks = (%d)", hpb->lu_num_blocks);
	INFO_MSG("regions_per_lu = (%d), subregions_per_lu = (%d)",
		 hpb->rgns_per_lu, hpb->srgns_per_lu);

	INFO_MSG("subregions_per_region = %d", hpb->srgns_per_rgn);
	INFO_MSG("entries_per_region (%u) shift (%u) mask (0x%X)",
		 entries_per_rgn, hpb->entries_per_rgn_shift,
		 hpb->entries_per_rgn_mask);
	INFO_MSG("entries_per_subregion (%u) shift (%u) mask (0x%X)",
		 hpb->entries_per_srgn, hpb->entries_per_srgn_shift,
		 hpb->entries_per_srgn_mask);
	INFO_MSG("mpages_per_subregion : (%d)", hpb->mpages_per_srgn);
	INFO_MSG("===================================");
}

static int ufsshpb_lu_hpb_init(struct ufsf_feature *ufsf, int lun)
{
	struct ufsshpb_lu *hpb = ufsf->hpb_lup[lun];
	struct ufsshpb_region *rgn_table, *rgn;
	struct ufsshpb_subregion *srgn;
	int rgn_idx, srgn_idx, total_srgn_cnt, srgn_cnt, i, ret = 0;

	ufsshpb_init_lu_constant(&ufsf->hpb_dev_info, hpb);

	hpb->hcm_disable = true;
	rgn_table = vzalloc(sizeof(struct ufsshpb_region) * hpb->rgns_per_lu);
	if (!rgn_table) {
		ret = -ENOMEM;
		goto out;
	}

	INFO_MSG("active_region_table bytes: (%lu)",
		 (sizeof(struct ufsshpb_region) * hpb->rgns_per_lu));

	hpb->rgn_tbl = rgn_table;

	spin_lock_init(&hpb->hpb_lock);
	spin_lock_init(&hpb->retry_list_lock);
	spin_lock_init(&hpb->rsp_list_lock);

	/* init lru information */
	INIT_LIST_HEAD(&hpb->lru_info.lh_lru_rgn);
	INIT_LIST_HEAD(&hpb->lru_info.lh_lru_hcm_region);
	hpb->lru_info.selection_type = LRU;

	INIT_LIST_HEAD(&hpb->lh_pinned_srgn);
	INIT_LIST_HEAD(&hpb->lh_act_srgn);
	INIT_LIST_HEAD(&hpb->lh_inact_rgn);

	INIT_LIST_HEAD(&hpb->lh_map_ctx_free);

	ufsshpb_init_jobs(hpb);

	ret = ufsshpb_map_req_mempool_init(hpb);
	if (ret) {
		ERR_MSG("map_req_mempool init fail!");
		goto release_rgn_table;
	}

	ret = ufsshpb_pre_req_mempool_init(hpb);
	if (ret) {
		ERR_MSG("pre_req_mempool init fail!");
		goto release_map_req_mempool;
	}

	ret = ufsshpb_table_mempool_init(hpb);
	if (ret) {
		ERR_MSG("ppn table mempool init fail!");
		ufsshpb_table_mempool_remove(hpb);
		goto release_pre_req_mempool;
	}

	total_srgn_cnt = hpb->srgns_per_lu;
	INFO_MSG("total_subregion_count: (%d)", total_srgn_cnt);
	for (rgn_idx = 0, srgn_cnt = 0; rgn_idx < hpb->rgns_per_lu;
	     rgn_idx++, total_srgn_cnt -= srgn_cnt) {
		rgn = rgn_table + rgn_idx;
		rgn->rgn_idx = rgn_idx;

		atomic_set(&rgn->reason, HPB_UPDATE_NONE);

		INIT_LIST_HEAD(&rgn->list_inact_rgn);

		/* init lru region information*/
		INIT_LIST_HEAD(&rgn->list_lru_rgn);

		srgn_cnt = min(total_srgn_cnt, hpb->srgns_per_rgn);

		ret = ufsshpb_alloc_subregion_tbl(hpb, rgn, srgn_cnt);
		if (ret)
			goto release_srgns;
		ufsshpb_init_subregion_tbl(hpb, rgn);

		if (ufsshpb_is_pinned_region(hpb, rgn_idx)) {
			ret = ufsshpb_init_pinned_active_region(hpb, rgn);
			if (ret)
				goto release_srgns;
		} else {
			rgn->rgn_state = HPB_RGN_INACTIVE;
		}
	}

	if (total_srgn_cnt != 0) {
		ERR_MSG("error total_subregion_count: %d",
			total_srgn_cnt);
		goto release_srgns;
	}

	atomic64_set(&hpb->lru_info.active_cnt, 0);
	/*
	 * even if creating sysfs failed, ufsshpb could run normally.
	 * so we don't deal with error handling
	 */
	ufsshpb_create_sysfs(ufsf, hpb);

	return 0;
release_srgns:
	for (i = 0; i < rgn_idx; i++) {
		rgn = rgn_table + i;
		if (rgn->srgn_tbl) {
			for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt;
			     srgn_idx++) {
				srgn = rgn->srgn_tbl + srgn_idx;
				if (srgn->mctx)
					ufsshpb_put_map_ctx(hpb, srgn->mctx);
			}
			kfree(rgn->srgn_tbl);
		}
	}

	ufsshpb_table_mempool_remove(hpb);
release_pre_req_mempool:
	ufsshpb_pre_req_mempool_remove(hpb);
release_map_req_mempool:
	ufsshpb_map_req_mempool_remove(hpb);
release_rgn_table:
	vfree(rgn_table);
out:
	return ret;
}

static inline int ufsshpb_version_check(struct ufsshpb_dev_info *hpb_dev_info)
{
	INFO_MSG("Support HPB Spec : Driver = (%.4X)  Device = (%.4X)",
		 UFSSHPB_VER, hpb_dev_info->version);

	INFO_MSG("HPB Driver Version : (%.6X%s)", UFSSHPB_DD_VER, UFSSHPB_DD_VER_POST);

	if (hpb_dev_info->version != UFSSHPB_VER) {
		ERR_MSG("ERROR: HPB Spec Version mismatch. So HPB disabled.");
		return -ENODEV;
	}
	return 0;
}

void ufsshpb_get_dev_info(struct ufsf_feature *ufsf, u8 *desc_buf)
{
	struct ufsshpb_dev_info *hpb_dev_info = &ufsf->hpb_dev_info;
	int ret;

	if (desc_buf[DEVICE_DESC_PARAM_UFS_FEAT] & UFS_FEATURE_SUPPORT_HPB_BIT)
		INFO_MSG("bUFSFeaturesSupport: HPB is set");
	else {
		INFO_MSG("bUFSFeaturesSupport: HPB not support");
		ufsshpb_set_state(ufsf, HPB_FAILED);
		return;
	}

	hpb_dev_info->version = LI_EN_16(desc_buf + DEVICE_DESC_PARAM_HPB_VER);

	ret = ufsshpb_version_check(hpb_dev_info);
	if (ret)
		ufsshpb_set_state(ufsf, HPB_FAILED);
}

void ufsshpb_get_geo_info(struct ufsf_feature *ufsf, u8 *geo_buf)
{
	struct ufsshpb_dev_info *hpb_dev_info = &ufsf->hpb_dev_info;
	int hpb_device_max_active_rgns = 0;

	hpb_dev_info->num_lu = geo_buf[GEOMETRY_DESC_HPB_NUMBER_LU];
	if (hpb_dev_info->num_lu == 0) {
		ERR_MSG("Don't have a lu for hpb.");
		ufsshpb_set_state(ufsf, HPB_FAILED);
		return;
	}

	hpb_dev_info->rgn_size = geo_buf[GEOMETRY_DESC_HPB_REGION_SIZE];
	hpb_dev_info->srgn_size = geo_buf[GEOMETRY_DESC_HPB_SUBREGION_SIZE];
	hpb_device_max_active_rgns =
		LI_EN_16(geo_buf + GEOMETRY_DESC_HPB_DEVICE_MAX_ACTIVE_REGIONS);

	INFO_MSG("[48] bHPBRegionSize (%u)", hpb_dev_info->rgn_size);
	INFO_MSG("[49] bHPBNumberLU (%u)", hpb_dev_info->num_lu);
	INFO_MSG("[4A] bHPBSubRegionSize (%u)", hpb_dev_info->srgn_size);
	INFO_MSG("[4B:4C] wDeviceMaxActiveHPBRegions (%u)",
		 hpb_device_max_active_rgns);

	if (hpb_dev_info->rgn_size == 0 || hpb_dev_info->srgn_size == 0 ||
	    hpb_device_max_active_rgns == 0) {
		ERR_MSG("HPB NOT normally supported by device");
		ufsshpb_set_state(ufsf, HPB_FAILED);
	}
}

void ufsshpb_get_lu_info(struct ufsf_feature *ufsf, int lun, u8 *unit_buf)
{
	struct ufsf_lu_desc lu_desc;
	struct ufsshpb_lu *hpb;

	lu_desc.lu_enable = unit_buf[UNIT_DESC_PARAM_LU_ENABLE];
	lu_desc.lu_queue_depth = unit_buf[UNIT_DESC_PARAM_LU_Q_DEPTH];
	lu_desc.lu_logblk_size = unit_buf[UNIT_DESC_PARAM_LOGICAL_BLK_SIZE];
	lu_desc.lu_logblk_cnt =
		LI_EN_64(unit_buf + UNIT_DESC_PARAM_LOGICAL_BLK_COUNT);
	lu_desc.lu_max_active_hpb_rgns =
		LI_EN_16(unit_buf + UNIT_DESC_HPB_LU_MAX_ACTIVE_REGIONS);
	lu_desc.lu_hpb_pinned_rgn_startidx =
		LI_EN_16(unit_buf + UNIT_DESC_HPB_LU_PIN_REGION_START_OFFSET);
	lu_desc.lu_num_hpb_pinned_rgns =
		LI_EN_16(unit_buf + UNIT_DESC_HPB_LU_NUM_PIN_REGIONS);

	if (lu_desc.lu_num_hpb_pinned_rgns > 0) {
		lu_desc.lu_hpb_pinned_end_offset =
			lu_desc.lu_hpb_pinned_rgn_startidx +
			lu_desc.lu_num_hpb_pinned_rgns - 1;
	} else
		lu_desc.lu_hpb_pinned_end_offset = PINNED_NOT_SET;

	INFO_MSG("LUN(%d) [0A] bLogicalBlockSize (%d)",
		 lun, lu_desc.lu_logblk_size);
	INFO_MSG("LUN(%d) [0B] qLogicalBlockCount (%llu)",
		 lun, lu_desc.lu_logblk_cnt);
	INFO_MSG("LUN(%d) [03] bLuEnable (%d)", lun, lu_desc.lu_enable);
	INFO_MSG("LUN(%d) [06] bLuQueueDepth (%d)", lun, lu_desc.lu_queue_depth);
	INFO_MSG("LUN(%d) [23:24] wLUMaxActiveHPBRegions (%d)",
		 lun, lu_desc.lu_max_active_hpb_rgns);
	INFO_MSG("LUN(%d) [25:26] wHPBPinnedRegionStartIdx (%d)",
		 lun, lu_desc.lu_hpb_pinned_rgn_startidx);
	INFO_MSG("LUN(%d) [27:28] wNumHPBPinnedRegions (%d)",
		 lun, lu_desc.lu_num_hpb_pinned_rgns);
	INFO_MSG("LUN(%d) PINNED Start (%d) End (%d)",
		 lun, lu_desc.lu_hpb_pinned_rgn_startidx,
		 lu_desc.lu_hpb_pinned_end_offset);

	ufsf->hpb_lup[lun] = NULL;

	if (lu_desc.lu_enable == 0x02 && lu_desc.lu_max_active_hpb_rgns) {
		ufsf->hpb_lup[lun] = kzalloc(sizeof(struct ufsshpb_lu),
					     GFP_KERNEL);
		if (!ufsf->hpb_lup[lun]) {
			ERR_MSG("hpb_lup[%d] alloc failed", lun);
			return;
		}

		hpb = ufsf->hpb_lup[lun];
		hpb->ufsf = ufsf;
		hpb->lun = lun;
		hpb->lu_num_blocks = lu_desc.lu_logblk_cnt;
		hpb->lu_max_active_rgns = lu_desc.lu_max_active_hpb_rgns;
		hpb->lru_info.max_lru_active_cnt =
			lu_desc.lu_max_active_hpb_rgns -
			lu_desc.lu_num_hpb_pinned_rgns;
		hpb->lu_pinned_rgn_startidx =
			lu_desc.lu_hpb_pinned_rgn_startidx;
		hpb->lu_pinned_end_offset = lu_desc.lu_hpb_pinned_end_offset;
	} else {
		INFO_MSG("===== LU (%d) is hpb-disabled.", lun);
	}
}

static void ufsshpb_error_handler(struct work_struct *work)
{
	struct ufsf_feature *ufsf;

	ufsf = container_of(work, struct ufsf_feature, hpb_eh_work);

	WARN_MSG("driver has failed. but UFSHCD can run without UFSSHPB");
	WARN_MSG("UFSSHPB will be removed from the kernel");

	ufsshpb_remove(ufsf, HPB_FAILED);
}

static int ufsshpb_init(struct ufsf_feature *ufsf)
{
	int lun, ret;
	struct ufsshpb_lu *hpb;
	int hpb_enabled_lun = 0;

	seq_scan_lu(lun) {
		if (!ufsf->hpb_lup[lun])
			continue;

		/*
		 * HPB need info about request queue in order to issue
		 * RB-CMD for pinned region.
		 */
		if (!ufsf->sdev_ufs_lu[lun]) {
			WARN_MSG("lun (%d) don't have scsi_device", lun);
			continue;
		}

		ret = ufsshpb_lu_hpb_init(ufsf, lun);
		if (ret) {
			kfree(ufsf->hpb_lup[lun]);
			continue;
		}
		hpb_enabled_lun++;
	}

	if (hpb_enabled_lun == 0) {
		ERR_MSG("No UFSSHPB LU to init");
		ret = -ENODEV;
		goto hpb_failed;
	}

	INIT_WORK(&ufsf->hpb_eh_work, ufsshpb_error_handler);

	kref_init(&ufsf->hpb_kref);
	ufsshpb_set_state(ufsf, HPB_PRESENT);

	seq_scan_lu(lun)
		if (ufsf->hpb_lup[lun]) {
			hpb = ufsf->hpb_lup[lun];

			INFO_MSG("UFSSHPB LU %d working", lun);
			if (hpb->lu_pinned_end_offset != PINNED_NOT_SET)
				schedule_work(&hpb->pinned_work);
		}

	return 0;
hpb_failed:
	ufsshpb_set_state(ufsf, HPB_FAILED);
	return ret;
}

static void ufsshpb_drop_retry_list(struct ufsshpb_lu *hpb)
{
	struct ufsshpb_req *map_req, *next;
	struct request *req;
	unsigned long flags;
	LIST_HEAD(lh_reqs);

	if (list_empty(&hpb->lh_map_req_retry))
		return;

	spin_lock_irqsave(&hpb->hpb_lock, flags);
	list_for_each_entry_safe(map_req, next, &hpb->lh_map_req_retry,
				 list_req) {
		INFO_MSG("drop map_req %p ( %d - %d )", map_req,
			 map_req->rb.rgn_idx, map_req->rb.srgn_idx);

		list_del_init(&map_req->list_req);

		/*
		 * blk_put_request() will grap the queue_lock.
		 * so requests add to a list.
		 * lock seq : queue_lock -> hpb_lock.
		 */
		list_add_tail(&map_req->req->queuelist, &lh_reqs);

		bio_put(map_req->bio);

		ufsshpb_put_map_req(hpb, map_req);
	}
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);

	list_for_each_entry(req, &lh_reqs, queuelist)
		blk_put_request(req);
}

static void ufsshpb_drop_rsp_lists(struct ufsshpb_lu *hpb)
{
	struct ufsshpb_region *rgn, *next_rgn;
	struct ufsshpb_subregion *srgn, *next_srgn;
	unsigned long flags;

	spin_lock_irqsave(&hpb->rsp_list_lock, flags);
	list_for_each_entry_safe(rgn, next_rgn, &hpb->lh_inact_rgn,
				 list_inact_rgn) {
		list_del_init(&rgn->list_inact_rgn);
	}

	list_for_each_entry_safe(srgn, next_srgn, &hpb->lh_act_srgn,
				 list_act_srgn) {
		list_del_init(&srgn->list_act_srgn);
	}
	spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
}

static void ufsshpb_destroy_subregion_tbl(struct ufsshpb_lu *hpb,
					 struct ufsshpb_region *rgn)
{
	int srgn_idx;

	for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) {
		struct ufsshpb_subregion *srgn;

		srgn = rgn->srgn_tbl + srgn_idx;
		srgn->srgn_state = HPB_SRGN_UNUSED;

		ufsshpb_put_map_ctx(hpb, srgn->mctx);
	}
}

static void ufsshpb_destroy_region_tbl(struct ufsshpb_lu *hpb)
{
	int rgn_idx;

	INFO_MSG("Start");

	for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) {
		struct ufsshpb_region *rgn;

		rgn = hpb->rgn_tbl + rgn_idx;
		if (rgn->rgn_state != HPB_RGN_INACTIVE) {
			rgn->rgn_state = HPB_RGN_INACTIVE;

			ufsshpb_destroy_subregion_tbl(hpb, rgn);
		}

		kfree(rgn->srgn_tbl);
	}

	ufsshpb_table_mempool_remove(hpb);
	vfree(hpb->rgn_tbl);

	INFO_MSG("End");
}

void ufsshpb_remove(struct ufsf_feature *ufsf, int state)
{
	struct ufsshpb_lu *hpb;
	int lun;

	INFO_MSG("start release");
	ufsshpb_set_state(ufsf, HPB_FAILED);

	INFO_MSG("kref count (%d)",
		 atomic_read(&ufsf->hpb_kref.refcount.refs));

	seq_scan_lu(lun) {
		hpb = ufsf->hpb_lup[lun];

		INFO_MSG("lun (%d) (%p)", lun, hpb);

		ufsf->hpb_lup[lun] = NULL;

		if (!hpb)
			continue;

		ufsshpb_cancel_jobs(hpb);

		ufsshpb_destroy_region_tbl(hpb);
		if (hpb->alloc_mctx != 0)
			WARN_MSG("warning: alloc_mctx (%d)", hpb->alloc_mctx);

		ufsshpb_map_req_mempool_remove(hpb);

		ufsshpb_pre_req_mempool_remove(hpb);

		ufsshpb_remove_sysfs(hpb);

		kfree(hpb);
	}

	ufsshpb_set_state(ufsf, state);

	INFO_MSG("end release");
}

void ufsshpb_reset(struct ufsf_feature *ufsf)
{
	ufsshpb_set_state(ufsf, HPB_PRESENT);
}

void ufsshpb_reset_host(struct ufsf_feature *ufsf)
{
	struct ufsshpb_lu *hpb;
	int lun;

	ufsshpb_set_state(ufsf, HPB_RESET);
	seq_scan_lu(lun) {
		hpb = ufsf->hpb_lup[lun];
		if (hpb) {
			INFO_MSG("UFSSHPB lun %d reset", lun);
			ufsshpb_cancel_jobs(hpb);
			ufsshpb_drop_retry_list(hpb);
			ufsshpb_drop_rsp_lists(hpb);
		}
	}
}

static inline int ufsshpb_probe_lun_done(struct ufsf_feature *ufsf)
{
	return (ufsf->num_lu == ufsf->slave_conf_cnt);
}

void ufsshpb_init_handler(struct work_struct *work)
{
	struct ufsf_feature *ufsf;
	int ret = 0;

	ufsf = container_of(work, struct ufsf_feature, hpb_init_work);

	INFO_MSG("probe_done(%d)", ufsshpb_probe_lun_done(ufsf));

	ufsshpb_set_state(ufsf, HPB_WAIT_INIT);

	ret = wait_event_timeout(ufsf->hpb_wait,
				 ufsshpb_probe_lun_done(ufsf),
				 msecs_to_jiffies(10000));
	if (ret == 0)
		ERR_MSG("Probing LU is not fully complete.");

	INFO_MSG("probe_done(%d) ret(%d)", ufsshpb_probe_lun_done(ufsf), ret);
	INFO_MSG("HPB_INIT_START");

	ret = ufsshpb_init(ufsf);
	if (ret)
		ERR_MSG("UFSSHPB driver init failed. (%d)", ret);
}

void ufsshpb_suspend(struct ufsf_feature *ufsf)
{
	struct ufsshpb_lu *hpb;
	int lun;

	seq_scan_lu(lun) {
		hpb = ufsf->hpb_lup[lun];
		if (hpb) {
			INFO_MSG("ufsshpb_lu %d goto suspend", lun);
			INFO_MSG("ufsshpb_lu %d changes suspend state", lun);
			ufsshpb_set_state(ufsf, HPB_SUSPEND);
			ufsshpb_cancel_jobs(hpb);
		}
	}
}

void ufsshpb_resume(struct ufsf_feature *ufsf)
{
	struct ufsshpb_lu *hpb;
	int lun;

	seq_scan_lu(lun) {
		hpb = ufsf->hpb_lup[lun];
		if (hpb) {
			bool do_task_work = false;
			bool do_retry_work = false;

			ufsshpb_set_state(ufsf, HPB_PRESENT);

			do_task_work = !ufsshpb_is_empty_rsp_lists(hpb);
			do_retry_work =
				!list_empty_careful(&hpb->lh_map_req_retry);

			INFO_MSG("ufsshpb_lu %d resume. do_task_work %d retry %d",
				 lun, do_task_work, do_retry_work);

			if (do_task_work)
				schedule_work(&hpb->task_work);
			if (do_retry_work)
				schedule_delayed_work(&hpb->retry_work,
						      msecs_to_jiffies(100));
		}
	}
}

static void ufsshpb_stat_init(struct ufsshpb_lu *hpb)
{
	atomic64_set(&hpb->hit, 0);
	atomic64_set(&hpb->miss, 0);
	atomic64_set(&hpb->rb_noti_cnt, 0);
	atomic64_set(&hpb->rb_active_cnt, 0);
	atomic64_set(&hpb->rb_inactive_cnt, 0);
	atomic64_set(&hpb->dev_inact_hcm_cnt, 0);
	atomic64_set(&hpb->map_req_cnt, 0);
	atomic64_set(&hpb->map_compl_cnt, 0);
	atomic64_set(&hpb->pre_req_cnt, 0);
	atomic64_set(&hpb->set_hcm_req_cnt, 0);
	atomic64_set(&hpb->unset_hcm_req_cnt, 0);
#if defined(CONFIG_HPB_DEBUG_SYSFS)
	ufsshpb_debug_sys_init(hpb);
#endif
}

/* SYSFS functions */
static ssize_t ufsshpb_sysfs_hcm_disable_show(struct ufsshpb_lu *hpb, char *buf)
{
	int ret;

	ret = snprintf(buf, PAGE_SIZE, "hcm_disable %d\n",
		       hpb->hcm_disable);

	INFO_MSG("hcm_disable %d", hpb->hcm_disable);
	return ret;
}

static ssize_t ufsshpb_sysfs_hcm_disable_store(struct ufsshpb_lu *hpb,
					       const char *buf, size_t cnt)
{
	unsigned long value;

	if (kstrtoul(buf, 0, &value))
		return -EINVAL;

	if (value > 1)
		return -EINVAL;

	if (value == 1)
		hpb->hcm_disable = true;
	else if (value == 0)
		hpb->hcm_disable = false;

	INFO_MSG("hcm_disable %d", hpb->hcm_disable);

	return cnt;
}

static ssize_t ufsshpb_sysfs_prep_disable_show(struct ufsshpb_lu *hpb, char *buf)
{
	int ret;

	ret = snprintf(buf, PAGE_SIZE, "force_hpb_read_disable %d\n",
		       hpb->force_disable);

	INFO_MSG("force_hpb_read_disable %d", hpb->force_disable);
	return ret;
}

static ssize_t ufsshpb_sysfs_prep_disable_store(struct ufsshpb_lu *hpb,
					       const char *buf, size_t cnt)
{
	unsigned long value;

	if (kstrtoul(buf, 0, &value))
		return -EINVAL;

	if (value > 1)
		return -EINVAL;

	if (value == 1)
		hpb->force_disable = true;
	else if (value == 0)
		hpb->force_disable = false;

	INFO_MSG("force_hpb_read_disable %d", hpb->force_disable);

	return cnt;
}

static ssize_t ufsshpb_sysfs_map_disable_show(struct ufsshpb_lu *hpb, char *buf)
{
	int ret;

	ret = snprintf(buf, PAGE_SIZE, "force_map_req_disable %d\n",
		       hpb->force_map_req_disable);

	INFO_MSG("force_map_req_disable %d", hpb->force_map_req_disable);

	return ret;
}

static ssize_t ufsshpb_sysfs_map_disable_store(struct ufsshpb_lu *hpb,
					      const char *buf, size_t cnt)
{
	unsigned long value;

	if (kstrtoul(buf, 0, &value))
		return -EINVAL;

	if (value > 1)
		return -EINVAL;

	if (value == 1)
		hpb->force_map_req_disable = true;
	else if (value == 0)
		hpb->force_map_req_disable = false;

	INFO_MSG("force_map_req_disable %d", hpb->force_map_req_disable);

	return cnt;
}

static ssize_t ufsshpb_sysfs_throttle_map_req_show(struct ufsshpb_lu *hpb,
						  char *buf)
{
	int ret;
#if defined(CONFIG_HPB_DEBUG)
	ret = snprintf(buf, PAGE_SIZE, "throttle_map_req %d (inflight: %d)\n",
		       hpb->throttle_map_req, hpb->num_inflight_map_req);
#else
	ret = snprintf(buf, PAGE_SIZE, "throttle_map_req %d\n",
		       hpb->throttle_map_req);
#endif

	INFO_MSG("throttle_map_req %d", hpb->throttle_map_req);

	return ret;
}

static ssize_t ufsshpb_sysfs_throttle_map_req_store(struct ufsshpb_lu *hpb,
						   const char *buf, size_t cnt)
{
	unsigned long throttle_map_req;

	if (kstrtoul(buf, 0, &throttle_map_req))
		return -EINVAL;

	if (throttle_map_req > hpb->qd)
		return -EINVAL;

	hpb->throttle_map_req = (int)throttle_map_req;

	INFO_MSG("throttle_map_req %d", hpb->throttle_map_req);

	return cnt;
}

static ssize_t ufsshpb_sysfs_throttle_pre_req_show(struct ufsshpb_lu *hpb,
						  char *buf)
{
	int ret;

#if defined(CONFIG_HPB_DEBUG)
	ret = snprintf(buf, PAGE_SIZE, "throttle_pre_req %d (inflight: %d)\n",
		       hpb->throttle_pre_req, hpb->num_inflight_pre_req);
#else
	ret = snprintf(buf, PAGE_SIZE, "throttle_pre_req %d\n",
		       hpb->throttle_pre_req);
#endif

	INFO_MSG("throttle_pre_req %d", hpb->throttle_pre_req);

	return ret;
}

static ssize_t ufsshpb_sysfs_throttle_pre_req_store(struct ufsshpb_lu *hpb,
						   const char *buf, size_t cnt)
{
	unsigned long throttle_pre_req;

	if (kstrtoul(buf, 0, &throttle_pre_req))
		return -EINVAL;

	if (throttle_pre_req > hpb->qd)
		return -EINVAL;

	hpb->throttle_pre_req = (int)throttle_pre_req;

	INFO_MSG("throttle_pre_req %d", hpb->throttle_pre_req);

	return cnt;
}

static ssize_t ufsshpb_sysfs_throttle_hcm_req_show(struct ufsshpb_lu *hpb, char *buf)
{
	int ret;

#if defined(CONFIG_HPB_DEBUG)
	ret = snprintf(buf, PAGE_SIZE, "throttle_hcm_req %d (inflight: %d)\n",
		       hpb->throttle_hcm_req, hpb->num_inflight_hcm_req);
#else
	ret = snprintf(buf, PAGE_SIZE, "throttle_hcm_req %d\n",
		       hpb->throttle_hcm_req);
#endif

	INFO_MSG("%s", buf);

	return ret;
}

static ssize_t ufsshpb_sysfs_throttle_hcm_req_store(struct ufsshpb_lu *hpb,
						   const char *buf, size_t cnt)
{
	unsigned long throttle_hcm_req;

	if (kstrtoul(buf, 0, &throttle_hcm_req))
		return -EINVAL;

	if (throttle_hcm_req > hpb->qd)
		return -EINVAL;

	if (throttle_hcm_req > 0 && throttle_hcm_req <= 32)
		hpb->throttle_hcm_req = (int)throttle_hcm_req;
	else
		INFO_MSG("value is wrong. throttle_hcm_req must be in [1~32] ");

	INFO_MSG("throttle_hcm_req %d", hpb->throttle_hcm_req);

	return cnt;
}

static ssize_t ufsshpb_sysfs_pre_req_min_tr_len_show(struct ufsshpb_lu *hpb,
						    char *buf)
{
	int ret;

	ret = snprintf(buf, PAGE_SIZE, "%d", hpb->pre_req_min_tr_len);
	INFO_MSG("pre_req min transfer len %d", hpb->pre_req_min_tr_len);

	return ret;
}

static ssize_t ufsshpb_sysfs_pre_req_min_tr_len_store(struct ufsshpb_lu *hpb,
						     const char *buf,
						     size_t count)
{
	unsigned long val;

	if (kstrtoul(buf, 0, &val))
		return -EINVAL;

	if (val < 0)
		val = 0;

	if (hpb->pre_req_max_tr_len < val || val < HPB_MULTI_CHUNK_LOW)
		INFO_MSG("value is wrong. pre_req transfer len %d ~ %d",
			 HPB_MULTI_CHUNK_LOW, hpb->pre_req_max_tr_len);
	else
		hpb->pre_req_min_tr_len = val;

	INFO_MSG("pre_req min transfer len %d", hpb->pre_req_min_tr_len);

	return count;
}

static ssize_t ufsshpb_sysfs_pre_req_max_tr_len_show(struct ufsshpb_lu *hpb,
						    char *buf)
{
	int ret;

	ret = snprintf(buf, PAGE_SIZE, "%d", hpb->pre_req_max_tr_len);
	INFO_MSG("pre_req max transfer len %d", hpb->pre_req_max_tr_len);

	return ret;
}

static ssize_t ufsshpb_sysfs_pre_req_max_tr_len_store(struct ufsshpb_lu *hpb,
						     const char *buf,
						     size_t count)
{
	unsigned long val;

	if (kstrtoul(buf, 0, &val))
		return -EINVAL;

	if (hpb->pre_req_min_tr_len > val || val > HPB_MULTI_CHUNK_HIGH)
		INFO_MSG("value is wrong. pre_req transfer len %d ~ %d",
			 hpb->pre_req_min_tr_len, HPB_MULTI_CHUNK_HIGH);
	else
		hpb->pre_req_max_tr_len = val;

	INFO_MSG("pre_req max transfer len %d", hpb->pre_req_max_tr_len);

	return count;
}

static ssize_t ufsshpb_sysfs_pre_req_timeout_ms_show(struct ufsshpb_lu *hpb,
						    char *buf)
{
	int ret;

	ret = snprintf(buf, PAGE_SIZE, "%u\n", hpb->requeue_timeout_ms);
	INFO_MSG("pre_req requeue timeout %u", hpb->requeue_timeout_ms);

	return ret;
}

static ssize_t ufsshpb_sysfs_pre_req_timeout_ms_store(struct ufsshpb_lu *hpb,
						     const char *buf,
						     size_t count)
{
	unsigned int val;

	if (kstrtouint(buf, 0, &val))
		return -EINVAL;

	hpb->requeue_timeout_ms = val;

	return count;
}

#if defined(CONFIG_HPB_DEBUG)
static ssize_t ufsshpb_sysfs_debug_show(struct ufsshpb_lu *hpb, char *buf)
{
	int ret;

	ret = snprintf(buf, PAGE_SIZE, "debug %d\n", hpb->debug);

	INFO_MSG("debug %d", hpb->debug);

	return ret;
}

static ssize_t ufsshpb_sysfs_debug_store(struct ufsshpb_lu *hpb,
					const char *buf, size_t cnt)
{
	unsigned long debug;

	if (kstrtoul(buf, 0, &debug))
		return -EINVAL;

	if (debug > 1)
		return -EINVAL;

	if (debug == 1)
		hpb->debug = 1;
	else
		hpb->debug = 0;

	INFO_MSG("debug %d", hpb->debug);

	return cnt;
}
#endif

#if defined(CONFIG_HPB_DEBUG_SYSFS)
static ssize_t ufsshpb_sysfs_lba_info_show(struct ufsshpb_lu *hpb, char *buf)
{
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;
	enum HPB_RGN_STATE rgn_state;
	enum HPB_SRGN_STATE srgn_state;
	unsigned long lpn, flags;
	int rgn_idx, srgn_idx, srgn_offset;
	int ppn_clean_offset, ppn_clean_pinned_cnt = 0, ppn_clean_active_cnt = 0;
	unsigned long long pinned_valid_size = 0, active_valid_size = 0;
	int ret;

	lpn = hpb->lba_rgns_info / SECTORS_PER_BLOCK;

	ufsshpb_get_pos_from_lpn(hpb, lpn, &rgn_idx, &srgn_idx, &srgn_offset);

	rgn = hpb->rgn_tbl + rgn_idx;

	spin_lock_irqsave(&hpb->hpb_lock, flags);
	rgn_state = rgn->rgn_state;
	for (srgn_idx = 0; srgn_idx < hpb->srgns_per_rgn; srgn_idx++) {
		srgn = rgn->srgn_tbl + srgn_idx;
		srgn_state = srgn->srgn_state;

		if (!ufsshpb_valid_srgn(rgn, srgn))
			continue;

		if (!srgn->mctx || !srgn->mctx->ppn_dirty)
			continue;

		ppn_clean_offset = 0;
		while (ppn_clean_offset < hpb->entries_per_srgn) {
			ppn_clean_offset =
				find_next_zero_bit((unsigned long *)srgn->mctx->ppn_dirty,
						   hpb->entries_per_srgn,
						   ppn_clean_offset);

			if (ppn_clean_offset >= hpb->entries_per_srgn)
				break;

			if (rgn_state == HPB_RGN_PINNED ||
			    rgn_state == HPB_RGN_HCM)
				ppn_clean_pinned_cnt++;
			else if (rgn_state == HPB_RGN_ACTIVE)
				ppn_clean_active_cnt++;

			ppn_clean_offset++;
		}
	}
	pinned_valid_size = ppn_clean_pinned_cnt * BLOCK_KB;
	active_valid_size = ppn_clean_active_cnt * BLOCK_KB;

	ret = snprintf(buf, PAGE_SIZE,
		       "[LBA] %d [Region State] %d (0-INACTIVE, 1-ACTIVE, 2-PINNED, 3-HCM)\n"
		       "[Pinned] hit_count(low) %lld hit_count(high) %lld miss_count(low) %lld miss_count(high) %lld\n"
		       "[Active] hit_count(low) %lld hit_count(high) %lld miss_count(low) %lld miss_count(high) %lld\n"
		       "[Inactive] read_count(low) %lld read_count(high) %lld\n"
		       "[Pinned_rb_cnt] %lld [Active_rb_cnt] %lld\n"
		       "[Current_pinned_valid_size] %llu [Current_active_valid_size] %llu\n",
		       hpb->lba_rgns_info, rgn_state,
		       (long long)atomic64_read(&rgn->rgn_pinned_low_hit),
		       (long long)atomic64_read(&rgn->rgn_pinned_high_hit),
		       (long long)atomic64_read(&rgn->rgn_pinned_low_miss),
		       (long long)atomic64_read(&rgn->rgn_pinned_high_miss),
		       (long long)atomic64_read(&rgn->rgn_active_low_hit),
		       (long long)atomic64_read(&rgn->rgn_active_high_hit),
		       (long long)atomic64_read(&rgn->rgn_active_low_miss),
		       (long long)atomic64_read(&rgn->rgn_active_high_miss),
		       (long long)atomic64_read(&rgn->rgn_inactive_low_read),
		       (long long)atomic64_read(&rgn->rgn_inactive_high_read),
		       (long long)atomic64_read(&rgn->rgn_pinned_rb_cnt),
		       (long long)atomic64_read(&rgn->rgn_active_rb_cnt),
		       pinned_valid_size, active_valid_size);

	INFO_MSG("%s", buf);
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	return ret;
}

static ssize_t ufsshpb_sysfs_lba_info_store(struct ufsshpb_lu *hpb,
					   const char *buf, size_t cnt)
{
	unsigned long value;

	if (kstrtoul(buf, 0, &value)) {
		ERR_MSG("kstrtoul error");
		return -EINVAL;
	}

	if (value > hpb->lu_num_blocks * SECTORS_PER_BLOCK) {
		ERR_MSG("value %lu > lu_num_blocks %d error",
			value, hpb->lu_num_blocks);
		return -EINVAL;
	}

	hpb->lba_rgns_info = (int)value;

	INFO_MSG("lba_info %d", hpb->lba_rgns_info);

	return cnt;
}

static ssize_t ufsshpb_sysfs_active_size_show(struct ufsshpb_lu *hpb, char *buf)
{
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;
	enum HPB_RGN_STATE rgn_state;
	enum HPB_SRGN_STATE srgn_state;
	int ret, rgn_idx, srgn_idx;
	int ppn_clean_offset, ppn_clean_pinned_cnt = 0, ppn_clean_active_cnt = 0;
	unsigned long flags;
	unsigned long long pinned_valid_size = 0, active_valid_size = 0;

	spin_lock_irqsave(&hpb->hpb_lock, flags);
	for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) {
		rgn = hpb->rgn_tbl + rgn_idx;
		rgn_state = rgn->rgn_state;

		for (srgn_idx = 0; srgn_idx < hpb->srgns_per_rgn; srgn_idx++) {
			srgn = rgn->srgn_tbl + srgn_idx;
			srgn_state = srgn->srgn_state;

			if (!ufsshpb_valid_srgn(rgn, srgn))
				continue;

			if (!srgn->mctx || !srgn->mctx->ppn_dirty)
				continue;

			ppn_clean_offset = 0;
			while (ppn_clean_offset < hpb->entries_per_srgn) {
				ppn_clean_offset =
					find_next_zero_bit((unsigned long *)srgn->mctx->ppn_dirty,
							   hpb->entries_per_srgn,
							   ppn_clean_offset);

				if (ppn_clean_offset >= hpb->entries_per_srgn)
					break;

				if (rgn_state == HPB_RGN_PINNED ||
				    rgn_state == HPB_RGN_HCM)
					ppn_clean_pinned_cnt++;
				else if (rgn_state == HPB_RGN_ACTIVE)
					ppn_clean_active_cnt++;

				ppn_clean_offset++;
			}
		}
	}
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	pinned_valid_size = ppn_clean_pinned_cnt * BLOCK_KB;
	active_valid_size = ppn_clean_active_cnt * BLOCK_KB;

	ret = snprintf(buf, PAGE_SIZE,
		       "[Pinned] current_valid_size %llu [Active] current_valid_size %llu\n",
		       pinned_valid_size, active_valid_size);

	INFO_MSG("%s", buf);

	return ret;
}

static ssize_t ufsshpb_sysfs_hpb_normal_read_count_show(struct ufsshpb_lu *hpb,
						       char *buf)
{
	long long pinned_low_hit_cnt, pinned_high_hit_cnt;
	long long pinned_low_miss_cnt, pinned_high_miss_cnt;
	long long active_low_hit_cnt, active_high_hit_cnt;
	long long active_low_miss_cnt, active_high_miss_cnt;
	long long inactive_low_read_cnt, inactive_high_read_cnt;
	int ret;

	pinned_low_hit_cnt = atomic64_read(&hpb->pinned_low_hit);
	pinned_high_hit_cnt = atomic64_read(&hpb->pinned_high_hit);
	pinned_low_miss_cnt = atomic64_read(&hpb->pinned_low_miss);
	pinned_high_miss_cnt = atomic64_read(&hpb->pinned_high_miss);
	active_low_hit_cnt = atomic64_read(&hpb->active_low_hit);
	active_high_hit_cnt = atomic64_read(&hpb->active_high_hit);
	active_low_miss_cnt = atomic64_read(&hpb->active_low_miss);
	active_high_miss_cnt = atomic64_read(&hpb->active_high_miss);
	inactive_low_read_cnt = atomic64_read(&hpb->inactive_low_read);
	inactive_high_read_cnt = atomic64_read(&hpb->inactive_high_read);

	ret = snprintf(buf, PAGE_SIZE,
		       "[Pinned] hit_count(low) %lld hit_count(high) %lld miss_count(low) %lld miss_count(high) %lld\n"
		       "[Active] hit_count(low) %lld hit_count(high) %lld miss_count(low) %lld miss_count(high) %lld\n"
		       "[Inactive] read_count(low) %lld read_count(high) %lld\n",
		       pinned_low_hit_cnt, pinned_high_hit_cnt, pinned_low_miss_cnt, pinned_high_miss_cnt,
		       active_low_hit_cnt, active_high_hit_cnt, active_low_miss_cnt, active_high_miss_cnt,
		       inactive_low_read_cnt, inactive_high_read_cnt);

	INFO_MSG("%s", buf);

	return ret;
}

static ssize_t ufsshpb_sysfs_rb_count_show(struct ufsshpb_lu *hpb, char *buf)
{
	long long pinned_rb_cnt, active_rb_cnt;
	int ret;

	pinned_rb_cnt = atomic64_read(&hpb->pinned_rb_cnt);
	active_rb_cnt = atomic64_read(&hpb->active_rb_cnt);

	ret = snprintf(buf, PAGE_SIZE,
		       "[Pinned] rb_count %lld [Active] rb_count %lld\n",
		       pinned_rb_cnt, active_rb_cnt);

	INFO_MSG("%s", buf);

	return ret;
}

static ssize_t ufsshpb_sysfs_rsp_list_cnt_show(struct ufsshpb_lu *hpb, char *buf)
{
	long long act_cnt, inact_cnt;
	int ret;
#if defined(CONFIG_HPB_DEBUG)
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;
	unsigned long flags;
	int cnt = 0;
#endif

	act_cnt = atomic64_read(&hpb->act_rsp_list_cnt);
	inact_cnt = atomic64_read(&hpb->inact_rsp_list_cnt);

	ret = snprintf(buf, PAGE_SIZE,
		       "act_rsp_list_cnt %lld inact_rsp_list_cnt %lld\n",
		       act_cnt, inact_cnt);

	INFO_MSG("%s", buf);

	if (!hpb->debug)
		return ret;

	spin_lock_irqsave(&hpb->rsp_list_lock, flags);
	if (act_cnt) {
		ret += snprintf(buf+ret, PAGE_SIZE - ret, "act_rsp_list:\n");
		cnt = 0;
		list_for_each_entry(srgn, &hpb->lh_act_srgn, list_act_srgn) {
			cnt++;
			if (cnt % 10 == 1)
				ret += snprintf(buf+ret, PAGE_SIZE - ret, "\t");
			ret += snprintf(buf+ret, PAGE_SIZE - ret, "%05ds%d ",
					srgn->rgn_idx, hpb->rgn_tbl[srgn->rgn_idx].rgn_state);
			if (cnt % 10 == 0)
				ret += snprintf(buf+ret, PAGE_SIZE - ret, "\n");

			if (ret > PAGE_SIZE) {
				ret = PAGE_SIZE - 10;
				ret += snprintf(buf+ret, PAGE_SIZE - ret,
						"... ");
				if (cnt % 10 == 0)
					ret += snprintf(buf+ret, PAGE_SIZE - ret, "\n");
				break;
			}
		}
		if (cnt % 10 != 0)
			ret += snprintf(buf+ret, PAGE_SIZE - ret, "\n");
	}
	if (inact_cnt) {
		ret += snprintf(buf+ret, PAGE_SIZE - ret, "inact_rsp_list:\n");
		cnt = 0;
		list_for_each_entry(rgn, &hpb->lh_inact_rgn, list_inact_rgn) {
			cnt++;
			if (cnt % 10 == 1)
				ret += snprintf(buf+ret, PAGE_SIZE - ret, "\t");
			ret += snprintf(buf+ret, PAGE_SIZE - ret, "%05ds%d ",
					rgn->rgn_idx, rgn->rgn_state);
			if (cnt % 10 == 0)
				ret += snprintf(buf+ret, PAGE_SIZE - ret, "\n");

			if (ret > PAGE_SIZE) {
				ret = PAGE_SIZE - 10;
				ret += snprintf(buf+ret, PAGE_SIZE - ret,
						"... ");
				if (cnt % 10 == 0)
					ret += snprintf(buf+ret, PAGE_SIZE - ret, "\n");
				break;
			}
		}
		if (cnt % 10 != 0)
			ret += snprintf(buf+ret, PAGE_SIZE - ret, "\n");
	}
	spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);

	return ret;
}
#endif

#if defined(CONFIG_HPB_ERR_INJECTION)
static ssize_t ufsshpb_sysfs_err_injection_select_show(struct ufsshpb_lu *hpb,
						      char *buf)
{
	char error_injection[8];
	int ret;

	if (hpb->err_injection_select == HPB_ERR_INJECTION_DISABLE)
		strlcpy(error_injection, "disable", 8);
	else if (hpb->err_injection_select == HPB_ERR_INJECTION_BITFLIP)
		strlcpy(error_injection, "bitflip", 8);
	else if (hpb->err_injection_select == HPB_ERR_INJECTION_OFFSET)
		strlcpy(error_injection, "offset", 7);
	else if (hpb->err_injection_select == HPB_ERR_INJECTION_RANDOM)
		strlcpy(error_injection, "random", 7);

	ret = snprintf(buf, PAGE_SIZE, "err_injection_select %s\n",
		       error_injection);

	INFO_MSG("err_injection_select %s", error_injection);

	return ret;
}

static ssize_t ufsshpb_sysfs_err_injection_select_store(struct ufsshpb_lu *hpb,
						       const char *buf,
						       size_t cnt)
{
	char error_injection[8];
	int ret;

	error_injection[7] = '\0';

	ret = sscanf(buf, "%7s", error_injection);
	if (!ret) {
		INFO_MSG("input failed...");
		return cnt;
	}

	if (!strncmp(error_injection, "disable", 7))
		hpb->err_injection_select = HPB_ERR_INJECTION_DISABLE;
	else if (!strncmp(error_injection, "bitflip", 7))
		hpb->err_injection_select = HPB_ERR_INJECTION_BITFLIP;
	else if (!strncmp(error_injection, "offset", 6))
		hpb->err_injection_select = HPB_ERR_INJECTION_OFFSET;
	else if (!strncmp(error_injection, "random", 6))
		hpb->err_injection_select = HPB_ERR_INJECTION_RANDOM;
	else {
		hpb->err_injection_select = HPB_ERR_INJECTION_DISABLE;
		strlcpy(error_injection, "disable", 8);
	}

	INFO_MSG("err_injection_select %s", error_injection);

	return cnt;
}

static ssize_t ufsshpb_sysfs_err_injection_bitflip_show(struct ufsshpb_lu *hpb,
						       char *buf)
{
	int ret;

	ret = snprintf(buf, PAGE_SIZE, "err_injection_bitflip %llx\n",
		       hpb->err_injection_bitflip);

	return ret;
}

static ssize_t ufsshpb_sysfs_err_injection_bitflip_store(struct ufsshpb_lu *hpb,
							const char *buf,
							size_t cnt)
{
	unsigned long long err_injection_bitflip;

	if (kstrtoull(buf, 0, &err_injection_bitflip))
		return -EINVAL;

	hpb->err_injection_bitflip = err_injection_bitflip;

	INFO_MSG("err_injection_bitflip %llx", hpb->err_injection_bitflip);

	return cnt;
}

static ssize_t ufsshpb_sysfs_err_injection_offset_show(struct ufsshpb_lu *hpb,
						      char *buf)
{
	int ret;

	ret = snprintf(buf, PAGE_SIZE, "err_injection_offset %lld\n",
		       hpb->err_injection_offset);

	INFO_MSG("err_injection_offset %lld", hpb->err_injection_offset);

	return ret;
}

static ssize_t ufsshpb_sysfs_err_injection_offset_store(struct ufsshpb_lu *hpb,
						       const char *buf,
						       size_t cnt)
{
	unsigned long long err_injection_offset;

	if (kstrtoull(buf, 0, &err_injection_offset))
		return -EINVAL;

	hpb->err_injection_offset = err_injection_offset;

	INFO_MSG("err_injection_offset %lld", hpb->err_injection_offset);

	return cnt;
}

static ssize_t ufsshpb_sysfs_err_injection_random_show(struct ufsshpb_lu *hpb,
						      char *buf)
{
	int ret;

	ret = snprintf(buf, PAGE_SIZE, "err_injection_random %d\n",
		       hpb->err_injection_random);

	INFO_MSG("err_injection_random %d", hpb->err_injection_random);

	return ret;
}

static ssize_t ufsshpb_sysfs_err_injection_random_store(struct ufsshpb_lu *hpb,
						       const char *buf,
						       size_t cnt)
{
	unsigned long err_injection_random;

	if (kstrtoul(buf, 0, &err_injection_random))
		return -EINVAL;


	if (err_injection_random > 1)
		return -EINVAL;

	hpb->err_injection_random = err_injection_random;

	INFO_MSG("err_injection_random %d", hpb->err_injection_random);

	return cnt;
}
#endif

static ssize_t ufsshpb_sysfs_version_show(struct ufsshpb_lu *hpb, char *buf)
{
	int ret;

	ret = snprintf(buf, PAGE_SIZE, "HPB version %.4X D/D version %.6X%s\n",
		       hpb->ufsf->hpb_dev_info.version,
		       UFSSHPB_DD_VER, UFSSHPB_DD_VER_POST);

	INFO_MSG("HPB version %.4X D/D version %.6X%s",
		 hpb->ufsf->hpb_dev_info.version,
		 UFSSHPB_DD_VER, UFSSHPB_DD_VER_POST);

	return ret;
}

static ssize_t ufsshpb_sysfs_hit_show(struct ufsshpb_lu *hpb, char *buf)
{
	long long hit_cnt;
	int ret;

	hit_cnt = atomic64_read(&hpb->hit);

	ret = snprintf(buf, PAGE_SIZE, "hit_count %lld\n", hit_cnt);

	INFO_MSG("hit_count %lld", hit_cnt);

	return ret;
}

static ssize_t ufsshpb_sysfs_miss_show(struct ufsshpb_lu *hpb, char *buf)
{
	long long miss_cnt;
	int ret;

	miss_cnt = atomic64_read(&hpb->miss);

	ret = snprintf(buf, PAGE_SIZE, "miss_count %lld\n", miss_cnt);

	INFO_MSG("miss_count %lld", miss_cnt);

	return ret;
}

static ssize_t ufsshpb_sysfs_map_req_show(struct ufsshpb_lu *hpb, char *buf)
{
	long long rb_noti_cnt, rb_active_cnt, rb_inactive_cnt, map_req_cnt;
	long long map_compl_cnt, dev_inact_hcm_cnt;
	int ret;

	rb_noti_cnt = atomic64_read(&hpb->rb_noti_cnt);
	rb_active_cnt = atomic64_read(&hpb->rb_active_cnt);
	rb_inactive_cnt = atomic64_read(&hpb->rb_inactive_cnt);
	map_req_cnt = atomic64_read(&hpb->map_req_cnt);
	map_compl_cnt = atomic64_read(&hpb->map_compl_cnt);
	dev_inact_hcm_cnt = atomic64_read(&hpb->dev_inact_hcm_cnt);

	ret = snprintf(buf, PAGE_SIZE,
		       "rb_noti %lld ACT %lld INACT %lld (hcm: %lld ) map_req_count %lld map_compl_count %lld\n",
		       rb_noti_cnt, rb_active_cnt, rb_inactive_cnt, dev_inact_hcm_cnt,
		       map_req_cnt, map_compl_cnt);

	INFO_MSG("rb_noti %lld ACT %lld INACT %lld map_req_count %lld",
		 rb_noti_cnt, rb_active_cnt, rb_inactive_cnt,
		 map_req_cnt);

	return ret;
}

static ssize_t ufsshpb_sysfs_pre_req_show(struct ufsshpb_lu *hpb, char *buf)
{
	long long pre_req_cnt;
	int ret;

	pre_req_cnt = atomic64_read(&hpb->pre_req_cnt);

	ret = snprintf(buf, PAGE_SIZE, "pre_req_count %lld\n", pre_req_cnt);

	INFO_MSG("pre_req_count %lld", pre_req_cnt);

	return ret;
}

static ssize_t ufsshpb_sysfs_hcm_req_show(struct ufsshpb_lu *hpb, char *buf)
{
	long long set_hcm_cnt, unset_hcm_cnt;
	int ret;

	set_hcm_cnt = atomic64_read(&hpb->set_hcm_req_cnt);
	unset_hcm_cnt = atomic64_read(&hpb->unset_hcm_req_cnt);

#if defined(CONFIG_HPB_DEBUG_SYSFS)
	ret = snprintf(buf, PAGE_SIZE,
		       "set_hcm_count %lld %lld unset_hcm_count %lld %lld v %lld\n",
		       set_hcm_cnt, atomic64_read(&hpb->fs_set_hcm_cnt),
		       unset_hcm_cnt, atomic64_read(&hpb->fs_unset_hcm_cnt),
		       atomic64_read(&hpb->unset_hcm_victim));
#else
	ret = snprintf(buf, PAGE_SIZE,
		       "set_hcm_count %lld unset_hcm_count %lld\n",
		       set_hcm_cnt, unset_hcm_cnt);
#endif

	INFO_MSG("set_hcm_count %lld unset_hcm_count %lld",
		 set_hcm_cnt, unset_hcm_cnt);

	return ret;
}

static ssize_t ufsshpb_sysfs_region_stat_show(struct ufsshpb_lu *hpb, char *buf)
{
	int ret = 0;
	int pin_cnt = 0, hcm_cnt = 0, act_cnt = 0, inact_cnt = 0;
	int rgn_idx;
	enum HPB_RGN_STATE state;
	unsigned long flags;

	for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) {
		state = hpb->rgn_tbl[rgn_idx].rgn_state;
		if (state == HPB_RGN_PINNED)
			pin_cnt++;
		else if (state == HPB_RGN_HCM)
			hcm_cnt++;
		else if (state == HPB_RGN_ACTIVE)
			act_cnt++;
		else if (state == HPB_RGN_INACTIVE)
			inact_cnt++;
	}

	ret = snprintf(buf, PAGE_SIZE,
		       "Total %d pinned %d hcm %d active %d inactive %d\n",
		       hpb->rgns_per_lu, pin_cnt, hcm_cnt, act_cnt,
		       inact_cnt);

	INFO_MSG("Total %d pinned %d hcm %d active %d inactive %d",
		 hpb->rgns_per_lu, pin_cnt, hcm_cnt, act_cnt, inact_cnt);

	if (!hpb->debug)
		return ret;

	act_cnt = 0, hcm_cnt = 0;
	spin_lock_irqsave(&hpb->hpb_lock, flags);
	for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) {
		state = hpb->rgn_tbl[rgn_idx].rgn_state;
		if (state != HPB_RGN_ACTIVE)
			continue;

		act_cnt++;
		if (act_cnt == 1)
			ret += snprintf(buf+ret, PAGE_SIZE - ret, "ACT:\n");

		if ((act_cnt % 10) == 1)
			ret += snprintf(buf+ret, PAGE_SIZE - ret, "\t");

		ret += snprintf(buf+ret, PAGE_SIZE - ret, "%05d   ", rgn_idx);

		if ((act_cnt % 10) == 0)
			ret += snprintf(buf+ret, PAGE_SIZE - ret, "\n");

		if (ret > PAGE_SIZE) {
			ret = PAGE_SIZE - 10;
			ret += snprintf(buf+ret, PAGE_SIZE - ret, "... ");
			if (hcm_cnt % 10 == 0)
				ret += snprintf(buf+ret, PAGE_SIZE - ret, "\n");
			break;
		}
	}
	if (act_cnt % 10)
		ret += snprintf(buf+ret, PAGE_SIZE - ret, "\n");

	for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) {
		state = hpb->rgn_tbl[rgn_idx].rgn_state;
		if (state == HPB_RGN_HCM) {
			hcm_cnt++;
			if (hcm_cnt == 1)
				ret += snprintf(buf+ret, PAGE_SIZE - ret, "HCM:\n");

			if ((hcm_cnt % 10) == 1)
				ret += snprintf(buf+ret, PAGE_SIZE - ret, "\t");

			ret += snprintf(buf+ret, PAGE_SIZE - ret, "%05d   ", rgn_idx);

			if ((hcm_cnt % 10) == 0)
				ret += snprintf(buf+ret, PAGE_SIZE - ret, "\n");

			if (ret > PAGE_SIZE) {
				ret = PAGE_SIZE - 10;
				ret += snprintf(buf+ret, PAGE_SIZE - ret, "... ");
				if (hcm_cnt % 10 == 0)
					ret += snprintf(buf+ret, PAGE_SIZE - ret, "\n");
				break;
			}
		}
	}

	if (hcm_cnt % 10)
		ret += snprintf(buf+ret, PAGE_SIZE - ret, "\n");

	spin_unlock_irqrestore(&hpb->hpb_lock, flags);

	return ret;
}

static ssize_t ufsshpb_sysfs_count_reset_store(struct ufsshpb_lu *hpb,
					      const char *buf, size_t cnt)
{
	unsigned long debug;

	if (kstrtoul(buf, 0, &debug))
		return -EINVAL;

	if (debug != 1)
		return cnt;

	INFO_MSG("Stat Init");

	ufsshpb_stat_init(hpb);

	return cnt;
}

static ssize_t ufsshpb_sysfs_info_lba_store(struct ufsshpb_lu *hpb,
					   const char *buf, size_t cnt)
{
	struct ufsshpb_region *rgn;
	struct ufsshpb_subregion *srgn;
	u64 ppn = 0;
	unsigned long value, lpn, flags;
	int rgn_idx, srgn_idx, srgn_offset, error = 0;

	if (kstrtoul(buf, 0, &value)) {
		ERR_MSG("kstrtoul error");
		return -EINVAL;
	}

	if (value >= hpb->lu_num_blocks * SECTORS_PER_BLOCK) {
		ERR_MSG("value %lu >= lu_num_blocks %d error",
			value, hpb->lu_num_blocks * SECTORS_PER_BLOCK);
		return -EINVAL;
	}

	lpn = value / SECTORS_PER_BLOCK;

	ufsshpb_get_pos_from_lpn(hpb, lpn, &rgn_idx, &srgn_idx, &srgn_offset);

	rgn = hpb->rgn_tbl + rgn_idx;
	srgn = rgn->srgn_tbl + srgn_idx;

	spin_lock_irqsave(&hpb->hpb_lock, flags);
	INFO_MSG("lba %lu lpn %lu region %d state %d subregion %d state %d",
		 value, lpn, rgn_idx, rgn->rgn_state, srgn_idx,
		 srgn->srgn_state);

	if (!ufsshpb_valid_srgn(rgn, srgn)) {
		INFO_MSG("[region %d subregion %d] has not valid hpb info.",
			 rgn_idx, srgn_idx);
		goto out;
	}

	if (!srgn->mctx) {
		INFO_MSG("mctx is NULL");
		goto out;
	}

	ppn = ufsshpb_get_ppn(srgn->mctx, srgn_offset, &error);
	if (error) {
		INFO_MSG("getting ppn is fail from a page.");
		goto out;
	}

	INFO_MSG("ppn %llx is_dirty %d", ppn,
		 ufsshpb_ppn_dirty_check(hpb, lpn, 1));
out:
	spin_unlock_irqrestore(&hpb->hpb_lock, flags);
	return cnt;
}

static ssize_t ufsshpb_sysfs_info_region_store(struct ufsshpb_lu *hpb,
					      const char *buf, size_t cnt)
{
	unsigned long rgn_idx;
	int srgn_idx;

	if (kstrtoul(buf, 0, &rgn_idx))
		return -EINVAL;

	if (rgn_idx >= hpb->rgns_per_lu)
		ERR_MSG("error region %ld max %d", rgn_idx, hpb->rgns_per_lu);
	else {
		INFO_MSG("(region state : PINNED=%d HCM=%d ACTIVE=%d INACTIVE=%d)",
			 HPB_RGN_PINNED, HPB_RGN_HCM, HPB_RGN_ACTIVE,
			 HPB_RGN_INACTIVE);

		INFO_MSG("region %ld state %d", rgn_idx,
			 hpb->rgn_tbl[rgn_idx].rgn_state);

		for (srgn_idx = 0; srgn_idx < hpb->rgn_tbl[rgn_idx].srgn_cnt;
		     srgn_idx++) {
			INFO_MSG("--- subregion %d state %d", srgn_idx,
				 hpb->rgn_tbl[rgn_idx].srgn_tbl[srgn_idx].srgn_state);
		}
	}

	return cnt;
}

static ssize_t ufsshpb_sysfs_ufsshpb_release_store(struct ufsshpb_lu *hpb,
						 const char *buf, size_t cnt)
{
	unsigned long value;

	INFO_MSG("start release function");

	if (kstrtoul(buf, 0, &value)) {
		ERR_MSG("kstrtoul error");
		return -EINVAL;
	}

	if (value == 0xab) {
		INFO_MSG("magic number %lu release start", value);
		goto err_out;
	} else
		INFO_MSG("wrong magic number %lu", value);

	return cnt;
err_out:
	INFO_MSG("ref_cnt %d",
		 atomic_read(&hpb->ufsf->hpb_kref.refcount.refs));
	ufsshpb_failed(hpb, __func__);

	return cnt;
}

static struct ufsshpb_sysfs_entry ufsshpb_sysfs_entries[] = {
	__ATTR(hcm_disable, 0644,
	       ufsshpb_sysfs_hcm_disable_show, ufsshpb_sysfs_hcm_disable_store),
	__ATTR(hpb_read_disable, 0644,
	       ufsshpb_sysfs_prep_disable_show, ufsshpb_sysfs_prep_disable_store),
	__ATTR(map_cmd_disable, 0644,
	       ufsshpb_sysfs_map_disable_show, ufsshpb_sysfs_map_disable_store),
	__ATTR(throttle_map_req, 0644,
	       ufsshpb_sysfs_throttle_map_req_show,
	       ufsshpb_sysfs_throttle_map_req_store),
	__ATTR(throttle_pre_req, 0644,
	       ufsshpb_sysfs_throttle_pre_req_show,
	       ufsshpb_sysfs_throttle_pre_req_store),
	__ATTR(throttle_hcm_req, 0644,
	       ufsshpb_sysfs_throttle_hcm_req_show,
	       ufsshpb_sysfs_throttle_hcm_req_store),
	__ATTR(pre_req_min_tr_len, 0644,
	       ufsshpb_sysfs_pre_req_min_tr_len_show,
	       ufsshpb_sysfs_pre_req_min_tr_len_store),
	__ATTR(pre_req_max_tr_len, 0644,
	       ufsshpb_sysfs_pre_req_max_tr_len_show,
	       ufsshpb_sysfs_pre_req_max_tr_len_store),
	__ATTR(pre_req_timeout_ms, 0644,
	       ufsshpb_sysfs_pre_req_timeout_ms_show,
	       ufsshpb_sysfs_pre_req_timeout_ms_store),
#if defined(CONFIG_HPB_DEBUG)
	__ATTR(debug, 0644,
	       ufsshpb_sysfs_debug_show, ufsshpb_sysfs_debug_store),
#endif
#if defined(CONFIG_HPB_DEBUG_SYSFS)
	__ATTR(lba_info, 0644,
	       ufsshpb_sysfs_lba_info_show, ufsshpb_sysfs_lba_info_store),
	__ATTR(active_size, 0444, ufsshpb_sysfs_active_size_show, NULL),
	__ATTR(read_count, 0444, ufsshpb_sysfs_hpb_normal_read_count_show, NULL),
	__ATTR(rb_count, 0444, ufsshpb_sysfs_rb_count_show, NULL),
	__ATTR(rsp_list_count, 0444, ufsshpb_sysfs_rsp_list_cnt_show, NULL),
#endif
#if defined(CONFIG_HPB_ERR_INJECTION)
	__ATTR(err_injection_select, 0644,
	       ufsshpb_sysfs_err_injection_select_show,
	       ufsshpb_sysfs_err_injection_select_store),
	__ATTR(err_injection_bitflip, 0644,
	       ufsshpb_sysfs_err_injection_bitflip_show,
	       ufsshpb_sysfs_err_injection_bitflip_store),
	__ATTR(err_injection_offset, 0644,
	       ufsshpb_sysfs_err_injection_offset_show,
	       ufsshpb_sysfs_err_injection_offset_store),
	__ATTR(err_injection_random, 0644,
	       ufsshpb_sysfs_err_injection_random_show,
	       ufsshpb_sysfs_err_injection_random_store),
#endif
	__ATTR(hpb_version, 0444, ufsshpb_sysfs_version_show, NULL),
	__ATTR(hit_count, 0444, ufsshpb_sysfs_hit_show, NULL),
	__ATTR(miss_count, 0444, ufsshpb_sysfs_miss_show, NULL),
	__ATTR(map_req_count, 0444, ufsshpb_sysfs_map_req_show, NULL),
	__ATTR(pre_req_count, 0444, ufsshpb_sysfs_pre_req_show, NULL),
	__ATTR(hcm_req_count, 0444, ufsshpb_sysfs_hcm_req_show, NULL),
	__ATTR(region_stat_count, 0444, ufsshpb_sysfs_region_stat_show, NULL),
	__ATTR(count_reset, 0200, NULL, ufsshpb_sysfs_count_reset_store),
	__ATTR(get_info_from_lba, 0200, NULL, ufsshpb_sysfs_info_lba_store),
	__ATTR(get_info_from_region, 0200, NULL,
	       ufsshpb_sysfs_info_region_store),
	__ATTR(release, 0200, NULL, ufsshpb_sysfs_ufsshpb_release_store),
	__ATTR_NULL
};

static ssize_t ufsshpb_attr_show(struct kobject *kobj, struct attribute *attr,
				char *page)
{
	struct ufsshpb_sysfs_entry *entry;
	struct ufsshpb_lu *hpb;
	ssize_t error;

	entry = container_of(attr, struct ufsshpb_sysfs_entry, attr);
	hpb = container_of(kobj, struct ufsshpb_lu, kobj);

	if (!entry->show)
		return -EIO;

	mutex_lock(&hpb->sysfs_lock);
	error = entry->show(hpb, page);
	mutex_unlock(&hpb->sysfs_lock);
	return error;
}

static ssize_t ufsshpb_attr_store(struct kobject *kobj, struct attribute *attr,
				 const char *page, size_t len)
{
	struct ufsshpb_sysfs_entry *entry;
	struct ufsshpb_lu *hpb;
	ssize_t error;

	entry = container_of(attr, struct ufsshpb_sysfs_entry, attr);
	hpb = container_of(kobj, struct ufsshpb_lu, kobj);

	if (!entry->store)
		return -EIO;

	mutex_lock(&hpb->sysfs_lock);
	error = entry->store(hpb, page, len);
	mutex_unlock(&hpb->sysfs_lock);
	return error;
}

static const struct sysfs_ops ufsshpb_sysfs_ops = {
	.show = ufsshpb_attr_show,
	.store = ufsshpb_attr_store,
};

static struct kobj_type ufsshpb_ktype = {
	.sysfs_ops = &ufsshpb_sysfs_ops,
	.release = NULL,
};

static int ufsshpb_create_sysfs(struct ufsf_feature *ufsf, struct ufsshpb_lu *hpb)
{
	struct device *dev = ufsf->hba->dev;
	struct ufsshpb_sysfs_entry *entry;
	int err;

	hpb->sysfs_entries = ufsshpb_sysfs_entries;

	ufsshpb_stat_init(hpb);
#if defined(CONFIG_HPB_DEBUG_SYSFS)
	atomic64_set(&hpb->act_rsp_list_cnt, 0);
	atomic64_set(&hpb->inact_rsp_list_cnt, 0);
#endif

	kobject_init(&hpb->kobj, &ufsshpb_ktype);
	mutex_init(&hpb->sysfs_lock);

	INFO_MSG("ufsshpb creates sysfs lu %d %p dev->kobj %p", hpb->lun,
		 &hpb->kobj, &dev->kobj);

	err = kobject_add(&hpb->kobj, kobject_get(&dev->kobj),
			  "ufsshpb_lu%d", hpb->lun);
	if (!err) {
		for (entry = hpb->sysfs_entries; entry->attr.name != NULL;
		     entry++) {
			INFO_MSG("ufsshpb_lu%d sysfs attr creates: %s",
				 hpb->lun, entry->attr.name);
			if (sysfs_create_file(&hpb->kobj, &entry->attr))
				break;
		}
		INFO_MSG("ufsshpb_lu%d sysfs adds uevent", hpb->lun);
		kobject_uevent(&hpb->kobj, KOBJ_ADD);
	}

	return err;
}

static inline void ufsshpb_remove_sysfs(struct ufsshpb_lu *hpb)
{
	kobject_uevent(&hpb->kobj, KOBJ_REMOVE);
	INFO_MSG("ufsshpb removes sysfs lu %d %p ", hpb->lun, &hpb->kobj);
	kobject_del(&hpb->kobj);
}

MODULE_LICENSE("GPL v2");