/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
 * Copyright (c) 2024, Microsoft Corporation. All rights reserved.
 */

#ifndef _DOORBELLS_H_
#define _DOORBELLS_H_

#include <util/udma_barrier.h>
#include <util/mmio.h>
#include "mana.h"

#define GDMA_CQE_OWNER_BITS 3
#define CQ_OWNER_MASK ((1 << (GDMA_CQE_OWNER_BITS)) - 1)

#define DOORBELL_OFFSET_SQ		0x0
#define DOORBELL_OFFSET_RQ		0x400
#define DOORBELL_OFFSET_RQ_CLIENT	0x408
#define DOORBELL_OFFSET_CQ		0x800

#define CQ_ARM_BIT 1
#define CQ_UNARM_BIT 0

union gdma_doorbell_entry {
	uint64_t     as_uint64;
	struct {
		uint64_t id   : 24;
		uint64_t reserved    : 8;
		uint64_t prod_idx    : 31;
		uint64_t arm     : 1;
	} cq;
	struct {
		uint32_t id   : 24;
		uint32_t wqe_cnt     : 8;
		uint32_t prod_idx;
	} rx;
	struct {
		uint32_t id   : 24;
		uint32_t reserved    : 8;
		uint32_t prod_idx;
	} tx;
	struct {
		uint64_t id : 24;
		uint64_t high : 8;
		uint64_t low : 32;
	} rqe_client;
}; /* HW DATA */

static inline void gdma_ring_recv_doorbell(struct mana_gdma_queue *wq, uint8_t wqe_cnt)
{
	union gdma_doorbell_entry e;

	e.as_uint64 = 0;
	e.rx.id = wq->id;
	e.rx.prod_idx = wq->prod_idx * GDMA_WQE_ALIGNMENT_UNIT_SIZE;
	e.rx.wqe_cnt = wqe_cnt;

	udma_to_device_barrier();
	mmio_write64(wq->db_page + DOORBELL_OFFSET_RQ, e.as_uint64);
	mmio_flush_writes();
}

static inline void gdma_ring_send_doorbell(struct mana_gdma_queue *wq)
{
	union gdma_doorbell_entry e;

	e.as_uint64 = 0;
	e.tx.id = wq->id;
	e.tx.prod_idx = wq->prod_idx * GDMA_WQE_ALIGNMENT_UNIT_SIZE;

	udma_to_device_barrier();
	mmio_write64(wq->db_page + DOORBELL_OFFSET_SQ, e.as_uint64);
	mmio_flush_writes();
}

static inline void gdma_arm_normal_cqe(struct mana_gdma_queue *wq, uint32_t psn)
{
	union gdma_doorbell_entry e;

	e.as_uint64 = 0;
	e.rqe_client.id = wq->id;
	e.rqe_client.high = 1;
	e.rqe_client.low = psn;

	udma_to_device_barrier();
	mmio_write64(wq->db_page + DOORBELL_OFFSET_RQ_CLIENT, e.as_uint64);
	mmio_flush_writes();
}

static inline void gdma_ring_cq_doorbell(struct mana_cq *cq, uint8_t arm)
{
	union gdma_doorbell_entry e;
	uint32_t prod_idx = cq->head;
	uint32_t max_credit = cq->cqe << (GDMA_CQE_OWNER_BITS - 1);

	if (cq->poll_credit >= max_credit) {
		// To address the use-case of ibv that re-arms the CQ without polling
		cq->poll_credit++;
	} else {
		// Set index of already polled CQE for unarm
		cq->poll_credit = max_credit - (arm ? 0 : 1);
	}
	// negative values are handled via mod 2^n
	prod_idx += (cq->poll_credit - max_credit);

	e.as_uint64 = 0;
	e.cq.id = cq->cqid;
	e.cq.prod_idx = prod_idx % (cq->cqe << GDMA_CQE_OWNER_BITS);
	e.cq.arm = arm;

	udma_to_device_barrier();
	mmio_write64(cq->db_page + DOORBELL_OFFSET_CQ, e.as_uint64);
	mmio_flush_writes();
}

#endif //_DOORBELLS_H_
