/*
 * offload engine driver for the NXP PNX833X series of STB platforms
 * Copyright © 2008, NXP Semiconductors.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 */

/*
 * This driver supports the asynchrounous DMA copy engine available
 * on the NXP PNX833x platform
 */

#include <linux/init.h>
#include <linux/module.h>
#include <linux/async_tx.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/platform_device.h>
#include <linux/memory.h>
#include <linux/ioport.h>
#include <linux/sched.h>
#include <adma.h>

/* #define NXP_ADMA_SELF_TEST */
/* #define NXP_ADMA_DEBUG */
#define NXP_ADMA_POLL

#define NXP_ADMA_NUM_DESCS 128
#define NXP_ADMA_INPUT_QUEUE_SIZE (NXP_ADMA_NUM_DESCS)

#define NXP_ADMA_NEXT_INDEX(a, b) (((a) == ((b) - 1)) ? 0 : ((a) + 1))
#define NXP_ADMA_PREV_INDEX(a, b) (((a) == 0) ? ((b) - 1) : ((a) - 1))

#define tx_to_slot(chan, tx) ((struct nxp_adma_desc_slot*)(tx)->phys)
#define slot_to_index(chan, x) ((x) - (chan).slot)
#define slot_is_cpi(x) (((x) == nxp_adma_dev.cpi_nda_a) || ((x) == nxp_adma_dev.cpi_nda_b))

#define NXP_ADMA_BIG_ENDIAN32(data) ((((data) & 0xff) << 24) | (((data) & 0xff00) << 8) | (((data) >> 8) & 0xff00) | (((data) >> 24) & 0xff))
#define NXP_ADMA_BIG_ENDIAN16(data) ((((data) & 0xff) << 8) | (((data) >> 8) & 0xff))

#define NXP_ADMA_KEY_WORD(key, x) ((((u32)(key[x])) << 24) | (((u32)(key[(x) + 1])) << 16) | (((u32)(key[(x) + 2])) << 8) | (((u32)(key[(x) + 3])) << 0)) 

#define NXP_ADMA_SET_WR_ITEM_SIZE(ctrl, wr_item_size)       ((ctrl) = (u8)(((ctrl) & 0x3f) | (((wr_item_size) & 3) << 6)))
#define NXP_ADMA_SET_LAST_DESCRIPTOR(ctrl, last_descriptor) ((ctrl) = (u8)(((ctrl) & 0xdf) | (last_descriptor ? (1 << 5) : 0)))
#define NXP_ADMA_SET_INTERRUPT(ctrl, interrupt)             ((ctrl) = (u8)(((ctrl) & 0xef) | (interrupt ? (1 << 4) : 0)))
#define NXP_ADMA_SET_DIRECTION(ctrl, direction)             ((ctrl) = (u8)(((ctrl) & 0xf3) | (((direction) & 3) << 2)))
#define NXP_ADMA_SET_ITEM_SIZE(ctrl, item_size)             ((ctrl) = (u8)(((ctrl) & 0xfc) | (((item_size) & 3) << 0)))

#ifdef NXP_ADMA_DEBUG
#define NXP_ADMA_REGISTER_READ(base, val)	do{unsigned long tmp = *(volatile unsigned long*)(base); \
						printk("read %p = %08x\n", (volatile unsigned long*)(base), tmp); \
						(val) = tmp; \
						} while(0)
#define NXP_ADMA_REGISTER_WRITE(base, val)	do{printk("write %p = %08x\n",(volatile unsigned long*)(base), (val)); \
						*(volatile unsigned long*)(base) = (val); \
						} while(0)
#else
#define NXP_ADMA_REGISTER_READ(base, val)	do{(val) = *(volatile unsigned long*)(base);}while(0)
#define NXP_ADMA_REGISTER_WRITE(base, val)	do{*(volatile unsigned long*)(base) = (val);}while(0)
#endif

#define NXP_ADMA_CMD_READ(base, val)           NXP_ADMA_REGISTER_READ(base + 0x300, val)
#define NXP_ADMA_NDA_A_READ(base, val)         NXP_ADMA_REGISTER_READ(base + 0x304, val)
#define NXP_ADMA_NDA_B_READ(base, val)         NXP_ADMA_REGISTER_READ(base + 0x308, val)
#define NXP_ADMA_STATUS_READ(base, val)        NXP_ADMA_REGISTER_READ(base + 0x30C, val)
#define NXP_ADMA_AES_CONFIG_READ(base, val)    NXP_ADMA_REGISTER_READ(base + 0x430, val)
#define NXP_ADMA_AES_CMD_READ(base, val)       NXP_ADMA_REGISTER_READ(base + 0x434, val)
#define NXP_ADMA_AES_16B_DATA_READ(base, val)  NXP_ADMA_REGISTER_READ(base + 0x438, val)
#define NXP_ADMA_AES_STATUS_READ(base, val)    NXP_ADMA_REGISTER_READ(base + 0x43C, val)
#define NXP_ADMA_AES_DATA_IN0_READ(base, val)  NXP_ADMA_REGISTER_READ(base + 0x440, val)
#define NXP_ADMA_AES_DATA_IN1_READ(base, val)  NXP_ADMA_REGISTER_READ(base + 0x444, val)
#define NXP_ADMA_AES_DATA_IN2_READ(base, val)  NXP_ADMA_REGISTER_READ(base + 0x448, val)
#define NXP_ADMA_AES_DATA_IN3_READ(base, val)  NXP_ADMA_REGISTER_READ(base + 0x44C, val)
#define NXP_ADMA_AES_PKT_SIZE_READ(base, val)  NXP_ADMA_REGISTER_READ(base + 0x468, val)
#define NXP_ADMA_INT_STATUS_READ(base, val)    NXP_ADMA_REGISTER_READ(base + 0xFE0, val)
#define NXP_ADMA_INT_ENABLE_READ(base, val)    NXP_ADMA_REGISTER_READ(base + 0xFE4, val)
#define NXP_ADMA_INT_CLEAR_READ(base, val)     NXP_ADMA_REGISTER_READ(base + 0xFE8, val)
#define NXP_ADMA_POWER_READ(base, val)         NXP_ADMA_REGISTER_READ(base + 0xFF4, val)
#define NXP_ADMA_MODULE_ID_READ(base, val)     NXP_ADMA_REGISTER_READ(base + 0xFFC, val)

#define NXP_ADMA_CMD_WRITE(base, val)          NXP_ADMA_REGISTER_WRITE(base + 0x300, val)
#define NXP_ADMA_NDA_A_WRITE(base, val)        NXP_ADMA_REGISTER_WRITE(base + 0x304, val)
#define NXP_ADMA_NDA_B_WRITE(base, val)        NXP_ADMA_REGISTER_WRITE(base + 0x308, val)
#define NXP_ADMA_STATUS_WRITE(base, val)       NXP_ADMA_REGISTER_WRITE(base + 0x30C, val)
#define NXP_ADMA_AES_CONFIG_WRITE(base, val)   NXP_ADMA_REGISTER_WRITE(base + 0x430, val)
#define NXP_ADMA_AES_CMD_WRITE(base, val)      NXP_ADMA_REGISTER_WRITE(base + 0x434, val)
#define NXP_ADMA_AES_16B_DATA_WRITE(base, val) NXP_ADMA_REGISTER_WRITE(base + 0x438, val)
#define NXP_ADMA_AES_STATUS_WRITE(base, val)   NXP_ADMA_REGISTER_WRITE(base + 0x43C, val)
#define NXP_ADMA_AES_DATA_IN0_WRITE(base, val) NXP_ADMA_REGISTER_WRITE(base + 0x440, val)
#define NXP_ADMA_AES_DATA_IN1_WRITE(base, val) NXP_ADMA_REGISTER_WRITE(base + 0x444, val)
#define NXP_ADMA_AES_DATA_IN2_WRITE(base, val) NXP_ADMA_REGISTER_WRITE(base + 0x448, val)
#define NXP_ADMA_AES_DATA_IN3_WRITE(base, val) NXP_ADMA_REGISTER_WRITE(base + 0x44C, val)
#define NXP_ADMA_AES_PKT_SIZE_WRITE(base, val) NXP_ADMA_REGISTER_WRITE(base + 0x468, val)
#define NXP_ADMA_INT_STATUS_WRITE(base, val)   NXP_ADMA_REGISTER_WRITE(base + 0xFE0, val)
#define NXP_ADMA_INT_ENABLE_WRITE(base, val)   NXP_ADMA_REGISTER_WRITE(base + 0xFE4, val)
#define NXP_ADMA_INT_CLEAR_WRITE(base, val)    NXP_ADMA_REGISTER_WRITE(base + 0xFE8, val)
#define NXP_ADMA_POWER_WRITE(base, val)        NXP_ADMA_REGISTER_WRITE(base + 0xFF4, val)
#define NXP_ADMA_MODULE_ID_WRITE(base, val)    NXP_ADMA_REGISTER_WRITE(base + 0xFFC, val)

#define NXP_ADMA_ISSUE_AES_CMD(x, y) do{unsigned long tmpStat; NXP_ADMA_AES_CMD_WRITE((x),(y)); do {NXP_ADMA_AES_CMD_READ((x), tmpStat);} while(tmpStat);}while(0)

#define NXP_ADMA_INT_NDA_B_FINISHED 	64
#define NXP_ADMA_INT_NDA_A_FINISHED 	32
#define NXP_ADMA_INT_NDA_B_PROG			16
#define NXP_ADMA_INT_NDA_A_PROG			8
#define NXP_ADMA_INT_NDA_B_STOP			4
#define NXP_ADMA_INT_NDA_A_STOP			2

#define NXP_ADMA_CMD_NDA_B_VALIDATE 	32
#define NXP_ADMA_CMD_NDA_A_VALIDATE 	16
#define NXP_ADMA_CMD_DISABLE			4
#define NXP_ADMA_CMD_ENABLE				2
#define NXP_ADMA_CMD_RESET				1

struct nxp_adma_desc_slot {
	u32 source;            /* Transfer Source Address                    */
	u32 destination;       /* Transfer Destination Address               */
	u16 destination_count; /* Destination Transfer size in byte minus 1  */
	u16 source_count;      /* Source Transfer size in byte minus 1       */
	u16 unused1;           
	u8  cmd_byte;          /* Used to select SGDESC module to go through */
	u8  ctrl;              /* Used for control bits:                     */

/*    bits 7:6    wr_item_size        Width in byte for write command
 *    bit 5       last_descriptor     Flag last descriptor entry              
 *    bit 4       interrupt           Raise interrupt when transfer completed
 *    bits 3:2    direction           Transfer direction
 *    bits 1:0    item_size           Width in byte for read command
 */
};

struct nxp_adma_device {
	u8                                  desc_used_head;
	u8                                  desc_submit_head;
	u8                                  desc_sequence_head;
	u8                                  desc_tail;
	u8                                  dma_input_head;
	u8                                  dma_submit_tail;
	volatile u8                         dma_complete_tail;
	volatile u8                         dma_cpi_head;
	unsigned long                       base;
	dma_cookie_t                        completed_cookie;
	spinlock_t                          lock;
	struct nxp_adma_desc_slot          *nda_a;
	struct nxp_adma_desc_slot          *nda_b;
	struct nxp_adma_desc_slot          *cpi_nda_a;
	struct nxp_adma_desc_slot          *cpi_nda_b;
	bool                                init;
	struct nxp_adma_desc_slot           slot[NXP_ADMA_NUM_DESCS];
	struct dma_async_tx_descriptor      tx[NXP_ADMA_NUM_DESCS];
	struct nxp_adma_desc_slot          *dma_input[NXP_ADMA_INPUT_QUEUE_SIZE];
	u8                                  end_of_sequence[NXP_ADMA_INPUT_QUEUE_SIZE];
	struct platform_device             *pdev;
	unsigned int                        irq;
	u32                                 irq_mask;
	void                              (*irq_handler)(u32);
	struct dma_device                   dma_dev;
	struct dma_chan                     dma_chan;
	bool                                pending;
};

static struct nxp_adma_device nxp_adma_dev;


static void nxp_adma_feed_hw(struct dma_chan *dchan)
{
	struct nxp_adma_desc_slot *slot = NULL;
	dma_addr_t addr;

	if (nxp_adma_dev.nda_a == NULL)
	{
		if (nxp_adma_dev.cpi_nda_a)
		{
			slot = nxp_adma_dev.cpi_nda_a;
			addr = (dma_addr_t)slot;
			nxp_adma_dev.irq_mask |= NXP_ADMA_INT_NDA_A_FINISHED;
		}
		else if (nxp_adma_dev.dma_input_head != nxp_adma_dev.dma_submit_tail)
		{
			slot = nxp_adma_dev.dma_input[nxp_adma_dev.dma_submit_tail];
			addr = dma_map_single(dchan->device->dev, slot, NXP_ADMA_NUM_DESCS * sizeof(struct nxp_adma_desc_slot), DMA_TO_DEVICE);
			nxp_adma_dev.dma_submit_tail = NXP_ADMA_NEXT_INDEX(nxp_adma_dev.dma_submit_tail, NXP_ADMA_INPUT_QUEUE_SIZE);
		}

		if (slot)
		{
			nxp_adma_dev.nda_a = slot;
			NXP_ADMA_NDA_A_WRITE(nxp_adma_dev.base, addr);
			NXP_ADMA_CMD_WRITE(nxp_adma_dev.base,  NXP_ADMA_CMD_NDA_A_VALIDATE | NXP_ADMA_CMD_ENABLE);
			slot = NULL;
		}
	}

	if (nxp_adma_dev.nda_b == NULL)
	{
		if (nxp_adma_dev.cpi_nda_b)
		{
			slot = nxp_adma_dev.cpi_nda_b;
			addr = (dma_addr_t)slot;
			nxp_adma_dev.irq_mask |= NXP_ADMA_INT_NDA_B_FINISHED;
		}
		else if (nxp_adma_dev.dma_input_head != nxp_adma_dev.dma_submit_tail)
		{
			slot = nxp_adma_dev.dma_input[nxp_adma_dev.dma_submit_tail];
			addr = dma_map_single(dchan->device->dev, slot, NXP_ADMA_NUM_DESCS * sizeof(struct nxp_adma_desc_slot), DMA_TO_DEVICE);
			nxp_adma_dev.dma_submit_tail = NXP_ADMA_NEXT_INDEX(nxp_adma_dev.dma_submit_tail, NXP_ADMA_INPUT_QUEUE_SIZE);
		}

		if (slot)
		{
			nxp_adma_dev.nda_b = slot;
			NXP_ADMA_NDA_B_WRITE(nxp_adma_dev.base, addr);
			NXP_ADMA_CMD_WRITE(nxp_adma_dev.base, NXP_ADMA_CMD_NDA_B_VALIDATE | NXP_ADMA_CMD_ENABLE);
		}
	}
}

static void nxp_adma_check_transaction_complete(void)
{
	bool cpi_required = false;
	u32 mask = 0;
	unsigned long status;

	NXP_ADMA_INT_STATUS_READ(nxp_adma_dev.base, status);

	if (status & NXP_ADMA_INT_NDA_A_FINISHED)
	{
		if (nxp_adma_dev.cpi_nda_a == nxp_adma_dev.nda_a)
		{
			nxp_adma_dev.cpi_nda_a = NULL;
			nxp_adma_dev.irq_mask &= ~NXP_ADMA_INT_NDA_A_FINISHED;
			cpi_required = true;
		}
		else
		{
			nxp_adma_dev.desc_tail = NXP_ADMA_NEXT_INDEX(nxp_adma_dev.end_of_sequence[nxp_adma_dev.dma_complete_tail], NXP_ADMA_NUM_DESCS);
			nxp_adma_dev.completed_cookie = nxp_adma_dev.tx[nxp_adma_dev.end_of_sequence[nxp_adma_dev.dma_complete_tail]].cookie;
			nxp_adma_dev.dma_complete_tail = NXP_ADMA_NEXT_INDEX(nxp_adma_dev.dma_complete_tail, NXP_ADMA_INPUT_QUEUE_SIZE);
			mask |= NXP_ADMA_INT_NDA_A_FINISHED | NXP_ADMA_INT_NDA_A_PROG | NXP_ADMA_INT_NDA_A_STOP;
		}
		nxp_adma_dev.nda_a = NULL;
	}

	if (status & NXP_ADMA_INT_NDA_B_FINISHED)
	{
		if (nxp_adma_dev.cpi_nda_b == nxp_adma_dev.nda_b)
		{
			nxp_adma_dev.cpi_nda_b = NULL;
			nxp_adma_dev.irq_mask &= ~NXP_ADMA_INT_NDA_B_FINISHED;
			cpi_required = true;
		}
		else
		{
			nxp_adma_dev.desc_tail = NXP_ADMA_NEXT_INDEX(nxp_adma_dev.end_of_sequence[nxp_adma_dev.dma_complete_tail], NXP_ADMA_NUM_DESCS);
			nxp_adma_dev.completed_cookie = nxp_adma_dev.tx[nxp_adma_dev.end_of_sequence[nxp_adma_dev.dma_complete_tail]].cookie;
			nxp_adma_dev.dma_complete_tail = NXP_ADMA_NEXT_INDEX(nxp_adma_dev.dma_complete_tail, NXP_ADMA_INPUT_QUEUE_SIZE);
			mask |= NXP_ADMA_INT_NDA_B_FINISHED | NXP_ADMA_INT_NDA_B_PROG | NXP_ADMA_INT_NDA_B_STOP;
		}
		nxp_adma_dev.nda_b = NULL;
	}

	NXP_ADMA_INT_CLEAR_WRITE(nxp_adma_dev.base, status);

	if (cpi_required)
	{
		if (nxp_adma_dev.irq_handler)
		{
			nxp_adma_dev.irq_handler(status & ~mask);
		}
	}

	nxp_adma_feed_hw(&nxp_adma_dev.dma_chan);
}

static enum dma_status nxp_adma_is_complete(struct dma_chan *dchan,
						dma_cookie_t cookie,
						dma_cookie_t *done,
						dma_cookie_t *used)
{
#ifdef NXP_ADMA_POLL
	spin_lock_bh(&nxp_adma_dev.lock);
	NXP_ADMA_INT_ENABLE_WRITE(nxp_adma_dev.base, 0);
	if (nxp_adma_dev.irq_mask == 0)
	{
		nxp_adma_check_transaction_complete();
	}
	NXP_ADMA_INT_ENABLE_WRITE(nxp_adma_dev.base, nxp_adma_dev.irq_mask);
	spin_unlock_bh(&nxp_adma_dev.lock);
#endif

	if (done)
	{
		*done = nxp_adma_dev.completed_cookie;
	}
	if (used)
	{
		*used = dchan->cookie;
	}
	return dma_async_is_complete(cookie, nxp_adma_dev.completed_cookie, dchan->cookie);
}

static irqreturn_t nxp_adma_interrupt(int irq, void *data)
{
	nxp_adma_check_transaction_complete();
	NXP_ADMA_INT_ENABLE_WRITE(nxp_adma_dev.base, nxp_adma_dev.irq_mask);
	return IRQ_HANDLED;
}

static void nxp_adma_issue_pending(struct dma_chan *dchan)
{
	struct nxp_adma_desc_slot *slot;
	int next_dma_input_head;
	int final_index;
	int submit_head = nxp_adma_dev.desc_submit_head;

	if (submit_head != nxp_adma_dev.desc_used_head)
	{
		nxp_adma_dev.pending = true;
	}
	else
	{
		nxp_adma_dev.pending = false;
		if (submit_head != nxp_adma_dev.desc_sequence_head)
		{
			next_dma_input_head = NXP_ADMA_NEXT_INDEX(nxp_adma_dev.dma_input_head, NXP_ADMA_INPUT_QUEUE_SIZE);

			if (next_dma_input_head != nxp_adma_dev.dma_complete_tail)
			{
				final_index = NXP_ADMA_PREV_INDEX(submit_head, NXP_ADMA_NUM_DESCS);
				slot = &nxp_adma_dev.slot[final_index];

				NXP_ADMA_SET_LAST_DESCRIPTOR(slot->ctrl, true);
				nxp_adma_dev.dma_input[nxp_adma_dev.dma_input_head] = &nxp_adma_dev.slot[nxp_adma_dev.desc_sequence_head];
				nxp_adma_dev.end_of_sequence[nxp_adma_dev.dma_input_head] = final_index;
				nxp_adma_dev.desc_sequence_head = submit_head;
				nxp_adma_dev.dma_input_head = next_dma_input_head;
			}
		}
	}
}

static void nxp_adma_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
{
	struct nxp_adma_desc_slot *slot = tx_to_slot(nxp_adma_dev, tx);

	slot->destination = NXP_ADMA_BIG_ENDIAN32(addr);
}

static inline dma_cookie_t nxp_desc_assign_cookie(void)
{
	dma_cookie_t cookie = nxp_adma_dev.dma_chan.cookie;

	cookie++;
	if (cookie < 0)
	{
		cookie = 1;
	}
	nxp_adma_dev.dma_chan.cookie = cookie;
	return cookie;
}

static dma_cookie_t nxp_adma_tx_submit(struct dma_async_tx_descriptor *tx)
{
	int next_desc_submit_head = NXP_ADMA_NEXT_INDEX(nxp_adma_dev.desc_submit_head, NXP_ADMA_NUM_DESCS);

#ifdef NXP_ADMA_DEBUG
	{
		struct nxp_adma_desc_slot *slot = tx_to_slot(nxp_adma_dev, tx);
		printk("submit ADMA descriptor 0x%p\n", slot);
		printk("source 0x%08lx dest 0x%08lx\n", slot->source, slot->destination);
		printk("dest_count %d source_count %d\n", slot->destination_count, slot->source_count);
		printk("cmd_byte = 0x%02x ctrl = 0x%02x\n", slot->cmd_byte, slot->ctrl);
	}
#endif
	nxp_adma_dev.desc_submit_head = next_desc_submit_head;

	if ((next_desc_submit_head == 0) || (nxp_adma_dev.pending == true))
	{
		nxp_adma_issue_pending(&nxp_adma_dev.dma_chan);
#ifdef NXP_ADMA_POLL
		spin_lock_bh(&nxp_adma_dev.lock);
		NXP_ADMA_INT_ENABLE_WRITE(nxp_adma_dev.base, 0);
		if (nxp_adma_dev.irq_mask == 0)
		{
			nxp_adma_check_transaction_complete();
		}
		NXP_ADMA_INT_ENABLE_WRITE(nxp_adma_dev.base, nxp_adma_dev.irq_mask);
		spin_unlock_bh(&nxp_adma_dev.lock);
#endif
	}

	return tx->cookie;
}

static void nxp_adma_memcpy_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
{
	struct nxp_adma_desc_slot *slot = tx_to_slot(nxp_adma_dev, tx);

	slot->source = NXP_ADMA_BIG_ENDIAN32(addr);
}

static struct dma_async_tx_descriptor *nxp_adma_prep_dma_memcpy(struct dma_chan *dchan, size_t len, int int_en)
{
	struct dma_async_tx_descriptor *ret;
	int new_desc_used_head;

	if (unlikely(!len))
	{
		return NULL;
	}

	new_desc_used_head = NXP_ADMA_NEXT_INDEX(nxp_adma_dev.desc_used_head, NXP_ADMA_NUM_DESCS);

	if (new_desc_used_head != nxp_adma_dev.desc_tail)
	{
		struct nxp_adma_desc_slot *slot = &nxp_adma_dev.slot[nxp_adma_dev.desc_used_head];
		u16 hw_len = NXP_ADMA_BIG_ENDIAN16(len - 1);

		slot->source_count = hw_len;
		slot->destination_count = hw_len;
		slot->cmd_byte = int_en;
		NXP_ADMA_SET_DIRECTION(slot->ctrl, (int_en) ? 0x03 : 0);
		ret = &nxp_adma_dev.tx[nxp_adma_dev.desc_used_head];
		nxp_adma_dev.desc_used_head = new_desc_used_head;
		ret->tx_set_src = nxp_adma_memcpy_set_src;
		ret->cookie = nxp_desc_assign_cookie();
	}
	else
	{
		nxp_adma_issue_pending(&nxp_adma_dev.dma_chan);
#ifdef NXP_ADMA_POLL
		spin_lock_bh(&nxp_adma_dev.lock);
		NXP_ADMA_INT_ENABLE_WRITE(nxp_adma_dev.base, 0);
		if (nxp_adma_dev.irq_mask == 0)
		{
			nxp_adma_check_transaction_complete();
		}
		NXP_ADMA_INT_ENABLE_WRITE(nxp_adma_dev.base, nxp_adma_dev.irq_mask);
		spin_unlock_bh(&nxp_adma_dev.lock);
#endif
		ret = NULL;
	}

	return ret;
}

static struct dma_async_tx_descriptor *nxp_adma_prep_dma_interrupt(struct dma_chan *dchan)
{
	return NULL;
}

static struct dma_async_tx_descriptor *nxp_adma_prep_dma_memset(struct dma_chan *dchan, int value, size_t len, int int_en)
{
	return NULL;
}

static struct dma_async_tx_descriptor *nxp_adma_prep_dma_xor(struct dma_chan *dchan, unsigned int src_cnt, size_t len, int int_en)
{
	return NULL;
}

static struct dma_async_tx_descriptor *nxp_adma_prep_dma_zero_sum(struct dma_chan *dchan, unsigned int src_cnt, size_t len, u32 *result, int int_en)
{
	return NULL;
}

static void nxp_adma_dependency_added(struct dma_chan *dchan)
{
}

static void nxp_adma_free_chan_resources(struct dma_chan *dchan)
{
}

dma_cookie_t dma_async_memcpy_buf_to_buf_crypt(struct dma_chan *chan, void *dest,
			void *src, size_t len)
{
	struct dma_device *dev = chan->device;
	struct dma_async_tx_descriptor *tx;
	dma_addr_t addr;
	dma_cookie_t cookie;
	int cpu;

	tx = dev->device_prep_dma_memcpy(chan, len, 10);
	if (!tx)
		return -ENOMEM;

	tx->ack = 1;
	tx->callback = NULL;
	addr = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE);
	tx->tx_set_src(addr, tx, 0);
	addr = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE);
	tx->tx_set_dest(addr, tx, 0);
	cookie = tx->tx_submit(tx);

	cpu = get_cpu();
	per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
	per_cpu_ptr(chan->local, cpu)->memcpy_count++;
	put_cpu();

	return cookie;
}
EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf_crypt);

void nxp_adma_aes_setup(bool encrypt, enum nxp_adma_aes_mode mode, u8 *key, u8 *iv, size_t len)
{
	u32 aesConfig = 0;

	NXP_ADMA_ISSUE_AES_CMD(nxp_adma_dev.base, 4); /* reset */

	if ((encrypt) | (mode == nxp_aes_mode_ctr))
	{
		aesConfig |= 1 << 2;        
	}

	if (mode == nxp_aes_mode_cbc)
	{
		aesConfig |= 1 << 4;        
	}

	if (key)
	{
		/* load software key */
		NXP_ADMA_AES_DATA_IN0_WRITE(nxp_adma_dev.base, NXP_ADMA_KEY_WORD(key,  0));
		NXP_ADMA_AES_DATA_IN1_WRITE(nxp_adma_dev.base, NXP_ADMA_KEY_WORD(key,  4));
		NXP_ADMA_AES_DATA_IN2_WRITE(nxp_adma_dev.base, NXP_ADMA_KEY_WORD(key,  8));
		NXP_ADMA_AES_DATA_IN3_WRITE(nxp_adma_dev.base, NXP_ADMA_KEY_WORD(key, 12));
	}
	else
	{
		/* use secret key */
		aesConfig |= 1 << 9;
	}
	NXP_ADMA_AES_CONFIG_WRITE(nxp_adma_dev.base,  aesConfig | (1 << 5));
	NXP_ADMA_ISSUE_AES_CMD(nxp_adma_dev.base, 1);

	if (mode == nxp_aes_mode_ctr)
	{
		/* load counter */
		aesConfig |= 1 << 8;
		NXP_ADMA_AES_DATA_IN0_WRITE(nxp_adma_dev.base, NXP_ADMA_KEY_WORD(iv,  0));
		NXP_ADMA_AES_DATA_IN1_WRITE(nxp_adma_dev.base, NXP_ADMA_KEY_WORD(iv,  4));
		NXP_ADMA_AES_DATA_IN2_WRITE(nxp_adma_dev.base, NXP_ADMA_KEY_WORD(iv,  8));
		NXP_ADMA_AES_DATA_IN3_WRITE(nxp_adma_dev.base, NXP_ADMA_KEY_WORD(iv, 12));
		NXP_ADMA_AES_CONFIG_WRITE(nxp_adma_dev.base, aesConfig | (1 << 5));
		NXP_ADMA_ISSUE_AES_CMD(nxp_adma_dev.base, 1);
		NXP_ADMA_AES_16B_DATA_WRITE(nxp_adma_dev.base, 0);
		NXP_ADMA_AES_PKT_SIZE_WRITE(nxp_adma_dev.base, len);
	}
	else
	{
		if (mode == nxp_aes_mode_cbc)
		{
			/* load iv */
			NXP_ADMA_AES_DATA_IN0_WRITE(nxp_adma_dev.base, NXP_ADMA_KEY_WORD(iv,  0));
			NXP_ADMA_AES_DATA_IN1_WRITE(nxp_adma_dev.base, NXP_ADMA_KEY_WORD(iv,  4));
			NXP_ADMA_AES_DATA_IN2_WRITE(nxp_adma_dev.base, NXP_ADMA_KEY_WORD(iv,  8));
			NXP_ADMA_AES_DATA_IN3_WRITE(nxp_adma_dev.base, NXP_ADMA_KEY_WORD(iv, 12));
			NXP_ADMA_AES_CONFIG_WRITE(nxp_adma_dev.base, aesConfig | (3 << 5));
			NXP_ADMA_ISSUE_AES_CMD(nxp_adma_dev.base, 1);
		}
		NXP_ADMA_AES_16B_DATA_WRITE(nxp_adma_dev.base, (len / 2) - 1);
	}

	NXP_ADMA_AES_CONFIG_WRITE(nxp_adma_dev.base, aesConfig | (1 << 7));
	NXP_ADMA_ISSUE_AES_CMD(nxp_adma_dev.base, 1);
	NXP_ADMA_AES_CMD_WRITE(nxp_adma_dev.base, 0x02); /* ready for block processing */
}
EXPORT_SYMBOL(nxp_adma_aes_setup);

void nxp_adma_cpi_submit(u32 descriptor_table, bool chan)
{
	if (chan)
	{
		nxp_adma_dev.cpi_nda_a = (struct nxp_adma_desc_slot*)descriptor_table;
	}
	else
	{
		nxp_adma_dev.cpi_nda_b = (struct nxp_adma_desc_slot*)descriptor_table;
	}

	spin_lock_bh(&nxp_adma_dev.lock);
	NXP_ADMA_INT_ENABLE_WRITE(nxp_adma_dev.base, 0);
	if (nxp_adma_dev.irq_mask == 0)
	{
		nxp_adma_check_transaction_complete();
	}
	NXP_ADMA_INT_ENABLE_WRITE(nxp_adma_dev.base, nxp_adma_dev.irq_mask);
	spin_unlock_bh(&nxp_adma_dev.lock);
}
EXPORT_SYMBOL(nxp_adma_cpi_submit);

void nxp_adma_cpi_register(void(*irq_handler)(u32))
{
	nxp_adma_dev.irq_handler = irq_handler;
}
EXPORT_SYMBOL(nxp_adma_cpi_register);

static int nxp_adma_alloc_chan_resources(struct dma_chan *dchan)
{
	if (!nxp_adma_dev.init)
	{
		struct dma_async_tx_descriptor *tx   = nxp_adma_dev.tx;
		struct nxp_adma_desc_slot      *slot = nxp_adma_dev.slot;
		int desc_index = NXP_ADMA_NUM_DESCS;
		do
		{
			dma_async_tx_descriptor_init(tx, dchan);
			tx->tx_submit   = nxp_adma_tx_submit;
			tx->tx_set_dest = nxp_adma_set_dest;
			INIT_LIST_HEAD(&tx->tx_list);
			tx->phys = (dma_addr_t)slot;
			tx++;
			slot++;
		} while (--desc_index);
		nxp_adma_dev.init = true;
	}

	return NXP_ADMA_NUM_DESCS;
}

#ifdef NXP_ADMA_SELF_TEST
#define NXP_ADMA_TEST_SIZE 2000
static int __devinit nxp_adma_self_test(struct nxp_adma_device *device)
{
	int i;
	void *src, *dest;
	dma_addr_t src_dma, dest_dma;
	struct dma_chan *dma_chan;
	dma_cookie_t cookie;
	struct dma_async_tx_descriptor *tx;
	int err = 0;

	dev_dbg(device->dma_dev.dev, "%s\n", __FUNCTION__);

	src = kzalloc(sizeof(u8) * NXP_ADMA_TEST_SIZE, GFP_KERNEL);
	if (!src)
	{
		return -ENOMEM;
	}
	dest = kzalloc(sizeof(u8) * NXP_ADMA_TEST_SIZE, GFP_KERNEL);
	if (!dest)
	{
		kfree(src);
		return -ENOMEM;
	}

	/* Fill in src buffer */
	for (i = 0; i < NXP_ADMA_TEST_SIZE; i++)
	{
		((u8 *) src)[i] = (u8)i;
	}

	memset(dest, 0, NXP_ADMA_TEST_SIZE);

	/* Start copy, using first DMA channel */
	dma_chan = &device->dma_chan;

	if (nxp_adma_alloc_chan_resources(dma_chan) < 1)
	{
		err = -ENODEV;
		goto out;
	}

	tx = nxp_adma_prep_dma_memcpy(dma_chan, NXP_ADMA_TEST_SIZE, 1);
	dest_dma = dma_map_single(dma_chan->device->dev, dest, NXP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
	nxp_adma_set_dest(dest_dma, tx, 0);
	src_dma = dma_map_single(dma_chan->device->dev, src, NXP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
	nxp_adma_memcpy_set_src(src_dma, tx, 0);
	cookie = nxp_adma_tx_submit(tx);
	nxp_adma_issue_pending(dma_chan);
	async_tx_ack(tx);
	msleep(10);

	if (nxp_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS)
	{
		printk("Self-test copy timed out, disabling\n");
		err = -ENODEV;
		goto free_resources;
	}

	dma_sync_single_for_cpu(&device->pdev->dev, dest_dma, NXP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
	if (memcmp(src, dest, NXP_ADMA_TEST_SIZE))
	{
		printk("Self-test copy failed compare, disabling\n");
		err = -ENODEV;
		goto free_resources;
	}

free_resources:
	nxp_adma_free_chan_resources(dma_chan);
out:
	kfree(src);
	kfree(dest);
	return err;
}
#endif

static int __devexit nxp_adma_remove(struct platform_device *dev)
{
	struct nxp_adma_device *device = platform_get_drvdata(dev);
	struct dma_chan *dchan, *_dchan;
	struct resource *res;
	unsigned int irq;

	dma_async_device_unregister(&nxp_adma_dev.dma_dev);

	irq = platform_get_irq(dev, 0);
	free_irq(irq, device);

	res = platform_get_resource(dev, IORESOURCE_MEM, 0);
	release_mem_region(res->start, res->end - res->start);

	list_for_each_entry_safe(dchan, _dchan, &nxp_adma_dev.dma_dev.channels, device_node)
	{
		list_del(&dchan->device_node);
	}

	return 0;
}

static int __devinit nxp_adma_probe(struct platform_device *pdev)
{
	struct resource *res;
	struct dma_device *dma_dev;
	struct nxp_adma_platform_data *plat_data = pdev->dev.platform_data;

	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	if (!res)
	{
		return -ENODEV;
	}

	{
		int length = (res->end - res->start) + 1;
		if (!request_mem_region((res->start & 0x1fffffff), length, pdev->name))
		{
			return -EBUSY;
		}

		nxp_adma_dev.base = (unsigned long)ioremap((res->start & 0x1fffffff), length);
		if (!nxp_adma_dev.base)
		{
			return -ENOMEM;
		}
	}

	nxp_adma_dev.init = false;
	dma_dev = &nxp_adma_dev.dma_dev;

	/* discover transaction capabilites from the platform data */
	dma_dev->cap_mask = plat_data->cap_mask;

	nxp_adma_dev.pdev = pdev;
	platform_set_drvdata(pdev, &nxp_adma_dev);

	INIT_LIST_HEAD(&dma_dev->channels);

	/* set base routines */
	dma_dev->device_alloc_chan_resources = nxp_adma_alloc_chan_resources;
	dma_dev->device_free_chan_resources = nxp_adma_free_chan_resources;
	dma_dev->device_is_tx_complete = nxp_adma_is_complete;
	dma_dev->device_issue_pending = nxp_adma_issue_pending;
	dma_dev->device_dependency_added = nxp_adma_dependency_added;
	dma_dev->dev = &pdev->dev;

	/* set prep routines based on capability */
	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
	{
		dma_dev->device_prep_dma_memcpy = nxp_adma_prep_dma_memcpy;
	}
	if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask))
	{
		dma_dev->device_prep_dma_memset = nxp_adma_prep_dma_memset;
	}
	if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
	{
		dma_dev->device_prep_dma_interrupt = nxp_adma_prep_dma_interrupt;
	}
	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask))
	{
		dma_dev->max_xor = 1;
		dma_dev->device_prep_dma_xor = nxp_adma_prep_dma_xor;
	}
	if (dma_has_cap(DMA_ZERO_SUM, dma_dev->cap_mask))
	{
		dma_dev->device_prep_dma_zero_sum = nxp_adma_prep_dma_zero_sum;
	}

	NXP_ADMA_POWER_WRITE(nxp_adma_dev.base, 0);
	NXP_ADMA_POWER_WRITE(nxp_adma_dev.base, 1);
	NXP_ADMA_CMD_WRITE(nxp_adma_dev.base, NXP_ADMA_CMD_RESET);

	nxp_adma_dev.irq = platform_get_irq(pdev, 0);
	nxp_adma_dev.irq_mask = 0;
	if (nxp_adma_dev.irq < 0)
	{
		return -ENXIO;
	}
	else
	{
		if (request_irq(nxp_adma_dev.irq, nxp_adma_interrupt, 0, pdev->name, &nxp_adma_dev))
		{
			return -ENXIO;
		}
		NXP_ADMA_INT_CLEAR_WRITE(nxp_adma_dev.base, NXP_ADMA_INT_NDA_A_FINISHED | NXP_ADMA_INT_NDA_B_FINISHED);
		NXP_ADMA_INT_ENABLE_WRITE(nxp_adma_dev.base, nxp_adma_dev.irq_mask);
	}

	nxp_adma_dev.pending = false;
	spin_lock_init(&nxp_adma_dev.lock);
	INIT_RCU_HEAD(&nxp_adma_dev.dma_chan.rcu);
	nxp_adma_dev.dma_chan.device = dma_dev;
	list_add_tail(&nxp_adma_dev.dma_chan.device_node, &dma_dev->channels);

	dma_async_device_register(dma_dev);
	printk("%s DMA driver irq %d base %08lx\n", pdev->name, nxp_adma_dev.irq, nxp_adma_dev.base);

#ifdef NXP_ADMA_SELF_TEST
	if (nxp_adma_self_test(&nxp_adma_dev) < 0)
	{
		printk("failed self test\n");
	}
#endif
	return 0;
}

static struct platform_driver nxp_adma_driver = {
	.probe		= nxp_adma_probe,
	.remove		= nxp_adma_remove,
	.driver		= {
		.owner	= THIS_MODULE,
		.name	= "nxp-adma",
	},
};

static int __init nxp_adma_init (void)
{
	return platform_driver_register(&nxp_adma_driver);
}

/* it's currently unsafe to unload this module */
static void __exit nxp_adma_exit (void)
{
	printk("Unsafe removal of NXP ADMA driver blocked\n");
/*
	platform_driver_unregister(&nxp_adma_driver);
*/	
	return;
}

module_init(nxp_adma_init);
module_exit(nxp_adma_exit);

MODULE_AUTHOR("NXP Semiconductors");
MODULE_DESCRIPTION("NXP DMA Engine Driver");
MODULE_LICENSE("GPL");
