#define XGENE_DMA_DESC_ELERR_POS 46
#define XGENE_DMA_DESC_RTYPE_POS 56
#define XGENE_DMA_DESC_LERR_POS 60
-#define XGENE_DMA_DESC_FLYBY_POS 4
#define XGENE_DMA_DESC_BUFLEN_POS 48
#define XGENE_DMA_DESC_HOENQ_NUM_POS 48
-
-#define XGENE_DMA_DESC_NV_SET(m) \
- (((u64 *)(m))[0] |= XGENE_DMA_DESC_NV_BIT)
-#define XGENE_DMA_DESC_IN_SET(m) \
- (((u64 *)(m))[0] |= XGENE_DMA_DESC_IN_BIT)
-#define XGENE_DMA_DESC_RTYPE_SET(m, v) \
- (((u64 *)(m))[0] |= ((u64)(v) << XGENE_DMA_DESC_RTYPE_POS))
-#define XGENE_DMA_DESC_BUFADDR_SET(m, v) \
- (((u64 *)(m))[0] |= (v))
-#define XGENE_DMA_DESC_BUFLEN_SET(m, v) \
- (((u64 *)(m))[0] |= ((u64)(v) << XGENE_DMA_DESC_BUFLEN_POS))
-#define XGENE_DMA_DESC_C_SET(m) \
- (((u64 *)(m))[1] |= XGENE_DMA_DESC_C_BIT)
-#define XGENE_DMA_DESC_FLYBY_SET(m, v) \
- (((u64 *)(m))[2] |= ((v) << XGENE_DMA_DESC_FLYBY_POS))
-#define XGENE_DMA_DESC_MULTI_SET(m, v, i) \
- (((u64 *)(m))[2] |= ((u64)(v) << (((i) + 1) * 8)))
-#define XGENE_DMA_DESC_DR_SET(m) \
- (((u64 *)(m))[2] |= XGENE_DMA_DESC_DR_BIT)
-#define XGENE_DMA_DESC_DST_ADDR_SET(m, v) \
- (((u64 *)(m))[3] |= (v))
-#define XGENE_DMA_DESC_H0ENQ_NUM_SET(m, v) \
- (((u64 *)(m))[3] |= ((u64)(v) << XGENE_DMA_DESC_HOENQ_NUM_POS))
#define XGENE_DMA_DESC_ELERR_RD(m) \
(((m) >> XGENE_DMA_DESC_ELERR_POS) & 0x3)
#define XGENE_DMA_DESC_LERR_RD(m) \
(((elerr) << 4) | (lerr))
/* X-Gene DMA descriptor empty s/w signature */
-#define XGENE_DMA_DESC_EMPTY_INDEX 0
#define XGENE_DMA_DESC_EMPTY_SIGNATURE ~0ULL
-#define XGENE_DMA_DESC_SET_EMPTY(m) \
- (((u64 *)(m))[XGENE_DMA_DESC_EMPTY_INDEX] = \
- XGENE_DMA_DESC_EMPTY_SIGNATURE)
-#define XGENE_DMA_DESC_IS_EMPTY(m) \
- (((u64 *)(m))[XGENE_DMA_DESC_EMPTY_INDEX] == \
- XGENE_DMA_DESC_EMPTY_SIGNATURE)
/* X-Gene DMA configurable parameters defines */
#define XGENE_DMA_RING_NUM 512
#define XGENE_DMA_XOR_ALIGNMENT 6 /* 64 Bytes */
#define XGENE_DMA_MAX_XOR_SRC 5
#define XGENE_DMA_16K_BUFFER_LEN_CODE 0x0
-#define XGENE_DMA_INVALID_LEN_CODE 0x7800
+#define XGENE_DMA_INVALID_LEN_CODE 0x7800000000000000ULL
/* X-Gene DMA descriptor error codes */
#define ERR_DESC_AXI 0x01
#define ERR_DESC_SRC_INT 0xB
/* X-Gene DMA flyby operation code */
-#define FLYBY_2SRC_XOR 0x8
-#define FLYBY_3SRC_XOR 0x9
-#define FLYBY_4SRC_XOR 0xA
-#define FLYBY_5SRC_XOR 0xB
+#define FLYBY_2SRC_XOR 0x80
+#define FLYBY_3SRC_XOR 0x90
+#define FLYBY_4SRC_XOR 0xA0
+#define FLYBY_5SRC_XOR 0xB0
/* X-Gene DMA SW descriptor flags */
#define XGENE_DMA_FLAG_64B_DESC BIT(0)
dev_err(chan->dev, "%s: " fmt, chan->name, ##arg)
struct xgene_dma_desc_hw {
- u64 m0;
- u64 m1;
- u64 m2;
- u64 m3;
+ __le64 m0;
+ __le64 m1;
+ __le64 m2;
+ __le64 m3;
};
enum xgene_dma_ring_cfgsize {
return !(val & XGENE_DMA_PQ_DISABLE_MASK);
}
-static void xgene_dma_cpu_to_le64(u64 *desc, int count)
-{
- int i;
-
- for (i = 0; i < count; i++)
- desc[i] = cpu_to_le64(desc[i]);
-}
-
-static u16 xgene_dma_encode_len(u32 len)
+static u64 xgene_dma_encode_len(size_t len)
{
return (len < XGENE_DMA_MAX_BYTE_CNT) ?
- len : XGENE_DMA_16K_BUFFER_LEN_CODE;
+ ((u64)len << XGENE_DMA_DESC_BUFLEN_POS) :
+ XGENE_DMA_16K_BUFFER_LEN_CODE;
}
static u8 xgene_dma_encode_xor_flyby(u32 src_cnt)
return XGENE_DMA_RING_DESC_CNT(ring_state);
}
-static void xgene_dma_set_src_buffer(void *ext8, size_t *len,
+static void xgene_dma_set_src_buffer(__le64 *ext8, size_t *len,
dma_addr_t *paddr)
{
size_t nbytes = (*len < XGENE_DMA_MAX_BYTE_CNT) ?
*len : XGENE_DMA_MAX_BYTE_CNT;
- XGENE_DMA_DESC_BUFADDR_SET(ext8, *paddr);
- XGENE_DMA_DESC_BUFLEN_SET(ext8, xgene_dma_encode_len(nbytes));
+ *ext8 |= cpu_to_le64(*paddr);
+ *ext8 |= cpu_to_le64(xgene_dma_encode_len(nbytes));
*len -= nbytes;
*paddr += nbytes;
}
-static void xgene_dma_invalidate_buffer(void *ext8)
+static void xgene_dma_invalidate_buffer(__le64 *ext8)
{
- XGENE_DMA_DESC_BUFLEN_SET(ext8, XGENE_DMA_INVALID_LEN_CODE);
+ *ext8 |= cpu_to_le64(XGENE_DMA_INVALID_LEN_CODE);
}
-static void *xgene_dma_lookup_ext8(u64 *desc, int idx)
+static __le64 *xgene_dma_lookup_ext8(struct xgene_dma_desc_hw *desc, int idx)
{
- return (idx % 2) ? (desc + idx - 1) : (desc + idx + 1);
+ switch (idx) {
+ case 0:
+ return &desc->m1;
+ case 1:
+ return &desc->m0;
+ case 2:
+ return &desc->m3;
+ case 3:
+ return &desc->m2;
+ default:
+ pr_err("Invalid dma descriptor index\n");
+ }
+
+ return NULL;
}
-static void xgene_dma_init_desc(void *desc, u16 dst_ring_num)
+static void xgene_dma_init_desc(struct xgene_dma_desc_hw *desc,
+ u16 dst_ring_num)
{
- XGENE_DMA_DESC_C_SET(desc); /* Coherent IO */
- XGENE_DMA_DESC_IN_SET(desc);
- XGENE_DMA_DESC_H0ENQ_NUM_SET(desc, dst_ring_num);
- XGENE_DMA_DESC_RTYPE_SET(desc, XGENE_DMA_RING_OWNER_DMA);
+ desc->m0 |= cpu_to_le64(XGENE_DMA_DESC_IN_BIT);
+ desc->m0 |= cpu_to_le64((u64)XGENE_DMA_RING_OWNER_DMA <<
+ XGENE_DMA_DESC_RTYPE_POS);
+ desc->m1 |= cpu_to_le64(XGENE_DMA_DESC_C_BIT);
+ desc->m3 |= cpu_to_le64((u64)dst_ring_num <<
+ XGENE_DMA_DESC_HOENQ_NUM_POS);
}
static void xgene_dma_prep_cpy_desc(struct xgene_dma_chan *chan,
dma_addr_t dst, dma_addr_t src,
size_t len)
{
- void *desc1, *desc2;
+ struct xgene_dma_desc_hw *desc1, *desc2;
int i;
/* Get 1st descriptor */
xgene_dma_init_desc(desc1, chan->tx_ring.dst_ring_num);
/* Set destination address */
- XGENE_DMA_DESC_DR_SET(desc1);
- XGENE_DMA_DESC_DST_ADDR_SET(desc1, dst);
+ desc1->m2 |= cpu_to_le64(XGENE_DMA_DESC_DR_BIT);
+ desc1->m3 |= cpu_to_le64(dst);
/* Set 1st source address */
- xgene_dma_set_src_buffer(desc1 + 8, &len, &src);
+ xgene_dma_set_src_buffer(&desc1->m1, &len, &src);
- if (len <= 0) {
- desc2 = NULL;
- goto skip_additional_src;
- }
+ if (!len)
+ return;
/*
* We need to split this source buffer,
* and need to use 2nd descriptor
*/
desc2 = &desc_sw->desc2;
- XGENE_DMA_DESC_NV_SET(desc1);
+ desc1->m0 |= cpu_to_le64(XGENE_DMA_DESC_NV_BIT);
/* Set 2nd to 5th source address */
for (i = 0; i < 4 && len; i++)
/* Updated flag that we have prepared 64B descriptor */
desc_sw->flags |= XGENE_DMA_FLAG_64B_DESC;
-
-skip_additional_src:
- /* Hardware stores descriptor in little endian format */
- xgene_dma_cpu_to_le64(desc1, 4);
- if (desc2)
- xgene_dma_cpu_to_le64(desc2, 4);
}
static void xgene_dma_prep_xor_desc(struct xgene_dma_chan *chan,
u32 src_cnt, size_t *nbytes,
const u8 *scf)
{
- void *desc1, *desc2;
+ struct xgene_dma_desc_hw *desc1, *desc2;
size_t len = *nbytes;
int i;
xgene_dma_init_desc(desc1, chan->tx_ring.dst_ring_num);
/* Set destination address */
- XGENE_DMA_DESC_DR_SET(desc1);
- XGENE_DMA_DESC_DST_ADDR_SET(desc1, *dst);
+ desc1->m2 |= cpu_to_le64(XGENE_DMA_DESC_DR_BIT);
+ desc1->m3 |= cpu_to_le64(*dst);
/* We have multiple source addresses, so need to set NV bit*/
- XGENE_DMA_DESC_NV_SET(desc1);
+ desc1->m0 |= cpu_to_le64(XGENE_DMA_DESC_NV_BIT);
/* Set flyby opcode */
- XGENE_DMA_DESC_FLYBY_SET(desc1, xgene_dma_encode_xor_flyby(src_cnt));
+ desc1->m2 |= cpu_to_le64(xgene_dma_encode_xor_flyby(src_cnt));
/* Set 1st to 5th source addresses */
for (i = 0; i < src_cnt; i++) {
len = *nbytes;
- xgene_dma_set_src_buffer((i == 0) ? (desc1 + 8) :
+ xgene_dma_set_src_buffer((i == 0) ? &desc1->m1 :
xgene_dma_lookup_ext8(desc2, i - 1),
&len, &src[i]);
- XGENE_DMA_DESC_MULTI_SET(desc1, scf[i], i);
+ desc1->m2 |= cpu_to_le64((scf[i] << ((i + 1) * 8)));
}
- /* Hardware stores descriptor in little endian format */
- xgene_dma_cpu_to_le64(desc1, 4);
- xgene_dma_cpu_to_le64(desc2, 4);
-
/* Update meta data */
*nbytes = len;
*dst += XGENE_DMA_MAX_BYTE_CNT;
* xgene_chan_xfer_ld_pending - push any pending transactions to hw
* @chan : X-Gene DMA channel
*
- * LOCKING: must hold chan->desc_lock
+ * LOCKING: must hold chan->lock
*/
static void xgene_chan_xfer_ld_pending(struct xgene_dma_chan *chan)
{
desc_hw = &ring->desc_hw[ring->head];
/* Check if this descriptor has been completed */
- if (unlikely(XGENE_DMA_DESC_IS_EMPTY(desc_hw)))
+ if (unlikely(le64_to_cpu(desc_hw->m0) ==
+ XGENE_DMA_DESC_EMPTY_SIGNATURE))
break;
if (++ring->head == ring->slots)
iowrite32(-1, ring->cmd);
/* Mark this hw descriptor as processed */
- XGENE_DMA_DESC_SET_EMPTY(desc_hw);
+ desc_hw->m0 = cpu_to_le64(XGENE_DMA_DESC_EMPTY_SIGNATURE);
xgene_dma_run_tx_complete_actions(chan, desc_sw);
* @chan: X-Gene DMA channel
* @list: the list to free
*
- * LOCKING: must hold chan->desc_lock
+ * LOCKING: must hold chan->lock
*/
static void xgene_dma_free_desc_list(struct xgene_dma_chan *chan,
struct list_head *list)
xgene_dma_clean_descriptor(chan, desc);
}
-static void xgene_dma_free_tx_desc_list(struct xgene_dma_chan *chan,
- struct list_head *list)
-{
- struct xgene_dma_desc_sw *desc, *_desc;
-
- list_for_each_entry_safe(desc, _desc, list, node)
- xgene_dma_clean_descriptor(chan, desc);
-}
-
static void xgene_dma_free_chan_resources(struct dma_chan *dchan)
{
struct xgene_dma_chan *chan = to_dma_chan(dchan);
if (!first)
return NULL;
- xgene_dma_free_tx_desc_list(chan, &first->tx_list);
+ xgene_dma_free_desc_list(chan, &first->tx_list);
return NULL;
}
if (!first)
return NULL;
- xgene_dma_free_tx_desc_list(chan, &first->tx_list);
+ xgene_dma_free_desc_list(chan, &first->tx_list);
return NULL;
}
if (!first)
return NULL;
- xgene_dma_free_tx_desc_list(chan, &first->tx_list);
+ xgene_dma_free_desc_list(chan, &first->tx_list);
return NULL;
}
if (!first)
return NULL;
- xgene_dma_free_tx_desc_list(chan, &first->tx_list);
+ xgene_dma_free_desc_list(chan, &first->tx_list);
return NULL;
}
{
void *ring_cfg = ring->state;
u64 addr = ring->desc_paddr;
- void *desc;
u32 i, val;
ring->slots = ring->size / XGENE_DMA_RING_WQ_DESC_SIZE;
/* Set empty signature to DMA Rx ring descriptors */
for (i = 0; i < ring->slots; i++) {
+ struct xgene_dma_desc_hw *desc;
+
desc = &ring->desc_hw[i];
- XGENE_DMA_DESC_SET_EMPTY(desc);
+ desc->m0 = cpu_to_le64(XGENE_DMA_DESC_EMPTY_SIGNATURE);
}
/* Enable DMA Rx ring interrupt */
projects / karo-tx-linux.git / commitdiff