1 /* NG2memcpy.S: Niagara-2 optimized memcpy.
3 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
7 #include <linux/linkage.h>
8 #include <asm/visasm.h>
10 #define GLOBAL_SPARE %g7
13 #define ASI_BLK_P 0xf0
14 #define ASI_BLK_INIT_QUAD_LDD_P 0xe2
17 #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
18 clr %g1; clr %g2; clr %g3; clr %g5; subcc %g0, %g0, %g0;
19 #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
21 #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
22 #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
24 #define GLOBAL_SPARE %g5
28 #ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
29 #define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P
31 #define STORE_ASI 0x80 /* ASI_P */
39 #define EX_LD_FP(x,y) x
46 #define EX_ST_FP(x,y) x
50 #define LOAD(type,addr,dest) type [addr], dest
54 #define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest
59 #define STORE(type,src,addr) type src, [addr]
61 #define STORE(type,src,addr) type##a src, [addr] 0x80
66 #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P
70 #define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI
74 #define FUNC_NAME NG2memcpy
85 #define FREG_FROB(x0, x1, x2, x3, x4, x5, x6, x7, x8) \
86 faligndata %x0, %x1, %f0; \
87 faligndata %x1, %x2, %f2; \
88 faligndata %x2, %x3, %f4; \
89 faligndata %x3, %x4, %f6; \
90 faligndata %x4, %x5, %f8; \
91 faligndata %x5, %x6, %f10; \
92 faligndata %x6, %x7, %f12; \
93 faligndata %x7, %x8, %f14;
95 #define FREG_MOVE_1(x0) \
97 #define FREG_MOVE_2(x0, x1) \
100 #define FREG_MOVE_3(x0, x1, x2) \
104 #define FREG_MOVE_4(x0, x1, x2, x3) \
109 #define FREG_MOVE_5(x0, x1, x2, x3, x4) \
115 #define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \
122 #define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \
130 #define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \
139 #define FREG_LOAD_1(base, x0) \
140 EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1)
141 #define FREG_LOAD_2(base, x0, x1) \
142 EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
143 EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1);
144 #define FREG_LOAD_3(base, x0, x1, x2) \
145 EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
146 EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
147 EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1);
148 #define FREG_LOAD_4(base, x0, x1, x2, x3) \
149 EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
150 EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
151 EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
152 EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1);
153 #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \
154 EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
155 EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
156 EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
157 EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
158 EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1);
159 #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \
160 EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
161 EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
162 EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
163 EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
164 EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
165 EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1);
166 #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \
167 EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
168 EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
169 EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
170 EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
171 EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
172 EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \
173 EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1);
175 .register %g2,#scratch
176 .register %g3,#scratch
180 #define EX_RETVAL(x) x
185 wr %g0, ASI_AIUS, %asi
187 ba,pt %xcc, __restore_asi
190 ENTRY(NG2_retl_o2_plus_1)
191 ba,pt %xcc, __restore_asi
193 ENDPROC(NG2_retl_o2_plus_1)
194 ENTRY(NG2_retl_o2_plus_4)
195 ba,pt %xcc, __restore_asi
197 ENDPROC(NG2_retl_o2_plus_4)
198 ENTRY(NG2_retl_o2_plus_8)
199 ba,pt %xcc, __restore_asi
201 ENDPROC(NG2_retl_o2_plus_8)
202 ENTRY(NG2_retl_o2_plus_o4_plus_1)
204 ba,pt %xcc, __restore_asi
206 ENDPROC(NG2_retl_o2_plus_o4_plus_1)
207 ENTRY(NG2_retl_o2_plus_o4_plus_8)
209 ba,pt %xcc, __restore_asi
211 ENDPROC(NG2_retl_o2_plus_o4_plus_8)
212 ENTRY(NG2_retl_o2_plus_o4_plus_16)
214 ba,pt %xcc, __restore_asi
216 ENDPROC(NG2_retl_o2_plus_o4_plus_16)
217 ENTRY(NG2_retl_o2_plus_g1_fp)
218 ba,pt %xcc, __restore_fp
220 ENDPROC(NG2_retl_o2_plus_g1_fp)
221 ENTRY(NG2_retl_o2_plus_g1_plus_64_fp)
223 ba,pt %xcc, __restore_fp
225 ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp)
226 ENTRY(NG2_retl_o2_plus_g1_plus_1)
228 ba,pt %xcc, __restore_asi
230 ENDPROC(NG2_retl_o2_plus_g1_plus_1)
231 ENTRY(NG2_retl_o2_and_7_plus_o4)
233 ba,pt %xcc, __restore_asi
235 ENDPROC(NG2_retl_o2_and_7_plus_o4)
236 ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8)
239 ba,pt %xcc, __restore_asi
241 ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8)
247 .type FUNC_NAME,#function
248 FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
256 or %o0, %o1, GLOBAL_SPARE
259 or GLOBAL_SPARE, %o2, GLOBAL_SPARE
261 /* 2 blocks (128 bytes) is the minimum we can do the block
262 * copy with. We need to ensure that we'll iterate at least
263 * once in the block copy loop. At worst we'll need to align
264 * the destination to a 64-byte boundary which can chew up
265 * to (64 - 1) bytes from the length before we perform the
268 * However, the cut-off point, performance wise, is around
273 andcc GLOBAL_SPARE, 0x7, %g0
277 * %o2: len (known to be >= 128)
279 * The block copy loops can use %o4, %g2, %g3 as
280 * temporaries while copying the data. %o5 must
281 * be preserved between VISEntryHalf and VISExitHalf
284 LOAD(prefetch, %o1 + 0x000, #one_read)
285 LOAD(prefetch, %o1 + 0x040, #one_read)
286 LOAD(prefetch, %o1 + 0x080, #one_read)
288 /* Align destination on 64-byte boundary. */
289 andcc %o0, (64 - 1), %o4
292 sub %g0, %o4, %o4 ! bytes to align dst
295 EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1)
296 EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1)
302 /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve
303 * o5 from here until we hit VISExitHalf.
308 alignaddr %o1, %g0, %g0
310 add %o1, (64 - 1), %o4
311 andn %o4, (64 - 1), %o4
312 andn %o2, (64 - 1), %g1
315 and %o1, (64 - 1), %g2
331 4: /* 32 <= low bits < 48 */
336 5: /* 0 < low bits < 32 */
344 6: /* 0 < low bits < 16 */
347 /* fall through for 0 < low bits < 8 */
348 110: sub %o4, 64, %g2
349 EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1)
350 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
351 EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
352 FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16)
353 EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
354 FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30)
358 LOAD(prefetch, %o4 + 64, #one_read)
362 120: sub %o4, 56, %g2
363 FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12)
364 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
365 EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
366 FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18)
367 EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
368 FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30)
372 LOAD(prefetch, %o4 + 64, #one_read)
376 130: sub %o4, 48, %g2
377 FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10)
378 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
379 EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
380 FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20)
381 EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
382 FREG_MOVE_6(f20, f22, f24, f26, f28, f30)
386 LOAD(prefetch, %o4 + 64, #one_read)
390 140: sub %o4, 40, %g2
391 FREG_LOAD_5(%g2, f0, f2, f4, f6, f8)
392 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
393 EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
394 FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22)
395 EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
396 FREG_MOVE_5(f22, f24, f26, f28, f30)
400 LOAD(prefetch, %o4 + 64, #one_read)
404 150: sub %o4, 32, %g2
405 FREG_LOAD_4(%g2, f0, f2, f4, f6)
406 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
407 EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
408 FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24)
409 EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
410 FREG_MOVE_4(f24, f26, f28, f30)
414 LOAD(prefetch, %o4 + 64, #one_read)
418 160: sub %o4, 24, %g2
419 FREG_LOAD_3(%g2, f0, f2, f4)
420 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
421 EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
422 FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26)
423 EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
424 FREG_MOVE_3(f26, f28, f30)
428 LOAD(prefetch, %o4 + 64, #one_read)
432 170: sub %o4, 16, %g2
433 FREG_LOAD_2(%g2, f0, f2)
434 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
435 EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
436 FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28)
437 EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
438 FREG_MOVE_2(f28, f30)
442 LOAD(prefetch, %o4 + 64, #one_read)
448 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
449 EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
450 FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30)
451 EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
456 LOAD(prefetch, %o4 + 64, #one_read)
461 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
463 EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64)
464 EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64)
467 LOAD(prefetch, %o4 + 64, #one_read)
475 /* %o2 contains any final bytes still needed to be copied
476 * over. If anything is left, we copy it one byte at a time.
479 sub %o0, %o1, GLOBAL_SPARE
484 75: /* 16 < len <= 64 */
486 sub %o0, %o1, GLOBAL_SPARE
491 1: subcc %o4, 0x10, %o4
492 EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16)
494 EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16)
496 EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16)
498 EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8)
501 73: andcc %o2, 0x8, %g0
505 EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8)
506 EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8)
508 1: andcc %o2, 0x4, %g0
512 EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4)
513 EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
529 EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1)
530 EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1)
534 2: add %o1, GLOBAL_SPARE, %o0
544 8: mov 64, GLOBAL_SPARE
546 EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2)
547 sub GLOBAL_SPARE, %g1, GLOBAL_SPARE
551 EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4)
553 srlx %g3, GLOBAL_SPARE, %o5
555 EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8)
565 sub %o0, %o1, GLOBAL_SPARE
568 80: /* 0 < len <= 16 */
569 andcc GLOBAL_SPARE, 0x3, %g0
571 sub %o0, %o1, GLOBAL_SPARE
575 EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4)
576 EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
581 mov EX_RETVAL(%o3), %o0
586 EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1)
587 EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1)
591 mov EX_RETVAL(%o3), %o0
593 .size FUNC_NAME, .-FUNC_NAME