2 * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
3 * instructions. This file contains accelerated part of ghash
4 * implementation. More information about PCLMULQDQ can be found at:
6 * http://software.intel.com/en-us/articles/carry-less-multiplication-and-its-usage-for-computing-the-gcm-mode/
8 * Copyright (c) 2009 Intel Corp.
9 * Author: Huang Ying <ying.huang@intel.com>
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU General Public License version 2 as published
16 * by the Free Software Foundation.
19 #include <linux/linkage.h>
24 .octa 0x000102030405060708090a0b0c0d0e0f
26 .octa 0xc2000000000000000000000000000001
28 .octa 0x00000001000000000000000000000001
41 * __clmul_gf128mul_ble: internal ABI
44 * SHASH: operand2, hash_key << 1 mod poly
46 * DATA: operand1 * operand2 mod poly
54 pshufd $0b01001110, DATA, T2
55 pshufd $0b01001110, SHASH, T3
59 PCLMULQDQ 0x00 SHASH DATA # DATA = a0 * b0
60 PCLMULQDQ 0x11 SHASH T1 # T1 = a1 * b1
61 PCLMULQDQ 0x00 T3 T2 # T2 = (a1 + a0) * (b1 + b0)
63 pxor T1, T2 # T2 = a0 * b1 + a1 * b0
69 pxor T2, T1 # <T1:DATA> is result of
70 # carry-less multiplication
72 # first phase of the reduction
85 # second phase of the reduction
96 /* void clmul_ghash_mul(char *dst, const be128 *shash) */
97 ENTRY(clmul_ghash_mul)
100 movaps .Lbswap_mask, BSWAP
101 PSHUFB_XMM BSWAP DATA
102 call __clmul_gf128mul_ble
103 PSHUFB_XMM BSWAP DATA
108 * void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
109 * const be128 *shash);
111 ENTRY(clmul_ghash_update)
113 jb .Lupdate_just_ret # check length
114 movaps .Lbswap_mask, BSWAP
117 PSHUFB_XMM BSWAP DATA
123 call __clmul_gf128mul_ble
128 PSHUFB_XMM BSWAP DATA
134 * void clmul_ghash_setkey(be128 *shash, const u8 *key);
136 * Calculate hash_key << 1 mod poly
138 ENTRY(clmul_ghash_setkey)
139 movaps .Lbswap_mask, BSWAP
141 PSHUFB_XMM BSWAP %xmm0
150 pshufd $0b00100100, %xmm2, %xmm1
151 pcmpeqd .Ltwo_one, %xmm1