drivers/staging/skein/skein_block.c

   1 /***********************************************************************
   2 **
   3 ** Implementation of the Skein block functions.
   4 **
   5 ** Source code author: Doug Whiting, 2008.
   6 **
   7 ** This algorithm and source code is released to the public domain.
   8 **
   9 ** Compile-time switches:
  10 **
  11 **  SKEIN_USE_ASM  -- set bits (256/512/1024) to select which
  12 **                    versions use ASM code for block processing
  13 **                    [default: use C for all block sizes]
  14 **
  15 ************************************************************************/
  16
  17 #include <linux/string.h>
  18 #include <linux/bitops.h>
  19 #include "skein_base.h"
  20 #include "skein_block.h"
  21
  22 #ifndef SKEIN_USE_ASM
  23 #define SKEIN_USE_ASM   (0) /* default is all C code (no ASM) */
  24 #endif
  25
  26 #ifndef SKEIN_LOOP
  27 #define SKEIN_LOOP 001 /* default: unroll 256 and 512, but not 1024 */
  28 #endif
  29
  30 #define BLK_BITS        (WCNT * 64) /* some useful definitions for code here */
  31 #define KW_TWK_BASE     (0)
  32 #define KW_KEY_BASE     (3)
  33 #define ks              (kw + KW_KEY_BASE)
  34 #define ts              (kw + KW_TWK_BASE)
  35
  36 #ifdef SKEIN_DEBUG
  37 #define debug_save_tweak(ctx)       \
  38 {                                   \
  39         ctx->h.tweak[0] = ts[0];    \
  40         ctx->h.tweak[1] = ts[1];    \
  41 }
  42 #else
  43 #define debug_save_tweak(ctx)
  44 #endif
  45
  46 #if !(SKEIN_USE_ASM & 256)
  47 #undef  RCNT
  48 #define RCNT (SKEIN_256_ROUNDS_TOTAL / 8)
  49 #ifdef SKEIN_LOOP /* configure how much to unroll the loop */
  50 #define SKEIN_UNROLL_256 (((SKEIN_LOOP) / 100) % 10)
  51 #else
  52 #define SKEIN_UNROLL_256 (0)
  53 #endif
  54
  55 #if SKEIN_UNROLL_256
  56 #if (RCNT % SKEIN_UNROLL_256)
  57 #error "Invalid SKEIN_UNROLL_256" /* sanity check on unroll count */
  58 #endif
  59 #endif
  60 #define ROUND256(p0, p1, p2, p3, ROT, r_num)         \
  61         do {                                         \
  62                 X##p0 += X##p1;                      \
  63                 X##p1 = rol64(X##p1, ROT##_0);       \
  64                 X##p1 ^= X##p0;                      \
  65                 X##p2 += X##p3;                      \
  66                 X##p3 = rol64(X##p3, ROT##_1);       \
  67                 X##p3 ^= X##p2;                      \
  68         } while (0)
  69
  70 #if SKEIN_UNROLL_256 == 0
  71 #define R256(p0, p1, p2, p3, ROT, r_num) /* fully unrolled */ \
  72         ROUND256(p0, p1, p2, p3, ROT, r_num)
  73
  74 #define I256(R)                                                         \
  75         do {                                                            \
  76                 /* inject the key schedule value */                     \
  77                 X0   += ks[((R) + 1) % 5];                              \
  78                 X1   += ks[((R) + 2) % 5] + ts[((R) + 1) % 3];          \
  79                 X2   += ks[((R) + 3) % 5] + ts[((R) + 2) % 3];          \
  80                 X3   += ks[((R) + 4) % 5] + (R) + 1;                    \
  81         } while (0)
  82 #else
  83 /* looping version */
  84 #define R256(p0, p1, p2, p3, ROT, r_num) ROUND256(p0, p1, p2, p3, ROT, r_num)
  85
  86 #define I256(R)                                         \
  87         do {                                            \
  88                 /* inject the key schedule value */     \
  89                 X0 += ks[r + (R) + 0];                  \
  90                 X1 += ks[r + (R) + 1] + ts[r + (R) + 0];\
  91                 X2 += ks[r + (R) + 2] + ts[r + (R) + 1];\
  92                 X3 += ks[r + (R) + 3] + r + (R);        \
  93                 /* rotate key schedule */               \
  94                 ks[r + (R) + 4] = ks[r + (R) - 1];      \
  95                 ts[r + (R) + 2] = ts[r + (R) - 1];      \
  96         } while (0)
  97 #endif
  98 #define R256_8_ROUNDS(R)                                \
  99         do {                                            \
 100                 R256(0, 1, 2, 3, R_256_0, 8 * (R) + 1); \
 101                 R256(0, 3, 2, 1, R_256_1, 8 * (R) + 2); \
 102                 R256(0, 1, 2, 3, R_256_2, 8 * (R) + 3); \
 103                 R256(0, 3, 2, 1, R_256_3, 8 * (R) + 4); \
 104                 I256(2 * (R));                          \
 105                 R256(0, 1, 2, 3, R_256_4, 8 * (R) + 5); \
 106                 R256(0, 3, 2, 1, R_256_5, 8 * (R) + 6); \
 107                 R256(0, 1, 2, 3, R_256_6, 8 * (R) + 7); \
 108                 R256(0, 3, 2, 1, R_256_7, 8 * (R) + 8); \
 109                 I256(2 * (R) + 1);                      \
 110         } while (0)
 111
 112 #define R256_UNROLL_R(NN)                     \
 113         ((SKEIN_UNROLL_256 == 0 &&            \
 114         SKEIN_256_ROUNDS_TOTAL / 8 > (NN)) || \
 115         (SKEIN_UNROLL_256 > (NN)))
 116
 117 #if  (SKEIN_UNROLL_256 > 14)
 118 #error  "need more unrolling in skein_256_process_block"
 119 #endif
 120 #endif
 121
 122 #if !(SKEIN_USE_ASM & 512)
 123 #undef  RCNT
 124 #define RCNT  (SKEIN_512_ROUNDS_TOTAL / 8)
 125
 126 #ifdef SKEIN_LOOP /* configure how much to unroll the loop */
 127 #define SKEIN_UNROLL_512 (((SKEIN_LOOP) / 10) % 10)
 128 #else
 129 #define SKEIN_UNROLL_512 (0)
 130 #endif
 131
 132 #if SKEIN_UNROLL_512
 133 #if (RCNT % SKEIN_UNROLL_512)
 134 #error "Invalid SKEIN_UNROLL_512" /* sanity check on unroll count */
 135 #endif
 136 #endif
 137 #define ROUND512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num)    \
 138         do {                                                    \
 139                 X##p0 += X##p1;                                 \
 140                 X##p1 = rol64(X##p1, ROT##_0);                  \
 141                 X##p1 ^= X##p0;                                 \
 142                 X##p2 += X##p3;                                 \
 143                 X##p3 = rol64(X##p3, ROT##_1);                  \
 144                 X##p3 ^= X##p2;                                 \
 145                 X##p4 += X##p5;                                 \
 146                 X##p5 = rol64(X##p5, ROT##_2);                  \
 147                 X##p5 ^= X##p4;                                 \
 148                 X##p6 += X##p7;                                 \
 149                 X##p7 = rol64(X##p7, ROT##_3);                  \
 150                 X##p7 ^= X##p6;                                 \
 151         } while (0)
 152
 153 #if SKEIN_UNROLL_512 == 0
 154 #define R512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num) /* unrolled */ \
 155         ROUND512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num)
 156
 157 #define I512(R)                                                         \
 158         do {                                                            \
 159                 /* inject the key schedule value */                     \
 160                 X0   += ks[((R) + 1) % 9];                              \
 161                 X1   += ks[((R) + 2) % 9];                              \
 162                 X2   += ks[((R) + 3) % 9];                              \
 163                 X3   += ks[((R) + 4) % 9];                              \
 164                 X4   += ks[((R) + 5) % 9];                              \
 165                 X5   += ks[((R) + 6) % 9] + ts[((R) + 1) % 3];          \
 166                 X6   += ks[((R) + 7) % 9] + ts[((R) + 2) % 3];          \
 167                 X7   += ks[((R) + 8) % 9] + (R) + 1;                    \
 168         } while (0)
 169
 170 #else /* looping version */
 171 #define R512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num)                 \
 172         ROUND512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num)             \
 173
 174 #define I512(R)                                                         \
 175         do {                                                            \
 176                 /* inject the key schedule value */                     \
 177                 X0   += ks[r + (R) + 0];                                \
 178                 X1   += ks[r + (R) + 1];                                \
 179                 X2   += ks[r + (R) + 2];                                \
 180                 X3   += ks[r + (R) + 3];                                \
 181                 X4   += ks[r + (R) + 4];                                \
 182                 X5   += ks[r + (R) + 5] + ts[r + (R) + 0];              \
 183                 X6   += ks[r + (R) + 6] + ts[r + (R) + 1];              \
 184                 X7   += ks[r + (R) + 7] + r + (R);                      \
 185                 /* rotate key schedule */                               \
 186                 ks[r + (R) + 8] = ks[r + (R) - 1];                      \
 187                 ts[r + (R) + 2] = ts[r + (R) - 1];                      \
 188         } while (0)
 189 #endif /* end of looped code definitions */
 190 #define R512_8_ROUNDS(R)  /* do 8 full rounds */                        \
 191         do {                                                            \
 192                 R512(0, 1, 2, 3, 4, 5, 6, 7, R_512_0, 8 * (R) + 1);     \
 193                 R512(2, 1, 4, 7, 6, 5, 0, 3, R_512_1, 8 * (R) + 2);     \
 194                 R512(4, 1, 6, 3, 0, 5, 2, 7, R_512_2, 8 * (R) + 3);     \
 195                 R512(6, 1, 0, 7, 2, 5, 4, 3, R_512_3, 8 * (R) + 4);     \
 196                 I512(2 * (R));                                          \
 197                 R512(0, 1, 2, 3, 4, 5, 6, 7, R_512_4, 8 * (R) + 5);     \
 198                 R512(2, 1, 4, 7, 6, 5, 0, 3, R_512_5, 8 * (R) + 6);     \
 199                 R512(4, 1, 6, 3, 0, 5, 2, 7, R_512_6, 8 * (R) + 7);     \
 200                 R512(6, 1, 0, 7, 2, 5, 4, 3, R_512_7, 8 * (R) + 8);     \
 201                 I512(2 * (R) + 1); /* and key injection */              \
 202         } while (0)
 203 #define R512_UNROLL_R(NN)                             \
 204                 ((SKEIN_UNROLL_512 == 0 &&            \
 205                 SKEIN_512_ROUNDS_TOTAL / 8 > (NN)) || \
 206                 (SKEIN_UNROLL_512 > (NN)))
 207
 208 #if  (SKEIN_UNROLL_512 > 14)
 209 #error  "need more unrolling in skein_512_process_block"
 210 #endif
 211 #endif
 212
 213 #if !(SKEIN_USE_ASM & 1024)
 214 #undef  RCNT
 215 #define RCNT  (SKEIN_1024_ROUNDS_TOTAL / 8)
 216 #ifdef SKEIN_LOOP /* configure how much to unroll the loop */
 217 #define SKEIN_UNROLL_1024 ((SKEIN_LOOP) % 10)
 218 #else
 219 #define SKEIN_UNROLL_1024 (0)
 220 #endif
 221
 222 #if (SKEIN_UNROLL_1024 != 0)
 223 #if (RCNT % SKEIN_UNROLL_1024)
 224 #error "Invalid SKEIN_UNROLL_1024" /* sanity check on unroll count */
 225 #endif
 226 #endif
 227 #define ROUND1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, \
 228                   pF, ROT, r_num)                                             \
 229         do {                                                                  \
 230                 X##p0 += X##p1;                                               \
 231                 X##p1 = rol64(X##p1, ROT##_0);                                \
 232                 X##p1 ^= X##p0;                                               \
 233                 X##p2 += X##p3;                                               \
 234                 X##p3 = rol64(X##p3, ROT##_1);                                \
 235                 X##p3 ^= X##p2;                                               \
 236                 X##p4 += X##p5;                                               \
 237                 X##p5 = rol64(X##p5, ROT##_2);                                \
 238                 X##p5 ^= X##p4;                                               \
 239                 X##p6 += X##p7;                                               \
 240                 X##p7 = rol64(X##p7, ROT##_3);                                \
 241                 X##p7 ^= X##p6;                                               \
 242                 X##p8 += X##p9;                                               \
 243                 X##p9 = rol64(X##p9, ROT##_4);                                \
 244                 X##p9 ^= X##p8;                                               \
 245                 X##pA += X##pB;                                               \
 246                 X##pB = rol64(X##pB, ROT##_5);                                \
 247                 X##pB ^= X##pA;                                               \
 248                 X##pC += X##pD;                                               \
 249                 X##pD = rol64(X##pD, ROT##_6);                                \
 250                 X##pD ^= X##pC;                                               \
 251                 X##pE += X##pF;                                               \
 252                 X##pF = rol64(X##pF, ROT##_7);                                \
 253                 X##pF ^= X##pE;                                               \
 254         } while (0)
 255
 256 #if SKEIN_UNROLL_1024 == 0
 257 #define R1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, pF, \
 258               ROT, rn)                                                        \
 259         ROUND1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, \
 260                   pF, ROT, rn)                                                \
 261
 262 #define I1024(R)                                                \
 263         do {                                                    \
 264                 /* inject the key schedule value */             \
 265                 X00 += ks[((R) + 1) % 17];                      \
 266                 X01 += ks[((R) + 2) % 17];                      \
 267                 X02 += ks[((R) + 3) % 17];                      \
 268                 X03 += ks[((R) + 4) % 17];                      \
 269                 X04 += ks[((R) + 5) % 17];                      \
 270                 X05 += ks[((R) + 6) % 17];                      \
 271                 X06 += ks[((R) + 7) % 17];                      \
 272                 X07 += ks[((R) + 8) % 17];                      \
 273                 X08 += ks[((R) + 9) % 17];                      \
 274                 X09 += ks[((R) + 10) % 17];                     \
 275                 X10 += ks[((R) + 11) % 17];                     \
 276                 X11 += ks[((R) + 12) % 17];                     \
 277                 X12 += ks[((R) + 13) % 17];                     \
 278                 X13 += ks[((R) + 14) % 17] + ts[((R) + 1) % 3]; \
 279                 X14 += ks[((R) + 15) % 17] + ts[((R) + 2) % 3]; \
 280                 X15 += ks[((R) + 16) % 17] + (R) + 1;           \
 281         } while (0)
 282 #else /* looping version */
 283 #define R1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, pF, \
 284               ROT, rn)                                                        \
 285         ROUND1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, \
 286                   pF, ROT, rn)                                                \
 287
 288 #define I1024(R)                                                        \
 289         do {                                                            \
 290                 /* inject the key schedule value */                     \
 291                 X00 += ks[r + (R) + 0];                                 \
 292                 X01 += ks[r + (R) + 1];                                 \
 293                 X02 += ks[r + (R) + 2];                                 \
 294                 X03 += ks[r + (R) + 3];                                 \
 295                 X04 += ks[r + (R) + 4];                                 \
 296                 X05 += ks[r + (R) + 5];                                 \
 297                 X06 += ks[r + (R) + 6];                                 \
 298                 X07 += ks[r + (R) + 7];                                 \
 299                 X08 += ks[r + (R) + 8];                                 \
 300                 X09 += ks[r + (R) + 9];                                 \
 301                 X10 += ks[r + (R) + 10];                                \
 302                 X11 += ks[r + (R) + 11];                                \
 303                 X12 += ks[r + (R) + 12];                                \
 304                 X13 += ks[r + (R) + 13] + ts[r + (R) + 0];              \
 305                 X14 += ks[r + (R) + 14] + ts[r + (R) + 1];              \
 306                 X15 += ks[r + (R) + 15] + r + (R);                      \
 307                 /* rotate key schedule */                               \
 308                 ks[r + (R) + 16] = ks[r + (R) - 1];                     \
 309                 ts[r + (R) + 2] = ts[r + (R) - 1];                      \
 310         } while (0)
 311
 312 #endif
 313 #define R1024_8_ROUNDS(R)                                                 \
 314         do {                                                              \
 315                 R1024(00, 01, 02, 03, 04, 05, 06, 07, 08, 09, 10, 11, 12, \
 316                       13, 14, 15, R1024_0, 8 * (R) + 1);                  \
 317                 R1024(00, 09, 02, 13, 06, 11, 04, 15, 10, 07, 12, 03, 14, \
 318                       05, 08, 01, R1024_1, 8 * (R) + 2);                  \
 319                 R1024(00, 07, 02, 05, 04, 03, 06, 01, 12, 15, 14, 13, 08, \
 320                       11, 10, 09, R1024_2, 8 * (R) + 3);                  \
 321                 R1024(00, 15, 02, 11, 06, 13, 04, 09, 14, 01, 08, 05, 10, \
 322                       03, 12, 07, R1024_3, 8 * (R) + 4);                  \
 323                 I1024(2 * (R));                                           \
 324                 R1024(00, 01, 02, 03, 04, 05, 06, 07, 08, 09, 10, 11, 12, \
 325                       13, 14, 15, R1024_4, 8 * (R) + 5);                  \
 326                 R1024(00, 09, 02, 13, 06, 11, 04, 15, 10, 07, 12, 03, 14, \
 327                       05, 08, 01, R1024_5, 8 * (R) + 6);                  \
 328                 R1024(00, 07, 02, 05, 04, 03, 06, 01, 12, 15, 14, 13, 08, \
 329                       11, 10, 09, R1024_6, 8  * (R) + 7);                 \
 330                 R1024(00, 15, 02, 11, 06, 13, 04, 09, 14, 01, 08, 05, 10, \
 331                       03, 12, 07, R1024_7, 8 * (R) + 8);                  \
 332                 I1024(2 * (R) + 1);                                        \
 333         } while (0)
 334
 335 #define R1024_UNROLL_R(NN)                              \
 336                 ((SKEIN_UNROLL_1024 == 0 &&             \
 337                 SKEIN_1024_ROUNDS_TOTAL / 8 > (NN)) ||  \
 338                 (SKEIN_UNROLL_1024 > (NN)))
 339
 340 #if  (SKEIN_UNROLL_1024 > 14)
 341 #error  "need more unrolling in Skein_1024_Process_Block"
 342 #endif
 343 #endif
 344
 345 /*****************************  SKEIN_256 ******************************/
 346 #if !(SKEIN_USE_ASM & 256)
 347 void skein_256_process_block(struct skein_256_ctx *ctx, const u8 *blk_ptr,
 348                              size_t blk_cnt, size_t byte_cnt_add)
 349 { /* do it in C */
 350         enum {
 351                 WCNT = SKEIN_256_STATE_WORDS
 352         };
 353         size_t r;
 354 #if SKEIN_UNROLL_256
 355         /* key schedule: chaining vars + tweak + "rot"*/
 356         u64  kw[WCNT + 4 + RCNT * 2];
 357 #else
 358         /* key schedule words : chaining vars + tweak */
 359         u64  kw[WCNT + 4];
 360 #endif
 361         u64  X0, X1, X2, X3; /* local copy of context vars, for speed */
 362         u64  w[WCNT]; /* local copy of input block */
 363 #ifdef SKEIN_DEBUG
 364         const u64 *X_ptr[4]; /* use for debugging (help cc put Xn in regs) */
 365
 366         X_ptr[0] = &X0;
 367         X_ptr[1] = &X1;
 368         X_ptr[2] = &X2;
 369         X_ptr[3] = &X3;
 370 #endif
 371         skein_assert(blk_cnt != 0); /* never call with blk_cnt == 0! */
 372         ts[0] = ctx->h.tweak[0];
 373         ts[1] = ctx->h.tweak[1];
 374         do  {
 375                 /*
 376                  * this implementation only supports 2**64 input bytes
 377                  * (no carry out here)
 378                  */
 379                 ts[0] += byte_cnt_add; /* update processed length */
 380
 381                 /* precompute the key schedule for this block */
 382                 ks[0] = ctx->x[0];
 383                 ks[1] = ctx->x[1];
 384                 ks[2] = ctx->x[2];
 385                 ks[3] = ctx->x[3];
 386                 ks[4] = ks[0] ^ ks[1] ^ ks[2] ^ ks[3] ^ SKEIN_KS_PARITY;
 387
 388                 ts[2] = ts[0] ^ ts[1];
 389
 390                 /* get input block in little-endian format */
 391                 skein_get64_lsb_first(w, blk_ptr, WCNT);
 392                 debug_save_tweak(ctx);
 393
 394                 /* do the first full key injection */
 395                 X0 = w[0] + ks[0];
 396                 X1 = w[1] + ks[1] + ts[0];
 397                 X2 = w[2] + ks[2] + ts[1];
 398                 X3 = w[3] + ks[3];
 399
 400                 blk_ptr += SKEIN_256_BLOCK_BYTES;
 401
 402                 /* run the rounds */
 403                 for (r = 1;
 404                         r < (SKEIN_UNROLL_256 ? 2 * RCNT : 2);
 405                         r += (SKEIN_UNROLL_256 ? 2 * SKEIN_UNROLL_256 : 1)) {
 406                         R256_8_ROUNDS(0);
 407 #if   R256_UNROLL_R(1)
 408                         R256_8_ROUNDS(1);
 409 #endif
 410 #if   R256_UNROLL_R(2)
 411                         R256_8_ROUNDS(2);
 412 #endif
 413 #if   R256_UNROLL_R(3)
 414                         R256_8_ROUNDS(3);
 415 #endif
 416 #if   R256_UNROLL_R(4)
 417                         R256_8_ROUNDS(4);
 418 #endif
 419 #if   R256_UNROLL_R(5)
 420                         R256_8_ROUNDS(5);
 421 #endif
 422 #if   R256_UNROLL_R(6)
 423                         R256_8_ROUNDS(6);
 424 #endif
 425 #if   R256_UNROLL_R(7)
 426                         R256_8_ROUNDS(7);
 427 #endif
 428 #if   R256_UNROLL_R(8)
 429                         R256_8_ROUNDS(8);
 430 #endif
 431 #if   R256_UNROLL_R(9)
 432                         R256_8_ROUNDS(9);
 433 #endif
 434 #if   R256_UNROLL_R(10)
 435                         R256_8_ROUNDS(10);
 436 #endif
 437 #if   R256_UNROLL_R(11)
 438                         R256_8_ROUNDS(11);
 439 #endif
 440 #if   R256_UNROLL_R(12)
 441                         R256_8_ROUNDS(12);
 442 #endif
 443 #if   R256_UNROLL_R(13)
 444                         R256_8_ROUNDS(13);
 445 #endif
 446 #if   R256_UNROLL_R(14)
 447                         R256_8_ROUNDS(14);
 448 #endif
 449                 }
 450                 /* do the final "feedforward" xor, update context chaining */
 451                 ctx->x[0] = X0 ^ w[0];
 452                 ctx->x[1] = X1 ^ w[1];
 453                 ctx->x[2] = X2 ^ w[2];
 454                 ctx->x[3] = X3 ^ w[3];
 455
 456                 ts[1] &= ~SKEIN_T1_FLAG_FIRST;
 457         } while (--blk_cnt);
 458         ctx->h.tweak[0] = ts[0];
 459         ctx->h.tweak[1] = ts[1];
 460 }
 461
 462 #if defined(SKEIN_CODE_SIZE) || defined(SKEIN_PERF)
 463 size_t skein_256_process_block_code_size(void)
 464 {
 465         return ((u8 *) skein_256_process_block_code_size) -
 466                 ((u8 *) skein_256_process_block);
 467 }
 468 unsigned int skein_256_unroll_cnt(void)
 469 {
 470         return SKEIN_UNROLL_256;
 471 }
 472 #endif
 473 #endif
 474
 475 /*****************************  SKEIN_512 ******************************/
 476 #if !(SKEIN_USE_ASM & 512)
 477 void skein_512_process_block(struct skein_512_ctx *ctx, const u8 *blk_ptr,
 478                              size_t blk_cnt, size_t byte_cnt_add)
 479 { /* do it in C */
 480         enum {
 481                 WCNT = SKEIN_512_STATE_WORDS
 482         };
 483         size_t  r;
 484 #if SKEIN_UNROLL_512
 485         /* key sched: chaining vars + tweak + "rot"*/
 486         u64  kw[WCNT + 4 + RCNT * 2];
 487 #else
 488         /* key schedule words : chaining vars + tweak */
 489         u64  kw[WCNT + 4];
 490 #endif
 491         u64  X0, X1, X2, X3, X4, X5, X6, X7; /* local copies, for speed */
 492         u64  w[WCNT]; /* local copy of input block */
 493 #ifdef SKEIN_DEBUG
 494         const u64 *X_ptr[8]; /* use for debugging (help cc put Xn in regs) */
 495
 496         X_ptr[0] = &X0;
 497         X_ptr[1] = &X1;
 498         X_ptr[2] = &X2;
 499         X_ptr[3] = &X3;
 500         X_ptr[4] = &X4;
 501         X_ptr[5] = &X5;
 502         X_ptr[6] = &X6;
 503         X_ptr[7] = &X7;
 504 #endif
 505
 506         skein_assert(blk_cnt != 0); /* never call with blk_cnt == 0! */
 507         ts[0] = ctx->h.tweak[0];
 508         ts[1] = ctx->h.tweak[1];
 509         do  {
 510                 /*
 511                  * this implementation only supports 2**64 input bytes
 512                  * (no carry out here)
 513                  */
 514                 ts[0] += byte_cnt_add; /* update processed length */
 515
 516                 /* precompute the key schedule for this block */
 517                 ks[0] = ctx->x[0];
 518                 ks[1] = ctx->x[1];
 519                 ks[2] = ctx->x[2];
 520                 ks[3] = ctx->x[3];
 521                 ks[4] = ctx->x[4];
 522                 ks[5] = ctx->x[5];
 523                 ks[6] = ctx->x[6];
 524                 ks[7] = ctx->x[7];
 525                 ks[8] = ks[0] ^ ks[1] ^ ks[2] ^ ks[3] ^
 526                         ks[4] ^ ks[5] ^ ks[6] ^ ks[7] ^ SKEIN_KS_PARITY;
 527
 528                 ts[2] = ts[0] ^ ts[1];
 529
 530                 /* get input block in little-endian format */
 531                 skein_get64_lsb_first(w, blk_ptr, WCNT);
 532                 debug_save_tweak(ctx);
 533
 534                 /* do the first full key injection */
 535                 X0 = w[0] + ks[0];
 536                 X1 = w[1] + ks[1];
 537                 X2 = w[2] + ks[2];
 538                 X3 = w[3] + ks[3];
 539                 X4 = w[4] + ks[4];
 540                 X5 = w[5] + ks[5] + ts[0];
 541                 X6 = w[6] + ks[6] + ts[1];
 542                 X7 = w[7] + ks[7];
 543
 544                 blk_ptr += SKEIN_512_BLOCK_BYTES;
 545
 546                 /* run the rounds */
 547                 for (r = 1;
 548                         r < (SKEIN_UNROLL_512 ? 2 * RCNT : 2);
 549                         r += (SKEIN_UNROLL_512 ? 2 * SKEIN_UNROLL_512 : 1)) {
 550
 551                         R512_8_ROUNDS(0);
 552
 553 #if   R512_UNROLL_R(1)
 554                         R512_8_ROUNDS(1);
 555 #endif
 556 #if   R512_UNROLL_R(2)
 557                         R512_8_ROUNDS(2);
 558 #endif
 559 #if   R512_UNROLL_R(3)
 560                         R512_8_ROUNDS(3);
 561 #endif
 562 #if   R512_UNROLL_R(4)
 563                         R512_8_ROUNDS(4);
 564 #endif
 565 #if   R512_UNROLL_R(5)
 566                         R512_8_ROUNDS(5);
 567 #endif
 568 #if   R512_UNROLL_R(6)
 569                         R512_8_ROUNDS(6);
 570 #endif
 571 #if   R512_UNROLL_R(7)
 572                         R512_8_ROUNDS(7);
 573 #endif
 574 #if   R512_UNROLL_R(8)
 575                         R512_8_ROUNDS(8);
 576 #endif
 577 #if   R512_UNROLL_R(9)
 578                         R512_8_ROUNDS(9);
 579 #endif
 580 #if   R512_UNROLL_R(10)
 581                         R512_8_ROUNDS(10);
 582 #endif
 583 #if   R512_UNROLL_R(11)
 584                         R512_8_ROUNDS(11);
 585 #endif
 586 #if   R512_UNROLL_R(12)
 587                         R512_8_ROUNDS(12);
 588 #endif
 589 #if   R512_UNROLL_R(13)
 590                         R512_8_ROUNDS(13);
 591 #endif
 592 #if   R512_UNROLL_R(14)
 593                         R512_8_ROUNDS(14);
 594 #endif
 595                 }
 596
 597                 /* do the final "feedforward" xor, update context chaining */
 598                 ctx->x[0] = X0 ^ w[0];
 599                 ctx->x[1] = X1 ^ w[1];
 600                 ctx->x[2] = X2 ^ w[2];
 601                 ctx->x[3] = X3 ^ w[3];
 602                 ctx->x[4] = X4 ^ w[4];
 603                 ctx->x[5] = X5 ^ w[5];
 604                 ctx->x[6] = X6 ^ w[6];
 605                 ctx->x[7] = X7 ^ w[7];
 606
 607                 ts[1] &= ~SKEIN_T1_FLAG_FIRST;
 608         } while (--blk_cnt);
 609         ctx->h.tweak[0] = ts[0];
 610         ctx->h.tweak[1] = ts[1];
 611 }
 612
 613 #if defined(SKEIN_CODE_SIZE) || defined(SKEIN_PERF)
 614 size_t skein_512_process_block_code_size(void)
 615 {
 616         return ((u8 *) skein_512_process_block_code_size) -
 617                 ((u8 *) skein_512_process_block);
 618 }
 619 unsigned int skein_512_unroll_cnt(void)
 620 {
 621         return SKEIN_UNROLL_512;
 622 }
 623 #endif
 624 #endif
 625
 626 /*****************************  SKEIN_1024 ******************************/
 627 #if !(SKEIN_USE_ASM & 1024)
 628 void skein_1024_process_block(struct skein_1024_ctx *ctx, const u8 *blk_ptr,
 629                               size_t blk_cnt, size_t byte_cnt_add)
 630 { /* do it in C, always looping (unrolled is bigger AND slower!) */
 631         enum {
 632                 WCNT = SKEIN_1024_STATE_WORDS
 633         };
 634         size_t  r;
 635 #if (SKEIN_UNROLL_1024 != 0)
 636         /* key sched: chaining vars + tweak + "rot" */
 637         u64  kw[WCNT + 4 + RCNT * 2];
 638 #else
 639         /* key schedule words : chaining vars + tweak */
 640         u64  kw[WCNT + 4];
 641 #endif
 642
 643         /* local copy of vars, for speed */
 644         u64  X00, X01, X02, X03, X04, X05, X06, X07,
 645              X08, X09, X10, X11, X12, X13, X14, X15;
 646         u64  w[WCNT]; /* local copy of input block */
 647
 648         skein_assert(blk_cnt != 0); /* never call with blk_cnt == 0! */
 649         ts[0] = ctx->h.tweak[0];
 650         ts[1] = ctx->h.tweak[1];
 651         do  {
 652                 /*
 653                  * this implementation only supports 2**64 input bytes
 654                  * (no carry out here)
 655                  */
 656                 ts[0] += byte_cnt_add; /* update processed length */
 657
 658                 /* precompute the key schedule for this block */
 659                 ks[0]  = ctx->x[0];
 660                 ks[1]  = ctx->x[1];
 661                 ks[2]  = ctx->x[2];
 662                 ks[3]  = ctx->x[3];
 663                 ks[4]  = ctx->x[4];
 664                 ks[5]  = ctx->x[5];
 665                 ks[6]  = ctx->x[6];
 666                 ks[7]  = ctx->x[7];
 667                 ks[8]  = ctx->x[8];
 668                 ks[9]  = ctx->x[9];
 669                 ks[10] = ctx->x[10];
 670                 ks[11] = ctx->x[11];
 671                 ks[12] = ctx->x[12];
 672                 ks[13] = ctx->x[13];
 673                 ks[14] = ctx->x[14];
 674                 ks[15] = ctx->x[15];
 675                 ks[16] =  ks[0] ^ ks[1] ^ ks[2] ^ ks[3] ^
 676                           ks[4] ^ ks[5] ^ ks[6] ^ ks[7] ^
 677                           ks[8] ^ ks[9] ^ ks[10] ^ ks[11] ^
 678                           ks[12] ^ ks[13] ^ ks[14] ^ ks[15] ^ SKEIN_KS_PARITY;
 679
 680                 ts[2] = ts[0] ^ ts[1];
 681
 682                 /* get input block in little-endian format */
 683                 skein_get64_lsb_first(w, blk_ptr, WCNT);
 684                 debug_save_tweak(ctx);
 685
 686                 /* do the first full key injection */
 687                 X00 = w[0] + ks[0];
 688                 X01 = w[1] + ks[1];
 689                 X02 = w[2] + ks[2];
 690                 X03 = w[3] + ks[3];
 691                 X04 = w[4] + ks[4];
 692                 X05 = w[5] + ks[5];
 693                 X06 = w[6] + ks[6];
 694                 X07 = w[7] + ks[7];
 695                 X08 = w[8] + ks[8];
 696                 X09 = w[9] + ks[9];
 697                 X10 = w[10] + ks[10];
 698                 X11 = w[11] + ks[11];
 699                 X12 = w[12] + ks[12];
 700                 X13 = w[13] + ks[13] + ts[0];
 701                 X14 = w[14] + ks[14] + ts[1];
 702                 X15 = w[15] + ks[15];
 703
 704                 for (r = 1;
 705                         r < (SKEIN_UNROLL_1024 ? 2 * RCNT : 2);
 706                         r += (SKEIN_UNROLL_1024 ? 2 * SKEIN_UNROLL_1024 : 1)) {
 707                         R1024_8_ROUNDS(0);
 708 #if   R1024_UNROLL_R(1)
 709                         R1024_8_ROUNDS(1);
 710 #endif
 711 #if   R1024_UNROLL_R(2)
 712                         R1024_8_ROUNDS(2);
 713 #endif
 714 #if   R1024_UNROLL_R(3)
 715                         R1024_8_ROUNDS(3);
 716 #endif
 717 #if   R1024_UNROLL_R(4)
 718                         R1024_8_ROUNDS(4);
 719 #endif
 720 #if   R1024_UNROLL_R(5)
 721                         R1024_8_ROUNDS(5);
 722 #endif
 723 #if   R1024_UNROLL_R(6)
 724                         R1024_8_ROUNDS(6);
 725 #endif
 726 #if   R1024_UNROLL_R(7)
 727                         R1024_8_ROUNDS(7);
 728 #endif
 729 #if   R1024_UNROLL_R(8)
 730                         R1024_8_ROUNDS(8);
 731 #endif
 732 #if   R1024_UNROLL_R(9)
 733                         R1024_8_ROUNDS(9);
 734 #endif
 735 #if   R1024_UNROLL_R(10)
 736                         R1024_8_ROUNDS(10);
 737 #endif
 738 #if   R1024_UNROLL_R(11)
 739                         R1024_8_ROUNDS(11);
 740 #endif
 741 #if   R1024_UNROLL_R(12)
 742                         R1024_8_ROUNDS(12);
 743 #endif
 744 #if   R1024_UNROLL_R(13)
 745                         R1024_8_ROUNDS(13);
 746 #endif
 747 #if   R1024_UNROLL_R(14)
 748                         R1024_8_ROUNDS(14);
 749 #endif
 750                 }
 751                 /* do the final "feedforward" xor, update context chaining */
 752
 753                 ctx->x[0] = X00 ^ w[0];
 754                 ctx->x[1] = X01 ^ w[1];
 755                 ctx->x[2] = X02 ^ w[2];
 756                 ctx->x[3] = X03 ^ w[3];
 757                 ctx->x[4] = X04 ^ w[4];
 758                 ctx->x[5] = X05 ^ w[5];
 759                 ctx->x[6] = X06 ^ w[6];
 760                 ctx->x[7] = X07 ^ w[7];
 761                 ctx->x[8] = X08 ^ w[8];
 762                 ctx->x[9] = X09 ^ w[9];
 763                 ctx->x[10] = X10 ^ w[10];
 764                 ctx->x[11] = X11 ^ w[11];
 765                 ctx->x[12] = X12 ^ w[12];
 766                 ctx->x[13] = X13 ^ w[13];
 767                 ctx->x[14] = X14 ^ w[14];
 768                 ctx->x[15] = X15 ^ w[15];
 769
 770                 ts[1] &= ~SKEIN_T1_FLAG_FIRST;
 771                 blk_ptr += SKEIN_1024_BLOCK_BYTES;
 772         } while (--blk_cnt);
 773         ctx->h.tweak[0] = ts[0];
 774         ctx->h.tweak[1] = ts[1];
 775 }
 776
 777 #if defined(SKEIN_CODE_SIZE) || defined(SKEIN_PERF)
 778 size_t skein_1024_process_block_code_size(void)
 779 {
 780         return ((u8 *) skein_1024_process_block_code_size) -
 781                 ((u8 *) skein_1024_process_block);
 782 }
 783 unsigned int skein_1024_unroll_cnt(void)
 784 {
 785         return SKEIN_UNROLL_1024;
 786 }
 787 #endif
 788 #endif