/*- * Copyright 2009 Colin Percival * Copyright 2012-2018 Alexander Peslyak * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * This file was originally written by Colin Percival as part of the Tarsnap * online backup system. */ #include "crypt-port.h" #if INCLUDE_yescrypt || INCLUDE_scrypt || INCLUDE_gost_yescrypt #pragma GCC diagnostic ignored "-Wcast-align" #pragma GCC diagnostic ignored "-Wconversion" #ifdef __clang__ #pragma GCC diagnostic ignored "-Wtautological-constant-out-of-range-compare" #endif /* * AVX and especially XOP speed up Salsa20 a lot, but this mostly matters for * classic scrypt and for YESCRYPT_WORM (which use 8 rounds of Salsa20 per * sub-block), and much less so for YESCRYPT_RW (which uses 2 rounds of Salsa20 * per block except during pwxform S-box initialization). */ #if 0 #ifdef __XOP__ #warning "Note: XOP is enabled. That's great." #elif defined(__AVX__) #warning "Note: AVX is enabled, which is great for classic scrypt and YESCRYPT_WORM, but is sometimes slightly slower than plain SSE2 for YESCRYPT_RW" #elif defined(__SSE2__) #warning "Note: AVX and XOP are not enabled, which is great for YESCRYPT_RW, but they would substantially improve performance at classic scrypt and YESCRYPT_WORM" #elif defined(__x86_64__) || defined(__i386__) #warning "SSE2 not enabled. Expect poor performance." #else #warning "Note: building generic code for non-x86. That's OK." #endif #endif /* * The SSE4 code version has fewer instructions than the generic SSE2 version, * but all of the instructions are SIMD, thereby wasting the scalar execution * units. Thus, the generic SSE2 version below actually runs faster on some * CPUs due to its balanced mix of SIMD and scalar instructions. */ #undef USE_SSE4_FOR_32BIT #ifdef __SSE2__ /* * GCC before 4.9 would by default unnecessarily use store/load (without * SSE4.1) or (V)PEXTR (with SSE4.1 or AVX) instead of simply (V)MOV. * This was tracked as GCC bug 54349. * "-mtune=corei7" works around this, but is only supported for GCC 4.6+. * We use inline asm for pre-4.6 GCC, further down this file. */ #if __GNUC__ == 4 && __GNUC_MINOR__ >= 6 && __GNUC_MINOR__ < 9 && \ !defined(__clang__) && !defined(__ICC) #pragma GCC target ("tune=corei7") #endif #include #ifdef __XOP__ #include #endif #elif defined(__SSE__) #include #endif #include #include #include #include #include #include "alg-sha256.h" #include "byteorder.h" #define YESCRYPT_INTERNAL #include "alg-yescrypt.h" #include "alg-yescrypt-platform.c" #if __STDC_VERSION__ >= 199901L /* Have restrict */ #elif defined(__GNUC__) #define restrict __restrict #else #define restrict #endif #ifdef __GNUC__ #define unlikely(exp) __builtin_expect(exp, 0) #else #define unlikely(exp) (exp) #endif #ifdef __SSE__ #define PREFETCH(x, hint) _mm_prefetch((x), (hint)); /* Older versions of clang have a bug in their xmmintrin.h that causes spurious -Wcast-qual warnings on uses of _mm_prefetch. */ # if defined __clang_major__ && __clang_major__ < 11 # pragma clang diagnostic ignored "-Wcast-qual" # endif #else #undef PREFETCH #endif typedef union { uint32_t w[16]; uint64_t d[8]; #ifdef __SSE2__ __m128i q[4]; #endif } salsa20_blk_t; static inline void salsa20_simd_shuffle(const salsa20_blk_t *Bin, salsa20_blk_t *Bout) { #define COMBINE(out, in1, in2) \ Bout->d[out] = Bin->w[in1 * 2] | ((uint64_t)Bin->w[in2 * 2 + 1] << 32); COMBINE(0, 0, 2) COMBINE(1, 5, 7) COMBINE(2, 2, 4) COMBINE(3, 7, 1) COMBINE(4, 4, 6) COMBINE(5, 1, 3) COMBINE(6, 6, 0) COMBINE(7, 3, 5) #undef COMBINE } static inline void salsa20_simd_unshuffle(const salsa20_blk_t *Bin, salsa20_blk_t *Bout) { #define UNCOMBINE(out, in1, in2) \ Bout->w[out * 2] = Bin->d[in1]; \ Bout->w[out * 2 + 1] = Bin->d[in2] >> 32; UNCOMBINE(0, 0, 6) UNCOMBINE(1, 5, 3) UNCOMBINE(2, 2, 0) UNCOMBINE(3, 7, 5) UNCOMBINE(4, 4, 2) UNCOMBINE(5, 1, 7) UNCOMBINE(6, 6, 4) UNCOMBINE(7, 3, 1) #undef UNCOMBINE } #ifdef __SSE2__ #define DECL_X \ __m128i X0, X1, X2, X3; #define DECL_Y \ __m128i Y0, Y1, Y2, Y3; #define READ_X(in) \ X0 = (in).q[0]; X1 = (in).q[1]; X2 = (in).q[2]; X3 = (in).q[3]; #define WRITE_X(out) \ (out).q[0] = X0; (out).q[1] = X1; (out).q[2] = X2; (out).q[3] = X3; #ifdef __XOP__ #define ARX(out, in1, in2, s) \ out = _mm_xor_si128(out, _mm_roti_epi32(_mm_add_epi32(in1, in2), s)); #else #define ARX(out, in1, in2, s) { \ __m128i tmp = _mm_add_epi32(in1, in2); \ out = _mm_xor_si128(out, _mm_slli_epi32(tmp, s)); \ out = _mm_xor_si128(out, _mm_srli_epi32(tmp, 32 - s)); \ } #endif #define SALSA20_2ROUNDS \ /* Operate on "columns" */ \ ARX(X1, X0, X3, 7) \ ARX(X2, X1, X0, 9) \ ARX(X3, X2, X1, 13) \ ARX(X0, X3, X2, 18) \ /* Rearrange data */ \ X1 = _mm_shuffle_epi32(X1, 0x93); \ X2 = _mm_shuffle_epi32(X2, 0x4E); \ X3 = _mm_shuffle_epi32(X3, 0x39); \ /* Operate on "rows" */ \ ARX(X3, X0, X1, 7) \ ARX(X2, X3, X0, 9) \ ARX(X1, X2, X3, 13) \ ARX(X0, X1, X2, 18) \ /* Rearrange data */ \ X1 = _mm_shuffle_epi32(X1, 0x39); \ X2 = _mm_shuffle_epi32(X2, 0x4E); \ X3 = _mm_shuffle_epi32(X3, 0x93); /** * Apply the Salsa20 core to the block provided in (X0 ... X3). */ #define SALSA20_wrapper(out, rounds) { \ __m128i Z0 = X0, Z1 = X1, Z2 = X2, Z3 = X3; \ rounds \ (out).q[0] = X0 = _mm_add_epi32(X0, Z0); \ (out).q[1] = X1 = _mm_add_epi32(X1, Z1); \ (out).q[2] = X2 = _mm_add_epi32(X2, Z2); \ (out).q[3] = X3 = _mm_add_epi32(X3, Z3); \ } /** * Apply the Salsa20/2 core to the block provided in X. */ #define SALSA20_2(out) \ SALSA20_wrapper(out, SALSA20_2ROUNDS) #define SALSA20_8ROUNDS \ SALSA20_2ROUNDS SALSA20_2ROUNDS SALSA20_2ROUNDS SALSA20_2ROUNDS #define XOR_X(in) \ X0 = _mm_xor_si128(X0, (in).q[0]); \ X1 = _mm_xor_si128(X1, (in).q[1]); \ X2 = _mm_xor_si128(X2, (in).q[2]); \ X3 = _mm_xor_si128(X3, (in).q[3]); #define XOR_X_2(in1, in2) \ X0 = _mm_xor_si128((in1).q[0], (in2).q[0]); \ X1 = _mm_xor_si128((in1).q[1], (in2).q[1]); \ X2 = _mm_xor_si128((in1).q[2], (in2).q[2]); \ X3 = _mm_xor_si128((in1).q[3], (in2).q[3]); #define XOR_X_WRITE_XOR_Y_2(out, in) \ (out).q[0] = Y0 = _mm_xor_si128((out).q[0], (in).q[0]); \ (out).q[1] = Y1 = _mm_xor_si128((out).q[1], (in).q[1]); \ (out).q[2] = Y2 = _mm_xor_si128((out).q[2], (in).q[2]); \ (out).q[3] = Y3 = _mm_xor_si128((out).q[3], (in).q[3]); \ X0 = _mm_xor_si128(X0, Y0); \ X1 = _mm_xor_si128(X1, Y1); \ X2 = _mm_xor_si128(X2, Y2); \ X3 = _mm_xor_si128(X3, Y3); /** * Apply the Salsa20/8 core to the block provided in X ^ in. */ #define SALSA20_8_XOR_MEM(in, out) \ XOR_X(in) \ SALSA20_wrapper(out, SALSA20_8ROUNDS) #define INTEGERIFY (uint32_t)_mm_cvtsi128_si32(X0) #else /* !defined(__SSE2__) */ #define DECL_X \ salsa20_blk_t X; #define DECL_Y \ salsa20_blk_t Y; #define COPY(out, in) \ (out).d[0] = (in).d[0]; \ (out).d[1] = (in).d[1]; \ (out).d[2] = (in).d[2]; \ (out).d[3] = (in).d[3]; \ (out).d[4] = (in).d[4]; \ (out).d[5] = (in).d[5]; \ (out).d[6] = (in).d[6]; \ (out).d[7] = (in).d[7]; #define READ_X(in) COPY(X, in) #define WRITE_X(out) COPY(out, X) /** * salsa20(B): * Apply the Salsa20 core to the provided block. */ static inline void salsa20(salsa20_blk_t *restrict B, salsa20_blk_t *restrict Bout, uint32_t doublerounds) { salsa20_blk_t X; #define x X.w salsa20_simd_unshuffle(B, &X); do { #define R(a,b) (((a) << (b)) | ((a) >> (32 - (b)))) /* Operate on columns */ x[ 4] ^= R(x[ 0]+x[12], 7); x[ 8] ^= R(x[ 4]+x[ 0], 9); x[12] ^= R(x[ 8]+x[ 4],13); x[ 0] ^= R(x[12]+x[ 8],18); x[ 9] ^= R(x[ 5]+x[ 1], 7); x[13] ^= R(x[ 9]+x[ 5], 9); x[ 1] ^= R(x[13]+x[ 9],13); x[ 5] ^= R(x[ 1]+x[13],18); x[14] ^= R(x[10]+x[ 6], 7); x[ 2] ^= R(x[14]+x[10], 9); x[ 6] ^= R(x[ 2]+x[14],13); x[10] ^= R(x[ 6]+x[ 2],18); x[ 3] ^= R(x[15]+x[11], 7); x[ 7] ^= R(x[ 3]+x[15], 9); x[11] ^= R(x[ 7]+x[ 3],13); x[15] ^= R(x[11]+x[ 7],18); /* Operate on rows */ x[ 1] ^= R(x[ 0]+x[ 3], 7); x[ 2] ^= R(x[ 1]+x[ 0], 9); x[ 3] ^= R(x[ 2]+x[ 1],13); x[ 0] ^= R(x[ 3]+x[ 2],18); x[ 6] ^= R(x[ 5]+x[ 4], 7); x[ 7] ^= R(x[ 6]+x[ 5], 9); x[ 4] ^= R(x[ 7]+x[ 6],13); x[ 5] ^= R(x[ 4]+x[ 7],18); x[11] ^= R(x[10]+x[ 9], 7); x[ 8] ^= R(x[11]+x[10], 9); x[ 9] ^= R(x[ 8]+x[11],13); x[10] ^= R(x[ 9]+x[ 8],18); x[12] ^= R(x[15]+x[14], 7); x[13] ^= R(x[12]+x[15], 9); x[14] ^= R(x[13]+x[12],13); x[15] ^= R(x[14]+x[13],18); #undef R } while (--doublerounds); #undef x { uint32_t i; salsa20_simd_shuffle(&X, Bout); for (i = 0; i < 16; i += 4) { B->w[i] = Bout->w[i] += B->w[i]; B->w[i + 1] = Bout->w[i + 1] += B->w[i + 1]; B->w[i + 2] = Bout->w[i + 2] += B->w[i + 2]; B->w[i + 3] = Bout->w[i + 3] += B->w[i + 3]; } } #if 0 /* Too expensive */ explicit_bzero(&X, sizeof(X)); #endif } /** * Apply the Salsa20/2 core to the block provided in X. */ #define SALSA20_2(out) \ salsa20(&X, &out, 1); #define XOR(out, in1, in2) \ (out).d[0] = (in1).d[0] ^ (in2).d[0]; \ (out).d[1] = (in1).d[1] ^ (in2).d[1]; \ (out).d[2] = (in1).d[2] ^ (in2).d[2]; \ (out).d[3] = (in1).d[3] ^ (in2).d[3]; \ (out).d[4] = (in1).d[4] ^ (in2).d[4]; \ (out).d[5] = (in1).d[5] ^ (in2).d[5]; \ (out).d[6] = (in1).d[6] ^ (in2).d[6]; \ (out).d[7] = (in1).d[7] ^ (in2).d[7]; #define XOR_X(in) XOR(X, X, in) #define XOR_X_2(in1, in2) XOR(X, in1, in2) #define XOR_X_WRITE_XOR_Y_2(out, in) \ XOR(Y, out, in) \ COPY(out, Y) \ XOR(X, X, Y) /** * Apply the Salsa20/8 core to the block provided in X ^ in. */ #define SALSA20_8_XOR_MEM(in, out) \ XOR_X(in); \ salsa20(&X, &out, 4); #define INTEGERIFY (uint32_t)X.d[0] #endif /** * blockmix_salsa8(Bin, Bout, r): * Compute Bout = BlockMix_{salsa20/8, r}(Bin). The input Bin must be 128r * bytes in length; the output Bout must also be the same size. */ static void blockmix_salsa8(const salsa20_blk_t *restrict Bin, salsa20_blk_t *restrict Bout, size_t r) { size_t i; DECL_X READ_X(Bin[r * 2 - 1]) for (i = 0; i < r; i++) { SALSA20_8_XOR_MEM(Bin[i * 2], Bout[i]) SALSA20_8_XOR_MEM(Bin[i * 2 + 1], Bout[r + i]) } } static uint32_t blockmix_salsa8_xor(const salsa20_blk_t *restrict Bin1, const salsa20_blk_t *restrict Bin2, salsa20_blk_t *restrict Bout, size_t r) { size_t i; DECL_X #ifdef PREFETCH PREFETCH(&Bin2[r * 2 - 1], _MM_HINT_T0) for (i = 0; i < r - 1; i++) { PREFETCH(&Bin2[i * 2], _MM_HINT_T0) PREFETCH(&Bin2[i * 2 + 1], _MM_HINT_T0) } PREFETCH(&Bin2[i * 2], _MM_HINT_T0) #endif XOR_X_2(Bin1[r * 2 - 1], Bin2[r * 2 - 1]) for (i = 0; i < r; i++) { XOR_X(Bin1[i * 2]) SALSA20_8_XOR_MEM(Bin2[i * 2], Bout[i]) XOR_X(Bin1[i * 2 + 1]) SALSA20_8_XOR_MEM(Bin2[i * 2 + 1], Bout[r + i]) } return INTEGERIFY; } /* This is tunable */ #define Swidth 8 /* Not tunable in this implementation, hard-coded in a few places */ #define PWXsimple 2 #define PWXgather 4 /* Derived values. Not tunable except via Swidth above. */ #define PWXbytes (PWXgather * PWXsimple * 8) #define Sbytes (3 * (1 << Swidth) * PWXsimple * 8) #define Smask (((1 << Swidth) - 1) * PWXsimple * 8) #define Smask2 (((uint64_t)Smask << 32) | Smask) #define DECL_SMASK2REG /* empty */ #define FORCE_REGALLOC_3 /* empty */ #define MAYBE_MEMORY_BARRIER /* empty */ #ifdef __SSE2__ /* * (V)PSRLDQ and (V)PSHUFD have higher throughput than (V)PSRLQ on some CPUs * starting with Sandy Bridge. Additionally, PSHUFD uses separate source and * destination registers, whereas the shifts would require an extra move * instruction for our code when building without AVX. Unfortunately, PSHUFD * is much slower on Conroe (4 cycles latency vs. 1 cycle latency for PSRLQ) * and somewhat slower on some non-Intel CPUs (luckily not including AMD * Bulldozer and Piledriver). */ #ifdef __AVX__ #define HI32(X) \ _mm_srli_si128((X), 4) #elif 1 /* As an option, check for __SSE4_1__ here not to hurt Conroe */ #define HI32(X) \ _mm_shuffle_epi32((X), _MM_SHUFFLE(2,3,0,1)) #else #define HI32(X) \ _mm_srli_epi64((X), 32) #endif #if defined(__x86_64__) && \ __GNUC__ == 4 && __GNUC_MINOR__ < 6 && !defined(__ICC) #ifdef __AVX__ #define MOVQ "vmovq" #else /* "movq" would be more correct, but "movd" is supported by older binutils * due to an error in AMD's spec for x86-64. */ #define MOVQ "movd" #endif #define EXTRACT64(X) ({ \ uint64_t result; \ __asm__(MOVQ " %1, %0" : "=r" (result) : "x" (X)); \ result; \ }) #elif defined(__x86_64__) && !defined(_MSC_VER) && !defined(__OPEN64__) /* MSVC and Open64 had bugs */ #define EXTRACT64(X) _mm_cvtsi128_si64(X) #elif defined(__x86_64__) && defined(__SSE4_1__) /* No known bugs for this intrinsic */ #include #define EXTRACT64(X) _mm_extract_epi64((X), 0) #elif defined(USE_SSE4_FOR_32BIT) && defined(__SSE4_1__) /* 32-bit */ #include #if 0 /* This is currently unused by the code below, which instead uses these two * intrinsics explicitly when (!defined(__x86_64__) && defined(__SSE4_1__)) */ #define EXTRACT64(X) \ ((uint64_t)(uint32_t)_mm_cvtsi128_si32(X) | \ ((uint64_t)(uint32_t)_mm_extract_epi32((X), 1) << 32)) #endif #else /* 32-bit or compilers with known past bugs in _mm_cvtsi128_si64() */ #define EXTRACT64(X) \ ((uint64_t)(uint32_t)_mm_cvtsi128_si32(X) | \ ((uint64_t)(uint32_t)_mm_cvtsi128_si32(HI32(X)) << 32)) #endif #if defined(__x86_64__) && (defined(__AVX__) || !defined(__GNUC__)) /* 64-bit with AVX */ /* Force use of 64-bit AND instead of two 32-bit ANDs */ #undef DECL_SMASK2REG #if defined(__GNUC__) && !defined(__ICC) #define DECL_SMASK2REG uint64_t Smask2reg = Smask2; /* Force use of lower-numbered registers to reduce number of prefixes, relying * on out-of-order execution and register renaming. */ #define FORCE_REGALLOC_1 \ __asm__("" : "=a" (x), "+d" (Smask2reg), "+S" (S0), "+D" (S1)); #define FORCE_REGALLOC_2 \ __asm__("" : : "c" (lo)); #else static volatile uint64_t Smask2var = Smask2; #define DECL_SMASK2REG uint64_t Smask2reg = Smask2var; #define FORCE_REGALLOC_1 /* empty */ #define FORCE_REGALLOC_2 /* empty */ #endif #define PWXFORM_SIMD(X) { \ uint64_t x; \ FORCE_REGALLOC_1 \ uint32_t lo = x = EXTRACT64(X) & Smask2reg; \ FORCE_REGALLOC_2 \ uint32_t hi = x >> 32; \ X = _mm_mul_epu32(HI32(X), X); \ X = _mm_add_epi64(X, *(__m128i *)(S0 + lo)); \ X = _mm_xor_si128(X, *(__m128i *)(S1 + hi)); \ } #elif defined(__x86_64__) /* 64-bit without AVX. This relies on out-of-order execution and register * renaming. It may actually be fastest on CPUs with AVX(2) as well - e.g., * it runs great on Haswell. */ #if 0 #warning "Note: using x86-64 inline assembly for YESCRYPT_RW. That's great." #endif /* We need a compiler memory barrier between sub-blocks to ensure that none of * the writes into what was S2 during processing of the previous sub-block are * postponed until after a read from S0 or S1 in the inline asm code below. */ #undef MAYBE_MEMORY_BARRIER #define MAYBE_MEMORY_BARRIER \ __asm__("" : : : "memory"); #ifdef __ILP32__ /* x32 */ #define REGISTER_PREFIX "e" #else #define REGISTER_PREFIX "r" #endif #define PWXFORM_SIMD(X) { \ __m128i H; \ __asm__( \ "movd %0, %%rax\n\t" \ "pshufd $0xb1, %0, %1\n\t" \ "andq %2, %%rax\n\t" \ "pmuludq %1, %0\n\t" \ "movl %%eax, %%ecx\n\t" \ "shrq $0x20, %%rax\n\t" \ "paddq (%3,%%" REGISTER_PREFIX "cx), %0\n\t" \ "pxor (%4,%%" REGISTER_PREFIX "ax), %0\n\t" \ : "+x" (X), "=x" (H) \ : "d" (Smask2), "S" (S0), "D" (S1) \ : "cc", "ax", "cx"); \ } #elif defined(USE_SSE4_FOR_32BIT) && defined(__SSE4_1__) /* 32-bit with SSE4.1 */ #define PWXFORM_SIMD(X) { \ __m128i x = _mm_and_si128(X, _mm_set1_epi64x(Smask2)); \ __m128i s0 = *(__m128i *)(S0 + (uint32_t)_mm_cvtsi128_si32(x)); \ __m128i s1 = *(__m128i *)(S1 + (uint32_t)_mm_extract_epi32(x, 1)); \ X = _mm_mul_epu32(HI32(X), X); \ X = _mm_add_epi64(X, s0); \ X = _mm_xor_si128(X, s1); \ } #else /* 32-bit without SSE4.1 */ #define PWXFORM_SIMD(X) { \ uint64_t x = EXTRACT64(X) & Smask2; \ __m128i s0 = *(__m128i *)(S0 + (uint32_t)x); \ __m128i s1 = *(__m128i *)(S1 + (x >> 32)); \ X = _mm_mul_epu32(HI32(X), X); \ X = _mm_add_epi64(X, s0); \ X = _mm_xor_si128(X, s1); \ } #endif #define PWXFORM_ROUND \ PWXFORM_SIMD(X0) \ PWXFORM_SIMD(X1) \ PWXFORM_SIMD(X2) \ PWXFORM_SIMD(X3) #if defined(__x86_64__) && defined(__GNUC__) && !defined(__ICC) #undef FORCE_REGALLOC_3 #define FORCE_REGALLOC_3 __asm__("" : : "b" (Sw)); #endif #else /* !defined(__SSE2__) */ #define PWXFORM_SIMD(x0, x1) { \ uint64_t x = x0 & Smask2; \ uint64_t *p0 = (uint64_t *)(S0 + (uint32_t)x); \ uint64_t *p1 = (uint64_t *)(S1 + (x >> 32)); \ x0 = ((x0 >> 32) * (uint32_t)x0 + p0[0]) ^ p1[0]; \ x1 = ((x1 >> 32) * (uint32_t)x1 + p0[1]) ^ p1[1]; \ } #define PWXFORM_ROUND \ PWXFORM_SIMD(X.d[0], X.d[1]) \ PWXFORM_SIMD(X.d[2], X.d[3]) \ PWXFORM_SIMD(X.d[4], X.d[5]) \ PWXFORM_SIMD(X.d[6], X.d[7]) #endif /* * This offset helps address the 256-byte write block via the single-byte * displacements encodable in x86(-64) instructions. It is needed because the * displacements are signed. Without it, we'd get 4-byte displacements for * half of the writes. Setting it to 0x80 instead of 0x7c would avoid needing * a displacement for one of the writes, but then the LEA instruction would * need a 4-byte displacement. */ #define PWXFORM_WRITE_OFFSET 0x7c #define PWXFORM_WRITE \ WRITE_X(*(salsa20_blk_t *)(Sw - PWXFORM_WRITE_OFFSET)) \ Sw += 64; #define PWXFORM { \ uint8_t *Sw = S2 + w + PWXFORM_WRITE_OFFSET; \ FORCE_REGALLOC_3 \ MAYBE_MEMORY_BARRIER \ PWXFORM_ROUND \ PWXFORM_ROUND PWXFORM_WRITE \ PWXFORM_ROUND PWXFORM_WRITE \ PWXFORM_ROUND PWXFORM_WRITE \ PWXFORM_ROUND PWXFORM_WRITE \ PWXFORM_ROUND \ w = (w + 64 * 4) & Smask2; \ { \ uint8_t *Stmp = S2; \ S2 = S1; \ S1 = S0; \ S0 = Stmp; \ } \ } typedef struct { uint8_t *S0, *S1, *S2; size_t w; } pwxform_ctx_t; #define Salloc (Sbytes + ((sizeof(pwxform_ctx_t) + 63) & ~63U)) /** * blockmix_pwxform(Bin, Bout, r, S): * Compute Bout = BlockMix_pwxform{salsa20/2, r, S}(Bin). The input Bin must * be 128r bytes in length; the output Bout must also be the same size. */ static void blockmix(const salsa20_blk_t *restrict Bin, salsa20_blk_t *restrict Bout, size_t r, pwxform_ctx_t *restrict ctx) { /* ctx MUST NOT be NULL */ assert(ctx != NULL); uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; size_t w = ctx->w; size_t i; DECL_X /* Convert count of 128-byte blocks to max index of 64-byte block */ r = r * 2 - 1; READ_X(Bin[r]) DECL_SMASK2REG i = 0; do { XOR_X(Bin[i]) PWXFORM if (unlikely(i >= r)) break; WRITE_X(Bout[i]) i++; } while (1); ctx->S0 = S0; ctx->S1 = S1; ctx->S2 = S2; ctx->w = w; SALSA20_2(Bout[i]) } static uint32_t blockmix_xor(const salsa20_blk_t *Bin1, const salsa20_blk_t *restrict Bin2, salsa20_blk_t *Bout, size_t r, int Bin2_in_ROM, pwxform_ctx_t *restrict ctx) { /* ctx MUST NOT be NULL */ assert(ctx != NULL); uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; size_t w = ctx->w; size_t i; DECL_X /* Convert count of 128-byte blocks to max index of 64-byte block */ r = r * 2 - 1; #ifdef PREFETCH if (Bin2_in_ROM) { PREFETCH(&Bin2[r], _MM_HINT_NTA) for (i = 0; i < r; i++) { PREFETCH(&Bin2[i], _MM_HINT_NTA) } } else { PREFETCH(&Bin2[r], _MM_HINT_T0) for (i = 0; i < r; i++) { PREFETCH(&Bin2[i], _MM_HINT_T0) } } #else (void)Bin2_in_ROM; /* unused */ #endif XOR_X_2(Bin1[r], Bin2[r]) DECL_SMASK2REG i = 0; r--; do { XOR_X(Bin1[i]) XOR_X(Bin2[i]) PWXFORM WRITE_X(Bout[i]) XOR_X(Bin1[i + 1]) XOR_X(Bin2[i + 1]) PWXFORM if (unlikely(i >= r)) break; WRITE_X(Bout[i + 1]) i += 2; } while (1); i++; ctx->S0 = S0; ctx->S1 = S1; ctx->S2 = S2; ctx->w = w; SALSA20_2(Bout[i]) return INTEGERIFY; } static uint32_t blockmix_xor_save(salsa20_blk_t *restrict Bin1out, salsa20_blk_t *restrict Bin2, size_t r, pwxform_ctx_t *restrict ctx) { /* ctx MUST NOT be NULL */ assert(ctx != NULL); uint8_t *S0 = ctx->S0, *S1 = ctx->S1, *S2 = ctx->S2; size_t w = ctx->w; size_t i; DECL_X DECL_Y /* Convert count of 128-byte blocks to max index of 64-byte block */ r = r * 2 - 1; #ifdef PREFETCH PREFETCH(&Bin2[r], _MM_HINT_T0) for (i = 0; i < r; i++) { PREFETCH(&Bin2[i], _MM_HINT_T0) } #endif XOR_X_2(Bin1out[r], Bin2[r]) DECL_SMASK2REG i = 0; r--; do { XOR_X_WRITE_XOR_Y_2(Bin2[i], Bin1out[i]) PWXFORM WRITE_X(Bin1out[i]) XOR_X_WRITE_XOR_Y_2(Bin2[i + 1], Bin1out[i + 1]) PWXFORM if (unlikely(i >= r)) break; WRITE_X(Bin1out[i + 1]) i += 2; } while (1); i++; ctx->S0 = S0; ctx->S1 = S1; ctx->S2 = S2; ctx->w = w; SALSA20_2(Bin1out[i]) return INTEGERIFY; } /** * integerify(B, r): * Return the result of parsing B_{2r-1} as a little-endian integer. */ static inline uint32_t integerify(const salsa20_blk_t *B, size_t r) { /* * Our 64-bit words are in host byte order, which is why we don't just read * w[0] here (would be wrong on big-endian). Also, our 32-bit words are * SIMD-shuffled (so the next 32 bits would be part of d[6]), but currently * this does not matter as we only care about the least significant 32 bits. */ return (uint32_t)B[2 * r - 1].d[0]; } /** * smix1(B, r, N, flags, V, NROM, VROM, XY, ctx): * Compute first loop of B = SMix_r(B, N). The input B must be 128r bytes in * length; the temporary storage V must be 128rN bytes in length; the temporary * storage XY must be 128r+64 bytes in length. N must be even and at least 4. * The array V must be aligned to a multiple of 64 bytes, and arrays B and XY * to a multiple of at least 16 bytes. */ static void smix1(uint8_t *B, size_t r, uint32_t N, yescrypt_flags_t flags, salsa20_blk_t *V, uint32_t NROM, const salsa20_blk_t *VROM, salsa20_blk_t *XY, pwxform_ctx_t *ctx) { size_t s = 2 * r; salsa20_blk_t *X = V, *Y = &V[s]; uint32_t i, j; for (i = 0; i < 2 * r; i++) { const salsa20_blk_t *src = (salsa20_blk_t *)&B[i * 64]; salsa20_blk_t *tmp = Y; salsa20_blk_t *dst = &X[i]; size_t k; for (k = 0; k < 16; k++) tmp->w[k] = le32dec((const uint8_t *) &src->w[k]); salsa20_simd_shuffle(tmp, dst); } if (VROM) { uint32_t n; const salsa20_blk_t *V_j; V_j = &VROM[(NROM - 1) * s]; j = blockmix_xor(X, V_j, Y, r, 1, ctx) & (NROM - 1); V_j = &VROM[j * s]; X = Y + s; j = blockmix_xor(Y, V_j, X, r, 1, ctx); for (n = 2; n < N; n <<= 1) { uint32_t m = (n < N / 2) ? n : (N - 1 - n); for (i = 1; i < m; i += 2) { j &= n - 1; j += i - 1; V_j = &V[j * s]; Y = X + s; j = blockmix_xor(X, V_j, Y, r, 0, ctx) & (NROM - 1); V_j = &VROM[j * s]; X = Y + s; j = blockmix_xor(Y, V_j, X, r, 1, ctx); } } n >>= 1; j &= n - 1; j += N - 2 - n; V_j = &V[j * s]; Y = X + s; j = blockmix_xor(X, V_j, Y, r, 0, ctx) & (NROM - 1); V_j = &VROM[j * s]; blockmix_xor(Y, V_j, XY, r, 1, ctx); } else if (flags & YESCRYPT_RW) { uint32_t n; salsa20_blk_t *V_j; blockmix(X, Y, r, ctx); X = Y + s; blockmix(Y, X, r, ctx); j = integerify(X, r); for (n = 2; n < N; n <<= 1) { uint32_t m = (n < N / 2) ? n : (N - 1 - n); for (i = 1; i < m; i += 2) { Y = X + s; j &= n - 1; j += i - 1; V_j = &V[j * s]; j = blockmix_xor(X, V_j, Y, r, 0, ctx); j &= n - 1; j += i; V_j = &V[j * s]; X = Y + s; j = blockmix_xor(Y, V_j, X, r, 0, ctx); } } n >>= 1; j &= n - 1; j += N - 2 - n; V_j = &V[j * s]; Y = X + s; j = blockmix_xor(X, V_j, Y, r, 0, ctx); j &= n - 1; j += N - 1 - n; V_j = &V[j * s]; blockmix_xor(Y, V_j, XY, r, 0, ctx); } else { N -= 2; do { blockmix_salsa8(X, Y, r); X = Y + s; blockmix_salsa8(Y, X, r); Y = X + s; } while ((N -= 2)); blockmix_salsa8(X, Y, r); blockmix_salsa8(Y, XY, r); } for (i = 0; i < 2 * r; i++) { const salsa20_blk_t *src = &XY[i]; salsa20_blk_t *tmp = &XY[s]; salsa20_blk_t *dst = (salsa20_blk_t *)&B[i * 64]; size_t k; for (k = 0; k < 16; k++) le32enc((uint8_t *)&tmp->w[k], src->w[k]); salsa20_simd_unshuffle(tmp, dst); } } /** * smix2(B, r, N, Nloop, flags, V, NROM, VROM, XY, ctx): * Compute second loop of B = SMix_r(B, N). The input B must be 128r bytes in * length; the temporary storage V must be 128rN bytes in length; the temporary * storage XY must be 256r bytes in length. N must be a power of 2 and at * least 2. Nloop must be even. The array V must be aligned to a multiple of * 64 bytes, and arrays B and XY to a multiple of at least 16 bytes. */ static void smix2(uint8_t *B, size_t r, uint32_t N, uint64_t Nloop, yescrypt_flags_t flags, salsa20_blk_t *V, uint32_t NROM, const salsa20_blk_t *VROM, salsa20_blk_t *XY, pwxform_ctx_t *ctx) { size_t s = 2 * r; salsa20_blk_t *X = XY, *Y = &XY[s]; uint32_t i, j; if (Nloop == 0) return; for (i = 0; i < 2 * r; i++) { const salsa20_blk_t *src = (salsa20_blk_t *)&B[i * 64]; salsa20_blk_t *tmp = Y; salsa20_blk_t *dst = &X[i]; size_t k; for (k = 0; k < 16; k++) tmp->w[k] = le32dec((const uint8_t *)&src->w[k]); salsa20_simd_shuffle(tmp, dst); } j = integerify(X, r) & (N - 1); /* * Normally, VROM implies YESCRYPT_RW, but we check for these separately * because our SMix resets YESCRYPT_RW for the smix2() calls operating on the * entire V when p > 1. */ if (VROM && (flags & YESCRYPT_RW)) { do { salsa20_blk_t *V_j = &V[j * s]; const salsa20_blk_t *VROM_j; j = blockmix_xor_save(X, V_j, r, ctx) & (NROM - 1); VROM_j = &VROM[j * s]; j = blockmix_xor(X, VROM_j, X, r, 1, ctx) & (N - 1); } while (Nloop -= 2); } else if (VROM) { do { const salsa20_blk_t *V_j = &V[j * s]; j = blockmix_xor(X, V_j, X, r, 0, ctx) & (NROM - 1); V_j = &VROM[j * s]; j = blockmix_xor(X, V_j, X, r, 1, ctx) & (N - 1); } while (Nloop -= 2); } else if (flags & YESCRYPT_RW) { do { salsa20_blk_t *V_j = &V[j * s]; j = blockmix_xor_save(X, V_j, r, ctx) & (N - 1); V_j = &V[j * s]; j = blockmix_xor_save(X, V_j, r, ctx) & (N - 1); } while (Nloop -= 2); } else if (ctx) { do { const salsa20_blk_t *V_j = &V[j * s]; j = blockmix_xor(X, V_j, X, r, 0, ctx) & (N - 1); V_j = &V[j * s]; j = blockmix_xor(X, V_j, X, r, 0, ctx) & (N - 1); } while (Nloop -= 2); } else { do { const salsa20_blk_t *V_j = &V[j * s]; j = blockmix_salsa8_xor(X, V_j, Y, r) & (N - 1); V_j = &V[j * s]; j = blockmix_salsa8_xor(Y, V_j, X, r) & (N - 1); } while (Nloop -= 2); } for (i = 0; i < 2 * r; i++) { const salsa20_blk_t *src = &X[i]; salsa20_blk_t *tmp = Y; salsa20_blk_t *dst = (salsa20_blk_t *)&B[i * 64]; size_t k; for (k = 0; k < 16; k++) le32enc((uint8_t *)&tmp->w[k], src->w[k]); salsa20_simd_unshuffle(tmp, dst); } } /** * p2floor(x): * Largest power of 2 not greater than argument. */ static uint64_t p2floor(uint64_t x) { uint64_t y; while ((y = x & (x - 1))) x = y; return x; } /** * smix(B, r, N, p, t, flags, V, NROM, VROM, XY, S, passwd): * Compute B = SMix_r(B, N). The input B must be 128rp bytes in length; the * temporary storage V must be 128rN bytes in length; the temporary storage * XY must be 256r or 256rp bytes in length (the larger size is required with * OpenMP-enabled builds). N must be a power of 2 and at least 4. The array V * must be aligned to a multiple of 64 bytes, and arrays B and XY to a multiple * of at least 16 bytes (aligning them to 64 bytes as well saves cache lines * and helps avoid false sharing in OpenMP-enabled builds when p > 1, but it * might also result in cache bank conflicts). */ static void smix(uint8_t *B, size_t r, uint32_t N, uint32_t p, uint32_t t, yescrypt_flags_t flags, salsa20_blk_t *V, uint32_t NROM, const salsa20_blk_t *VROM, salsa20_blk_t *XY, uint8_t *S, uint8_t *passwd) { size_t s = 2 * r; uint32_t Nchunk; uint64_t Nloop_all, Nloop_rw; uint32_t i; Nchunk = N / p; Nloop_all = Nchunk; if (flags & YESCRYPT_RW) { if (t <= 1) { if (t) Nloop_all *= 2; /* 2/3 */ Nloop_all = (Nloop_all + 2) / 3; /* 1/3, round up */ } else { Nloop_all *= t - 1; } } else if (t) { if (t == 1) Nloop_all += (Nloop_all + 1) / 2; /* 1.5, round up */ Nloop_all *= t; } Nloop_rw = 0; if (flags & YESCRYPT_INIT_SHARED) Nloop_rw = Nloop_all; else if (flags & YESCRYPT_RW) Nloop_rw = Nloop_all / p; Nchunk &= ~(uint32_t)1; /* round down to even */ Nloop_all++; Nloop_all &= ~(uint64_t)1; /* round up to even */ Nloop_rw++; Nloop_rw &= ~(uint64_t)1; /* round up to even */ #ifdef _OPENMP #pragma omp parallel if (p > 1) default(none) private(i) shared(B, r, N, p, flags, V, NROM, VROM, XY, S, passwd, s, Nchunk, Nloop_all, Nloop_rw) { #pragma omp for #endif for (i = 0; i < p; i++) { uint32_t Vchunk = i * Nchunk; uint32_t Np = (i < p - 1) ? Nchunk : (N - Vchunk); uint8_t *Bp = &B[128 * r * i]; salsa20_blk_t *Vp = &V[Vchunk * s]; #ifdef _OPENMP salsa20_blk_t *XYp = &XY[i * (2 * s)]; #else salsa20_blk_t *XYp = XY; #endif pwxform_ctx_t *ctx_i = NULL; if (flags & YESCRYPT_RW) { uint8_t *Si = S + i * Salloc; smix1(Bp, 1, Sbytes / 128, 0 /* no flags */, (salsa20_blk_t *)Si, 0, NULL, XYp, NULL); ctx_i = (pwxform_ctx_t *)(Si + Sbytes); ctx_i->S2 = Si; ctx_i->S1 = Si + Sbytes / 3; ctx_i->S0 = Si + Sbytes / 3 * 2; ctx_i->w = 0; if (i == 0) HMAC_SHA256_Buf(Bp + (128 * r - 64), 64, passwd, 32, passwd); } smix1(Bp, r, Np, flags, Vp, NROM, VROM, XYp, ctx_i); smix2(Bp, r, p2floor(Np), Nloop_rw, flags, Vp, NROM, VROM, XYp, ctx_i); } if (Nloop_all > Nloop_rw) { #ifdef _OPENMP #pragma omp for #endif for (i = 0; i < p; i++) { uint8_t *Bp = &B[128 * r * i]; #ifdef _OPENMP salsa20_blk_t *XYp = &XY[i * (2 * s)]; #else salsa20_blk_t *XYp = XY; #endif pwxform_ctx_t *ctx_i = NULL; if (flags & YESCRYPT_RW) { uint8_t *Si = S + i * Salloc; ctx_i = (pwxform_ctx_t *)(Si + Sbytes); } smix2(Bp, r, N, Nloop_all - Nloop_rw, flags & (yescrypt_flags_t)~YESCRYPT_RW, V, NROM, VROM, XYp, ctx_i); } } #ifdef _OPENMP } #endif } /** * yescrypt_kdf_body(shared, local, passwd, passwdlen, salt, saltlen, * flags, N, r, p, t, NROM, buf, buflen): * Compute scrypt(passwd[0 .. passwdlen - 1], salt[0 .. saltlen - 1], N, r, * p, buflen), or a revision of scrypt as requested by flags and shared, and * write the result into buf. * * shared and flags may request special modes as described in yescrypt.h. * * local is the thread-local data structure, allowing to preserve and reuse a * memory allocation across calls, thereby reducing its overhead. * * t controls computation time while not affecting peak memory usage. * * Return 0 on success; or -1 on error. * * This optimized implementation currently limits N to the range from 4 to * 2^31, but other implementations might not. */ static int yescrypt_kdf_body(const yescrypt_shared_t *shared, yescrypt_local_t *local, const uint8_t *passwd, size_t passwdlen, const uint8_t *salt, size_t saltlen, yescrypt_flags_t flags, uint64_t N, uint32_t r, uint32_t p, uint32_t t, uint64_t NROM, uint8_t *buf, size_t buflen) { yescrypt_region_t tmp; const salsa20_blk_t *VROM; size_t B_size, V_size, XY_size, need; uint8_t *B, *S; salsa20_blk_t *V, *XY; uint8_t sha256[32]; uint8_t dk[sizeof(sha256)], *dkp = buf; /* Sanity-check parameters */ switch (flags & YESCRYPT_MODE_MASK) { case 0: /* classic scrypt - can't have anything non-standard */ if (flags || t || NROM) goto out_EINVAL; break; case YESCRYPT_WORM: if (flags != YESCRYPT_WORM || NROM) goto out_EINVAL; break; case YESCRYPT_RW: if (flags != (flags & YESCRYPT_KNOWN_FLAGS)) goto out_EINVAL; #if PWXsimple == 2 && PWXgather == 4 && Sbytes == 12288 if ((flags & YESCRYPT_RW_FLAVOR_MASK) == (YESCRYPT_ROUNDS_6 | YESCRYPT_GATHER_4 | YESCRYPT_SIMPLE_2 | YESCRYPT_SBOX_12K)) break; #else #error "Unsupported pwxform settings" #endif /* FALLTHRU */ default: goto out_EINVAL; } #if SIZE_MAX > UINT32_MAX if (buflen > (((uint64_t)1 << 32) - 1) * 32) goto out_EINVAL; #endif if ((uint64_t)r * (uint64_t)p >= 1 << 30) goto out_EINVAL; if (N > UINT32_MAX) goto out_EINVAL; if ((N & (N - 1)) != 0 || N <= 3 || r < 1 || p < 1) goto out_EINVAL; if (r > SIZE_MAX / 256 / p || N > SIZE_MAX / 128 / r) goto out_EINVAL; if (flags & YESCRYPT_RW) { /* p cannot be greater than SIZE_MAX/Salloc on 64-bit systems, but it can on 32-bit systems. */ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wtype-limits" if (N / p <= 3 || p > SIZE_MAX / Salloc) goto out_EINVAL; #pragma GCC diagnostic pop } #ifdef _OPENMP else if (N > SIZE_MAX / 128 / (r * p)) { goto out_EINVAL; } #endif VROM = NULL; if (shared) { uint64_t expected_size = (size_t)128 * r * NROM; if ((NROM & (NROM - 1)) != 0 || NROM <= 1 || NROM > UINT32_MAX || shared->aligned_size < expected_size) goto out_EINVAL; if (!(flags & YESCRYPT_INIT_SHARED)) { uint64_t *tag = (uint64_t *) ((uint8_t *)shared->aligned + expected_size - 48); if (tag[0] != YESCRYPT_ROM_TAG1 || tag[1] != YESCRYPT_ROM_TAG2) goto out_EINVAL; } VROM = shared->aligned; } else { if (NROM) goto out_EINVAL; } /* Allocate memory */ V = NULL; V_size = (size_t)128 * r * N; #ifdef _OPENMP if (!(flags & YESCRYPT_RW)) V_size *= p; #endif need = V_size; if (flags & YESCRYPT_INIT_SHARED) { if (local->aligned_size < need) { if (local->base || local->aligned || local->base_size || local->aligned_size) goto out_EINVAL; if (!alloc_region(local, need)) return -1; } if (flags & YESCRYPT_ALLOC_ONLY) return -2; /* expected "failure" */ V = (salsa20_blk_t *)local->aligned; need = 0; } B_size = (size_t)128 * r * p; need += B_size; if (need < B_size) goto out_EINVAL; XY_size = (size_t)256 * r; #ifdef _OPENMP XY_size *= p; #endif need += XY_size; if (need < XY_size) goto out_EINVAL; if (flags & YESCRYPT_RW) { size_t S_size = (size_t)Salloc * p; need += S_size; if (need < S_size) goto out_EINVAL; } if (flags & YESCRYPT_INIT_SHARED) { if (!alloc_region(&tmp, need)) return -1; B = (uint8_t *)tmp.aligned; XY = (salsa20_blk_t *)((uint8_t *)B + B_size); } else { init_region(&tmp); if (local->aligned_size < need) { if (free_region(local)) return -1; if (!alloc_region(local, need)) return -1; } if (flags & YESCRYPT_ALLOC_ONLY) return -3; /* expected "failure" */ B = (uint8_t *)local->aligned; V = (salsa20_blk_t *)((uint8_t *)B + B_size); XY = (salsa20_blk_t *)((uint8_t *)V + V_size); } S = NULL; if (flags & YESCRYPT_RW) S = (uint8_t *)XY + XY_size; if (flags) { HMAC_SHA256_Buf("yescrypt-prehash", (flags & YESCRYPT_PREHASH) ? 16 : 8, passwd, passwdlen, sha256); passwd = sha256; passwdlen = sizeof(sha256); } PBKDF2_SHA256(passwd, passwdlen, salt, saltlen, 1, B, B_size); if (flags) memcpy(sha256, B, sizeof(sha256)); if (p == 1 || (flags & YESCRYPT_RW)) { smix(B, r, N, p, t, flags, V, NROM, VROM, XY, S, sha256); } else { uint32_t i; #ifdef _OPENMP #pragma omp parallel for default(none) private(i) shared(B, r, N, p, t, flags, V, NROM, VROM, XY, S) #endif for (i = 0; i < p; i++) { #ifdef _OPENMP smix(&B[(size_t)128 * r * i], r, N, 1, t, flags, &V[(size_t)2 * r * i * N], NROM, VROM, &XY[(size_t)4 * r * i], NULL, NULL); #else smix(&B[(size_t)128 * r * i], r, N, 1, t, flags, V, NROM, VROM, XY, NULL, NULL); #endif } } dkp = buf; if (flags && buflen < sizeof(dk)) { PBKDF2_SHA256(passwd, passwdlen, B, B_size, 1, dk, sizeof(dk)); dkp = dk; } PBKDF2_SHA256(passwd, passwdlen, B, B_size, 1, buf, buflen); /* * Except when computing classic scrypt, allow all computation so far * to be performed on the client. The final steps below match those of * SCRAM (RFC 5802), so that an extension of SCRAM (with the steps so * far in place of SCRAM's use of PBKDF2 and with SHA-256 in place of * SCRAM's use of SHA-1) would be usable with yescrypt hashes. */ if (flags && !(flags & YESCRYPT_PREHASH)) { /* Compute ClientKey */ HMAC_SHA256_Buf(dkp, sizeof(dk), "Client Key", 10, sha256); /* Compute StoredKey */ { size_t clen = buflen; if (clen > sizeof(dk)) clen = sizeof(dk); SHA256_Buf(sha256, sizeof(sha256), dk); memcpy(buf, dk, clen); } } if (flags) { explicit_bzero(sha256, sizeof(sha256)); explicit_bzero(dk, sizeof(dk)); } if (free_region(&tmp)) { explicit_bzero(buf, buflen); /* must preserve errno */ return -1; } /* Success! */ return 0; out_EINVAL: errno = EINVAL; return -1; } /** * yescrypt_kdf(shared, local, passwd, passwdlen, salt, saltlen, params, * buf, buflen): * Compute scrypt or its revision as requested by the parameters. The inputs * to this function are the same as those for yescrypt_kdf_body() above, with * the addition of g, which controls hash upgrades (0 for no upgrades so far). */ int yescrypt_kdf(const yescrypt_shared_t *shared, yescrypt_local_t *local, const uint8_t *passwd, size_t passwdlen, const uint8_t *salt, size_t saltlen, const yescrypt_params_t *params, uint8_t *buf, size_t buflen) { yescrypt_flags_t flags = params->flags; uint64_t N = params->N; uint32_t r = params->r; uint32_t p = params->p; uint32_t t = params->t; uint32_t g = params->g; uint64_t NROM = params->NROM; uint8_t dk[32]; int retval; /* Support for hash upgrades has been temporarily removed */ if (g) { errno = EINVAL; return -1; } if ((flags & (YESCRYPT_RW | YESCRYPT_INIT_SHARED)) == YESCRYPT_RW && p >= 1 && N / p >= 0x100 && N / p * r >= 0x20000) { if (yescrypt_kdf_body(shared, local, passwd, passwdlen, salt, saltlen, flags | YESCRYPT_ALLOC_ONLY, N, r, p, t, NROM, buf, buflen) != -3) { errno = EINVAL; return -1; } if ((retval = yescrypt_kdf_body(shared, local, passwd, passwdlen, salt, saltlen, flags | YESCRYPT_PREHASH, N >> 6, r, p, 0, NROM, dk, sizeof(dk)))) return retval; passwd = dk; passwdlen = sizeof(dk); } retval = yescrypt_kdf_body(shared, local, passwd, passwdlen, salt, saltlen, flags, N, r, p, t, NROM, buf, buflen); #ifndef SKIP_MEMZERO if (passwd == dk) explicit_bzero(dk, sizeof(dk)); #endif return retval; } int yescrypt_init_shared(yescrypt_shared_t *shared, const uint8_t *seed, size_t seedlen, const yescrypt_params_t *params) { yescrypt_params_t subparams; yescrypt_shared_t half1, half2; uint8_t salt[32]; uint64_t *tag; subparams = *params; subparams.flags |= YESCRYPT_INIT_SHARED; subparams.N = params->NROM; subparams.NROM = 0; if (!(params->flags & YESCRYPT_RW) || params->N || params->g) return -1; if (params->flags & YESCRYPT_SHARED_PREALLOCATED) { if (!shared->aligned || !shared->aligned_size) return -1; /* Overwrite a possible old ROM tag before we overwrite the rest */ tag = (uint64_t *) ((uint8_t *)shared->aligned + shared->aligned_size - 48); memset(tag, 0, 48); } else { init_region(shared); subparams.flags |= YESCRYPT_ALLOC_ONLY; if (yescrypt_kdf(NULL, shared, NULL, 0, NULL, 0, &subparams, NULL, 0) != -2 || !shared->aligned) return -1; subparams.flags -= YESCRYPT_ALLOC_ONLY; } subparams.N /= 2; half1 = *shared; half1.aligned_size /= 2; half2 = half1; half2.aligned = (uint8_t *)half2.aligned + half1.aligned_size; if (yescrypt_kdf(NULL, &half1, seed, seedlen, (const uint8_t *)"yescrypt-ROMhash", 16, &subparams, salt, sizeof(salt))) goto fail; subparams.NROM = subparams.N; if (yescrypt_kdf(&half1, &half2, seed, seedlen, salt, sizeof(salt), &subparams, salt, sizeof(salt))) goto fail; if (yescrypt_kdf(&half2, &half1, seed, seedlen, salt, sizeof(salt), &subparams, salt, sizeof(salt))) goto fail; tag = (uint64_t *) ((uint8_t *)shared->aligned + shared->aligned_size - 48); tag[0] = YESCRYPT_ROM_TAG1; tag[1] = YESCRYPT_ROM_TAG2; tag[2] = le64dec(salt); tag[3] = le64dec(salt + 8); tag[4] = le64dec(salt + 16); tag[5] = le64dec(salt + 24); explicit_bzero(salt, sizeof(salt)); return 0; fail: explicit_bzero(salt, sizeof(salt)); if (!(params->flags & YESCRYPT_SHARED_PREALLOCATED)) free_region(shared); return -1; } yescrypt_binary_t *yescrypt_digest_shared(yescrypt_shared_t *shared) { static yescrypt_binary_t digest; uint64_t *tag; if (shared->aligned_size < 48) return NULL; tag = (uint64_t *) ((uint8_t *)shared->aligned + shared->aligned_size - 48); if (tag[0] != YESCRYPT_ROM_TAG1 || tag[1] != YESCRYPT_ROM_TAG2) return NULL; le64enc(digest.uc, tag[2]); le64enc(digest.uc + 8, tag[3]); le64enc(digest.uc + 16, tag[4]); le64enc(digest.uc + 24, tag[5]); return &digest; } int yescrypt_free_shared(yescrypt_shared_t *shared) { return free_region(shared); } int yescrypt_init_local(yescrypt_local_t *local) { init_region(local); return 0; } int yescrypt_free_local(yescrypt_local_t *local) { return free_region(local); } #endif /* INCLUDE_yescrypt || INCLUDE_scrypt || INCLUDE_gost_yescrypt */