1#if !defined(CURAND_KERNEL_H_)
2# define CURAND_KERNEL_H_
9# define SKIPAHEAD_BLOCKSIZE (4)
10# define SKIPAHEAD_MASK ((1 << SKIPAHEAD_BLOCKSIZE) - 1)
11# define CURAND_2POW32_INV (2.3283064e-10f)
12# define CURAND_2POW32_INV_DOUBLE (2.3283064365386963e-10)
13# define CURAND_2POW53_INV_DOUBLE (1.1102230246251565e-16)
14# define CURAND_2POW32_INV_2PI (2.3283064e-10f * 6.2831855f)
15# define CURAND_2POW53_INV_2PI_DOUBLE \
16 (1.1102230246251565e-16 * 6.2831853071795860)
17# define CURAND_SQRT2 (-1.4142135f)
18# define CURAND_SQRT2_DOUBLE (-1.4142135623730951)
19# define PRECALC_NUM_MATRICES (8)
20# define PRECALC_BLOCK_SIZE (2)
21# define PRECALC_BLOCK_MASK ((1 << PRECALC_BLOCK_SIZE) - 1)
22# define XORWOW_SEQUENCE_SPACING (67)
26# if !defined(QUALIFIERS)
27# define QUALIFIERS static inline __device__
117 unsigned long long i,
x;
134 unsigned long long i,
x,
c;
159 unsigned int *vector,
160 unsigned int *matrix,
161 unsigned int *result,
164 for (
int i = 0; i < n; i++)
168 for (
int i = 0; i < n; i++)
170 for (
int j = 0; j < 32; j++)
172 if (vector[i] & (1 << j))
174 for (
int k = 0; k < n; k++)
176 result[k] ^= matrix[n * (i * 32 + j) + k];
187 for (
int i = 0; i < n * 32; i++)
189 for (
int j = 0; j < n; j++)
194 matrix[i * n + j] = (1 << r);
198 matrix[i * n + j] = 0;
210 for (
int i = 0; i < n * 32; i++)
213 for (
int j = 0; j < n; j++)
215 matrixA[i * n + j] = result[j];
224 for (
int i = 0; i < n; i++)
226 vector[i] = vectorA[i];
234 for (
int i = 0; i < n * n * 32; i++)
236 matrix[i] = matrixA[i];
242 unsigned int *matrix,
243 unsigned int *matrixA,
244 unsigned long long p,
293# if __CUDA_ARCH__ > 0
294 return __int_as_float(x);
295# elif !defined(__CUDA_ARCH__)
319 unsigned long long seed,
320 unsigned long long subsequence,
321 unsigned long long offset,
324 state->
v = (
unsigned int)(seed * 3) + (
unsigned int)(subsequence * 31337) +
325 (
unsigned int)offset;
330 unsigned int r = state->
v++;
336 state->
v += (
unsigned int)n;
341template <
typename T,
int n>
349 for (
int i = 0; i < 32 * n; i++)
352 for (
int j = 0; j < n; j++)
356 state.v[i / 32] = (1 << (i & 31));
358 for (
int j = 0; j < n; j++)
360 matrix[i * n + j] = state.v[j];
365template <
typename T,
int n>
370 unsigned int *matrix = scratch;
372 unsigned int *matrixA = scratch + (n * n * 32);
374 unsigned int *vector = scratch + (n * n * 32) + (n * n * 32);
376 unsigned int *result = scratch + (n * n * 32) + (n * n * 32) + n;
377 unsigned long long p = x;
378 for (
int i = 0; i < n; i++)
380 vector[i] = state->v[i];
389 vector, precalc_xorwow_offset_matrix[matrix_num], result, n);
430 for (
int i = 0; i < n; i++)
432 state->v[i] = vector[i];
434 state->d += 362437 * (
unsigned int)x;
437template <
typename T,
int n>
439 unsigned long long x,
441 unsigned int *scratch)
444 unsigned int *matrix = scratch;
446 unsigned int *matrixA = scratch + (n * n * 32);
448 unsigned int *vector = scratch + (n * n * 32) + (n * n * 32);
450 unsigned int *result = scratch + (n * n * 32) + (n * n * 32) + n;
451 unsigned long long p = x;
452 for (
int i = 0; i < n; i++)
454 vector[i] = state->v[i];
503 for (
int i = 0; i < n; i++)
505 state->v[i] = vector[i];
523 unsigned int scratch[5 * 5 * 32 * 2 + 5 * 2];
544 unsigned int scratch[5 * 5 * 32 * 2 + 5 * 2];
546 n, state, (
unsigned int *)scratch);
550 unsigned long long seed,
551 unsigned long long subsequence,
552 unsigned long long offset,
554 unsigned int *scratch)
558 unsigned int s0 = ((
unsigned int)seed) ^ 0xaad26b49UL;
560 s1 = (
unsigned int)(seed >> 32) ^ 0xf7dcefddUL;
563 unsigned int t0 = 1099087573UL * s0;
564 unsigned int t1 = 2591861531UL * s1;
565 state->
d = 6615241 + t1 + t0;
566 state->
v[0] = 123456789UL + t0;
567 state->
v[1] = 362436069UL ^ t0;
568 state->
v[2] = 521288629UL + t1;
569 state->
v[3] = 88675123UL ^ t1;
570 state->
v[4] = 5783321UL + t0;
573 subsequence, state, scratch);
596 unsigned long long seed,
597 unsigned long long subsequence,
598 unsigned long long offset,
601 unsigned int scratch[5 * 5 * 32 * 2 + 5 * 2];
603 seed, subsequence, offset, state, (
unsigned int *)scratch);
619 t = (state->
v[0] ^ (state->
v[0] >> 2));
620 state->
v[0] = state->
v[1];
621 state->
v[1] = state->
v[2];
622 state->
v[2] = state->
v[3];
623 state->
v[3] = state->
v[4];
624 state->
v[4] = (state->
v[4] ^ (state->
v[4] << 4)) ^ (t ^ (t << 1));
626 return state->
v[4] + state->
d;
647 i_gray = state->i ^ (state->i >> 1);
648 for (
unsigned int k = 0; k < 32; k++)
650 if (i_gray & (1 << k))
652 state->x ^= state->direction_vectors[k];
671 unsigned long long i_gray;
674 i_gray = state->i ^ (state->i >> 1);
675 for (
unsigned k = 0; k < 64; k++)
677 if (i_gray & (1ULL << k))
679 state->x ^= state->direction_vectors[k];
746template <
typename XT>
749# if __CUDA_ARCH__ > 0
750 unsigned long long z = x;
751 int y = __ffsll(~z) | 0x40;
752 return (y - 1) & 0x3F;
754 unsigned long long z = x;
784 unsigned long long offset,
788 for (
int i = 0; i < 64; i++)
796template <
typename PT>
800 unsigned int shifted_i = state->i >> n_log2;
801 state->x ^= state->direction_vectors[n_log2 - 1];
804 state->i += 1 << n_log2;
826 unsigned long long scramble_c,
827 unsigned long long offset,
831 for (
int i = 0; i < 64; i++)
835 state->
x = scramble_c;
855 unsigned int res = state->
x;
877 unsigned int res = state->
x;
900 unsigned long long res = state->
x;
923 unsigned long long res = state->
x;
939 t = (
unsigned int)(x >> 32);
956 unsigned long long z =
957 (
unsigned long long)x ^ ((
unsigned long long)y << (53 - 32));
963 unsigned long long z;
964 z = (
unsigned long long)x >> (64 - 53);
double _curand_uniform_double(unsigned int x)
Definition curand_kernel.h:943
static unsigned int precalc_xorwow_matrix[8][2]
Definition curand_kernel.h:24
void __curand_matidentity(unsigned int *matrix, int n)
Definition curand_kernel.h:184
double _curand_uniform_double_hq(unsigned int x, unsigned int y)
Definition curand_kernel.h:954
#define PRECALC_BLOCK_SIZE
Definition curand_kernel.h:20
double curand_uniform_double(curandStateTest_t *state)
Definition curand_kernel.h:973
struct curandStateXORWOW curandState_t
Default RNG.
Definition curand_kernel.h:146
struct curandStateXORWOW curandState
Definition curand_kernel.h:147
#define SKIPAHEAD_BLOCKSIZE
Definition curand_kernel.h:9
int __curand_find_trailing_zero(XT x)
Initialize Sobol32 state.
Definition curand_kernel.h:747
float _curand_uniform(unsigned int x)
Definition curand_kernel.h:931
#define CURAND_2POW53_INV_DOUBLE
Definition curand_kernel.h:13
void _skipahead_sequence_scratch(unsigned long long x, T *state, unsigned int *scratch)
Definition curand_kernel.h:438
void __curand_veccopy(unsigned int *vector, unsigned int *vectorA, int n)
Definition curand_kernel.h:222
#define MAX_XOR_N
Definition curand_kernel.h:8
void __curand_matmat(unsigned int *matrixA, unsigned int *matrixB, int n)
Definition curand_kernel.h:207
#define PRECALC_BLOCK_MASK
Definition curand_kernel.h:21
double _curand_uniform_double_64(unsigned long long x)
Definition curand_kernel.h:961
struct curandStateXORWOW curandStateXORWOW_t
CURAND XORWOW state.
Definition curand_kernel.h:63
void __curand_generate_skipahead_matrix_xor(unsigned int matrix[])
Definition curand_kernel.h:343
void _curand_init_scratch(unsigned long long seed, unsigned long long subsequence, unsigned long long offset, curandStateXORWOW_t *state, unsigned int *scratch)
Definition curand_kernel.h:549
double __curand_hilouint32AsDouble(unsigned int hi, unsigned int lo)
Definition curand_kernel.h:277
#define CURAND_2POW32_INV_DOUBLE
Definition curand_kernel.h:12
struct curandStateScrambledSobol64 curandStateScrambledSobol64_t
CURAND Scrambled Sobol64 state.
Definition curand_kernel.h:141
unsigned int curand(curandStateTest_t *state)
Definition curand_kernel.h:328
void __curand_matpow(unsigned int *matrix, unsigned int *matrixA, unsigned long long p, int n)
Definition curand_kernel.h:241
void skipahead(unsigned long long n, curandStateTest_t *state)
Definition curand_kernel.h:334
struct curandStateTest curandStateTest_t
Definition curand_kernel.h:41
void _skipahead_scratch(unsigned long long x, T *state, unsigned int *scratch)
Definition curand_kernel.h:367
void skipahead_sequence(unsigned long long n, curandStateXORWOW_t *state)
Update XORWOW state to skip ahead n subsequences.
Definition curand_kernel.h:542
void _skipahead_stride(int n_log2, PT state)
Definition curand_kernel.h:797
void __curand_matvec(unsigned int *vector, unsigned int *matrix, unsigned int *result, int n)
Definition curand_kernel.h:158
float curand_uniform(curandStateTest_t *state)
Definition curand_kernel.h:968
struct curandStateSobol64 curandStateSobol64_t
CURAND Sobol64 state.
Definition curand_kernel.h:124
struct curandStateSobol32 curandStateSobol32_t
CURAND Sobol32 state.
Definition curand_kernel.h:90
#define CURAND_2POW32_INV
Definition curand_kernel.h:11
#define PRECALC_NUM_MATRICES
Definition curand_kernel.h:19
struct curandStateScrambledSobol32 curandStateScrambledSobol32_t
CURAND Scrambled Sobol32 state.
Definition curand_kernel.h:107
void curand_init(unsigned long long seed, unsigned long long subsequence, unsigned long long offset, curandStateTest_t *state)
Definition curand_kernel.h:318
float __curand_uint32_as_float(unsigned int x)
Definition curand_kernel.h:291
void __curand_matcopy(unsigned int *matrix, unsigned int *matrixA, int n)
Definition curand_kernel.h:232
float __curand_uint32AsFloat(unsigned int i)
Definition curand_kernel.h:264
#define SKIPAHEAD_MASK
Definition curand_kernel.h:10
#define T
Definition exp2f.c:24
unsigned long long curandDirectionVectors64_t[64]
CURAND array of 64-bit direction vectors.
Definition curand.h:145
Definition curand_kernel.h:99
unsigned int x
Definition curand_kernel.h:100
unsigned int c
Definition curand_kernel.h:100
unsigned int i
Definition curand_kernel.h:100
unsigned int direction_vectors[32]
Definition curand_kernel.h:101
Definition curand_kernel.h:133
unsigned long long i
Definition curand_kernel.h:134
unsigned long long x
Definition curand_kernel.h:134
unsigned long long c
Definition curand_kernel.h:134
unsigned long long direction_vectors[64]
Definition curand_kernel.h:135
Definition curand_kernel.h:82
unsigned int direction_vectors[32]
Definition curand_kernel.h:84
unsigned int x
Definition curand_kernel.h:83
unsigned int i
Definition curand_kernel.h:83
Definition curand_kernel.h:116
unsigned long long i
Definition curand_kernel.h:117
unsigned long long direction_vectors[64]
Definition curand_kernel.h:118
unsigned long long x
Definition curand_kernel.h:117
Definition curand_kernel.h:37
unsigned int v
Definition curand_kernel.h:38
Definition curand_kernel.h:67
double boxmuller_extra_double
Definition curand_kernel.h:71
int boxmuller_flag
Definition curand_kernel.h:69
unsigned int d
Definition curand_kernel.h:68
unsigned int v[5]
Definition curand_kernel.h:68
float boxmuller_extra
Definition curand_kernel.h:70