|
|
@ -66,7 +66,6 @@ extern "C" { |
|
|
|
|
|
|
|
#define PAD(x,b) ( ( (b) - ((x)%(b)) ) & ((b)-1) ) /* b must be power of 2 */ |
|
|
|
#define ROUND(x,b) ( (x) + PAD( (x), (b) ) ) |
|
|
|
#define MINONE(x) ((x < 1) ? (1) : (x)) |
|
|
|
|
|
|
|
/* The size is in bytes. Note that this is without the extra overhead from Lace. |
|
|
|
The value must be greater than or equal to the maximum size of your tasks. |
|
|
@ -184,7 +183,7 @@ struct __lace_common_fields_only { TASK_COMMON_FIELDS(_Task) }; |
|
|
|
|
|
|
|
typedef struct _Task { |
|
|
|
TASK_COMMON_FIELDS(_Task); |
|
|
|
char p1[MINONE(PAD(LACE_COMMON_FIELD_SIZE, P_SZ))]; |
|
|
|
char p1[PAD(LACE_COMMON_FIELD_SIZE, P_SZ)]; |
|
|
|
char d[LACE_TASKSIZE]; |
|
|
|
char p2[PAD(ROUND(LACE_COMMON_FIELD_SIZE, P_SZ) + LACE_TASKSIZE, LINE_SIZE)]; |
|
|
|
} Task; |
|
|
@ -227,7 +226,7 @@ typedef struct _WorkerP { |
|
|
|
uint32_t seed; // my random seed (for lace_steal_random) |
|
|
|
} WorkerP; |
|
|
|
|
|
|
|
#define LACE_TYPEDEF_CB(t, f, ...) typedef t (*f)(WorkerP *, Task *, ##__VA_ARGS__) |
|
|
|
#define LACE_TYPEDEF_CB(t, f, ...) typedef t (*f)(WorkerP *, Task *, ##__VA_ARGS__); |
|
|
|
LACE_TYPEDEF_CB(void, lace_startup_cb, void*); |
|
|
|
|
|
|
|
/** |
|
|
@ -649,8 +648,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head ) |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head+1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -762,7 +760,7 @@ RTYPE NAME##_CALL(WorkerP *w, Task *__dq_head ) |
|
|
|
static inline __attribute__((always_inline)) \ |
|
|
|
RTYPE NAME##_WORK(WorkerP *__lace_worker __attribute__((unused)), Task *__lace_dq_head __attribute__((unused)) )\ |
|
|
|
|
|
|
|
#define TASK_0(RTYPE, NAME) TASK_DECL_0(RTYPE, NAME); TASK_IMPL_0(RTYPE, NAME) |
|
|
|
#define TASK_0(RTYPE, NAME) TASK_DECL_0(RTYPE, NAME) TASK_IMPL_0(RTYPE, NAME) |
|
|
|
|
|
|
|
#define VOID_TASK_DECL_0(NAME) \ |
|
|
|
\ |
|
|
@ -800,8 +798,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head ) |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -954,8 +951,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head , ATYPE_1 arg_1) |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -1105,8 +1101,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head , ATYPE_1 arg_1) |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -1259,8 +1254,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head , ATYPE_1 arg_1, ATYPE_2 arg_2) |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -1410,8 +1404,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head , ATYPE_1 arg_1, ATYPE_2 arg_2) |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -1564,8 +1557,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head , ATYPE_1 arg_1, ATYPE_2 arg_2, AT |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -1715,8 +1707,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head , ATYPE_1 arg_1, ATYPE_2 arg_2, AT |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -1869,8 +1860,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head , ATYPE_1 arg_1, ATYPE_2 arg_2, AT |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -2020,8 +2010,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head , ATYPE_1 arg_1, ATYPE_2 arg_2, AT |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -2174,8 +2163,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head , ATYPE_1 arg_1, ATYPE_2 arg_2, AT |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -2325,8 +2313,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head , ATYPE_1 arg_1, ATYPE_2 arg_2, AT |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -2479,8 +2466,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head , ATYPE_1 arg_1, ATYPE_2 arg_2, AT |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
@ -2630,8 +2616,7 @@ void NAME##_SPAWN(WorkerP *w, Task *__dq_head , ATYPE_1 arg_1, ATYPE_2 arg_2, AT |
|
|
|
if (unlikely(w->allstolen)) { \ |
|
|
|
if (wt->movesplit) wt->movesplit = 0; \ |
|
|
|
head = __dq_head - w->dq; \ |
|
|
|
ts.ts.tail = head; \ |
|
|
|
ts.ts.split = head + 1; \ |
|
|
|
ts = (TailSplit){{head,head+1}}; \ |
|
|
|
wt->ts.v = ts.v; \ |
|
|
|
compiler_barrier(); \ |
|
|
|
wt->allstolen = 0; \ |
|
|
|