| 1303 |
|
} |
| 1304 |
|
|
| 1305 |
|
/** |
| 1306 |
– |
* Performs secondary initialization, called when plock is zero. |
| 1307 |
– |
* Creates workQueue array and sets plock to a valid value. The |
| 1308 |
– |
* lock body must be exception-free (so no try/finally) so we |
| 1309 |
– |
* optimistically allocate new array outside the lock and throw |
| 1310 |
– |
* away if (very rarely) not needed. (A similar tactic is used in |
| 1311 |
– |
* fullExternalPush.) Because the plock seq value can eventually |
| 1312 |
– |
* wrap around zero, this method harmlessly fails to reinitialize |
| 1313 |
– |
* if workQueues exists, while still advancing plock. |
| 1314 |
– |
* |
| 1315 |
– |
* Additionally tries to create the first worker. |
| 1316 |
– |
*/ |
| 1317 |
– |
private void initWorkers() { |
| 1318 |
– |
WorkQueue[] ws, nws; int ps; |
| 1319 |
– |
int p = config & SMASK; // find power of two table size |
| 1320 |
– |
int n = (p > 1) ? p - 1 : 1; // ensure at least 2 slots |
| 1321 |
– |
n |= n >>> 1; n |= n >>> 2; n |= n >>> 4; n |= n >>> 8; n |= n >>> 16; |
| 1322 |
– |
n = (n + 1) << 1; |
| 1323 |
– |
if ((ws = workQueues) == null || ws.length == 0) |
| 1324 |
– |
nws = new WorkQueue[n]; |
| 1325 |
– |
else |
| 1326 |
– |
nws = null; |
| 1327 |
– |
if (((ps = plock) & PL_LOCK) != 0 || |
| 1328 |
– |
!U.compareAndSwapInt(this, PLOCK, ps, ps += PL_LOCK)) |
| 1329 |
– |
ps = acquirePlock(); |
| 1330 |
– |
if (((ws = workQueues) == null || ws.length == 0) && nws != null) |
| 1331 |
– |
workQueues = nws; |
| 1332 |
– |
int nps = (ps & SHUTDOWN) | ((ps + PL_LOCK) & ~SHUTDOWN); |
| 1333 |
– |
if (!U.compareAndSwapInt(this, PLOCK, ps, nps)) |
| 1334 |
– |
releasePlock(nps); |
| 1335 |
– |
tryAddWorker(); |
| 1336 |
– |
} |
| 1337 |
– |
|
| 1338 |
– |
/** |
| 1306 |
|
* Tries to create and start one worker if fewer than target |
| 1307 |
|
* parallelism level exist. Adjusts counts etc on failure. |
| 1308 |
|
*/ |
| 1487 |
|
/** |
| 1488 |
|
* Full version of externalPush. This method is called, among |
| 1489 |
|
* other times, upon the first submission of the first task to the |
| 1490 |
< |
* pool, so must perform secondary initialization (via |
| 1491 |
< |
* initWorkers). It also detects first submission by an external |
| 1492 |
< |
* thread by looking up its ThreadLocal, and creates a new shared |
| 1493 |
< |
* queue if the one at index if empty or contended. The plock lock |
| 1494 |
< |
* body must be exception-free (so no try/finally) so we |
| 1495 |
< |
* optimistically allocate new queues outside the lock and throw |
| 1496 |
< |
* them away if (very rarely) not needed. |
| 1490 |
> |
* pool, so must perform secondary initialization. It also |
| 1491 |
> |
* detects first submission by an external thread by looking up |
| 1492 |
> |
* its ThreadLocal, and creates a new shared queue if the one at |
| 1493 |
> |
* index if empty or contended. The plock lock body must be |
| 1494 |
> |
* exception-free (so no try/finally) so we optimistically |
| 1495 |
> |
* allocate new queues outside the lock and throw them away if |
| 1496 |
> |
* (very rarely) not needed. |
| 1497 |
> |
* |
| 1498 |
> |
* Secondary initialization occurs when plock is zero, to create |
| 1499 |
> |
* workQueue array and set plock to a valid value. This lock body |
| 1500 |
> |
* must also be exception-free. Because the plock seq value can |
| 1501 |
> |
* eventually wrap around zero, this method harmlessly fails to |
| 1502 |
> |
* reinitialize if workQueues exists, while still advancing plock. |
| 1503 |
|
*/ |
| 1504 |
|
private void fullExternalPush(ForkJoinTask<?> task) { |
| 1505 |
|
int r = 0; // random index seed |
| 1510 |
|
r += SEED_INCREMENT) && r != 0) |
| 1511 |
|
submitters.set(z = new Submitter(r)); |
| 1512 |
|
} |
| 1513 |
< |
else if (r == 0) { // move to a different index |
| 1513 |
> |
else if (r == 0) { // move to a different index |
| 1514 |
|
r = z.seed; |
| 1515 |
< |
r ^= r << 13; // same xorshift as WorkQueues |
| 1515 |
> |
r ^= r << 13; // same xorshift as WorkQueues |
| 1516 |
|
r ^= r >>> 17; |
| 1517 |
|
z.seed = r ^ (r << 5); |
| 1518 |
|
} |
| 1519 |
|
else if ((ps = plock) < 0) |
| 1520 |
|
throw new RejectedExecutionException(); |
| 1521 |
|
else if (ps == 0 || (ws = workQueues) == null || |
| 1522 |
< |
(m = ws.length - 1) < 0) |
| 1523 |
< |
initWorkers(); |
| 1522 |
> |
(m = ws.length - 1) < 0) { // initialize workQueues |
| 1523 |
> |
int p = config & SMASK; // find power of two table size |
| 1524 |
> |
int n = (p > 1) ? p - 1 : 1; // ensure at least 2 slots |
| 1525 |
> |
n |= n >>> 1; n |= n >>> 2; n |= n >>> 4; |
| 1526 |
> |
n |= n >>> 8; n |= n >>> 16; n = (n + 1) << 1; |
| 1527 |
> |
WorkQueue[] nws = ((ws = workQueues) == null || ws.length == 0 ? |
| 1528 |
> |
new WorkQueue[n] : null); |
| 1529 |
> |
if (((ps = plock) & PL_LOCK) != 0 || |
| 1530 |
> |
!U.compareAndSwapInt(this, PLOCK, ps, ps += PL_LOCK)) |
| 1531 |
> |
ps = acquirePlock(); |
| 1532 |
> |
if (((ws = workQueues) == null || ws.length == 0) && nws != null) |
| 1533 |
> |
workQueues = nws; |
| 1534 |
> |
int nps = (ps & SHUTDOWN) | ((ps + PL_LOCK) & ~SHUTDOWN); |
| 1535 |
> |
if (!U.compareAndSwapInt(this, PLOCK, ps, nps)) |
| 1536 |
> |
releasePlock(nps); |
| 1537 |
> |
} |
| 1538 |
|
else if ((q = ws[k = r & m & SQMASK]) != null) { |
| 1539 |
|
if (q.qlock == 0 && U.compareAndSwapInt(q, QLOCK, 0, 1)) { |
| 1540 |
|
ForkJoinTask<?>[] a = q.array; |
| 1716 |
|
else if ((int)(c >> AC_SHIFT) == 1 - (config & SMASK)) |
| 1717 |
|
idleAwaitWork(w, nc, c); |
| 1718 |
|
} |
| 1719 |
< |
else if (w.eventCount < 0 && !tryTerminate(false, false) && |
| 1733 |
< |
ctl == c) { // block |
| 1719 |
> |
else if (w.eventCount < 0 && ctl == c) { |
| 1720 |
|
Thread wt = Thread.currentThread(); |
| 1721 |
|
Thread.interrupted(); // clear status |
| 1722 |
|
U.putObject(wt, PARKBLOCKER, this); |
| 1723 |
|
w.parker = wt; // emulate LockSupport.park |
| 1724 |
|
if (w.eventCount < 0) // recheck |
| 1725 |
< |
U.park(false, 0L); |
| 1725 |
> |
U.park(false, 0L); // block |
| 1726 |
|
w.parker = null; |
| 1727 |
|
U.putObject(wt, PARKBLOCKER, null); |
| 1728 |
|
} |
| 1731 |
|
(ws = workQueues) != null && h < ws.length && |
| 1732 |
|
(q = ws[h]) != null) { // signal others before retry |
| 1733 |
|
WorkQueue v; Thread p; int u, i, s; |
| 1734 |
< |
for (int n = (config & SMASK) >>> 1;;) { |
| 1734 |
> |
for (int n = (config & SMASK) - 1;;) { |
| 1735 |
|
int idleCount = (w.eventCount < 0) ? 0 : -1; |
| 1736 |
|
if (((s = idleCount - q.base + q.top) <= n && |
| 1737 |
|
(n = s) <= 0) || |
| 1771 |
|
*/ |
| 1772 |
|
private void idleAwaitWork(WorkQueue w, long currentCtl, long prevCtl) { |
| 1773 |
|
if (w != null && w.eventCount < 0 && |
| 1774 |
< |
!tryTerminate(false, false) && (int)prevCtl != 0) { |
| 1774 |
> |
!tryTerminate(false, false) && (int)prevCtl != 0 && |
| 1775 |
> |
ctl == currentCtl) { |
| 1776 |
|
int dc = -(short)(currentCtl >>> TC_SHIFT); |
| 1777 |
|
long parkTime = dc < 0 ? FAST_IDLE_TIMEOUT: (dc + 1) * IDLE_TIMEOUT; |
| 1778 |
|
long deadline = System.nanoTime() + parkTime - TIMEOUT_SLOP; |
| 1790 |
|
if (deadline - System.nanoTime() <= 0L && |
| 1791 |
|
U.compareAndSwapLong(this, CTL, currentCtl, prevCtl)) { |
| 1792 |
|
w.eventCount = (w.eventCount + E_SEQ) | E_MASK; |
| 1793 |
+ |
w.hint = -1; |
| 1794 |
|
w.qlock = -1; // shrink |
| 1795 |
|
break; |
| 1796 |
|
} |
| 2084 |
|
|
| 2085 |
|
/** |
| 2086 |
|
* Returns a (probably) non-empty steal queue, if one is found |
| 2087 |
< |
* during a random, then cyclic scan, else null. This method must |
| 2088 |
< |
* be retried by caller if, by the time it tries to use the queue, |
| 2101 |
< |
* it is empty. |
| 2087 |
> |
* during a scan, else null. This method must be retried by |
| 2088 |
> |
* caller if, by the time it tries to use the queue, it is empty. |
| 2089 |
|
* @param r a (random) seed for scanning |
| 2090 |
|
*/ |
| 2091 |
|
private WorkQueue findNonEmptyStealQueue(int r) { |
| 2092 |
< |
for (WorkQueue[] ws;;) { |
| 2093 |
< |
int ps = plock, m; |
| 2094 |
< |
if ((ws = workQueues) == null || (m = ws.length - 1) < 1) |
| 2095 |
< |
return null; |
| 2096 |
< |
for (int j = (m + 1) << 2; ;) { |
| 2097 |
< |
WorkQueue q = ws[(((r + j) << 1) | 1) & m]; |
| 2098 |
< |
if (q != null && q.base - q.top < 0) |
| 2112 |
< |
return q; |
| 2113 |
< |
else if (--j < 0) { |
| 2114 |
< |
if (plock == ps) |
| 2115 |
< |
return null; |
| 2116 |
< |
break; |
| 2092 |
> |
for (;;) { |
| 2093 |
> |
int ps = plock, m; WorkQueue[] ws; WorkQueue q; |
| 2094 |
> |
if ((ws = workQueues) != null && (m = ws.length - 1) >= 0) { |
| 2095 |
> |
for (int j = (m + 1) << 2; j >= 0; --j) { |
| 2096 |
> |
if ((q = ws[(((r + j) << 1) | 1) & m]) != null && |
| 2097 |
> |
q.base - q.top < 0) |
| 2098 |
> |
return q; |
| 2099 |
|
} |
| 2100 |
|
} |
| 2101 |
+ |
if (plock == ps) |
| 2102 |
+ |
return null; |
| 2103 |
|
} |
| 2104 |
|
} |
| 2105 |
|
|
| 2111 |
|
*/ |
| 2112 |
|
final void helpQuiescePool(WorkQueue w) { |
| 2113 |
|
for (boolean active = true;;) { |
| 2114 |
< |
ForkJoinTask<?> localTask; // exhaust local queue |
| 2115 |
< |
while ((localTask = w.nextLocalTask()) != null) |
| 2116 |
< |
localTask.doExec(); |
| 2117 |
< |
// Similar to loop in scan(), but ignoring submissions |
| 2118 |
< |
WorkQueue q = findNonEmptyStealQueue(w.nextSeed()); |
| 2119 |
< |
if (q != null) { |
| 2120 |
< |
ForkJoinTask<?> t; int b; |
| 2114 |
> |
long c; WorkQueue q; ForkJoinTask<?> t; int b; |
| 2115 |
> |
while ((t = w.nextLocalTask()) != null) { |
| 2116 |
> |
if (w.base - w.top < 0) |
| 2117 |
> |
signalWork(w); |
| 2118 |
> |
t.doExec(); |
| 2119 |
> |
} |
| 2120 |
> |
if ((q = findNonEmptyStealQueue(w.nextSeed())) != null) { |
| 2121 |
|
if (!active) { // re-establish active count |
| 2138 |
– |
long c; |
| 2122 |
|
active = true; |
| 2123 |
|
do {} while (!U.compareAndSwapLong |
| 2124 |
|
(this, CTL, c = ctl, c + AC_UNIT)); |
| 2129 |
|
w.runSubtask(t); |
| 2130 |
|
} |
| 2131 |
|
} |
| 2132 |
< |
else { |
| 2133 |
< |
long c; |
| 2134 |
< |
if (active) { // decrement active count without queuing |
| 2132 |
> |
else if (active) { // decrement active count without queuing |
| 2133 |
> |
long nc = (c = ctl) - AC_UNIT; |
| 2134 |
> |
if ((int)(nc >> AC_SHIFT) + (config & SMASK) == 0) |
| 2135 |
> |
return; // bypass decrement-then-increment |
| 2136 |
> |
if (U.compareAndSwapLong(this, CTL, c, nc)) |
| 2137 |
|
active = false; |
| 2153 |
– |
do {} while (!U.compareAndSwapLong |
| 2154 |
– |
(this, CTL, c = ctl, c -= AC_UNIT)); |
| 2155 |
– |
} |
| 2156 |
– |
else |
| 2157 |
– |
c = ctl; // re-increment on exit |
| 2158 |
– |
if ((int)(c >> AC_SHIFT) + (config & SMASK) == 0) { |
| 2159 |
– |
do {} while (!U.compareAndSwapLong |
| 2160 |
– |
(this, CTL, c = ctl, c + AC_UNIT)); |
| 2161 |
– |
break; |
| 2162 |
– |
} |
| 2138 |
|
} |
| 2139 |
+ |
else if ((int)((c = ctl) >> AC_SHIFT) + (config & SMASK) == 0 && |
| 2140 |
+ |
U.compareAndSwapLong(this, CTL, c, c + AC_UNIT)) |
| 2141 |
+ |
return; |
| 2142 |
|
} |
| 2143 |
|
} |
| 2144 |
|
|
| 2240 |
|
* @return true if now terminating or terminated |
| 2241 |
|
*/ |
| 2242 |
|
private boolean tryTerminate(boolean now, boolean enable) { |
| 2243 |
< |
if (this == commonPool) // cannot shut down |
| 2243 |
> |
int ps; |
| 2244 |
> |
if (this == commonPool) // cannot shut down |
| 2245 |
|
return false; |
| 2246 |
+ |
if ((ps = plock) >= 0) { // enable by setting plock |
| 2247 |
+ |
if (!enable) |
| 2248 |
+ |
return false; |
| 2249 |
+ |
if ((ps & PL_LOCK) != 0 || |
| 2250 |
+ |
!U.compareAndSwapInt(this, PLOCK, ps, ps += PL_LOCK)) |
| 2251 |
+ |
ps = acquirePlock(); |
| 2252 |
+ |
int nps = ((ps + PL_LOCK) & ~SHUTDOWN) | SHUTDOWN; |
| 2253 |
+ |
if (!U.compareAndSwapInt(this, PLOCK, ps, nps)) |
| 2254 |
+ |
releasePlock(nps); |
| 2255 |
+ |
} |
| 2256 |
|
for (long c;;) { |
| 2257 |
< |
if (((c = ctl) & STOP_BIT) != 0) { // already terminating |
| 2257 |
> |
if (((c = ctl) & STOP_BIT) != 0) { // already terminating |
| 2258 |
|
if ((short)(c >>> TC_SHIFT) == -(config & SMASK)) { |
| 2259 |
|
synchronized (this) { |
| 2260 |
< |
notifyAll(); // signal when 0 workers |
| 2260 |
> |
notifyAll(); // signal when 0 workers |
| 2261 |
|
} |
| 2262 |
|
} |
| 2263 |
|
return true; |
| 2264 |
|
} |
| 2265 |
< |
if (plock >= 0) { // not yet enabled |
| 2266 |
< |
int ps; |
| 2267 |
< |
if (!enable) |
| 2279 |
< |
return false; |
| 2280 |
< |
if (((ps = plock) & PL_LOCK) != 0 || |
| 2281 |
< |
!U.compareAndSwapInt(this, PLOCK, ps, ps += PL_LOCK)) |
| 2282 |
< |
ps = acquirePlock(); |
| 2283 |
< |
if (!U.compareAndSwapInt(this, PLOCK, ps, SHUTDOWN)) |
| 2284 |
< |
releasePlock(SHUTDOWN); |
| 2285 |
< |
} |
| 2286 |
< |
if (!now) { // check if idle & no tasks |
| 2287 |
< |
if ((int)(c >> AC_SHIFT) != -(config & SMASK) || |
| 2288 |
< |
hasQueuedSubmissions()) |
| 2265 |
> |
if (!now) { // check if idle & no tasks |
| 2266 |
> |
WorkQueue[] ws; WorkQueue w; |
| 2267 |
> |
if ((int)(c >> AC_SHIFT) != -(config & SMASK)) |
| 2268 |
|
return false; |
| 2269 |
< |
// Check for unqueued inactive workers. One pass suffices. |
| 2270 |
< |
WorkQueue[] ws = workQueues; WorkQueue w; |
| 2271 |
< |
if (ws != null) { |
| 2272 |
< |
for (int i = 1; i < ws.length; i += 2) { |
| 2273 |
< |
if ((w = ws[i]) != null && w.eventCount >= 0) |
| 2274 |
< |
return false; |
| 2269 |
> |
if ((ws = workQueues) != null) { |
| 2270 |
> |
for (int i = 0; i < ws.length; ++i) { |
| 2271 |
> |
if ((w = ws[i]) != null) { |
| 2272 |
> |
if (!w.isEmpty()) { // signal unprocessed tasks |
| 2273 |
> |
signalWork(w); |
| 2274 |
> |
return false; |
| 2275 |
> |
} |
| 2276 |
> |
if ((i & 1) != 0 && w.eventCount >= 0) |
| 2277 |
> |
return false; // unqueued inactive worker |
| 2278 |
> |
} |
| 2279 |
|
} |
| 2280 |
|
} |
| 2281 |
|
} |
| 2282 |
|
if (U.compareAndSwapLong(this, CTL, c, c | STOP_BIT)) { |
| 2283 |
|
for (int pass = 0; pass < 3; ++pass) { |
| 2284 |
< |
WorkQueue[] ws = workQueues; |
| 2285 |
< |
if (ws != null) { |
| 2303 |
< |
WorkQueue w; Thread wt; |
| 2284 |
> |
WorkQueue[] ws; WorkQueue w; Thread wt; |
| 2285 |
> |
if ((ws = workQueues) != null) { |
| 2286 |
|
int n = ws.length; |
| 2287 |
|
for (int i = 0; i < n; ++i) { |
| 2288 |
|
if ((w = ws[i]) != null) { |
| 2293 |
|
if (!wt.isInterrupted()) { |
| 2294 |
|
try { |
| 2295 |
|
wt.interrupt(); |
| 2296 |
< |
} catch (SecurityException ignore) { |
| 2296 |
> |
} catch (Throwable ignore) { |
| 2297 |
|
} |
| 2298 |
|
} |
| 2299 |
|
U.unpark(wt); |
| 2304 |
|
// Wake up workers parked on event queue |
| 2305 |
|
int i, e; long cc; Thread p; |
| 2306 |
|
while ((e = (int)(cc = ctl) & E_MASK) != 0 && |
| 2307 |
< |
(i = e & SMASK) < n && |
| 2307 |
> |
(i = e & SMASK) < n && i >= 0 && |
| 2308 |
|
(w = ws[i]) != null) { |
| 2309 |
|
long nc = ((long)(w.nextWait & E_MASK) | |
| 2310 |
|
((cc + AC_UNIT) & AC_MASK) | |
| 3346 |
|
commonPool = new ForkJoinPool(par, ct, fac, handler); |
| 3347 |
|
} |
| 3348 |
|
|
| 3349 |
+ |
|
| 3350 |
|
/** |
| 3351 |
|
* Returns a sun.misc.Unsafe. Suitable for use in a 3rd party package. |
| 3352 |
|
* Replace with a simple call to Unsafe.getUnsafe when integrating |
