ViewVC Help

View File | Revision Log | Show Annotations | Download File | Root Listing

root/jsr166/jsr166/src/jsr166y/LinkedTransferQueue.java

Comparing jsr166/src/jsr166y/LinkedTransferQueue.java (file contents):
Revision 1.43 by jsr166, Sat Aug 1 21:24:01 2009 UTC vs.
Revision 1.78 by dl, Thu Sep 9 16:52:49 2010 UTC

#	Line 15 | Line 15 | import java.util.Iterator;
15		import java.util.NoSuchElementException;
16		import java.util.Queue;
17		import java.util.concurrent.locks.LockSupport;
18	–	import java.util.concurrent.atomic.AtomicReference;
18
19		/**
20		* An unbounded {@link TransferQueue} based on linked nodes.
#	Line 54 | Line 53 | public class LinkedTransferQueue<E> exte
53		private static final long serialVersionUID = -3223113410248163686L;
54
55		/*
56	<	* This class extends the approach used in FIFO-mode
58	<	* SynchronousQueues. See the internal documentation, as well as
59	<	* the PPoPP 2006 paper "Scalable Synchronous Queues" by Scherer,
60	<	* Lea & Scott
61	<	* (http://www.cs.rice.edu/~wns1/papers/2006-PPoPP-SQ.pdf)
56	>	* *** Overview of Dual Queues with Slack ***
57		*
58	<	* The main extension is to provide different Wait modes for the
59	<	* main "xfer" method that puts or takes items.  These don't
60	<	* impact the basic dual-queue logic, but instead control whether
61	<	* or how threads block upon insertion of request or data nodes
62	<	* into the dual queue. It also uses slightly different
63	<	* conventions for tracking whether nodes are off-list or
64	<	* cancelled.
65	<	*/
66	<
67	<	// Wait modes for xfer method
68	<	static final int NOWAIT  = 0;
69	<	static final int TIMEOUT = 1;
70	<	static final int WAIT    = 2;
71	<
72	<	/** The number of CPUs, for spin control */
73	<	static final int NCPUS = Runtime.getRuntime().availableProcessors();
74	<
75	<	/**
76	<	* The number of times to spin before blocking in timed waits.
77	<	* The value is empirically derived -- it works well across a
78	<	* variety of processors and OSes. Empirically, the best value
79	<	* seems not to vary with number of CPUs (beyond 2) so is just
80	<	* a constant.
81	<	*/
82	<	static final int maxTimedSpins = (NCPUS < 2) ? 0 : 32;
83	<
84	<	/**
85	<	* The number of times to spin before blocking in untimed waits.
86	<	* This is greater than timed value because untimed waits spin
87	<	* faster since they don't need to check times on each spin.
88	<	*/
89	<	static final int maxUntimedSpins = maxTimedSpins * 16;
58	>	* Dual Queues, introduced by Scherer and Scott
59	>	* (http://www.cs.rice.edu/~wns1/papers/2004-DISC-DDS.pdf) are
60	>	* (linked) queues in which nodes may represent either data or
61	>	* requests.  When a thread tries to enqueue a data node, but
62	>	* encounters a request node, it instead "matches" and removes it;
63	>	* and vice versa for enqueuing requests. Blocking Dual Queues
64	>	* arrange that threads enqueuing unmatched requests block until
65	>	* other threads provide the match. Dual Synchronous Queues (see
66	>	* Scherer, Lea, & Scott
67	>	* http://www.cs.rochester.edu/u/scott/papers/2009_Scherer_CACM_SSQ.pdf)
68	>	* additionally arrange that threads enqueuing unmatched data also
69	>	* block.  Dual Transfer Queues support all of these modes, as
70	>	* dictated by callers.
71	>	*
72	>	* A FIFO dual queue may be implemented using a variation of the
73	>	* Michael & Scott (M&S) lock-free queue algorithm
74	>	* (http://www.cs.rochester.edu/u/scott/papers/1996_PODC_queues.pdf).
75	>	* It maintains two pointer fields, "head", pointing to a
76	>	* (matched) node that in turn points to the first actual
77	>	* (unmatched) queue node (or null if empty); and "tail" that
78	>	* points to the last node on the queue (or again null if
79	>	* empty). For example, here is a possible queue with four data
80	>	* elements:
81	>	*
82	>	*  head                tail
83	>	*    |                   |
84	>	*    v                   v
85	>	*    M -> U -> U -> U -> U
86	>	*
87	>	* The M&S queue algorithm is known to be prone to scalability and
88	>	* overhead limitations when maintaining (via CAS) these head and
89	>	* tail pointers. This has led to the development of
90	>	* contention-reducing variants such as elimination arrays (see
91	>	* Moir et al http://portal.acm.org/citation.cfm?id=1074013) and
92	>	* optimistic back pointers (see Ladan-Mozes & Shavit
93	>	* http://people.csail.mit.edu/edya/publications/OptimisticFIFOQueue-journal.pdf).
94	>	* However, the nature of dual queues enables a simpler tactic for
95	>	* improving M&S-style implementations when dual-ness is needed.
96	>	*
97	>	* In a dual queue, each node must atomically maintain its match
98	>	* status. While there are other possible variants, we implement
99	>	* this here as: for a data-mode node, matching entails CASing an
100	>	* "item" field from a non-null data value to null upon match, and
101	>	* vice-versa for request nodes, CASing from null to a data
102	>	* value. (Note that the linearization properties of this style of
103	>	* queue are easy to verify -- elements are made available by
104	>	* linking, and unavailable by matching.) Compared to plain M&S
105	>	* queues, this property of dual queues requires one additional
106	>	* successful atomic operation per enq/deq pair. But it also
107	>	* enables lower cost variants of queue maintenance mechanics. (A
108	>	* variation of this idea applies even for non-dual queues that
109	>	* support deletion of interior elements, such as
110	>	* j.u.c.ConcurrentLinkedQueue.)
111	>	*
112	>	* Once a node is matched, its match status can never again
113	>	* change.  We may thus arrange that the linked list of them
114	>	* contain a prefix of zero or more matched nodes, followed by a
115	>	* suffix of zero or more unmatched nodes. (Note that we allow
116	>	* both the prefix and suffix to be zero length, which in turn
117	>	* means that we do not use a dummy header.)  If we were not
118	>	* concerned with either time or space efficiency, we could
119	>	* correctly perform enqueue and dequeue operations by traversing
120	>	* from a pointer to the initial node; CASing the item of the
121	>	* first unmatched node on match and CASing the next field of the
122	>	* trailing node on appends. (Plus some special-casing when
123	>	* initially empty).  While this would be a terrible idea in
124	>	* itself, it does have the benefit of not requiring ANY atomic
125	>	* updates on head/tail fields.
126	>	*
127	>	* We introduce here an approach that lies between the extremes of
128	>	* never versus always updating queue (head and tail) pointers.
129	>	* This offers a tradeoff between sometimes requiring extra
130	>	* traversal steps to locate the first and/or last unmatched
131	>	* nodes, versus the reduced overhead and contention of fewer
132	>	* updates to queue pointers. For example, a possible snapshot of
133	>	* a queue is:
134	>	*
135	>	*  head           tail
136	>	*    |              |
137	>	*    v              v
138	>	*    M -> M -> U -> U -> U -> U
139	>	*
140	>	* The best value for this "slack" (the targeted maximum distance
141	>	* between the value of "head" and the first unmatched node, and
142	>	* similarly for "tail") is an empirical matter. We have found
143	>	* that using very small constants in the range of 1-3 work best
144	>	* over a range of platforms. Larger values introduce increasing
145	>	* costs of cache misses and risks of long traversal chains, while
146	>	* smaller values increase CAS contention and overhead.
147	>	*
148	>	* Dual queues with slack differ from plain M&S dual queues by
149	>	* virtue of only sometimes updating head or tail pointers when
150	>	* matching, appending, or even traversing nodes; in order to
151	>	* maintain a targeted slack.  The idea of "sometimes" may be
152	>	* operationalized in several ways. The simplest is to use a
153	>	* per-operation counter incremented on each traversal step, and
154	>	* to try (via CAS) to update the associated queue pointer
155	>	* whenever the count exceeds a threshold. Another, that requires
156	>	* more overhead, is to use random number generators to update
157	>	* with a given probability per traversal step.
158	>	*
159	>	* In any strategy along these lines, because CASes updating
160	>	* fields may fail, the actual slack may exceed targeted
161	>	* slack. However, they may be retried at any time to maintain
162	>	* targets.  Even when using very small slack values, this
163	>	* approach works well for dual queues because it allows all
164	>	* operations up to the point of matching or appending an item
165	>	* (hence potentially allowing progress by another thread) to be
166	>	* read-only, thus not introducing any further contention. As
167	>	* described below, we implement this by performing slack
168	>	* maintenance retries only after these points.
169	>	*
170	>	* As an accompaniment to such techniques, traversal overhead can
171	>	* be further reduced without increasing contention of head
172	>	* pointer updates: Threads may sometimes shortcut the "next" link
173	>	* path from the current "head" node to be closer to the currently
174	>	* known first unmatched node, and similarly for tail. Again, this
175	>	* may be triggered with using thresholds or randomization.
176	>	*
177	>	* These ideas must be further extended to avoid unbounded amounts
178	>	* of costly-to-reclaim garbage caused by the sequential "next"
179	>	* links of nodes starting at old forgotten head nodes: As first
180	>	* described in detail by Boehm
181	>	* (http://portal.acm.org/citation.cfm?doid=503272.503282) if a GC
182	>	* delays noticing that any arbitrarily old node has become
183	>	* garbage, all newer dead nodes will also be unreclaimed.
184	>	* (Similar issues arise in non-GC environments.)  To cope with
185	>	* this in our implementation, upon CASing to advance the head
186	>	* pointer, we set the "next" link of the previous head to point
187	>	* only to itself; thus limiting the length of connected dead lists.
188	>	* (We also take similar care to wipe out possibly garbage
189	>	* retaining values held in other Node fields.)  However, doing so
190	>	* adds some further complexity to traversal: If any "next"
191	>	* pointer links to itself, it indicates that the current thread
192	>	* has lagged behind a head-update, and so the traversal must
193	>	* continue from the "head".  Traversals trying to find the
194	>	* current tail starting from "tail" may also encounter
195	>	* self-links, in which case they also continue at "head".
196	>	*
197	>	* It is tempting in slack-based scheme to not even use CAS for
198	>	* updates (similarly to Ladan-Mozes & Shavit). However, this
199	>	* cannot be done for head updates under the above link-forgetting
200	>	* mechanics because an update may leave head at a detached node.
201	>	* And while direct writes are possible for tail updates, they
202	>	* increase the risk of long retraversals, and hence long garbage
203	>	* chains, which can be much more costly than is worthwhile
204	>	* considering that the cost difference of performing a CAS vs
205	>	* write is smaller when they are not triggered on each operation
206	>	* (especially considering that writes and CASes equally require
207	>	* additional GC bookkeeping ("write barriers") that are sometimes
208	>	* more costly than the writes themselves because of contention).
209	>	*
210	>	* *** Overview of implementation ***
211	>	*
212	>	* We use a threshold-based approach to updates, with a slack
213	>	* threshold of two -- that is, we update head/tail when the
214	>	* current pointer appears to be two or more steps away from the
215	>	* first/last node. The slack value is hard-wired: a path greater
216	>	* than one is naturally implemented by checking equality of
217	>	* traversal pointers except when the list has only one element,
218	>	* in which case we keep slack threshold at one. Avoiding tracking
219	>	* explicit counts across method calls slightly simplifies an
220	>	* already-messy implementation. Using randomization would
221	>	* probably work better if there were a low-quality dirt-cheap
222	>	* per-thread one available, but even ThreadLocalRandom is too
223	>	* heavy for these purposes.
224	>	*
225	>	* With such a small slack threshold value, it is not worthwhile
226	>	* to augment this with path short-circuiting (i.e., unsplicing
227	>	* interior nodes) except in the case of cancellation/removal (see
228	>	* below).
229	>	*
230	>	* We allow both the head and tail fields to be null before any
231	>	* nodes are enqueued; initializing upon first append.  This
232	>	* simplifies some other logic, as well as providing more
233	>	* efficient explicit control paths instead of letting JVMs insert
234	>	* implicit NullPointerExceptions when they are null.  While not
235	>	* currently fully implemented, we also leave open the possibility
236	>	* of re-nulling these fields when empty (which is complicated to
237	>	* arrange, for little benefit.)
238	>	*
239	>	* All enqueue/dequeue operations are handled by the single method
240	>	* "xfer" with parameters indicating whether to act as some form
241	>	* of offer, put, poll, take, or transfer (each possibly with
242	>	* timeout). The relative complexity of using one monolithic
243	>	* method outweighs the code bulk and maintenance problems of
244	>	* using separate methods for each case.
245	>	*
246	>	* Operation consists of up to three phases. The first is
247	>	* implemented within method xfer, the second in tryAppend, and
248	>	* the third in method awaitMatch.
249	>	*
250	>	* 1. Try to match an existing node
251	>	*
252	>	*    Starting at head, skip already-matched nodes until finding
253	>	*    an unmatched node of opposite mode, if one exists, in which
254	>	*    case matching it and returning, also if necessary updating
255	>	*    head to one past the matched node (or the node itself if the
256	>	*    list has no other unmatched nodes). If the CAS misses, then
257	>	*    a loop retries advancing head by two steps until either
258	>	*    success or the slack is at most two. By requiring that each
259	>	*    attempt advances head by two (if applicable), we ensure that
260	>	*    the slack does not grow without bound. Traversals also check
261	>	*    if the initial head is now off-list, in which case they
262	>	*    start at the new head.
263	>	*
264	>	*    If no candidates are found and the call was untimed
265	>	*    poll/offer, (argument "how" is NOW) return.
266	>	*
267	>	* 2. Try to append a new node (method tryAppend)
268	>	*
269	>	*    Starting at current tail pointer, find the actual last node
270	>	*    and try to append a new node (or if head was null, establish
271	>	*    the first node). Nodes can be appended only if their
272	>	*    predecessors are either already matched or are of the same
273	>	*    mode. If we detect otherwise, then a new node with opposite
274	>	*    mode must have been appended during traversal, so we must
275	>	*    restart at phase 1. The traversal and update steps are
276	>	*    otherwise similar to phase 1: Retrying upon CAS misses and
277	>	*    checking for staleness.  In particular, if a self-link is
278	>	*    encountered, then we can safely jump to a node on the list
279	>	*    by continuing the traversal at current head.
280	>	*
281	>	*    On successful append, if the call was ASYNC, return.
282	>	*
283	>	* 3. Await match or cancellation (method awaitMatch)
284	>	*
285	>	*    Wait for another thread to match node; instead cancelling if
286	>	*    the current thread was interrupted or the wait timed out. On
287	>	*    multiprocessors, we use front-of-queue spinning: If a node
288	>	*    appears to be the first unmatched node in the queue, it
289	>	*    spins a bit before blocking. In either case, before blocking
290	>	*    it tries to unsplice any nodes between the current "head"
291	>	*    and the first unmatched node.
292	>	*
293	>	*    Front-of-queue spinning vastly improves performance of
294	>	*    heavily contended queues. And so long as it is relatively
295	>	*    brief and "quiet", spinning does not much impact performance
296	>	*    of less-contended queues.  During spins threads check their
297	>	*    interrupt status and generate a thread-local random number
298	>	*    to decide to occasionally perform a Thread.yield. While
299	>	*    yield has underdefined specs, we assume that might it help,
300	>	*    and will not hurt in limiting impact of spinning on busy
301	>	*    systems.  We also use smaller (1/2) spins for nodes that are
302	>	*    not known to be front but whose predecessors have not
303	>	*    blocked -- these "chained" spins avoid artifacts of
304	>	*    front-of-queue rules which otherwise lead to alternating
305	>	*    nodes spinning vs blocking. Further, front threads that
306	>	*    represent phase changes (from data to request node or vice
307	>	*    versa) compared to their predecessors receive additional
308	>	*    chained spins, reflecting longer paths typically required to
309	>	*    unblock threads during phase changes.
310	>	*
311	>	*
312	>	* ** Unlinking removed interior nodes **
313	>	*
314	>	* In addition to minimizing garbage retention via self-linking
315	>	* described above, we also unlink removed interior nodes. These
316	>	* may arise due to timed out or interrupted waits, or calls to
317	>	* remove(x) or Iterator.remove.  Normally, given a node that was
318	>	* at one time known to be the predecessor of some node s that is
319	>	* to be removed, we can unsplice s by CASing the next field of
320	>	* its predecessor if it still points to s (otherwise s must
321	>	* already have been removed or is now offlist). But there are two
322	>	* situations in which we cannot guarantee to make node s
323	>	* unreachable in this way: (1) If s is the trailing node of list
324	>	* (i.e., with null next), then it is pinned as the target node
325	>	* for appends, so can only be removed later after other nodes are
326	>	* appended. (2) We cannot necessarily unlink s given a
327	>	* predecessor node that is matched (including the case of being
328	>	* cancelled): the predecessor may already be unspliced, in which
329	>	* case some previous reachable node may still point to s.
330	>	* (For further explanation see Herlihy & Shavit "The Art of
331	>	* Multiprocessor Programming" chapter 9).  Although, in both
332	>	* cases, we can rule out the need for further action if either s
333	>	* or its predecessor are (or can be made to be) at, or fall off
334	>	* from, the head of list.
335	>	*
336	>	* Without taking these into account, it would be possible for an
337	>	* unbounded number of supposedly removed nodes to remain
338	>	* reachable.  Situations leading to such buildup are uncommon but
339	>	* can occur in practice; for example when a series of short timed
340	>	* calls to poll repeatedly time out but never otherwise fall off
341	>	* the list because of an untimed call to take at the front of the
342	>	* queue.
343	>	*
344	>	* When these cases arise, rather than always retraversing the
345	>	* entire list to find an actual predecessor to unlink (which
346	>	* won't help for case (1) anyway), we record a conservative
347	>	* estimate of possible unsplice failures (in "sweepVotes").
348	>	* We trigger a full sweep when the estimate exceeds a threshold
349	>	* ("SWEEP_THRESHOLD") indicating the maximum number of estimated
350	>	* removal failures to tolerate before sweeping through, unlinking
351	>	* cancelled nodes that were not unlinked upon initial removal.
352	>	* We perform sweeps by the thread hitting threshold (rather than
353	>	* background threads or by spreading work to other threads)
354	>	* because in the main contexts in which removal occurs, the
355	>	* caller is already timed-out, cancelled, or performing a
356	>	* potentially O(n) operation (e.g. remove(x)), none of which are
357	>	* time-critical enough to warrant the overhead that alternatives
358	>	* would impose on other threads.
359	>	*
360	>	* Because the sweepVotes estimate is conservative, and because
361	>	* nodes become unlinked "naturally" as they fall off the head of
362	>	* the queue, and because we allow votes to accumulate even while
363	>	* sweeps are in progress, there are typically significantly fewer
364	>	* such nodes than estimated.  Choice of a threshold value
365	>	* balances the likelihood of wasted effort and contention, versus
366	>	* providing a worst-case bound on retention of interior nodes in
367	>	* quiescent queues. The value defined below was chosen
368	>	* empirically to balance these under various timeout scenarios.
369	>	*
370	>	* Note that we cannot self-link unlinked interior nodes during
371	>	* sweeps. However, the associated garbage chains terminate when
372	>	* some successor ultimately falls off the head of the list and is
373	>	* self-linked.
374	>	*/
375	>
376	>	/** True if on multiprocessor */
377	>	private static final boolean MP =
378	>	Runtime.getRuntime().availableProcessors() > 1;
379	>
380	>	/**
381	>	* The number of times to spin (with randomly interspersed calls
382	>	* to Thread.yield) on multiprocessor before blocking when a node
383	>	* is apparently the first waiter in the queue.  See above for
384	>	* explanation. Must be a power of two. The value is empirically
385	>	* derived -- it works pretty well across a variety of processors,
386	>	* numbers of CPUs, and OSes.
387	>	*/
388	>	private static final int FRONT_SPINS   = 1 << 7;
389	>
390	>	/**
391	>	* The number of times to spin before blocking when a node is
392	>	* preceded by another node that is apparently spinning.  Also
393	>	* serves as an increment to FRONT_SPINS on phase changes, and as
394	>	* base average frequency for yielding during spins. Must be a
395	>	* power of two.
396	>	*/
397	>	private static final int CHAINED_SPINS = FRONT_SPINS >>> 1;
398	>
399	>	/**
400	>	* The maximum number of estimated removal failures (sweepVotes)
401	>	* to tolerate before sweeping through the queue unlinking
402	>	* cancelled nodes that were not unlinked upon initial
403	>	* removal. See above for explanation. The value must be at least
404	>	* two to avoid useless sweeps when removing trailing nodes.
405	>	*/
406	>	static final int SWEEP_THRESHOLD = 32;
407	>
408	>	/**
409	>	* Queue nodes. Uses Object, not E, for items to allow forgetting
410	>	* them after use.  Relies heavily on Unsafe mechanics to minimize
411	>	* unnecessary ordering constraints: Writes that are intrinsically
412	>	* ordered wrt other accesses or CASes use simple relaxed forms.
413	>	*/
414	>	static final class Node {
415	>	final boolean isData;   // false if this is a request node
416	>	volatile Object item;   // initially non-null if isData; CASed to match
417	>	volatile Node next;
418	>	volatile Thread waiter; // null until waiting
419
420	<	/**
421	<	* The number of nanoseconds for which it is faster to spin
422	<	* rather than to use timed park. A rough estimate suffices.
423	<	*/
100	<	static final long spinForTimeoutThreshold = 1000L;
420	>	// CAS methods for fields
421	>	final boolean casNext(Node cmp, Node val) {
422	>	return UNSAFE.compareAndSwapObject(this, nextOffset, cmp, val);
423	>	}
424
425	<	/**
426	<	* Node class for LinkedTransferQueue. Opportunistically
427	<	* subclasses from AtomicReference to represent item. Uses Object,
428	<	* not E, to allow setting item to "this" after use, to avoid
106	<	* garbage retention. Similarly, setting the next field to this is
107	<	* used as sentinel that node is off list.
108	<	*/
109	<	static final class Node<E> extends AtomicReference<Object> {
110	<	volatile Node<E> next;
111	<	volatile Thread waiter;       // to control park/unpark
112	<	final boolean isData;
425	>	final boolean casItem(Object cmp, Object val) {
426	>	//            assert cmp == null || cmp.getClass() != Node.class;
427	>	return UNSAFE.compareAndSwapObject(this, itemOffset, cmp, val);
428	>	}
429
430	<	Node(E item, boolean isData) {
431	<	super(item);
430	>	/**
431	>	* Constructs a new node.  Uses relaxed write because item can
432	>	* only be seen after publication via casNext.
433	>	*/
434	>	Node(Object item, boolean isData) {
435	>	UNSAFE.putObject(this, itemOffset, item); // relaxed write
436		this.isData = isData;
437		}
438
439	<	// Unsafe mechanics
439	>	/**
440	>	* Links node to itself to avoid garbage retention.  Called
441	>	* only after CASing head field, so uses relaxed write.
442	>	*/
443	>	final void forgetNext() {
444	>	UNSAFE.putObject(this, nextOffset, this);
445	>	}
446
447	<	private static final sun.misc.Unsafe UNSAFE = getUnsafe();
448	<	private static final long nextOffset =
449	<	objectFieldOffset(UNSAFE, "next", Node.class);
447	>	/**
448	>	* Sets item to self and waiter to null, to avoid garbage
449	>	* retention after matching or cancelling. Uses relaxed writes
450	>	* because order is already constrained in the only calling
451	>	* contexts: item is forgotten only after volatile/atomic
452	>	* mechanics that extract items.  Similarly, clearing waiter
453	>	* follows either CAS or return from park (if ever parked;
454	>	* else we don't care).
455	>	*/
456	>	final void forgetContents() {
457	>	UNSAFE.putObject(this, itemOffset, this);
458	>	UNSAFE.putObject(this, waiterOffset, null);
459	>	}
460
461	<	final boolean casNext(Node<E> cmp, Node<E> val) {
462	<	return UNSAFE.compareAndSwapObject(this, nextOffset, cmp, val);
461	>	/**
462	>	* Returns true if this node has been matched, including the
463	>	* case of artificial matches due to cancellation.
464	>	*/
465	>	final boolean isMatched() {
466	>	Object x = item;
467	>	return (x == this) || ((x == null) == isData);
468		}
469
470	<	final void clearNext() {
471	<	UNSAFE.putOrderedObject(this, nextOffset, this);
470	>	/**
471	>	* Returns true if this is an unmatched request node.
472	>	*/
473	>	final boolean isUnmatchedRequest() {
474	>	return !isData && item == null;
475		}
476
477		/**
478	<	* Returns a sun.misc.Unsafe.  Suitable for use in a 3rd party package.
479	<	* Replace with a simple call to Unsafe.getUnsafe when integrating
480	<	* into a jdk.
137	<	*
138	<	* @return a sun.misc.Unsafe
478	>	* Returns true if a node with the given mode cannot be
479	>	* appended to this node because this node is unmatched and
480	>	* has opposite data mode.
481		*/
482	<	private static sun.misc.Unsafe getUnsafe() {
483	<	try {
484	<	return sun.misc.Unsafe.getUnsafe();
485	<	} catch (SecurityException se) {
486	<	try {
487	<	return java.security.AccessController.doPrivileged
488	<	(new java.security
489	<	.PrivilegedExceptionAction<sun.misc.Unsafe>() {
490	<	public sun.misc.Unsafe run() throws Exception {
491	<	java.lang.reflect.Field f = sun.misc
492	<	.Unsafe.class.getDeclaredField("theUnsafe");
493	<	f.setAccessible(true);
494	<	return (sun.misc.Unsafe) f.get(null);
495	<	}});
496	<	} catch (java.security.PrivilegedActionException e) {
155	<	throw new RuntimeException("Could not initialize intrinsics",
156	<	e.getCause());
157	<	}
482	>	final boolean cannotPrecede(boolean haveData) {
483	>	boolean d = isData;
484	>	Object x;
485	>	return d != haveData && (x = item) != this && (x != null) == d;
486	>	}
487	>
488	>	/**
489	>	* Tries to artificially match a data node -- used by remove.
490	>	*/
491	>	final boolean tryMatchData() {
492	>	//            assert isData;
493	>	Object x = item;
494	>	if (x != null && x != this && casItem(x, null)) {
495	>	LockSupport.unpark(waiter);
496	>	return true;
497		}
498	+	return false;
499		}
500
501	+	// Unsafe mechanics
502	+	private static final sun.misc.Unsafe UNSAFE = getUnsafe();
503	+	private static final long nextOffset =
504	+	objectFieldOffset(UNSAFE, "next", Node.class);
505	+	private static final long itemOffset =
506	+	objectFieldOffset(UNSAFE, "item", Node.class);
507	+	private static final long waiterOffset =
508	+	objectFieldOffset(UNSAFE, "waiter", Node.class);
509	+
510		private static final long serialVersionUID = -3375979862319811754L;
511		}
512
513	<	/**
514	<	* Padded version of AtomicReference used for head, tail and
166	<	* cleanMe, to alleviate contention across threads CASing one vs
167	<	* the other.
168	<	*/
169	<	static final class PaddedAtomicReference<T> extends AtomicReference<T> {
170	<	// enough padding for 64bytes with 4byte refs
171	<	Object p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pa, pb, pc, pd, pe;
172	<	PaddedAtomicReference(T r) { super(r); }
173	<	private static final long serialVersionUID = 8170090609809740854L;
174	<	}
513	>	/** head of the queue; null until first enqueue */
514	>	transient volatile Node head;
515
516	+	/** tail of the queue; null until first append */
517	+	private transient volatile Node tail;
518
519	<	/** head of the queue */
520	<	private transient final PaddedAtomicReference<Node<E>> head;
519	>	/** The number of apparent failures to unsplice removed nodes */
520	>	private transient volatile int sweepVotes;
521
522	<	/** tail of the queue */
523	<	private transient final PaddedAtomicReference<Node<E>> tail;
522	>	// CAS methods for fields
523	>	private boolean casTail(Node cmp, Node val) {
524	>	return UNSAFE.compareAndSwapObject(this, tailOffset, cmp, val);
525	>	}
526
527	<	/**
528	<	* Reference to a cancelled node that might not yet have been
529	<	* unlinked from queue because it was the last inserted node
186	<	* when it cancelled.
187	<	*/
188	<	private transient final PaddedAtomicReference<Node<E>> cleanMe;
527	>	private boolean casHead(Node cmp, Node val) {
528	>	return UNSAFE.compareAndSwapObject(this, headOffset, cmp, val);
529	>	}
530
531	<	/**
532	<	* Tries to cas nh as new head; if successful, unlink
533	<	* old head's next node to avoid garbage retention.
531	>	private boolean casSweepVotes(int cmp, int val) {
532	>	return UNSAFE.compareAndSwapInt(this, sweepVotesOffset, cmp, val);
533	>	}
534	>
535	>	/*
536	>	* Possible values for "how" argument in xfer method.
537		*/
538	<	private boolean advanceHead(Node<E> h, Node<E> nh) {
539	<	if (h == head.get() && head.compareAndSet(h, nh)) {
540	<	h.clearNext(); // forget old next
541	<	return true;
542	<	}
543	<	return false;
538	>	private static final int NOW   = 0; // for untimed poll, tryTransfer
539	>	private static final int ASYNC = 1; // for offer, put, add
540	>	private static final int SYNC  = 2; // for transfer, take
541	>	private static final int TIMED = 3; // for timed poll, tryTransfer
542	>
543	>	@SuppressWarnings("unchecked")
544	>	static <E> E cast(Object item) {
545	>	//        assert item == null || item.getClass() != Node.class;
546	>	return (E) item;
547		}
548
549		/**
550	<	* Puts or takes an item. Used for most queue operations (except
551	<	* poll() and tryTransfer()). See the similar code in
552	<	* SynchronousQueue for detailed explanation.
553	<	*
554	<	* @param e the item or if null, signifies that this is a take
555	<	* @param mode the wait mode: NOWAIT, TIMEOUT, WAIT
556	<	* @param nanos timeout in nanosecs, used only if mode is TIMEOUT
557	<	* @return an item, or null on failure
558	<	*/
559	<	private E xfer(E e, int mode, long nanos) {
560	<	boolean isData = (e != null);
561	<	Node<E> s = null;
562	<	final PaddedAtomicReference<Node<E>> head = this.head;
216	<	final PaddedAtomicReference<Node<E>> tail = this.tail;
550	>	* Implements all queuing methods. See above for explanation.
551	>	*
552	>	* @param e the item or null for take
553	>	* @param haveData true if this is a put, else a take
554	>	* @param how NOW, ASYNC, SYNC, or TIMED
555	>	* @param nanos timeout in nanosecs, used only if mode is TIMED
556	>	* @return an item if matched, else e
557	>	* @throws NullPointerException if haveData mode but e is null
558	>	*/
559	>	private E xfer(E e, boolean haveData, int how, long nanos) {
560	>	if (haveData && (e == null))
561	>	throw new NullPointerException();
562	>	Node s = null;                        // the node to append, if needed
563
564	<	for (;;) {
219	<	Node<E> t = tail.get();
220	<	Node<E> h = head.get();
564	>	retry: for (;;) {                     // restart on append race
565
566	<	if (t != null && (t == h || t.isData == isData)) {
567	<	if (s == null)
568	<	s = new Node<E>(e, isData);
569	<	Node<E> last = t.next;
570	<	if (last != null) {
571	<	if (t == tail.get())
572	<	tail.compareAndSet(t, last);
573	<	}
574	<	else if (t.casNext(null, s)) {
575	<	tail.compareAndSet(t, s);
576	<	return awaitFulfill(t, s, e, mode, nanos);
577	<	}
578	<	}
579	<
580	<	else if (h != null) {
581	<	Node<E> first = h.next;
582	<	if (t == tail.get() && first != null &&
583	<	advanceHead(h, first)) {
584	<	Object x = first.get();
241	<	if (x != first && first.compareAndSet(x, e)) {
242	<	LockSupport.unpark(first.waiter);
243	<	return isData ? e : (E) x;
566	>	for (Node h = head, p = h; p != null;) { // find & match first node
567	>	boolean isData = p.isData;
568	>	Object item = p.item;
569	>	if (item != p && (item != null) == isData) { // unmatched
570	>	if (isData == haveData)   // can't match
571	>	break;
572	>	if (p.casItem(item, e)) { // match
573	>	for (Node q = p; q != h;) {
574	>	Node n = q.next;  // update by 2 unless singleton
575	>	if (head == h && casHead(h, n == null? q : n)) {
576	>	h.forgetNext();
577	>	break;
578	>	}                 // advance and retry
579	>	if ((h = head)   == null ||
580	>	(q = h.next) == null || !q.isMatched())
581	>	break;        // unless slack < 2
582	>	}
583	>	LockSupport.unpark(p.waiter);
584	>	return this.<E>cast(item);
585		}
586		}
587	+	Node n = p.next;
588	+	p = (p != n) ? n : (h = head); // Use head if p offlist
589		}
590	+
591	+	if (how != NOW) {                 // No matches available
592	+	if (s == null)
593	+	s = new Node(e, haveData);
594	+	Node pred = tryAppend(s, haveData);
595	+	if (pred == null)
596	+	continue retry;           // lost race vs opposite mode
597	+	if (how != ASYNC)
598	+	return awaitMatch(s, pred, e, (how == TIMED), nanos);
599	+	}
600	+	return e; // not waiting
601		}
602		}
603
250	–
604		/**
605	<	* Version of xfer for poll() and tryTransfer, which
606	<	* simplifies control paths both here and in xfer.
607	<	*/
608	<	private E fulfill(E e) {
609	<	boolean isData = (e != null);
610	<	final PaddedAtomicReference<Node<E>> head = this.head;
611	<	final PaddedAtomicReference<Node<E>> tail = this.tail;
612	<
613	<	for (;;) {
614	<	Node<E> t = tail.get();
615	<	Node<E> h = head.get();
616	<
617	<	if (t != null && (t == h || t.isData == isData)) {
618	<	Node<E> last = t.next;
619	<	if (t == tail.get()) {
620	<	if (last != null)
621	<	tail.compareAndSet(t, last);
622	<	else
623	<	return null;
624	<	}
625	<	}
626	<	else if (h != null) {
627	<	Node<E> first = h.next;
628	<	if (t == tail.get() &&
629	<	first != null &&
630	<	advanceHead(h, first)) {
631	<	Object x = first.get();
632	<	if (x != first && first.compareAndSet(x, e)) {
280	<	LockSupport.unpark(first.waiter);
281	<	return isData ? e : (E) x;
282	<	}
605	>	* Tries to append node s as tail.
606	>	*
607	>	* @param s the node to append
608	>	* @param haveData true if appending in data mode
609	>	* @return null on failure due to losing race with append in
610	>	* different mode, else s's predecessor, or s itself if no
611	>	* predecessor
612	>	*/
613	>	private Node tryAppend(Node s, boolean haveData) {
614	>	for (Node t = tail, p = t;;) {        // move p to last node and append
615	>	Node n, u;                        // temps for reads of next & tail
616	>	if (p == null && (p = head) == null) {
617	>	if (casHead(null, s))
618	>	return s;                 // initialize
619	>	}
620	>	else if (p.cannotPrecede(haveData))
621	>	return null;                  // lost race vs opposite mode
622	>	else if ((n = p.next) != null)    // not last; keep traversing
623	>	p = p != t && t != (u = tail) ? (t = u) : // stale tail
624	>	(p != n) ? n : null;      // restart if off list
625	>	else if (!p.casNext(null, s))
626	>	p = p.next;                   // re-read on CAS failure
627	>	else {
628	>	if (p != t) {                 // update if slack now >= 2
629	>	while ((tail != t || !casTail(t, s)) &&
630	>	(t = tail)   != null &&
631	>	(s = t.next) != null && // advance and retry
632	>	(s = s.next) != null && s != t);
633		}
634	+	return p;
635		}
636		}
637		}
638
639		/**
640	<	* Spins/blocks until node s is fulfilled or caller gives up,
290	<	* depending on wait mode.
640	>	* Spins/yields/blocks until node s is matched or caller gives up.
641		*
292	–	* @param pred the predecessor of waiting node
642		* @param s the waiting node
643	+	* @param pred the predecessor of s, or s itself if it has no
644	+	* predecessor, or null if unknown (the null case does not occur
645	+	* in any current calls but may in possible future extensions)
646		* @param e the comparison value for checking match
647	<	* @param mode mode
648	<	* @param nanos timeout value
649	<	* @return matched item, or null if cancelled
650	<	*/
651	<	private E awaitFulfill(Node<E> pred, Node<E> s, E e,
652	<	int mode, long nanos) {
301	<	if (mode == NOWAIT)
302	<	return null;
303	<
304	<	long lastTime = (mode == TIMEOUT) ? System.nanoTime() : 0;
647	>	* @param timed if true, wait only until timeout elapses
648	>	* @param nanos timeout in nanosecs, used only if timed is true
649	>	* @return matched item, or e if unmatched on interrupt or timeout
650	>	*/
651	>	private E awaitMatch(Node s, Node pred, E e, boolean timed, long nanos) {
652	>	long lastTime = timed ? System.nanoTime() : 0L;
653		Thread w = Thread.currentThread();
654	<	int spins = -1; // set to desired spin count below
654	>	int spins = -1; // initialized after first item and cancel checks
655	>	ThreadLocalRandom randomYields = null; // bound if needed
656	>
657		for (;;) {
658	<	if (w.isInterrupted())
659	<	s.compareAndSet(e, s);
660	<	Object x = s.get();
661	<	if (x != e) {                 // Node was matched or cancelled
662	<	advanceHead(pred, s);     // unlink if head
663	<	if (x == s) {             // was cancelled
664	<	clean(pred, s);
665	<	return null;
666	<	}
667	<	else if (x != null) {
668	<	s.set(s);             // avoid garbage retention
669	<	return (E) x;
670	<	}
671	<	else
672	<	return e;
658	>	Object item = s.item;
659	>	if (item != e) {                  // matched
660	>	//                assert item != s;
661	>	s.forgetContents();           // avoid garbage
662	>	return this.<E>cast(item);
663	>	}
664	>	if ((w.isInterrupted() || (timed && nanos <= 0)) &&
665	>	s.casItem(e, s)) {        // cancel
666	>	unsplice(pred, s);
667	>	return e;
668	>	}
669	>
670	>	if (spins < 0) {                  // establish spins at/near front
671	>	if ((spins = spinsFor(pred, s.isData)) > 0)
672	>	randomYields = ThreadLocalRandom.current();
673	>	}
674	>	else if (spins > 0) {             // spin
675	>	--spins;
676	>	if (randomYields.nextInt(CHAINED_SPINS) == 0)
677	>	Thread.yield();           // occasionally yield
678	>	}
679	>	else if (s.waiter == null) {
680	>	s.waiter = w;                 // request unpark then recheck
681		}
682	<	if (mode == TIMEOUT) {
682	>	else if (timed) {
683		long now = System.nanoTime();
684	<	nanos -= now - lastTime;
684	>	if ((nanos -= now - lastTime) > 0)
685	>	LockSupport.parkNanos(this, nanos);
686		lastTime = now;
328	–	if (nanos <= 0) {
329	–	s.compareAndSet(e, s); // try to cancel
330	–	continue;
331	–	}
332	–	}
333	–	if (spins < 0) {
334	–	Node<E> h = head.get(); // only spin if at head
335	–	spins = ((h != null && h.next == s) ?
336	–	((mode == TIMEOUT) ?
337	–	maxTimedSpins : maxUntimedSpins) : 0);
687		}
688	<	if (spins > 0)
340	<	--spins;
341	<	else if (s.waiter == null)
342	<	s.waiter = w;
343	<	else if (mode != TIMEOUT) {
688	>	else {
689		LockSupport.park(this);
345	–	s.waiter = null;
346	–	spins = -1;
347	–	}
348	–	else if (nanos > spinForTimeoutThreshold) {
349	–	LockSupport.parkNanos(this, nanos);
350	–	s.waiter = null;
351	–	spins = -1;
690		}
691		}
692		}
693
694		/**
695	<	* Returns validated tail for use in cleaning methods.
695	>	* Returns spin/yield value for a node with given predecessor and
696	>	* data mode. See above for explanation.
697		*/
698	<	private Node<E> getValidatedTail() {
699	<	for (;;) {
700	<	Node<E> h = head.get();
701	<	Node<E> first = h.next;
702	<	if (first != null && first.get() == first) { // help advance
703	<	advanceHead(h, first);
704	<	continue;
705	<	}
367	<	Node<E> t = tail.get();
368	<	Node<E> last = t.next;
369	<	if (t == tail.get()) {
370	<	if (last != null)
371	<	tail.compareAndSet(t, last); // help advance
372	<	else
373	<	return t;
374	<	}
698	>	private static int spinsFor(Node pred, boolean haveData) {
699	>	if (MP && pred != null) {
700	>	if (pred.isData != haveData)      // phase change
701	>	return FRONT_SPINS + CHAINED_SPINS;
702	>	if (pred.isMatched())             // probably at front
703	>	return FRONT_SPINS;
704	>	if (pred.waiter == null)          // pred apparently spinning
705	>	return CHAINED_SPINS;
706		}
707	+	return 0;
708		}
709
710	+	/* -------------- Traversal methods -------------- */
711	+
712		/**
713	<	* Gets rid of cancelled node s with original predecessor pred.
714	<	*
715	<	* @param pred predecessor of cancelled node
716	<	* @param s the cancelled node
713	>	* Returns the successor of p, or the head node if p.next has been
714	>	* linked to self, which will only be true if traversing with a
715	>	* stale pointer that is now off the list.
716	>	*/
717	>	final Node succ(Node p) {
718	>	Node next = p.next;
719	>	return (p == next) ? head : next;
720	>	}
721	>
722	>	/**
723	>	* Returns the first unmatched node of the given mode, or null if
724	>	* none.  Used by methods isEmpty, hasWaitingConsumer.
725		*/
726	<	private void clean(Node<E> pred, Node<E> s) {
727	<	Thread w = s.waiter;
728	<	if (w != null) {             // Wake up thread
729	<	s.waiter = null;
730	<	if (w != Thread.currentThread())
731	<	LockSupport.unpark(w);
726	>	private Node firstOfMode(boolean isData) {
727	>	for (Node p = head; p != null; p = succ(p)) {
728	>	if (!p.isMatched())
729	>	return (p.isData == isData) ? p : null;
730	>	}
731	>	return null;
732	>	}
733	>
734	>	/**
735	>	* Returns the item in the first unmatched node with isData; or
736	>	* null if none.  Used by peek.
737	>	*/
738	>	private E firstDataItem() {
739	>	for (Node p = head; p != null; p = succ(p)) {
740	>	Object item = p.item;
741	>	if (p.isData) {
742	>	if (item != null && item != p)
743	>	return this.<E>cast(item);
744	>	}
745	>	else if (item == null)
746	>	return null;
747		}
748	+	return null;
749	+	}
750
751	<	if (pred == null)
752	<	return;
751	>	/**
752	>	* Traverses and counts unmatched nodes of the given mode.
753	>	* Used by methods size and getWaitingConsumerCount.
754	>	*/
755	>	private int countOfMode(boolean data) {
756	>	int count = 0;
757	>	for (Node p = head; p != null; ) {
758	>	if (!p.isMatched()) {
759	>	if (p.isData != data)
760	>	return 0;
761	>	if (++count == Integer.MAX_VALUE) // saturated
762	>	break;
763	>	}
764	>	Node n = p.next;
765	>	if (n != p)
766	>	p = n;
767	>	else {
768	>	count = 0;
769	>	p = head;
770	>	}
771	>	}
772	>	return count;
773	>	}
774	>
775	>	final class Itr implements Iterator<E> {
776	>	private Node nextNode;   // next node to return item for
777	>	private E nextItem;      // the corresponding item
778	>	private Node lastRet;    // last returned node, to support remove
779	>	private Node lastPred;   // predecessor to unlink lastRet
780
781	<	/*
782	<	* At any given time, exactly one node on list cannot be
397	<	* deleted -- the last inserted node. To accommodate this, if
398	<	* we cannot delete s, we save its predecessor as "cleanMe",
399	<	* processing the previously saved version first. At least one
400	<	* of node s or the node previously saved can always be
401	<	* processed, so this always terminates.
781	>	/**
782	>	* Moves to next node after prev, or first node if prev null.
783		*/
784	<	while (pred.next == s) {
785	<	Node<E> oldpred = reclean();  // First, help get rid of cleanMe
786	<	Node<E> t = getValidatedTail();
787	<	if (s != t) {               // If not tail, try to unsplice
788	<	Node<E> sn = s.next;      // s.next == s means s already off list
789	<	if (sn == s || pred.casNext(s, sn))
784	>	private void advance(Node prev) {
785	>	lastPred = lastRet;
786	>	lastRet = prev;
787	>	for (Node p = (prev == null) ? head : succ(prev);
788	>	p != null; p = succ(p)) {
789	>	Object item = p.item;
790	>	if (p.isData) {
791	>	if (item != null && item != p) {
792	>	nextItem = LinkedTransferQueue.this.<E>cast(item);
793	>	nextNode = p;
794	>	return;
795	>	}
796	>	}
797	>	else if (item == null)
798		break;
799		}
800	<	else if (oldpred == pred || // Already saved
801	<	(oldpred == null && cleanMe.compareAndSet(null, pred)))
802	<	break;                  // Postpone cleaning
800	>	nextNode = null;
801	>	}
802	>
803	>	Itr() {
804	>	advance(null);
805	>	}
806	>
807	>	public final boolean hasNext() {
808	>	return nextNode != null;
809	>	}
810	>
811	>	public final E next() {
812	>	Node p = nextNode;
813	>	if (p == null) throw new NoSuchElementException();
814	>	E e = nextItem;
815	>	advance(p);
816	>	return e;
817	>	}
818	>
819	>	public final void remove() {
820	>	Node p = lastRet;
821	>	if (p == null) throw new IllegalStateException();
822	>	if (p.tryMatchData())
823	>	unsplice(lastPred, p);
824		}
825		}
826
827	+	/* -------------- Removal methods -------------- */
828	+
829		/**
830	<	* Tries to unsplice the cancelled node held in cleanMe that was
831	<	* previously uncleanable because it was at tail.
830	>	* Unsplices (now or later) the given deleted/cancelled node with
831	>	* the given predecessor.
832		*
833	<	* @return current cleanMe node (or null)
833	>	* @param pred a node that was at one time known to be the
834	>	* predecessor of s, or null or s itself if s is/was at head
835	>	* @param s the node to be unspliced
836		*/
837	<	private Node<E> reclean() {
837	>	final void unsplice(Node pred, Node s) {
838	>	s.forgetContents(); // forget unneeded fields
839		/*
840	<	* cleanMe is, or at one time was, predecessor of cancelled
841	<	* node s that was the tail so could not be unspliced.  If s
842	<	* is no longer the tail, try to unsplice if necessary and
843	<	* make cleanMe slot available.  This differs from similar
844	<	* code in clean() because we must check that pred still
430	<	* points to a cancelled node that must be unspliced -- if
431	<	* not, we can (must) clear cleanMe without unsplicing.
432	<	* This can loop only due to contention on casNext or
433	<	* clearing cleanMe.
840	>	* See above for rationale. Briefly: if pred still points to
841	>	* s, try to unlink s.  If s cannot be unlinked, because it is
842	>	* trailing node or pred might be unlinked, and neither pred
843	>	* nor s are head or offlist, add to sweepVotes, and if enough
844	>	* votes have accumulated, sweep.
845		*/
846	<	Node<E> pred;
847	<	while ((pred = cleanMe.get()) != null) {
848	<	Node<E> t = getValidatedTail();
849	<	Node<E> s = pred.next;
850	<	if (s != t) {
851	<	Node<E> sn;
852	<	if (s == null || s == pred || s.get() != s ||
853	<	(sn = s.next) == s || pred.casNext(s, sn))
854	<	cleanMe.compareAndSet(pred, null);
846	>	if (pred != null && pred != s && pred.next == s) {
847	>	Node n = s.next;
848	>	if (n == null ||
849	>	(n != s && pred.casNext(s, n) && pred.isMatched())) {
850	>	for (;;) {               // check if at, or could be, head
851	>	Node h = head;
852	>	if (h == pred || h == s || h == null)
853	>	return;          // at head or list empty
854	>	if (!h.isMatched())
855	>	break;
856	>	Node hn = h.next;
857	>	if (hn == null)
858	>	return;          // now empty
859	>	if (hn != h && casHead(h, hn))
860	>	h.forgetNext();  // advance head
861	>	}
862	>	if (pred.next != pred && s.next != s) { // recheck if offlist
863	>	for (;;) {           // sweep now if enough votes
864	>	int v = sweepVotes;
865	>	if (v < SWEEP_THRESHOLD) {
866	>	if (casSweepVotes(v, v + 1))
867	>	break;
868	>	}
869	>	else if (casSweepVotes(v, 0)) {
870	>	sweep();
871	>	break;
872	>	}
873	>	}
874	>	}
875		}
876	<	else // s is still tail; cannot clean
876	>	}
877	>	}
878	>
879	>	/**
880	>	* Unlinks matched (typically cancelled) nodes encountered in a
881	>	* traversal from head.
882	>	*/
883	>	private void sweep() {
884	>	for (Node p = head, s, n; p != null && (s = p.next) != null; ) {
885	>	if (p == s)                    // stale
886	>	p = head;
887	>	else if (!s.isMatched())
888	>	p = s;
889	>	else if ((n = s.next) == null || s == n) // trailing node is pinned
890		break;
891	+	else
892	+	p.casNext(s, n);
893		}
448	–	return pred;
894		}
895
896		/**
897	+	* Main implementation of remove(Object)
898	+	*/
899	+	private boolean findAndRemove(Object e) {
900	+	if (e != null) {
901	+	for (Node pred = null, p = head; p != null; ) {
902	+	Object item = p.item;
903	+	if (p.isData) {
904	+	if (item != null && item != p && e.equals(item) &&
905	+	p.tryMatchData()) {
906	+	unsplice(pred, p);
907	+	return true;
908	+	}
909	+	}
910	+	else if (item == null)
911	+	break;
912	+	pred = p;
913	+	if ((p = p.next) == pred) { // stale
914	+	pred = null;
915	+	p = head;
916	+	}
917	+	}
918	+	}
919	+	return false;
920	+	}
921	+
922	+
923	+	/**
924		* Creates an initially empty {@code LinkedTransferQueue}.
925		*/
926		public LinkedTransferQueue() {
455	–	Node<E> dummy = new Node<E>(null, false);
456	–	head = new PaddedAtomicReference<Node<E>>(dummy);
457	–	tail = new PaddedAtomicReference<Node<E>>(dummy);
458	–	cleanMe = new PaddedAtomicReference<Node<E>>(null);
927		}
928
929		/**
#	Line 479 | Line 947 | public class LinkedTransferQueue<E> exte
947		* @throws NullPointerException if the specified element is null
948		*/
949		public void put(E e) {
950	<	offer(e);
950	>	xfer(e, true, ASYNC, 0);
951		}
952
953		/**
#	Line 492 | Line 960 | public class LinkedTransferQueue<E> exte
960		* @throws NullPointerException if the specified element is null
961		*/
962		public boolean offer(E e, long timeout, TimeUnit unit) {
963	<	return offer(e);
963	>	xfer(e, true, ASYNC, 0);
964	>	return true;
965		}
966
967		/**
#	Line 504 | Line 973 | public class LinkedTransferQueue<E> exte
973		* @throws NullPointerException if the specified element is null
974		*/
975		public boolean offer(E e) {
976	<	if (e == null) throw new NullPointerException();
508	<	xfer(e, NOWAIT, 0);
976	>	xfer(e, true, ASYNC, 0);
977		return true;
978		}
979
#	Line 518 | Line 986 | public class LinkedTransferQueue<E> exte
986		* @throws NullPointerException if the specified element is null
987		*/
988		public boolean add(E e) {
989	<	return offer(e);
989	>	xfer(e, true, ASYNC, 0);
990	>	return true;
991		}
992
993		/**
#	Line 532 | Line 1001 | public class LinkedTransferQueue<E> exte
1001		* @throws NullPointerException if the specified element is null
1002		*/
1003		public boolean tryTransfer(E e) {
1004	<	if (e == null) throw new NullPointerException();
536	<	return fulfill(e) != null;
1004	>	return xfer(e, true, NOW, 0) == null;
1005		}
1006
1007		/**
#	Line 548 | Line 1016 | public class LinkedTransferQueue<E> exte
1016		* @throws NullPointerException if the specified element is null
1017		*/
1018		public void transfer(E e) throws InterruptedException {
1019	<	if (e == null) throw new NullPointerException();
1020	<	if (xfer(e, WAIT, 0) == null) {
553	<	Thread.interrupted();
1019	>	if (xfer(e, true, SYNC, 0) != null) {
1020	>	Thread.interrupted(); // failure possible only due to interrupt
1021		throw new InterruptedException();
1022		}
1023		}
#	Line 571 | Line 1038 | public class LinkedTransferQueue<E> exte
1038		*/
1039		public boolean tryTransfer(E e, long timeout, TimeUnit unit)
1040		throws InterruptedException {
1041	<	if (e == null) throw new NullPointerException();
575	<	if (xfer(e, TIMEOUT, unit.toNanos(timeout)) != null)
1041	>	if (xfer(e, true, TIMED, unit.toNanos(timeout)) == null)
1042		return true;
1043		if (!Thread.interrupted())
1044		return false;
#	Line 580 | Line 1046 | public class LinkedTransferQueue<E> exte
1046		}
1047
1048		public E take() throws InterruptedException {
1049	<	E e = xfer(null, WAIT, 0);
1049	>	E e = xfer(null, false, SYNC, 0);
1050		if (e != null)
1051		return e;
1052		Thread.interrupted();
#	Line 588 | Line 1054 | public class LinkedTransferQueue<E> exte
1054		}
1055
1056		public E poll(long timeout, TimeUnit unit) throws InterruptedException {
1057	<	E e = xfer(null, TIMEOUT, unit.toNanos(timeout));
1057	>	E e = xfer(null, false, TIMED, unit.toNanos(timeout));
1058		if (e != null || !Thread.interrupted())
1059		return e;
1060		throw new InterruptedException();
1061		}
1062
1063		public E poll() {
1064	<	return fulfill(null);
1064	>	return xfer(null, false, NOW, 0);
1065		}
1066
1067		/**
#	Line 634 | Line 1100 | public class LinkedTransferQueue<E> exte
1100		return n;
1101		}
1102
637	–	// Traversal-based methods
638	–
639	–	/**
640	–	* Returns head after performing any outstanding helping steps.
641	–	*/
642	–	private Node<E> traversalHead() {
643	–	for (;;) {
644	–	Node<E> t = tail.get();
645	–	Node<E> h = head.get();
646	–	if (h != null && t != null) {
647	–	Node<E> last = t.next;
648	–	Node<E> first = h.next;
649	–	if (t == tail.get()) {
650	–	if (last != null)
651	–	tail.compareAndSet(t, last);
652	–	else if (first != null) {
653	–	Object x = first.get();
654	–	if (x == first)
655	–	advanceHead(h, first);
656	–	else
657	–	return h;
658	–	}
659	–	else
660	–	return h;
661	–	}
662	–	}
663	–	reclean();
664	–	}
665	–	}
666	–
1103		/**
1104		* Returns an iterator over the elements in this queue in proper
1105		* sequence, from head to tail.
#	Line 681 | Line 1117 | public class LinkedTransferQueue<E> exte
1117		return new Itr();
1118		}
1119
684	–	/**
685	–	* Iterators. Basic strategy is to traverse list, treating
686	–	* non-data (i.e., request) nodes as terminating list.
687	–	* Once a valid data node is found, the item is cached
688	–	* so that the next call to next() will return it even
689	–	* if subsequently removed.
690	–	*/
691	–	class Itr implements Iterator<E> {
692	–	Node<E> next;        // node to return next
693	–	Node<E> pnext;       // predecessor of next
694	–	Node<E> curr;        // last returned node, for remove()
695	–	Node<E> pcurr;       // predecessor of curr, for remove()
696	–	E nextItem;          // Cache of next item, once committed to in next
697	–
698	–	Itr() {
699	–	advance();
700	–	}
701	–
702	–	/**
703	–	* Moves to next valid node and returns item to return for
704	–	* next(), or null if no such.
705	–	*/
706	–	private E advance() {
707	–	pcurr = pnext;
708	–	curr = next;
709	–	E item = nextItem;
710	–
711	–	for (;;) {
712	–	pnext = (next == null) ? traversalHead() : next;
713	–	next = pnext.next;
714	–	if (next == pnext) {
715	–	next = null;
716	–	continue;  // restart
717	–	}
718	–	if (next == null)
719	–	break;
720	–	Object x = next.get();
721	–	if (x != null && x != next) {
722	–	nextItem = (E) x;
723	–	break;
724	–	}
725	–	}
726	–	return item;
727	–	}
728	–
729	–	public boolean hasNext() {
730	–	return next != null;
731	–	}
732	–
733	–	public E next() {
734	–	if (next == null)
735	–	throw new NoSuchElementException();
736	–	return advance();
737	–	}
738	–
739	–	public void remove() {
740	–	Node<E> p = curr;
741	–	if (p == null)
742	–	throw new IllegalStateException();
743	–	Object x = p.get();
744	–	if (x != null && x != p && p.compareAndSet(x, p))
745	–	clean(pcurr, p);
746	–	}
747	–	}
748	–
1120		public E peek() {
1121	<	for (;;) {
751	<	Node<E> h = traversalHead();
752	<	Node<E> p = h.next;
753	<	if (p == null)
754	<	return null;
755	<	Object x = p.get();
756	<	if (p != x) {
757	<	if (!p.isData)
758	<	return null;
759	<	if (x != null)
760	<	return (E) x;
761	<	}
762	<	}
1121	>	return firstDataItem();
1122		}
1123
1124		/**
#	Line 768 | Line 1127 | public class LinkedTransferQueue<E> exte
1127		* @return {@code true} if this queue contains no elements
1128		*/
1129		public boolean isEmpty() {
1130	<	for (;;) {
1131	<	Node<E> h = traversalHead();
1132	<	Node<E> p = h.next;
774	<	if (p == null)
775	<	return true;
776	<	Object x = p.get();
777	<	if (p != x) {
778	<	if (!p.isData)
779	<	return true;
780	<	if (x != null)
781	<	return false;
782	<	}
1130	>	for (Node p = head; p != null; p = succ(p)) {
1131	>	if (!p.isMatched())
1132	>	return !p.isData;
1133		}
1134	+	return true;
1135		}
1136
1137		public boolean hasWaitingConsumer() {
1138	<	for (;;) {
788	<	Node<E> h = traversalHead();
789	<	Node<E> p = h.next;
790	<	if (p == null)
791	<	return false;
792	<	Object x = p.get();
793	<	if (p != x)
794	<	return !p.isData;
795	<	}
1138	>	return firstOfMode(false) != null;
1139		}
1140
1141		/**
#	Line 808 | Line 1151 | public class LinkedTransferQueue<E> exte
1151		* @return the number of elements in this queue
1152		*/
1153		public int size() {
1154	<	for (;;) {
812	<	int count = 0;
813	<	Node<E> pred = traversalHead();
814	<	for (;;) {
815	<	Node<E> q = pred.next;
816	<	if (q == pred) // restart
817	<	break;
818	<	if (q == null || !q.isData)
819	<	return count;
820	<	Object x = q.get();
821	<	if (x != null && x != q) {
822	<	if (++count == Integer.MAX_VALUE) // saturated
823	<	return count;
824	<	}
825	<	pred = q;
826	<	}
827	<	}
1154	>	return countOfMode(true);
1155		}
1156
1157		public int getWaitingConsumerCount() {
1158	<	// converse of size -- count valid non-data nodes
832	<	for (;;) {
833	<	int count = 0;
834	<	Node<E> pred = traversalHead();
835	<	for (;;) {
836	<	Node<E> q = pred.next;
837	<	if (q == pred) // restart
838	<	break;
839	<	if (q == null || q.isData)
840	<	return count;
841	<	Object x = q.get();
842	<	if (x == null) {
843	<	if (++count == Integer.MAX_VALUE) // saturated
844	<	return count;
845	<	}
846	<	pred = q;
847	<	}
848	<	}
1158	>	return countOfMode(false);
1159		}
1160
1161		/**
#	Line 860 | Line 1170 | public class LinkedTransferQueue<E> exte
1170		* @return {@code true} if this queue changed as a result of the call
1171		*/
1172		public boolean remove(Object o) {
1173	<	if (o == null)
864	<	return false;
865	<	for (;;) {
866	<	Node<E> pred = traversalHead();
867	<	for (;;) {
868	<	Node<E> q = pred.next;
869	<	if (q == pred) // restart
870	<	break;
871	<	if (q == null || !q.isData)
872	<	return false;
873	<	Object x = q.get();
874	<	if (x != null && x != q && o.equals(x) &&
875	<	q.compareAndSet(x, q)) {
876	<	clean(pred, q);
877	<	return true;
878	<	}
879	<	pred = q;
880	<	}
881	<	}
1173	>	return findAndRemove(o);
1174		}
1175
1176		/**
#	Line 893 | Line 1185 | public class LinkedTransferQueue<E> exte
1185		}
1186
1187		/**
1188	<	* Save the state to a stream (that is, serialize it).
1188	>	* Saves the state to a stream (that is, serializes it).
1189		*
1190		* @serialData All of the elements (each an {@code E}) in
1191		* the proper order, followed by a null
#	Line 909 | Line 1201 | public class LinkedTransferQueue<E> exte
1201		}
1202
1203		/**
1204	<	* Reconstitute the Queue instance from a stream (that is,
1205	<	* deserialize it).
1204	>	* Reconstitutes the Queue instance from a stream (that is,
1205	>	* deserializes it).
1206		*
1207		* @param s the stream
1208		*/
1209		private void readObject(java.io.ObjectInputStream s)
1210		throws java.io.IOException, ClassNotFoundException {
1211		s.defaultReadObject();
920	–	resetHeadAndTail();
1212		for (;;) {
1213		@SuppressWarnings("unchecked") E item = (E) s.readObject();
1214		if (item == null)
#	Line 927 | Line 1218 | public class LinkedTransferQueue<E> exte
1218		}
1219		}
1220
930	–	// Support for resetting head/tail while deserializing
931	–	private void resetHeadAndTail() {
932	–	Node<E> dummy = new Node<E>(null, false);
933	–	UNSAFE.putObjectVolatile(this, headOffset,
934	–	new PaddedAtomicReference<Node<E>>(dummy));
935	–	UNSAFE.putObjectVolatile(this, tailOffset,
936	–	new PaddedAtomicReference<Node<E>>(dummy));
937	–	UNSAFE.putObjectVolatile(this, cleanMeOffset,
938	–	new PaddedAtomicReference<Node<E>>(null));
939	–	}
940	–
1221		// Unsafe mechanics
1222
1223		private static final sun.misc.Unsafe UNSAFE = getUnsafe();
#	Line 945 | Line 1225 | public class LinkedTransferQueue<E> exte
1225		objectFieldOffset(UNSAFE, "head", LinkedTransferQueue.class);
1226		private static final long tailOffset =
1227		objectFieldOffset(UNSAFE, "tail", LinkedTransferQueue.class);
1228	<	private static final long cleanMeOffset =
1229	<	objectFieldOffset(UNSAFE, "cleanMe", LinkedTransferQueue.class);
950	<
1228	>	private static final long sweepVotesOffset =
1229	>	objectFieldOffset(UNSAFE, "sweepVotes", LinkedTransferQueue.class);
1230
1231		static long objectFieldOffset(sun.misc.Unsafe UNSAFE,
1232		String field, Class<?> klazz) {
#	Line 968 | Line 1247 | public class LinkedTransferQueue<E> exte
1247		*
1248		* @return a sun.misc.Unsafe
1249		*/
1250	<	private static sun.misc.Unsafe getUnsafe() {
1250	>	static sun.misc.Unsafe getUnsafe() {
1251		try {
1252		return sun.misc.Unsafe.getUnsafe();
1253		} catch (SecurityException se) {
#	Line 988 | Line 1267 | public class LinkedTransferQueue<E> exte
1267		}
1268		}
1269		}
1270	+
1271		}

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines