util/Collection/RemoveMicroBenchmark.java

/*
 * Copyright (c) 2007, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */

/*
 * @test
 * @summary micro-benchmark correctness mode
 * @run main RemoveMicroBenchmark iterations=1 size=8 warmup=0
 */

import java.lang.ref.WeakReference;
import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Deque;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Spliterator;
import java.util.Vector;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.LinkedTransferQueue;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.function.Supplier;

/**
 * Usage: [iterations=N] [size=N] [filter=REGEXP] [warmup=SECONDS]
 *
 * To run this in micro-benchmark mode, simply run as a normal java program.
 * Be patient; this program runs for a very long time.
 * For faster runs, restrict execution using command line args.
 *
 * @author Martin Buchholz
 */
public class RemoveMicroBenchmark {
    abstract static class Job {
        private final String name;
        public Job(String name) { this.name = name; }
        public String name() { return name; }
        public abstract void work() throws Throwable;
    }

    int iterations;
    int size;
    double warmupSeconds;
    Pattern filter;

    // --------------- GC finalization infrastructure ---------------

    /** No guarantees, but effective in practice. */
    static void forceFullGc() {
        CountDownLatch finalizeDone = new CountDownLatch(1);
        WeakReference<?> ref = new WeakReference<Object>(new Object() {
            protected void finalize() { finalizeDone.countDown(); }});
        try {
            for (int i = 0; i < 10; i++) {
                System.gc();
                if (finalizeDone.await(1L, TimeUnit.SECONDS) && ref.get() == null) {
                    System.runFinalization(); // try to pick up stragglers
                    return;
                }
            }
        } catch (InterruptedException unexpected) {
            throw new AssertionError("unexpected InterruptedException");
        }
        throw new AssertionError("failed to do a \"full\" gc");
    }

    /**
     * Runs each job for long enough that all the runtime compilers
     * have had plenty of time to warm up, i.e. get around to
     * compiling everything worth compiling.
     * Returns array of average times per job per run.
     */
    long[] time0(List<Job> jobs) throws Throwable {
        final long warmupNanos = (long) (warmupSeconds * 1000L * 1000L * 1000L);
        final int size = jobs.size();
        long[] nanoss = new long[size];
        for (int i = 0; i < size; i++) {
            if (warmupNanos > 0) forceFullGc();
            long t0 = System.nanoTime();
            long t;
            int j = 0;
            do { jobs.get(i).work(); j++; }
            while ((t = System.nanoTime() - t0) < warmupNanos);
            nanoss[i] = t/j;
        }
        return nanoss;
    }

    void time(List<Job> jobs) throws Throwable {
        if (warmupSeconds > 0) time0(jobs); // Warm up run
        final int size = jobs.size();
        final long[] nanoss = time0(jobs); // Real timing run
        final long[] milliss = new long[size];
        final double[] ratios = new double[size];

        final String nameHeader   = "Method";
        final String millisHeader = "Millis";
        final String ratioHeader  = "Ratio";

        int nameWidth   = nameHeader.length();
        int millisWidth = millisHeader.length();
        int ratioWidth  = ratioHeader.length();

        for (int i = 0; i < size; i++) {
            nameWidth = Math.max(nameWidth, jobs.get(i).name().length());

            milliss[i] = nanoss[i]/(1000L * 1000L);
            millisWidth = Math.max(millisWidth,
                                   String.format("%d", milliss[i]).length());

            ratios[i] = (double) nanoss[i] / (double) nanoss[0];
            ratioWidth = Math.max(ratioWidth,
                                  String.format("%.3f", ratios[i]).length());
        }

        String format = String.format("%%-%ds %%%dd %%%d.3f%%n",
                                      nameWidth, millisWidth, ratioWidth);
        String headerFormat = String.format("%%-%ds %%%ds %%%ds%%n",
                                            nameWidth, millisWidth, ratioWidth);
        System.out.printf(headerFormat, "Method", "Millis", "Ratio");

        // Print out absolute and relative times, calibrated against first job
        for (int i = 0; i < size; i++)
            System.out.printf(format, jobs.get(i).name(), milliss[i], ratios[i]);
    }

    private static String keywordValue(String[] args, String keyword) {
        for (String arg : args)
            if (arg.startsWith(keyword))
                return arg.substring(keyword.length() + 1);
        return null;
    }

    private static int intArg(String[] args, String keyword, int defaultValue) {
        String val = keywordValue(args, keyword);
        return (val == null) ? defaultValue : Integer.parseInt(val);
    }

    private static double doubleArg(String[] args, String keyword, double defaultValue) {
        String val = keywordValue(args, keyword);
        return (val == null) ? defaultValue : Double.parseDouble(val);
    }

    private static Pattern patternArg(String[] args, String keyword) {
        String val = keywordValue(args, keyword);
        return (val == null) ? null : Pattern.compile(val);
    }

    private static List<Job> filter(Pattern filter, List<Job> jobs) {
        if (filter == null) return jobs;
        ArrayList<Job> newJobs = new ArrayList<>();
        for (Job job : jobs)
            if (filter.matcher(job.name()).find())
                newJobs.add(job);
        return newJobs;
    }

    private static void deoptimize(int sum) {
        if (sum == 42)
            System.out.println("the answer");
    }

    private static <T> List<T> asSubList(List<T> list) {
        return list.subList(0, list.size());
    }

    private static <T> Iterable<T> backwards(final List<T> list) {
        return new Iterable<T>() {
            public Iterator<T> iterator() {
                return new Iterator<T>() {
                    final ListIterator<T> it = list.listIterator(list.size());
                    public boolean hasNext() { return it.hasPrevious(); }
                    public T next()          { return it.previous(); }
                    public void remove()     {        it.remove(); }};}};
    }

    // Checks for correctness *and* prevents loop optimizations
    class Check {
        private int sum;
        public void sum(int sum) {
            if (this.sum == 0)
                this.sum = sum;
            if (this.sum != sum)
                throw new AssertionError("Sum mismatch");
        }
    }
    volatile Check check = new Check();

    public static void main(String[] args) throws Throwable {
        new RemoveMicroBenchmark().run(args);
    }

    void run(String[] args) throws Throwable {
        iterations    = intArg(args, "iterations", 10_000);
        size          = intArg(args, "size", 1000);
        warmupSeconds = doubleArg(args, "warmup", 5);
        filter        = patternArg(args, "filter");
//         System.out.printf(
//             "iterations=%d size=%d, warmup=%1g, filter=\"%s\"%n",
//             iterations, size, warmupSeconds, filter);

        final ArrayList<Integer> al = new ArrayList<Integer>(size);

        // Populate collections with random data
        final ThreadLocalRandom rnd = ThreadLocalRandom.current();
        for (int i = 0; i < size; i++)
            al.add(rnd.nextInt(size));

        ArrayList<Job> jobs = new ArrayList<>();

        List.<Collection<Integer>>of(
            new ArrayList<>(),
            new LinkedList<>(),
            new Vector<>(),
            new ArrayDeque<>(),
            new ArrayBlockingQueue<>(al.size()),
            new ConcurrentLinkedQueue<>(),
            new ConcurrentLinkedDeque<>(),
            new LinkedBlockingQueue<>(),
            new LinkedBlockingDeque<>(),
            new LinkedTransferQueue<>())
            .stream().forEach(
                x -> {
                    String klazz = x.getClass().getSimpleName();
                    jobs.addAll(collectionJobs(klazz, () -> x, al));
                    if (x instanceof Deque) {
                        Deque<Integer> deque = (Deque<Integer>) x;
                        jobs.addAll(dequeJobs(klazz, () -> deque, al));
                    }
                    if (x instanceof List) {
                        List<Integer> list = (List<Integer>) x;
                        jobs.addAll(
                            collectionJobs(
                                klazz + " subList",
                                () -> list.subList(0, x.size()),
                                al));
                    }
                });

        time(filter(filter, jobs));
    }

    Collection<Integer> universeRecorder(int[] sum) {
        return new ArrayList<>() {
            public boolean contains(Object x) {
                sum[0] += (Integer) x;
                return true;
            }};
    }

    Collection<Integer> emptyRecorder(int[] sum) {
        return new ArrayList<>() {
            public boolean contains(Object x) {
                sum[0] += (Integer) x;
                return false;
            }};
    }

    List<Job> collectionJobs(
        String description,
        Supplier<Collection<Integer>> supplier,
        ArrayList<Integer> al) {
        return List.of(
            new Job(description + " .removeIf") {
                public void work() throws Throwable {
                    Collection<Integer> x = supplier.get();
                    int[] sum = new int[1];
                    for (int i = 0; i < iterations; i++) {
                        sum[0] = 0;
                        x.addAll(al);
                        x.removeIf(n -> { sum[0] += n; return true; });
                        check.sum(sum[0]);}}},
            new Job(description + " .removeAll") {
                public void work() throws Throwable {
                    Collection<Integer> x = supplier.get();
                    int[] sum = new int[1];
                    Collection<Integer> universe = universeRecorder(sum);
                    for (int i = 0; i < iterations; i++) {
                        sum[0] = 0;
                        x.addAll(al);
                        x.removeAll(universe);
                        check.sum(sum[0]);}}},
            new Job(description + " .retainAll") {
                public void work() throws Throwable {
                    Collection<Integer> x = supplier.get();
                    int[] sum = new int[1];
                    Collection<Integer> empty = emptyRecorder(sum);
                    for (int i = 0; i < iterations; i++) {
                        sum[0] = 0;
                        x.addAll(al);
                        x.retainAll(empty);
                        check.sum(sum[0]);}}},
            new Job(description + " Iterator.remove") {
                public void work() throws Throwable {
                    Collection<Integer> x = supplier.get();
                    int[] sum = new int[1];
                    for (int i = 0; i < iterations; i++) {
                        sum[0] = 0;
                        x.addAll(al);
                        Iterator<Integer> it = x.iterator();
                        while (it.hasNext()) {
                            sum[0] += it.next();
                            it.remove();
                        }
                        check.sum(sum[0]);}}},
            new Job(description + " clear") {
                public void work() throws Throwable {
                    Collection<Integer> x = supplier.get();
                    int[] sum = new int[1];
                    for (int i = 0; i < iterations; i++) {
                        sum[0] = 0;
                        x.addAll(al);
                        x.forEach(e -> sum[0] += e);
                        x.clear();
                        check.sum(sum[0]);}}});
    }

    List<Job> dequeJobs(
        String description,
        Supplier<Deque<Integer>> supplier,
        ArrayList<Integer> al) {
        return List.of(
            new Job(description + " descendingIterator().remove") {
                public void work() throws Throwable {
                    Deque<Integer> x = supplier.get();
                    int[] sum = new int[1];
                    for (int i = 0; i < iterations; i++) {
                        sum[0] = 0;
                        x.addAll(al);
                        Iterator<Integer> it = x.descendingIterator();
                        while (it.hasNext()) {
                            sum[0] += it.next();
                            it.remove();
                        }
                        check.sum(sum[0]);}}});
    }
}
Revision:	1.1
Committed:	Sun Nov 13 18:57:46 2016 UTC (7 years, 6 months ago) by jsr166
Branch:	MAIN
Log Message:	add RemoveMicroBenchmark
#	User	Rev	Content
1	jsr166	1.1	/*
2			* Copyright (c) 2007, Oracle and/or its affiliates. All rights reserved.
3			* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4			*
5			* This code is free software; you can redistribute it and/or modify it
6			* under the terms of the GNU General Public License version 2 only, as
7			* published by the Free Software Foundation.
8			*
9			* This code is distributed in the hope that it will be useful, but WITHOUT
10			* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11			* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12			* version 2 for more details (a copy is included in the LICENSE file that
13			* accompanied this code).
14			*
15			* You should have received a copy of the GNU General Public License version
16			* 2 along with this work; if not, write to the Free Software Foundation,
17			* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18			*
19			* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20			* or visit www.oracle.com if you need additional information or have any
21			* questions.
22			*/
23
24			/*
25			* @test
26			* @summary micro-benchmark correctness mode
27			* @run main RemoveMicroBenchmark iterations=1 size=8 warmup=0
28			*/
29
30			import java.lang.ref.WeakReference;
31			import java.util.ArrayDeque;
32			import java.util.Arrays;
33			import java.util.ArrayList;
34			import java.util.Collection;
35			import java.util.Deque;
36			import java.util.Enumeration;
37			import java.util.Iterator;
38			import java.util.LinkedList;
39			import java.util.List;
40			import java.util.ListIterator;
41			import java.util.Map;
42			import java.util.Spliterator;
43			import java.util.Vector;
44			import java.util.concurrent.ArrayBlockingQueue;
45			import java.util.concurrent.ConcurrentLinkedDeque;
46			import java.util.concurrent.ConcurrentLinkedQueue;
47			import java.util.concurrent.LinkedBlockingDeque;
48			import java.util.concurrent.LinkedBlockingQueue;
49			import java.util.concurrent.LinkedTransferQueue;
50			import java.util.concurrent.ConcurrentSkipListMap;
51			import java.util.concurrent.CountDownLatch;
52			import java.util.concurrent.ThreadLocalRandom;
53			import java.util.concurrent.TimeUnit;
54			import java.util.regex.Pattern;
55			import java.util.function.Supplier;
56
57			/**
58			* Usage: [iterations=N] [size=N] [filter=REGEXP] [warmup=SECONDS]
59			*
60			* To run this in micro-benchmark mode, simply run as a normal java program.
61			* Be patient; this program runs for a very long time.
62			* For faster runs, restrict execution using command line args.
63			*
64			* @author Martin Buchholz
65			*/
66			public class RemoveMicroBenchmark {
67			abstract static class Job {
68			private final String name;
69			public Job(String name) { this.name = name; }
70			public String name() { return name; }
71			public abstract void work() throws Throwable;
72			}
73
74			int iterations;
75			int size;
76			double warmupSeconds;
77			Pattern filter;
78
79			// --------------- GC finalization infrastructure ---------------
80
81			/** No guarantees, but effective in practice. */
82			static void forceFullGc() {
83			CountDownLatch finalizeDone = new CountDownLatch(1);
84			WeakReference<?> ref = new WeakReference<Object>(new Object() {
85			protected void finalize() { finalizeDone.countDown(); }});
86			try {
87			for (int i = 0; i < 10; i++) {
88			System.gc();
89			if (finalizeDone.await(1L, TimeUnit.SECONDS) && ref.get() == null) {
90			System.runFinalization(); // try to pick up stragglers
91			return;
92			}
93			}
94			} catch (InterruptedException unexpected) {
95			throw new AssertionError("unexpected InterruptedException");
96			}
97			throw new AssertionError("failed to do a \"full\" gc");
98			}
99
100			/**
101			* Runs each job for long enough that all the runtime compilers
102			* have had plenty of time to warm up, i.e. get around to
103			* compiling everything worth compiling.
104			* Returns array of average times per job per run.
105			*/
106			long[] time0(List<Job> jobs) throws Throwable {
107			final long warmupNanos = (long) (warmupSeconds * 1000L * 1000L * 1000L);
108			final int size = jobs.size();
109			long[] nanoss = new long[size];
110			for (int i = 0; i < size; i++) {
111			if (warmupNanos > 0) forceFullGc();
112			long t0 = System.nanoTime();
113			long t;
114			int j = 0;
115			do { jobs.get(i).work(); j++; }
116			while ((t = System.nanoTime() - t0) < warmupNanos);
117			nanoss[i] = t/j;
118			}
119			return nanoss;
120			}
121
122			void time(List<Job> jobs) throws Throwable {
123			if (warmupSeconds > 0) time0(jobs); // Warm up run
124			final int size = jobs.size();
125			final long[] nanoss = time0(jobs); // Real timing run
126			final long[] milliss = new long[size];
127			final double[] ratios = new double[size];
128
129			final String nameHeader = "Method";
130			final String millisHeader = "Millis";
131			final String ratioHeader = "Ratio";
132
133			int nameWidth = nameHeader.length();
134			int millisWidth = millisHeader.length();
135			int ratioWidth = ratioHeader.length();
136
137			for (int i = 0; i < size; i++) {
138			nameWidth = Math.max(nameWidth, jobs.get(i).name().length());
139
140			milliss[i] = nanoss[i]/(1000L * 1000L);
141			millisWidth = Math.max(millisWidth,
142			String.format("%d", milliss[i]).length());
143
144			ratios[i] = (double) nanoss[i] / (double) nanoss[0];
145			ratioWidth = Math.max(ratioWidth,
146			String.format("%.3f", ratios[i]).length());
147			}
148
149			String format = String.format("%%-%ds %%%dd %%%d.3f%%n",
150			nameWidth, millisWidth, ratioWidth);
151			String headerFormat = String.format("%%-%ds %%%ds %%%ds%%n",
152			nameWidth, millisWidth, ratioWidth);
153			System.out.printf(headerFormat, "Method", "Millis", "Ratio");
154
155			// Print out absolute and relative times, calibrated against first job
156			for (int i = 0; i < size; i++)
157			System.out.printf(format, jobs.get(i).name(), milliss[i], ratios[i]);
158			}
159
160			private static String keywordValue(String[] args, String keyword) {
161			for (String arg : args)
162			if (arg.startsWith(keyword))
163			return arg.substring(keyword.length() + 1);
164			return null;
165			}
166
167			private static int intArg(String[] args, String keyword, int defaultValue) {
168			String val = keywordValue(args, keyword);
169			return (val == null) ? defaultValue : Integer.parseInt(val);
170			}
171
172			private static double doubleArg(String[] args, String keyword, double defaultValue) {
173			String val = keywordValue(args, keyword);
174			return (val == null) ? defaultValue : Double.parseDouble(val);
175			}
176
177			private static Pattern patternArg(String[] args, String keyword) {
178			String val = keywordValue(args, keyword);
179			return (val == null) ? null : Pattern.compile(val);
180			}
181
182			private static List<Job> filter(Pattern filter, List<Job> jobs) {
183			if (filter == null) return jobs;
184			ArrayList<Job> newJobs = new ArrayList<>();
185			for (Job job : jobs)
186			if (filter.matcher(job.name()).find())
187			newJobs.add(job);
188			return newJobs;
189			}
190
191			private static void deoptimize(int sum) {
192			if (sum == 42)
193			System.out.println("the answer");
194			}
195
196			private static <T> List<T> asSubList(List<T> list) {
197			return list.subList(0, list.size());
198			}
199
200			private static <T> Iterable<T> backwards(final List<T> list) {
201			return new Iterable<T>() {
202			public Iterator<T> iterator() {
203			return new Iterator<T>() {
204			final ListIterator<T> it = list.listIterator(list.size());
205			public boolean hasNext() { return it.hasPrevious(); }
206			public T next() { return it.previous(); }
207			public void remove() { it.remove(); }};}};
208			}
209
210			// Checks for correctness and prevents loop optimizations
211			class Check {
212			private int sum;
213			public void sum(int sum) {
214			if (this.sum == 0)
215			this.sum = sum;
216			if (this.sum != sum)
217			throw new AssertionError("Sum mismatch");
218			}
219			}
220			volatile Check check = new Check();
221
222			public static void main(String[] args) throws Throwable {
223			new RemoveMicroBenchmark().run(args);
224			}
225
226			void run(String[] args) throws Throwable {
227			iterations = intArg(args, "iterations", 10_000);
228			size = intArg(args, "size", 1000);
229			warmupSeconds = doubleArg(args, "warmup", 5);
230			filter = patternArg(args, "filter");
231			// System.out.printf(
232			// "iterations=%d size=%d, warmup=%1g, filter=\"%s\"%n",
233			// iterations, size, warmupSeconds, filter);
234
235			final ArrayList<Integer> al = new ArrayList<Integer>(size);
236
237			// Populate collections with random data
238			final ThreadLocalRandom rnd = ThreadLocalRandom.current();
239			for (int i = 0; i < size; i++)
240			al.add(rnd.nextInt(size));
241
242			ArrayList<Job> jobs = new ArrayList<>();
243
244			List.<Collection<Integer>>of(
245			new ArrayList<>(),
246			new LinkedList<>(),
247			new Vector<>(),
248			new ArrayDeque<>(),
249			new ArrayBlockingQueue<>(al.size()),
250			new ConcurrentLinkedQueue<>(),
251			new ConcurrentLinkedDeque<>(),
252			new LinkedBlockingQueue<>(),
253			new LinkedBlockingDeque<>(),
254			new LinkedTransferQueue<>())
255			.stream().forEach(
256			x -> {
257			String klazz = x.getClass().getSimpleName();
258			jobs.addAll(collectionJobs(klazz, () -> x, al));
259			if (x instanceof Deque) {
260			Deque<Integer> deque = (Deque<Integer>) x;
261			jobs.addAll(dequeJobs(klazz, () -> deque, al));
262			}
263			if (x instanceof List) {
264			List<Integer> list = (List<Integer>) x;
265			jobs.addAll(
266			collectionJobs(
267			klazz + " subList",
268			() -> list.subList(0, x.size()),
269			al));
270			}
271			});
272
273			time(filter(filter, jobs));
274			}
275
276			Collection<Integer> universeRecorder(int[] sum) {
277			return new ArrayList<>() {
278			public boolean contains(Object x) {
279			sum[0] += (Integer) x;
280			return true;
281			}};
282			}
283
284			Collection<Integer> emptyRecorder(int[] sum) {
285			return new ArrayList<>() {
286			public boolean contains(Object x) {
287			sum[0] += (Integer) x;
288			return false;
289			}};
290			}
291
292			List<Job> collectionJobs(
293			String description,
294			Supplier<Collection<Integer>> supplier,
295			ArrayList<Integer> al) {
296			return List.of(
297			new Job(description + " .removeIf") {
298			public void work() throws Throwable {
299			Collection<Integer> x = supplier.get();
300			int[] sum = new int[1];
301			for (int i = 0; i < iterations; i++) {
302			sum[0] = 0;
303			x.addAll(al);
304			x.removeIf(n -> { sum[0] += n; return true; });
305			check.sum(sum[0]);}}},
306			new Job(description + " .removeAll") {
307			public void work() throws Throwable {
308			Collection<Integer> x = supplier.get();
309			int[] sum = new int[1];
310			Collection<Integer> universe = universeRecorder(sum);
311			for (int i = 0; i < iterations; i++) {
312			sum[0] = 0;
313			x.addAll(al);
314			x.removeAll(universe);
315			check.sum(sum[0]);}}},
316			new Job(description + " .retainAll") {
317			public void work() throws Throwable {
318			Collection<Integer> x = supplier.get();
319			int[] sum = new int[1];
320			Collection<Integer> empty = emptyRecorder(sum);
321			for (int i = 0; i < iterations; i++) {
322			sum[0] = 0;
323			x.addAll(al);
324			x.retainAll(empty);
325			check.sum(sum[0]);}}},
326			new Job(description + " Iterator.remove") {
327			public void work() throws Throwable {
328			Collection<Integer> x = supplier.get();
329			int[] sum = new int[1];
330			for (int i = 0; i < iterations; i++) {
331			sum[0] = 0;
332			x.addAll(al);
333			Iterator<Integer> it = x.iterator();
334			while (it.hasNext()) {
335			sum[0] += it.next();
336			it.remove();
337			}
338			check.sum(sum[0]);}}},
339			new Job(description + " clear") {
340			public void work() throws Throwable {
341			Collection<Integer> x = supplier.get();
342			int[] sum = new int[1];
343			for (int i = 0; i < iterations; i++) {
344			sum[0] = 0;
345			x.addAll(al);
346			x.forEach(e -> sum[0] += e);
347			x.clear();
348			check.sum(sum[0]);}}});
349			}
350
351			List<Job> dequeJobs(
352			String description,
353			Supplier<Deque<Integer>> supplier,
354			ArrayList<Integer> al) {
355			return List.of(
356			new Job(description + " descendingIterator().remove") {
357			public void work() throws Throwable {
358			Deque<Integer> x = supplier.get();
359			int[] sum = new int[1];
360			for (int i = 0; i < iterations; i++) {
361			sum[0] = 0;
362			x.addAll(al);
363			Iterator<Integer> it = x.descendingIterator();
364			while (it.hasNext()) {
365			sum[0] += it.next();
366			it.remove();
367			}
368			check.sum(sum[0]);}}});
369			}
370			}