ViewVC Help
View File | Revision Log | Show Annotations | Download File | Root Listing
root/jsr166/jsr166/src/test/loops/MapMicroBenchmark.java
(Generate patch)

Comparing jsr166/src/test/loops/MapMicroBenchmark.java (file contents):
Revision 1.1 by dl, Wed Jul 1 11:24:49 2009 UTC vs.
Revision 1.14 by jsr166, Wed Mar 5 17:31:06 2014 UTC

# Line 1 | Line 1
1   /*
2   * Written by Doug Lea with assistance from members of JCP JSR-166
3   * Expert Group and released to the public domain, as explained at
4 < * http://creativecommons.org/licenses/publicdomain
4 > * http://creativecommons.org/publicdomain/zero/1.0/
5   */
6  
7   import java.util.*;
8 + import java.io.*;
9   import java.math.*;
10  
11   /**
12   * A micro-benchmark with key types and operation mixes roughly
13   * corresponding to some real programs.
14 < *
14 > *
15   * The main results are a table of approximate nanoseconds per
16   * element-operation (averaged across get, put etc) for each type,
17 < * across a range of map sizes.
17 > * across a range of map sizes. It also includes category "Mixed"
18 > * that includes elements of multiple types including those with
19 > * identical hash codes.
20   *
21   * The program includes a bunch of microbenchmarking safeguards that
22   * might underestimate typical performance. For example, by using many
# Line 21 | Line 24 | import java.math.*;
24   * dynamic type specialization.  Some test classes, like Float and
25   * BigDecimal are included not because they are commonly used as keys,
26   * but because they can be problematic for some map implementations.
27 < *
27 > *
28   * By default, it creates and inserts in order dense numerical keys
29 < * and searches for keys in scrambled order. Use "r" as second arg to
30 < * instead use random numerical values, and "s" as third arg to search
31 < * in insertion order.
29 > * and searches for keys in scrambled order. Use "s" as second arg to
30 > * instead insert and search in unscrambled order.
31 > *
32 > * For String keys, the program tries to use file "testwords.txt", which
33 > * is best used with real words.  We can't check in this file, but you
34 > * can create one from a real dictionary (1 line per word) and then run
35 > * linux "shuf" to randomize entries. If no file exists, it uses
36 > * String.valueOf(i) for element i.
37   */
38   public class MapMicroBenchmark {
39 +    static final String wordFile = "testwords.txt";
40 +
41      static Class mapClass;
42      static boolean randomSearches = true;
33    static boolean randomKeys = false;
43  
44 <    static final long NANOS_PER_JOB = 8L * 1000L*1000L*1000L;
44 >    // Nanoseconds per run
45 >    static final long NANOS_PER_JOB = 6L * 1000L*1000L*1000L;
46 >    static final long NANOS_PER_WARMUP = 100L*1000L*1000L;
47  
48      // map operations per item per iteration -- change if job.work changed
49      static final int OPS_PER_ITER = 11;
50 <    static final int MIN_ITERS_PER_TEST = 2;
51 <    static final int MAX_ITERS_PER_TEST = 1000000;
50 >    static final int MIN_ITERS_PER_TEST = 3; // must be > 1
51 >    static final int MAX_ITERS_PER_TEST = 1000000; // avoid runaway
52  
53      // sizes are at halfway points for HashMap default resizes
54 <    static final int firstSize = 36;
54 >    static final int firstSize = 9;
55      static final int sizeStep = 4; // each size 4X last
56 <    static final int nsizes = 8;
56 >    static final int nsizes = 9;
57      static final int[] sizes = new int[nsizes];
58  
48    static final class Missing {} // class for guaranteed non-matches
49    static final Object MISSING = new Missing();
50
51    static Map newMap() {
52        try {
53            return (Map)mapClass.newInstance();
54        } catch(Exception e) {
55            throw new RuntimeException("Can't instantiate " + mapClass + ": " + e);
56        }
57    }
58
59      public static void main(String[] args) throws Throwable {
60          if (args.length == 0) {
61              System.out.println("Usage: java MapMicroBenchmark className [r|s]keys [r|s]searches");
62              return;
63          }
64 <            
64 >
65          mapClass = Class.forName(args[0]);
66  
67          if (args.length > 1) {
68              if (args[1].startsWith("s"))
69                randomKeys = false;
70            else if (args[1].startsWith("r"))
71                randomKeys = true;
72        }
73        if (args.length > 2) {
74            if (args[2].startsWith("s"))
69                  randomSearches = false;
70 <            else if (args[2].startsWith("r"))
70 >            else if (args[1].startsWith("r"))
71                  randomSearches = true;
72          }
73  
74          System.out.print("Class " + mapClass.getName());
81        if (randomKeys)
82            System.out.print(" random keys");
83        else
84            System.out.print(" sequential keys");
75          if (randomSearches)
76              System.out.print(" randomized searches");
77          else
# Line 96 | Line 86 | public class MapMicroBenchmark {
86          }
87          sizes[nsizes - 1] = n;
88  
89 <        Object[] ss = new Object[n];
89 >        int njobs = 10;
90 >        Job[] jobs = new Job[njobs];
91 >
92          Object[] os = new Object[n];
93 +        for (int i = 0; i < n; i++) os[i] = new Object();
94 +        jobs[0] = new Job("Object    ", os, Object.class);
95 +
96 +        Object[] ss = new Object[n];
97 +        initStringKeys(ss, n);
98 +        jobs[1] = new Job("String    ", ss, String.class);
99 +
100          Object[] is = new Object[n];
101 +        for (int i = 0; i < n; i++) is[i] = Integer.valueOf(i);
102 +        jobs[2] = new Job("Integer   ", is, Integer.class);
103 +
104          Object[] ls = new Object[n];
105 +        for (int i = 0; i < n; i++) ls[i] = Long.valueOf((long) i);
106 +        jobs[3] = new Job("Long      ", ls, Long.class);
107 +
108          Object[] fs = new Object[n];
109 +        for (int i = 0; i < n; i++) fs[i] = Float.valueOf((float) i);
110 +        jobs[4] = new Job("Float     ", fs, Float.class);
111 +
112          Object[] ds = new Object[n];
113 +        for (int i = 0; i < n; i++) ds[i] = Double.valueOf((double) i);
114 +        jobs[5] = new Job("Double    ", ds, Double.class);
115 +
116          Object[] bs = new Object[n];
117 <        Object[] es = new Object[n];
117 >        long b = -n; // include some negatives
118 >        for (int i = 0; i < n; i++) bs[i] = BigInteger.valueOf(b += 3);
119 >        jobs[6] = new Job("BigInteger", bs, BigInteger.class);
120  
121 <        // To guarantee uniqueness, use xorshift for "random" versions
122 <        int j = randomKeys? 1234567 : 0;
123 <        for (int i = 0; i < n; i++) {
124 <            ss[i] = String.valueOf(j);
125 <            os[i] = new Object();
126 <            is[i] = Integer.valueOf(j);
127 <            ls[i] = Long.valueOf((long)j);
128 <            fs[i] = Float.valueOf((float)i); // can't use random for float
129 <            ds[i] = Double.valueOf((double)j);
130 <            bs[i] = BigInteger.valueOf(j);
131 <            es[i] = BigDecimal.valueOf(j);
132 <            j = randomKeys? xorshift(j) : j + 1;
133 <        }
134 <
135 <        List<Job> list = new ArrayList<Job>();
136 <        list.add(new Job("BigDecimal", es));
137 <        list.add(new Job("BigInteger", bs));
138 <        list.add(new Job("String    ", ss));
139 <        list.add(new Job("Double    ", ds));
140 <        list.add(new Job("Float     ", fs));
141 <        list.add(new Job("Long      ", ls));
142 <        list.add(new Job("Integer   ", is));
143 <        list.add(new Job("Object    ", os));
144 <
145 <        Job[] jobs = list.toArray(new Job[0]);
146 <        warmup(jobs);
134 <        warmup(jobs);
135 <        time(jobs);
121 >        Object[] es = new Object[n];
122 >        long d = Integer.MAX_VALUE; // include crummy codes
123 >        for (int i = 0; i < n; i++) es[i] = BigDecimal.valueOf(d += 65536);
124 >        jobs[7] = new Job("BigDecimal", es, BigDecimal.class);
125 >
126 >        Object[] rs = new Object[n];
127 >        for (int i = 0; i < n; i++) rs[i] = new RandomInt();
128 >        jobs[8] = new Job("RandomInt ", rs, RandomInt.class);
129 >
130 >        Object[] ms = new Object[n];
131 >        for (int i = 0; i < n; i += 2) {
132 >            int r = rng.nextInt(njobs - 1);
133 >            ms[i] = jobs[r].items[i];
134 >            // include some that will have same hash but not .equal
135 >            if (++r >= njobs - 1) r = 0;
136 >            ms[i+1] = jobs[r].items[i];
137 >        }
138 >        jobs[9] = new Job("Mixed     ", ms, Object.class);
139 >        Job mixed = jobs[9];
140 >
141 >        warmup1(mixed);
142 >        warmup2(jobs);
143 >        warmup1(mixed);
144 >        warmup3(jobs);
145 >        Thread.sleep(500);
146 >        time(jobs);
147      }
148  
149 <    static void runWork(Job[] jobs, int maxIters) throws Throwable {
150 <        for (int k = 0; k <  nsizes; ++k) {
149 >    static void runWork(Job[] jobs, int minIters, int maxIters, long timeLimit) throws Throwable {
150 >        for (int k = 0; k < nsizes; ++k) {
151              int len = sizes[k];
152              for (int i = 0; i < jobs.length; i++) {
142                jobs[i].setup(len);
153                  Thread.sleep(50);
154 <                jobs[i].nanos[k] = jobs[i].work(len, maxIters);
154 >                jobs[i].nanos[k] = jobs[i].work(len, minIters, maxIters, timeLimit);
155                  System.out.print(".");
156              }
157          }
158          System.out.println();
159      }
160  
161 <    static void warmup(Job[] jobs) throws Throwable {
161 >    // First warmup -- run only mixed job to discourage type specialization
162 >    static void warmup1(Job job) throws Throwable {
163 >        for (int k = 0; k < nsizes; ++k)
164 >            job.work(sizes[k], 1, 1, 0);
165 >    }
166 >
167 >    // Second, run each once
168 >    static void warmup2(Job[] jobs) throws Throwable {
169          System.out.print("warm up");
170 <        runWork(jobs, MIN_ITERS_PER_TEST);
170 >        runWork(jobs, 1, 1, 0);
171          long ck = jobs[0].checkSum;
172 <        for (int i = 1; i < jobs.length; i++) {
172 >        for (int i = 1; i < jobs.length - 1; i++) {
173              if (jobs[i].checkSum != ck)
174                  throw new Error("CheckSum");
175          }
176      }
177  
178 +    // Third: short timed runs
179 +    static void warmup3(Job[] jobs) throws Throwable {
180 +        System.out.print("warm up");
181 +        runWork(jobs, 1, MAX_ITERS_PER_TEST, NANOS_PER_WARMUP);
182 +    }
183 +
184      static void time(Job[] jobs) throws Throwable {
185          System.out.print("running");
186 <        runWork(jobs, MAX_ITERS_PER_TEST);
186 >        runWork(jobs, MIN_ITERS_PER_TEST, MAX_ITERS_PER_TEST, NANOS_PER_JOB);
187  
188          System.out.print("Type/Size:");
189          for (int k = 0; k < nsizes; ++k)
190 <            System.out.printf("%8d", sizes[k]);
190 >            System.out.printf("%7d", sizes[k]);
191          System.out.println();
192  
193          long[] aves = new long[nsizes];
194          int njobs = jobs.length;
195  
196 <        for (int i = 0; i < njobs; i++) {
196 >        for (int i = 0; i < njobs; i++) {
197              System.out.print(jobs[i].name);
198              for (int k = 0; k < nsizes; ++k) {
199                  long nanos = jobs[i].nanos[k];
200 <                System.out.printf("%8d", nanos);
200 >                System.out.printf("%7d", nanos);
201                  aves[k] += nanos;
202              }
203              System.out.println();
# Line 183 | Line 206 | public class MapMicroBenchmark {
206          System.out.println();
207          System.out.print("average   ");
208          for (int k = 0; k < nsizes; ++k)
209 <            System.out.printf("%8d", (aves[k] / njobs));
210 <        System.out.println();
209 >            System.out.printf("%7d", (aves[k] / njobs));
210 >        System.out.println("\n");
211      }
212  
213  
214      static final class Job {
215 <        final String name;
215 >        final String name;
216 >        final Class elementClass;
217          long[] nanos = new long[nsizes];
218          final Object[] items;
219 <        final Object[] dups;
196 <        Object[] reordered;
219 >        Object[] searches;
220          volatile long checkSum;
221          volatile int lastSum;
222 <        Job(String name, Object[] items) {
222 >        Job(String name, Object[] items, Class elementClass) {
223              this.name = name;
224              this.items = items;
225 <            if (randomSearches)
203 <                this.dups = new Object[items.length];
204 <            else
205 <                this.dups = items;
206 <        }
207 <
208 <        public void setup(int len) {
225 >            this.elementClass = elementClass;
226              if (randomSearches) {
227 <                System.arraycopy(items, 0, dups, 0, len);
228 <                shuffle(dups, len);
229 <                reordered = dups;
227 >                scramble(items);
228 >                this.searches = new Object[items.length];
229 >                System.arraycopy(items, 0, searches, 0, items.length);
230 >                scramble(searches);
231              }
232              else
233 <                reordered = items;
233 >                this.searches = items;
234          }
235  
236 <        public long work(int len, int maxIters) {
236 >        public long work(int len, int minIters, int maxIters, long timeLimit) {
237 >            Map m;
238 >            try {
239 >                m = (Map) mapClass.newInstance();
240 >            } catch (Exception e) {
241 >                throw new RuntimeException("Can't instantiate " + mapClass + ": " + e);
242 >            }
243              Object[] ins = items;
244 <            Object[] keys = reordered;
244 >            Object[] keys = searches;
245  
246              if (ins.length < len || keys.length < len)
247                  throw new Error(name);
248              int half = len / 2;
249 <            Class eclass = ins[0].getClass();
249 >            int quarter = half / 2;
250              int sum = lastSum;
251              long startTime = System.nanoTime();
252              long elapsed;
229            Map m = newMap();
253              int j = 0;
254              for (;;) {
255                  for (int i = 0; i < half; ++i) {
# Line 235 | Line 258 | public class MapMicroBenchmark {
258                          ++sum;
259                  }
260                  checkSum += sum ^ (sum << 1); // help avoid loop merging
261 +                sum += len - half;
262                  for (int i = 0; i < len; ++i) {
263                      Object x = keys[i];
264 <                    Object v = m.get(x);
265 <                    if (v != null && v.getClass() == eclass) // touch v
266 <                        sum += 2;
264 >                    Object v = m.get(x);
265 >                    if (elementClass.isInstance(v)) // touch v
266 >                        ++sum;
267                  }
268                  checkSum += sum ^ (sum << 2);
269                  for (int i = half; i < len; ++i) {
# Line 249 | Line 273 | public class MapMicroBenchmark {
273                  }
274                  checkSum += sum ^ (sum << 3);
275                  for (Object e : m.keySet()) {
276 <                    if (e.getClass() == eclass)
276 >                    if (elementClass.isInstance(e))
277                          ++sum;
278                  }
279                  checkSum += sum ^ (sum << 4);
280                  for (Object e : m.values()) {
281 <                    if (e.getClass() == eclass)
281 >                    if (elementClass.isInstance(e))
282                          ++sum;
283                  }
284                  checkSum += sum ^ (sum << 5);
285                  for (int i = len - 1; i >= 0; --i) {
286                      Object x = keys[i];
287                      Object v = m.get(x);
288 <                    if (v != null && v.getClass() == eclass)
288 >                    if (elementClass.isInstance(v))
289                          ++sum;
290                  }
291                  checkSum += sum ^ (sum << 6);
292                  for (int i = 0; i < len; ++i) {
293                      Object x = ins[i];
294                      Object v = m.get(x);
295 <                    if (v != null && v.getClass() == eclass)
295 >                    if (elementClass.isInstance(v))
296                          ++sum;
297                  }
298                  checkSum += sum ^ (sum << 7);
# Line 282 | Line 306 | public class MapMicroBenchmark {
306                  for (int i = 0; i < len; ++i) {
307                      Object x = keys[i];
308                      Object v = ins[i];
309 <                    if (v.equals(m.get(x)))
309 >                    if (v == m.get(x))
310                          ++sum;
311                  }
312                  checkSum += sum ^ (sum << 9);
313                  for (int i = len - 1; i >= 0; --i) {
314                      Object x = ins[i];
315 <                    if (m.get(x) != MISSING)
315 >                    Object v = m.get(x);
316 >                    if (elementClass.isInstance(v))
317                          ++sum;
318                  }
319                  checkSum += sum ^ (sum << 10);
320                  for (int i = len - 1; i >= 0; --i) {
321                      Object x = keys[i];
322                      Object v = ins[i];
323 <                    if (v.equals(m.get(x)))
323 >                    if (v == m.get(x))
324                          ++sum;
325                  }
326                  checkSum += sum ^ (sum << 11);
327 <                if ((j & 1) == 1) { // lower remove rate
328 <                    for (int i = 0; i < len; ++i) {
329 <                        Object x = keys[i];
330 <                        if (m.remove(x) != null)
331 <                            ++sum;
332 <                    }
333 <                }
334 <                else {
335 <                    m.clear();
311 <                    sum += len;
327 >                for (int i = 0; i < quarter; ++i) {
328 >                    Object x = keys[i];
329 >                    if (m.remove(x) != null)
330 >                        ++sum;
331 >                }
332 >                for (int i = 0; i < quarter; ++i) {
333 >                    Object x = keys[i];
334 >                    if (m.put(x, x) == null)
335 >                        ++sum;
336                  }
337 +                /*  // uncomment to avoid calling clear()
338 +                for (int i = 0; i < len; ++i) {
339 +                    Object x = keys[i];
340 +                    m.remove(x);
341 +                }
342 +                */
343 +                m.clear();
344 +                sum += len - (quarter * 2);
345                  checkSum += sum ^ (sum << 12);
346  
347 +                if (j == 0 && sum != lastSum + len * OPS_PER_ITER)
348 +                    throw new Error(name);
349 +
350                  elapsed = System.nanoTime() - startTime;
351                  ++j;
352 <                if (j >= MIN_ITERS_PER_TEST &&
353 <                    (j >= maxIters || elapsed >= NANOS_PER_JOB))
352 >                if (j >= minIters &&
353 >                    (j >= maxIters || elapsed >= timeLimit))
354                      break;
355 +                // non-warmup - swap some keys for next insert
356 +                if (minIters != 1 && randomSearches)
357 +                    shuffleSome(ins, len, len >>> 3);
358              }
359 <            long ops = ((long)j) * len * OPS_PER_ITER;
322 <            if (sum != lastSum + (int)ops)
323 <                throw new Error(name);
359 >            long ops = ((long) j) * len * OPS_PER_ITER;
360              lastSum = sum;
361              return elapsed / ops;
362          }
363  
364      }
365  
330    static final int xorshift(int seed) {
331        seed ^= seed << 1;
332        seed ^= seed >>> 3;
333        seed ^= seed << 10;
334        return seed;
335    }
366  
367 +    static final Random rng = new Random(3122688);
368  
369 <    static final Random rng = new Random(3152688);
369 >    // Shuffle the subarrays for each size. This doesn't fully
370 >    // randomize, but the remaining partial locality is arguably a bit
371 >    // more realistic
372 >    static void scramble(Object[] a) {
373 >        for (int k = 0; k < sizes.length; ++k) {
374 >            int origin = (k == 0) ? 0 : sizes[k-1];
375 >            for (int i = sizes[k]; i > origin + 1; i--) {
376 >                Object t = a[i-1];
377 >                int r = rng.nextInt(i - origin) + origin;
378 >                a[i-1] = a[r];
379 >                a[r] = t;
380 >            }
381 >        }
382 >    }
383  
384 +    // plain array shuffle
385      static void shuffle(Object[] a, int size) {
386 <        for (int i= size; i>1; i--) {
386 >        for (int i = size; i > 1; i--) {
387              Object t = a[i-1];
388              int r = rng.nextInt(i);
389              a[i-1] = a[r];
# Line 346 | Line 391 | public class MapMicroBenchmark {
391          }
392      }
393  
394 < }
394 >    // swap nswaps elements
395 >    static void shuffleSome(Object[] a, int size, int nswaps) {
396 >        for (int s = 0; s < nswaps; ++s) {
397 >            int i = rng.nextInt(size);
398 >            int r = rng.nextInt(size);
399 >            Object t = a[i];
400 >            a[i] = a[r];
401 >            a[r] = t;
402 >        }
403 >    }
404  
405 +    // Integer-like class with random hash codes
406 +    static final class RandomInt {
407 +        static int seed = 3122688;
408 +        static int next() { // a non-xorshift, 2^32-period RNG
409 +            int x = seed;
410 +            int lo = 16807 * (x & 0xFFFF);
411 +            int hi = 16807 * (x >>> 16);
412 +            lo += (hi & 0x7FFF) << 16;
413 +            if ((lo & 0x80000000) != 0) {
414 +                lo &= 0x7fffffff;
415 +                ++lo;
416 +            }
417 +            lo += hi >>> 15;
418 +            if (lo == 0 || (lo & 0x80000000) != 0) {
419 +                lo &= 0x7fffffff;
420 +                ++lo;
421 +            }
422 +            seed = lo;
423 +            return x;
424 +        }
425 +        final int value;
426 +        RandomInt() { value = next(); }
427 +        public int hashCode() { return value; }
428 +        public boolean equals(Object x) {
429 +            return (x instanceof RandomInt) && ((RandomInt)x).value == value;
430 +        }
431 +    }
432 +
433 +    // Read in String keys from file if possible
434 +    static void initStringKeys(Object[] keys, int n) throws Exception {
435 +        FileInputStream fr = null;
436 +        try {
437 +            fr = new FileInputStream(wordFile);
438 +        } catch (IOException ex) {
439 +            System.out.println("No word file. Using String.valueOf(i)");
440 +            for (int i = 0; i < n; i++)
441 +                keys[i] = String.valueOf(i);
442 +            return;
443 +        }
444 +
445 +        BufferedInputStream in = new BufferedInputStream(fr);
446 +        int k = 0;
447 +        outer: while (k < n) {
448 +            StringBuffer sb = new StringBuffer();
449 +            for (;;) {
450 +                int c = in.read();
451 +                if (c < 0)
452 +                    break outer;
453 +                char ch = (char) c;
454 +                if (ch == '\n') {
455 +                    keys[k++] = sb.toString();
456 +                    break;
457 +                }
458 +                if (!Character.isWhitespace(ch))
459 +                    sb.append(ch);
460 +            }
461 +        }
462 +        in.close();
463 +
464 +        // fill up remaining keys with path-like compounds of previous pairs
465 +        int j = 0;
466 +        while (k < n)
467 +            keys[k++] = (String) keys[j++] + "/" + (String) keys[j];
468 +    }
469 +
470 + }

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines