ViewVC Help
View File | Revision Log | Show Annotations | Download File | Root Listing
root/jsr166/jsr166/src/test/loops/MapMicroBenchmark.java
(Generate patch)

Comparing jsr166/src/test/loops/MapMicroBenchmark.java (file contents):
Revision 1.1 by dl, Wed Jul 1 11:24:49 2009 UTC vs.
Revision 1.17 by jsr166, Thu Jan 15 18:34:19 2015 UTC

# Line 1 | Line 1
1   /*
2   * Written by Doug Lea with assistance from members of JCP JSR-166
3   * Expert Group and released to the public domain, as explained at
4 < * http://creativecommons.org/licenses/publicdomain
4 > * http://creativecommons.org/publicdomain/zero/1.0/
5   */
6  
7 < import java.util.*;
7 > import java.io.*;
8   import java.math.*;
9 + import java.util.*;
10  
11   /**
12   * A micro-benchmark with key types and operation mixes roughly
13   * corresponding to some real programs.
14 < *
14 > *
15   * The main results are a table of approximate nanoseconds per
16   * element-operation (averaged across get, put etc) for each type,
17 < * across a range of map sizes.
17 > * across a range of map sizes. It also includes category "Mixed"
18 > * that includes elements of multiple types including those with
19 > * identical hash codes.
20   *
21   * The program includes a bunch of microbenchmarking safeguards that
22   * might underestimate typical performance. For example, by using many
# Line 21 | Line 24 | import java.math.*;
24   * dynamic type specialization.  Some test classes, like Float and
25   * BigDecimal are included not because they are commonly used as keys,
26   * but because they can be problematic for some map implementations.
27 < *
27 > *
28   * By default, it creates and inserts in order dense numerical keys
29 < * and searches for keys in scrambled order. Use "r" as second arg to
30 < * instead use random numerical values, and "s" as third arg to search
31 < * in insertion order.
29 > * and searches for keys in scrambled order. Use "s" as second arg to
30 > * instead insert and search in unscrambled order.
31 > *
32 > * For String keys, the program tries to use file "testwords.txt", which
33 > * is best used with real words.  We can't check in this file, but you
34 > * can create one from a real dictionary (1 line per word) and then run
35 > * linux "shuf" to randomize entries. If no file exists, it uses
36 > * String.valueOf(i) for element i.
37   */
38   public class MapMicroBenchmark {
39 <    static Class mapClass;
39 >    static final String wordFile = "testwords.txt";
40 >
41 >    static Class<?> mapClass;
42      static boolean randomSearches = true;
33    static boolean randomKeys = false;
43  
44 <    static final long NANOS_PER_JOB = 8L * 1000L*1000L*1000L;
44 >    // Nanoseconds per run
45 >    static final long NANOS_PER_JOB = 6L * 1000L*1000L*1000L;
46 >    static final long NANOS_PER_WARMUP = 100L*1000L*1000L;
47  
48      // map operations per item per iteration -- change if job.work changed
49      static final int OPS_PER_ITER = 11;
50 <    static final int MIN_ITERS_PER_TEST = 2;
51 <    static final int MAX_ITERS_PER_TEST = 1000000;
50 >    static final int MIN_ITERS_PER_TEST = 3; // must be > 1
51 >    static final int MAX_ITERS_PER_TEST = 1000000; // avoid runaway
52  
53      // sizes are at halfway points for HashMap default resizes
54 <    static final int firstSize = 36;
54 >    static final int firstSize = 9;
55      static final int sizeStep = 4; // each size 4X last
56 <    static final int nsizes = 8;
56 >    static final int nsizes = 9;
57      static final int[] sizes = new int[nsizes];
58  
48    static final class Missing {} // class for guaranteed non-matches
49    static final Object MISSING = new Missing();
50
51    static Map newMap() {
52        try {
53            return (Map)mapClass.newInstance();
54        } catch(Exception e) {
55            throw new RuntimeException("Can't instantiate " + mapClass + ": " + e);
56        }
57    }
58
59      public static void main(String[] args) throws Throwable {
60          if (args.length == 0) {
61              System.out.println("Usage: java MapMicroBenchmark className [r|s]keys [r|s]searches");
62              return;
63          }
64 <            
64 >
65          mapClass = Class.forName(args[0]);
66  
67          if (args.length > 1) {
68              if (args[1].startsWith("s"))
69                randomKeys = false;
70            else if (args[1].startsWith("r"))
71                randomKeys = true;
72        }
73        if (args.length > 2) {
74            if (args[2].startsWith("s"))
69                  randomSearches = false;
70 <            else if (args[2].startsWith("r"))
70 >            else if (args[1].startsWith("r"))
71                  randomSearches = true;
72          }
73  
74          System.out.print("Class " + mapClass.getName());
81        if (randomKeys)
82            System.out.print(" random keys");
83        else
84            System.out.print(" sequential keys");
75          if (randomSearches)
76              System.out.print(" randomized searches");
77          else
# Line 96 | Line 86 | public class MapMicroBenchmark {
86          }
87          sizes[nsizes - 1] = n;
88  
89 <        Object[] ss = new Object[n];
89 >        int njobs = 10;
90 >        Job[] jobs = new Job[njobs];
91 >
92          Object[] os = new Object[n];
93 +        for (int i = 0; i < n; i++) os[i] = new Object();
94 +        jobs[0] = new Job("Object    ", os, Object.class);
95 +
96 +        Object[] ss = new Object[n];
97 +        initStringKeys(ss, n);
98 +        jobs[1] = new Job("String    ", ss, String.class);
99 +
100          Object[] is = new Object[n];
101 +        for (int i = 0; i < n; i++) is[i] = Integer.valueOf(i);
102 +        jobs[2] = new Job("Integer   ", is, Integer.class);
103 +
104          Object[] ls = new Object[n];
105 +        for (int i = 0; i < n; i++) ls[i] = Long.valueOf((long) i);
106 +        jobs[3] = new Job("Long      ", ls, Long.class);
107 +
108          Object[] fs = new Object[n];
109 +        for (int i = 0; i < n; i++) fs[i] = Float.valueOf((float) i);
110 +        jobs[4] = new Job("Float     ", fs, Float.class);
111 +
112          Object[] ds = new Object[n];
113 +        for (int i = 0; i < n; i++) ds[i] = Double.valueOf((double) i);
114 +        jobs[5] = new Job("Double    ", ds, Double.class);
115 +
116          Object[] bs = new Object[n];
117 <        Object[] es = new Object[n];
117 >        long b = -n; // include some negatives
118 >        for (int i = 0; i < n; i++) bs[i] = BigInteger.valueOf(b += 3);
119 >        jobs[6] = new Job("BigInteger", bs, BigInteger.class);
120  
121 <        // To guarantee uniqueness, use xorshift for "random" versions
122 <        int j = randomKeys? 1234567 : 0;
123 <        for (int i = 0; i < n; i++) {
124 <            ss[i] = String.valueOf(j);
125 <            os[i] = new Object();
126 <            is[i] = Integer.valueOf(j);
127 <            ls[i] = Long.valueOf((long)j);
128 <            fs[i] = Float.valueOf((float)i); // can't use random for float
129 <            ds[i] = Double.valueOf((double)j);
130 <            bs[i] = BigInteger.valueOf(j);
131 <            es[i] = BigDecimal.valueOf(j);
132 <            j = randomKeys? xorshift(j) : j + 1;
133 <        }
134 <
135 <        List<Job> list = new ArrayList<Job>();
136 <        list.add(new Job("BigDecimal", es));
137 <        list.add(new Job("BigInteger", bs));
138 <        list.add(new Job("String    ", ss));
139 <        list.add(new Job("Double    ", ds));
140 <        list.add(new Job("Float     ", fs));
141 <        list.add(new Job("Long      ", ls));
142 <        list.add(new Job("Integer   ", is));
143 <        list.add(new Job("Object    ", os));
144 <
145 <        Job[] jobs = list.toArray(new Job[0]);
146 <        warmup(jobs);
134 <        warmup(jobs);
135 <        time(jobs);
121 >        Object[] es = new Object[n];
122 >        long d = Integer.MAX_VALUE; // include crummy codes
123 >        for (int i = 0; i < n; i++) es[i] = BigDecimal.valueOf(d += 65536);
124 >        jobs[7] = new Job("BigDecimal", es, BigDecimal.class);
125 >
126 >        Object[] rs = new Object[n];
127 >        for (int i = 0; i < n; i++) rs[i] = new RandomInt();
128 >        jobs[8] = new Job("RandomInt ", rs, RandomInt.class);
129 >
130 >        Object[] ms = new Object[n];
131 >        for (int i = 0; i < n; i += 2) {
132 >            int r = rng.nextInt(njobs - 1);
133 >            ms[i] = jobs[r].items[i];
134 >            // include some that will have same hash but not .equal
135 >            if (++r >= njobs - 1) r = 0;
136 >            ms[i+1] = jobs[r].items[i];
137 >        }
138 >        jobs[9] = new Job("Mixed     ", ms, Object.class);
139 >        Job mixed = jobs[9];
140 >
141 >        warmup1(mixed);
142 >        warmup2(jobs);
143 >        warmup1(mixed);
144 >        warmup3(jobs);
145 >        Thread.sleep(500);
146 >        time(jobs);
147      }
148  
149 <    static void runWork(Job[] jobs, int maxIters) throws Throwable {
150 <        for (int k = 0; k <  nsizes; ++k) {
149 >    static void runWork(Job[] jobs, int minIters, int maxIters, long timeLimit) throws Throwable {
150 >        for (int k = 0; k < nsizes; ++k) {
151              int len = sizes[k];
152              for (int i = 0; i < jobs.length; i++) {
142                jobs[i].setup(len);
153                  Thread.sleep(50);
154 <                jobs[i].nanos[k] = jobs[i].work(len, maxIters);
154 >                jobs[i].nanos[k] = jobs[i].work(len, minIters, maxIters, timeLimit);
155                  System.out.print(".");
156              }
157          }
158          System.out.println();
159      }
160  
161 <    static void warmup(Job[] jobs) throws Throwable {
161 >    // First warmup -- run only mixed job to discourage type specialization
162 >    static void warmup1(Job job) throws Throwable {
163 >        for (int k = 0; k < nsizes; ++k)
164 >            job.work(sizes[k], 1, 1, 0);
165 >    }
166 >
167 >    // Second, run each once
168 >    static void warmup2(Job[] jobs) throws Throwable {
169          System.out.print("warm up");
170 <        runWork(jobs, MIN_ITERS_PER_TEST);
170 >        runWork(jobs, 1, 1, 0);
171          long ck = jobs[0].checkSum;
172 <        for (int i = 1; i < jobs.length; i++) {
172 >        for (int i = 1; i < jobs.length - 1; i++) {
173              if (jobs[i].checkSum != ck)
174                  throw new Error("CheckSum");
175          }
176      }
177  
178 +    // Third: short timed runs
179 +    static void warmup3(Job[] jobs) throws Throwable {
180 +        System.out.print("warm up");
181 +        runWork(jobs, 1, MAX_ITERS_PER_TEST, NANOS_PER_WARMUP);
182 +    }
183 +
184      static void time(Job[] jobs) throws Throwable {
185          System.out.print("running");
186 <        runWork(jobs, MAX_ITERS_PER_TEST);
186 >        runWork(jobs, MIN_ITERS_PER_TEST, MAX_ITERS_PER_TEST, NANOS_PER_JOB);
187  
188          System.out.print("Type/Size:");
189          for (int k = 0; k < nsizes; ++k)
190 <            System.out.printf("%8d", sizes[k]);
190 >            System.out.printf("%7d", sizes[k]);
191          System.out.println();
192  
193          long[] aves = new long[nsizes];
194          int njobs = jobs.length;
195  
196 <        for (int i = 0; i < njobs; i++) {
196 >        for (int i = 0; i < njobs; i++) {
197              System.out.print(jobs[i].name);
198              for (int k = 0; k < nsizes; ++k) {
199                  long nanos = jobs[i].nanos[k];
200 <                System.out.printf("%8d", nanos);
200 >                System.out.printf("%7d", nanos);
201                  aves[k] += nanos;
202              }
203              System.out.println();
# Line 183 | Line 206 | public class MapMicroBenchmark {
206          System.out.println();
207          System.out.print("average   ");
208          for (int k = 0; k < nsizes; ++k)
209 <            System.out.printf("%8d", (aves[k] / njobs));
210 <        System.out.println();
209 >            System.out.printf("%7d", (aves[k] / njobs));
210 >        System.out.println("\n");
211      }
212  
190
213      static final class Job {
214 <        final String name;
214 >        final String name;
215 >        final Class<?> elementClass;
216          long[] nanos = new long[nsizes];
217          final Object[] items;
218 <        final Object[] dups;
196 <        Object[] reordered;
218 >        Object[] searches;
219          volatile long checkSum;
220          volatile int lastSum;
221 <        Job(String name, Object[] items) {
221 >        Job(String name, Object[] items, Class<?> elementClass) {
222              this.name = name;
223              this.items = items;
224 <            if (randomSearches)
203 <                this.dups = new Object[items.length];
204 <            else
205 <                this.dups = items;
206 <        }
207 <
208 <        public void setup(int len) {
224 >            this.elementClass = elementClass;
225              if (randomSearches) {
226 <                System.arraycopy(items, 0, dups, 0, len);
227 <                shuffle(dups, len);
228 <                reordered = dups;
226 >                scramble(items);
227 >                this.searches = new Object[items.length];
228 >                System.arraycopy(items, 0, searches, 0, items.length);
229 >                scramble(searches);
230              }
231              else
232 <                reordered = items;
232 >                this.searches = items;
233          }
234  
235 <        public long work(int len, int maxIters) {
235 >        public long work(int len, int minIters, int maxIters, long timeLimit) {
236 >            Map m;
237 >            try {
238 >                m = (Map) mapClass.newInstance();
239 >            } catch (Exception e) {
240 >                throw new RuntimeException("Can't instantiate " + mapClass + ": " + e);
241 >            }
242              Object[] ins = items;
243 <            Object[] keys = reordered;
243 >            Object[] keys = searches;
244  
245              if (ins.length < len || keys.length < len)
246                  throw new Error(name);
247              int half = len / 2;
248 <            Class eclass = ins[0].getClass();
248 >            int quarter = half / 2;
249              int sum = lastSum;
250              long startTime = System.nanoTime();
251              long elapsed;
229            Map m = newMap();
252              int j = 0;
253              for (;;) {
254                  for (int i = 0; i < half; ++i) {
# Line 235 | Line 257 | public class MapMicroBenchmark {
257                          ++sum;
258                  }
259                  checkSum += sum ^ (sum << 1); // help avoid loop merging
260 +                sum += len - half;
261                  for (int i = 0; i < len; ++i) {
262                      Object x = keys[i];
263 <                    Object v = m.get(x);
264 <                    if (v != null && v.getClass() == eclass) // touch v
265 <                        sum += 2;
263 >                    Object v = m.get(x);
264 >                    if (elementClass.isInstance(v)) // touch v
265 >                        ++sum;
266                  }
267                  checkSum += sum ^ (sum << 2);
268                  for (int i = half; i < len; ++i) {
# Line 249 | Line 272 | public class MapMicroBenchmark {
272                  }
273                  checkSum += sum ^ (sum << 3);
274                  for (Object e : m.keySet()) {
275 <                    if (e.getClass() == eclass)
275 >                    if (elementClass.isInstance(e))
276                          ++sum;
277                  }
278                  checkSum += sum ^ (sum << 4);
279                  for (Object e : m.values()) {
280 <                    if (e.getClass() == eclass)
280 >                    if (elementClass.isInstance(e))
281                          ++sum;
282                  }
283                  checkSum += sum ^ (sum << 5);
284                  for (int i = len - 1; i >= 0; --i) {
285                      Object x = keys[i];
286                      Object v = m.get(x);
287 <                    if (v != null && v.getClass() == eclass)
287 >                    if (elementClass.isInstance(v))
288                          ++sum;
289                  }
290                  checkSum += sum ^ (sum << 6);
291                  for (int i = 0; i < len; ++i) {
292                      Object x = ins[i];
293                      Object v = m.get(x);
294 <                    if (v != null && v.getClass() == eclass)
294 >                    if (elementClass.isInstance(v))
295                          ++sum;
296                  }
297                  checkSum += sum ^ (sum << 7);
# Line 282 | Line 305 | public class MapMicroBenchmark {
305                  for (int i = 0; i < len; ++i) {
306                      Object x = keys[i];
307                      Object v = ins[i];
308 <                    if (v.equals(m.get(x)))
308 >                    if (v == m.get(x))
309                          ++sum;
310                  }
311                  checkSum += sum ^ (sum << 9);
312                  for (int i = len - 1; i >= 0; --i) {
313                      Object x = ins[i];
314 <                    if (m.get(x) != MISSING)
314 >                    Object v = m.get(x);
315 >                    if (elementClass.isInstance(v))
316                          ++sum;
317                  }
318                  checkSum += sum ^ (sum << 10);
319                  for (int i = len - 1; i >= 0; --i) {
320                      Object x = keys[i];
321                      Object v = ins[i];
322 <                    if (v.equals(m.get(x)))
322 >                    if (v == m.get(x))
323                          ++sum;
324                  }
325                  checkSum += sum ^ (sum << 11);
326 <                if ((j & 1) == 1) { // lower remove rate
327 <                    for (int i = 0; i < len; ++i) {
328 <                        Object x = keys[i];
329 <                        if (m.remove(x) != null)
330 <                            ++sum;
331 <                    }
332 <                }
333 <                else {
334 <                    m.clear();
335 <                    sum += len;
326 >                for (int i = 0; i < quarter; ++i) {
327 >                    Object x = keys[i];
328 >                    if (m.remove(x) != null)
329 >                        ++sum;
330 >                }
331 >                for (int i = 0; i < quarter; ++i) {
332 >                    Object x = keys[i];
333 >                    if (m.put(x, x) == null)
334 >                        ++sum;
335 >                }
336 >                /*  // uncomment to avoid calling clear()
337 >                for (int i = 0; i < len; ++i) {
338 >                    Object x = keys[i];
339 >                    m.remove(x);
340                  }
341 +                */
342 +                m.clear();
343 +                sum += len - (quarter * 2);
344                  checkSum += sum ^ (sum << 12);
345  
346 +                if (j == 0 && sum != lastSum + len * OPS_PER_ITER)
347 +                    throw new Error(name);
348 +
349                  elapsed = System.nanoTime() - startTime;
350                  ++j;
351 <                if (j >= MIN_ITERS_PER_TEST &&
352 <                    (j >= maxIters || elapsed >= NANOS_PER_JOB))
351 >                if (j >= minIters &&
352 >                    (j >= maxIters || elapsed >= timeLimit))
353                      break;
354 +                // non-warmup - swap some keys for next insert
355 +                if (minIters != 1 && randomSearches)
356 +                    shuffleSome(ins, len, len >>> 3);
357              }
358 <            long ops = ((long)j) * len * OPS_PER_ITER;
322 <            if (sum != lastSum + (int)ops)
323 <                throw new Error(name);
358 >            long ops = ((long) j) * len * OPS_PER_ITER;
359              lastSum = sum;
360              return elapsed / ops;
361          }
362  
363      }
364  
365 <    static final int xorshift(int seed) {
331 <        seed ^= seed << 1;
332 <        seed ^= seed >>> 3;
333 <        seed ^= seed << 10;
334 <        return seed;
335 <    }
365 >    static final Random rng = new Random(3122688);
366  
367 +    // Shuffle the subarrays for each size. This doesn't fully
368 +    // randomize, but the remaining partial locality is arguably a bit
369 +    // more realistic
370 +    static void scramble(Object[] a) {
371 +        for (int k = 0; k < sizes.length; ++k) {
372 +            int origin = (k == 0) ? 0 : sizes[k-1];
373 +            for (int i = sizes[k]; i > origin + 1; i--) {
374 +                Object t = a[i-1];
375 +                int r = rng.nextInt(i - origin) + origin;
376 +                a[i-1] = a[r];
377 +                a[r] = t;
378 +            }
379 +        }
380 +    }
381  
382 <    static final Random rng = new Random(3152688);
339 <
382 >    // plain array shuffle
383      static void shuffle(Object[] a, int size) {
384 <        for (int i= size; i>1; i--) {
384 >        for (int i = size; i > 1; i--) {
385              Object t = a[i-1];
386              int r = rng.nextInt(i);
387              a[i-1] = a[r];
# Line 346 | Line 389 | public class MapMicroBenchmark {
389          }
390      }
391  
392 < }
392 >    // swap nswaps elements
393 >    static void shuffleSome(Object[] a, int size, int nswaps) {
394 >        for (int s = 0; s < nswaps; ++s) {
395 >            int i = rng.nextInt(size);
396 >            int r = rng.nextInt(size);
397 >            Object t = a[i];
398 >            a[i] = a[r];
399 >            a[r] = t;
400 >        }
401 >    }
402 >
403 >    // Integer-like class with random hash codes
404 >    static final class RandomInt {
405 >        static int seed = 3122688;
406 >        static int next() { // a non-xorshift, 2^32-period RNG
407 >            int x = seed;
408 >            int lo = 16807 * (x & 0xFFFF);
409 >            int hi = 16807 * (x >>> 16);
410 >            lo += (hi & 0x7FFF) << 16;
411 >            if ((lo & 0x80000000) != 0) {
412 >                lo &= 0x7fffffff;
413 >                ++lo;
414 >            }
415 >            lo += hi >>> 15;
416 >            if (lo == 0 || (lo & 0x80000000) != 0) {
417 >                lo &= 0x7fffffff;
418 >                ++lo;
419 >            }
420 >            seed = lo;
421 >            return x;
422 >        }
423 >        final int value;
424 >        RandomInt() { value = next(); }
425 >        public int hashCode() { return value; }
426 >        public boolean equals(Object x) {
427 >            return (x instanceof RandomInt) && ((RandomInt)x).value == value;
428 >        }
429 >    }
430 >
431 >    // Read in String keys from file if possible
432 >    static void initStringKeys(Object[] keys, int n) throws Exception {
433 >        FileInputStream fr = null;
434 >        try {
435 >            fr = new FileInputStream(wordFile);
436 >        } catch (IOException ex) {
437 >            System.out.println("No word file. Using String.valueOf(i)");
438 >            for (int i = 0; i < n; i++)
439 >                keys[i] = String.valueOf(i);
440 >            return;
441 >        }
442 >
443 >        BufferedInputStream in = new BufferedInputStream(fr);
444 >        int k = 0;
445 >        outer: while (k < n) {
446 >            StringBuffer sb = new StringBuffer();
447 >            for (;;) {
448 >                int c = in.read();
449 >                if (c < 0)
450 >                    break outer;
451 >                char ch = (char) c;
452 >                if (ch == '\n') {
453 >                    keys[k++] = sb.toString();
454 >                    break;
455 >                }
456 >                if (!Character.isWhitespace(ch))
457 >                    sb.append(ch);
458 >            }
459 >        }
460 >        in.close();
461  
462 +        // fill up remaining keys with path-like compounds of previous pairs
463 +        int j = 0;
464 +        while (k < n)
465 +            keys[k++] = (String) keys[j++] + "/" + (String) keys[j];
466 +    }
467 +
468 + }

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines