Go Through The Source Code of HyperLogLog

Last program we present TinyHyperLogLog that used to count distinct elements of a set which is inspired by HyperLogLog. Today we gonna go through HyperLogLog. Actually, TinyHyperLogLog is pretty much close to HyperLogLog except optimization techniques.

There are two fundamental and outstandingly changes.

  • HyperLogLog uses a serial of constants to optimize the results. Two important factors are alpha and linear counting which are mathematical knowledge. Here I’m not going to talk
    too much. If you are interested, you can go through the author’s paper.

  • HyperLogLog introduce the idea of register to conserve memory space. In TinyHyperLogLog we use integer for each bucket counter. We all know that the size of integer is 32, apart from the bucket id, the longest one is only (32-BIT_LENGTH_FOR_BUCKET_ID). So we can use at most 5 bits to represent it if bucketSize>=1.

For a long type, apart from the bucket id, the longest one is only (64-BIT_LENGTH_FOR_BUCKET_ID). So we can use at most 6 bits to represent it if bucketSize>=1.

To conserve memory, each integer is divided into BIT_LENGTH_FOR_BUCKET_VALUE_RANGE buckets. The ultimate purpose is to make memory fully used because the length of the longest leading 0s cannot be very long.

The register comes out. Register is a group of buckets, use a single integer to represent a bucket is too expensive for memory. We can divide it to save a considerable amount of memory.

the generated hash length, we recommend to use long type since it’s able to represent much more distinct elements than integer.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
public class HyperLogLog<T> {

private final int hashLen = 64;
// the leading BIT_LENGTH_FOR_BUCKET_ID bits represent the bucket id
private final int BIT_LENGTH_FOR_BUCKET_ID;
private RegisterSet registerSet;

/**
* accuracy = 1.04/sqrt(2^BIT_LENGTH_FOR_BUCKET_ID)
*
* @param BIT_LENGTH_FOR_BUCKET_ID
*/
public HyperLogLog(int BIT_LENGTH_FOR_BUCKET_ID) {
this.BIT_LENGTH_FOR_BUCKET_ID = BIT_LENGTH_FOR_BUCKET_ID;

this.registerSet = new RegisterSet(1 << BIT_LENGTH_FOR_BUCKET_ID, hashLen);
}

public static void main(String[] args) {
HyperLogLog<String> hyperLogLog = new HyperLogLog(16);

int count = 0;
while (true) {
for (int i = 0; i < 100000; i++) {
hyperLogLog.offer(i + "adf");
}

if (count > 10) {
break;
}

count++;
}

System.out.println(hyperLogLog.cardinality());
}

protected static double getAlpha(final int p) {

// generate a optimization factor

int m = 1 << p;
// See the paper.
switch (p) {
case 4:
return 0.673 * m * m;
case 5:
return 0.697 * m * m;
case 6:
return 0.709 * m * m;
default:
return (0.7213 / (1 + 1.079 / m)) * m * m;
}
}

protected static double linearCounting(int bucketNum, double zeroNumOfBuckets) {
return bucketNum * Math.log(bucketNum / zeroNumOfBuckets);
}

/**
* Add element to the HyperLogLog counter
*
* @param o inserted element
* @return
*/
public boolean offer(T o) {
if (hashLen == 32) {
// we generate a integer hash sequence
return offerHashed(MurmurHash.hash32(o.toString()));
}

// we generate a long hash sequence so that we can count more distinct elements than Integer can be.
return offerHashed(MurmurHash.hash64(o.toString()));
}

public boolean offerHashed(int hashedValue) {
// the leading BIT_LENGTH_FOR_BUCKET_ID bits represents the bucket id
int bid = hashedValue >>> (Integer.SIZE - BIT_LENGTH_FOR_BUCKET_ID);

// hashValue << BIT_LENGTH_FOR_BUCKET_ID means the rest bits other than the bucket id
// if the hashedValue is 0000,0000,0000,0000, at least you should stop counting at the (32-BIT_LENGTH_FOR_BUCKET_ID)
// position because trailing bits represents nothing which are introduced because of shift process.
final int leading0s = Long.numberOfLeadingZeros((hashedValue << BIT_LENGTH_FOR_BUCKET_ID) | (1 << (BIT_LENGTH_FOR_BUCKET_ID - 1)) + 1) + 1;

return registerSet.updateIfGreater(bid, leading0s);
}

public boolean offerHashed(long hashedValue) {
int bid = (int) (hashedValue >>> (Long.SIZE - BIT_LENGTH_FOR_BUCKET_ID));

final int leading0s = Long.numberOfLeadingZeros((hashedValue << BIT_LENGTH_FOR_BUCKET_ID) | (1 << (BIT_LENGTH_FOR_BUCKET_ID - 1)) + 1) + 1;

return registerSet.updateIfGreater(bid, leading0s);
}

public long cardinality() {
double sum = 0;
double zeros = 0.0;

for (int j = 0; j < registerSet.bucketNum; j++) {
int val = registerSet.get(j);

sum += 1.0 / (1 << val);

if (val == 0) {
zeros++;
}
}

double estimate = getAlpha(BIT_LENGTH_FOR_BUCKET_ID) * (1 / sum);

if (estimate <= (5.0 / 2.0) * registerSet.bucketNum) {
return Math.round(linearCounting(registerSet.bucketNum, zeros));
} else {
return Math.round(estimate);
}
}
}

class RegisterSet {

// its length is 5 if hash type is integer, or else 6 if hash type is long
public static int BIT_LENGTH_FOR_BUCKET_VALUE_RANGE = 5;

// BUCKET_NUM_PER_REGISTER = 32 / BIT_LENGTH_FOR_BUCKET_VALUE_RANGE;
public static int BUCKET_NUM_PER_REGISTER = 6;

public int BUCKET_MASK = (1 << (BIT_LENGTH_FOR_BUCKET_VALUE_RANGE)) - 1;

public int bucketNum;
public int registerNum;

private int[] registers;

public RegisterSet(int bucketNum, int hashLen) {
this.bucketNum = bucketNum;

// default hash type is int, 32
if (hashLen == 64) {
BIT_LENGTH_FOR_BUCKET_VALUE_RANGE = 6;

// the hash length may be 32 (integer) or 64 (long)
this.BUCKET_NUM_PER_REGISTER = 32 / BIT_LENGTH_FOR_BUCKET_VALUE_RANGE;

this.BUCKET_MASK = (1 << (BIT_LENGTH_FOR_BUCKET_VALUE_RANGE)) - 1;
}

// if cannot be exactly divided, we still need to allocate memory for the rest buckets
this.registerNum = bucketNum / BUCKET_NUM_PER_REGISTER + 1;

this.registers = new int[this.registerNum];
}

public boolean set(int bucketId, int value) {
// register index
int regId = bucketId / BUCKET_NUM_PER_REGISTER;

int shift = BIT_LENGTH_FOR_BUCKET_VALUE_RANGE * (bucketId % BUCKET_NUM_PER_REGISTER);

this.registers[regId] = (this.registers[regId] & ~(BUCKET_MASK << shift)) | (value << shift);

return true;
}

public int get(int bucketId) {
int regId = bucketId / BUCKET_NUM_PER_REGISTER;

int shift = BIT_LENGTH_FOR_BUCKET_VALUE_RANGE * (bucketId - (regId * BUCKET_NUM_PER_REGISTER));

return (this.registers[regId] & (BUCKET_MASK << shift)) >>> shift;
}

public boolean updateIfGreater(int bucketId, int value) {
int regId = bucketId / BUCKET_NUM_PER_REGISTER;

int shift = BIT_LENGTH_FOR_BUCKET_VALUE_RANGE * (bucketId % BUCKET_NUM_PER_REGISTER);

int mask = BUCKET_MASK << shift;

long curVal = this.registers[regId] & mask;
long newVal = value << shift;

if (curVal < newVal) {
return set(bucketId, value);
}

return false;
}
}

/**
* murmur hash 2.0.
* <p>
* The murmur hash is a relatively fast hash function from
* http://murmurhash.googlepages.com/ for platforms with efficient
* multiplication.
* <p>
* This is a re-implementation of the original C code plus some
* additional features.
* <p>
* Public domain.
*
* @author Viliam Holub
* @version 1.0.2
*/
final class MurmurHash {
// all methods static; private constructor.
private MurmurHash() {
}

/**
* Generates 32 bit hash from byte array of the given length and
* seed.
*
* @param data byte array to hash
* @param length length of the array to hash
* @param seed initial seed value
* @return 32 bit hash of the given array
*/
public static int hash32(final byte[] data, int length, int seed) {

// 'm' and 'r' are mixing constants generated offline.
// They're not really 'magic', they just happen to work well.

final int m = 0x5bd1e995;
final int r = 24;

// Initialize the hash to a random value
int h = seed ^ length;
int length4 = length / 4;
for (int i = 0; i < length4; i++) {
final int i4 = i * 4;
int k = (data[i4 + 0] & 0xff) + ((data[i4 + 1] & 0xff) << 8)
+ ((data[i4 + 2] & 0xff) << 16) + ((data[i4 + 3] & 0xff) << 24);
k *= m;
k ^= k >>> r;
k *= m;
h *= m;
h ^= k;
}

// Handle the last few bytes of the input array
switch (length % 4) {
case 3:
h ^= (data[(length & ~3) + 2] & 0xff) << 16;
case 2:
h ^= (data[(length & ~3) + 1] & 0xff) << 8;
case 1:
h ^= (data[length & ~3] & 0xff);
h *= m;
}
h ^= h >>> 13;
h *= m;
h ^= h >>> 15;
return h;
}

/**
* Generates 32 bit hash from byte array with default seed value.
*
* @param data byte array to hash
* @param length length of the array to hash
* @return 32 bit hash of the given array
*/
public static int hash32(final byte[] data, int length) {
return hash32(data, length, 0x9747b28c);
}

/**
* Generates 32 bit hash from a string.
*
* @param text string to hash
* @return 32 bit hash of the given string
*/
public static int hash32(final String text) {
final byte[] bytes = text.getBytes();
return hash32(bytes, bytes.length);
}

/**
* Generates 32 bit hash from a substring.
*
* @param text string to hash
* @param from starting index
* @param length length of the substring to hash
* @return 32 bit hash of the given string
*/
public static int hash32(final String text, int from, int length) {
return hash32(text.substring(from, from + length));
}

/**
* Generates 64 bit hash from byte array of the given length and seed.
*
* @param data byte array to hash
* @param length length of the array to hash
* @param seed initial seed value
* @return 64 bit hash of the given array
*/
public static long hash64(final byte[] data, int length, int seed) {
final long m = 0xc6a4a7935bd1e995L;
final int r = 47;
long h = (seed & 0xffffffffl) ^ (length * m);
int length8 = length / 8;
for (int i = 0; i < length8; i++) {
final int i8 = i * 8;
long k = ((long) data[i8 + 0] & 0xff) + (((long) data[i8 + 1] & 0xff) << 8)
+ (((long) data[i8 + 2] & 0xff) << 16) + (((long) data[i8 + 3] & 0xff) << 24)
+ (((long) data[i8 + 4] & 0xff) << 32) + (((long) data[i8 + 5] & 0xff) << 40)
+ (((long) data[i8 + 6] & 0xff) << 48) + (((long) data[i8 + 7] & 0xff) << 56);
k *= m;
k ^= k >>> r;
k *= m;
h ^= k;
h *= m;
}
switch (length % 8) {
case 7:
h ^= (long) (data[(length & ~7) + 6] & 0xff) << 48;
case 6:
h ^= (long) (data[(length & ~7) + 5] & 0xff) << 40;
case 5:
h ^= (long) (data[(length & ~7) + 4] & 0xff) << 32;
case 4:
h ^= (long) (data[(length & ~7) + 3] & 0xff) << 24;
case 3:
h ^= (long) (data[(length & ~7) + 2] & 0xff) << 16;
case 2:
h ^= (long) (data[(length & ~7) + 1] & 0xff) << 8;
case 1:
h ^= (long) (data[length & ~7] & 0xff);
h *= m;
}
;
h ^= h >>> r;
h *= m;
h ^= h >>> r;
return h;
}

/**
* Generates 64 bit hash from byte array with default seed value.
*
* @param data byte array to hash
* @param length length of the array to hash
* @return 64 bit hash of the given string
*/
public static long hash64(final byte[] data, int length) {
return hash64(data, length, 0xe17a1465);
}

/**
* Generates 64 bit hash from a string.
*
* @param text string to hash
* @return 64 bit hash of the given string
*/
public static long hash64(final String text) {
final byte[] bytes = text.getBytes();
return hash64(bytes, bytes.length);
}

/**
* Generates 64 bit hash from a substring.
*
* @param text string to hash
* @param from starting index
* @param length length of the substring to hash
* @return 64 bit hash of the given array
*/
public static long hash64(final String text, int from, int length) {
return hash64(text.substring(from, from + length));
}
}

References

  1. https://odino.org/my-favorite-data-structure-hyperloglog/
  2. http://content.research.neustar.biz/blog/hll.html
  3. https://www.jianshu.com/p/55defda6dcd2
  4. https://github.com/tnm/murmurhash-java