Last program we present TinyHyperLogLog that used to count distinct elements of a set which is inspired by HyperLogLog. Today we gonna go through HyperLogLog. Actually, TinyHyperLogLog is pretty much close to HyperLogLog except optimization techniques.

There are two fundamental and outstandingly changes.

HyperLogLog uses a serial of constants to optimize the results. Two important factors are alpha and linear counting which are mathematical knowledge. Here I’m not going to talk too much. If you are interested, you can go through the author’s paper.

HyperLogLog introduce the idea of register to conserve memory space. In TinyHyperLogLog we use integer for each bucket counter. We all know that the size of integer is 32, apart from the bucket id, the longest one is only (32-BIT_LENGTH_FOR_BUCKET_ID). So we can use at most 5 bits to represent it if bucketSize>=1.

For a long type, apart from the bucket id, the longest one is only (64-BIT_LENGTH_FOR_BUCKET_ID). So we can use at most 6 bits to represent it if bucketSize>=1.

To conserve memory, each integer is divided into BIT_LENGTH_FOR_BUCKET_VALUE_RANGE buckets. The ultimate purpose is to make memory fully used because the length of the longest leading 0s cannot be very long.

The register comes out. Register is a group of buckets, use a single integer to represent a bucket is too expensive for memory. We can divide it to save a considerable amount of memory.

the generated hash length, we recommend to use long type since it’s able to represent much more distinct elements than integer.

privatefinalint hashLen = 64; // the leading BIT_LENGTH_FOR_BUCKET_ID bits represent the bucket id privatefinalint BIT_LENGTH_FOR_BUCKET_ID; private RegisterSet registerSet;

this.registerSet = new RegisterSet(1 << BIT_LENGTH_FOR_BUCKET_ID, hashLen); }

publicstaticvoidmain(String[] args){ HyperLogLog<String> hyperLogLog = new HyperLogLog(16);

int count = 0; while (true) { for (int i = 0; i < 100000; i++) { hyperLogLog.offer(i + "adf"); }

if (count > 10) { break; }

count++; }

System.out.println(hyperLogLog.cardinality()); }

protectedstaticdoublegetAlpha(finalint p){

// generate a optimization factor

int m = 1 << p; // See the paper. switch (p) { case4: return0.673 * m * m; case5: return0.697 * m * m; case6: return0.709 * m * m; default: return (0.7213 / (1 + 1.079 / m)) * m * m; } }

/** * Add element to the HyperLogLog counter * * @param o inserted element * @return */ publicbooleanoffer(T o){ if (hashLen == 32) { // we generate a integer hash sequence return offerHashed(MurmurHash.hash32(o.toString())); }

// we generate a long hash sequence so that we can count more distinct elements than Integer can be. return offerHashed(MurmurHash.hash64(o.toString())); }

publicbooleanofferHashed(int hashedValue){ // the leading BIT_LENGTH_FOR_BUCKET_ID bits represents the bucket id int bid = hashedValue >>> (Integer.SIZE - BIT_LENGTH_FOR_BUCKET_ID);

// hashValue << BIT_LENGTH_FOR_BUCKET_ID means the rest bits other than the bucket id // if the hashedValue is 0000,0000,0000,0000, at least you should stop counting at the (32-BIT_LENGTH_FOR_BUCKET_ID) // position because trailing bits represents nothing which are introduced because of shift process. finalint leading0s = Long.numberOfLeadingZeros((hashedValue << BIT_LENGTH_FOR_BUCKET_ID) | (1 << (BIT_LENGTH_FOR_BUCKET_ID - 1)) + 1) + 1;

publicbooleanupdateIfGreater(int bucketId, int value){ int regId = bucketId / BUCKET_NUM_PER_REGISTER;

int shift = BIT_LENGTH_FOR_BUCKET_VALUE_RANGE * (bucketId % BUCKET_NUM_PER_REGISTER);

int mask = BUCKET_MASK << shift;

long curVal = this.registers[regId] & mask; long newVal = value << shift;

if (curVal < newVal) { return set(bucketId, value); }

returnfalse; } }

/** * murmur hash 2.0. * <p> * The murmur hash is a relatively fast hash function from * http://murmurhash.googlepages.com/ for platforms with efficient * multiplication. * <p> * This is a re-implementation of the original C code plus some * additional features. * <p> * Public domain. * * @author Viliam Holub * @version 1.0.2 */ finalclassMurmurHash{ // all methods static; private constructor. privateMurmurHash(){ }

/** * Generates 32 bit hash from byte array of the given length and * seed. * * @param data byte array to hash * @param length length of the array to hash * @param seed initial seed value * @return 32 bit hash of the given array */ publicstaticinthash32(finalbyte[] data, int length, int seed){

// 'm' and 'r' are mixing constants generated offline. // They're not really 'magic', they just happen to work well.

finalint m = 0x5bd1e995; finalint r = 24;

// Initialize the hash to a random value int h = seed ^ length; int length4 = length / 4; for (int i = 0; i < length4; i++) { finalint i4 = i * 4; int k = (data[i4 + 0] & 0xff) + ((data[i4 + 1] & 0xff) << 8) + ((data[i4 + 2] & 0xff) << 16) + ((data[i4 + 3] & 0xff) << 24); k *= m; k ^= k >>> r; k *= m; h *= m; h ^= k; }

// Handle the last few bytes of the input array switch (length % 4) { case3: h ^= (data[(length & ~3) + 2] & 0xff) << 16; case2: h ^= (data[(length & ~3) + 1] & 0xff) << 8; case1: h ^= (data[length & ~3] & 0xff); h *= m; } h ^= h >>> 13; h *= m; h ^= h >>> 15; return h; }

/** * Generates 32 bit hash from byte array with default seed value. * * @param data byte array to hash * @param length length of the array to hash * @return 32 bit hash of the given array */ publicstaticinthash32(finalbyte[] data, int length){ return hash32(data, length, 0x9747b28c); }

/** * Generates 32 bit hash from a string. * * @param text string to hash * @return 32 bit hash of the given string */ publicstaticinthash32(final String text){ finalbyte[] bytes = text.getBytes(); return hash32(bytes, bytes.length); }

/** * Generates 32 bit hash from a substring. * * @param text string to hash * @param from starting index * @param length length of the substring to hash * @return 32 bit hash of the given string */ publicstaticinthash32(final String text, int from, int length){ return hash32(text.substring(from, from + length)); }

/** * Generates 64 bit hash from byte array of the given length and seed. * * @param data byte array to hash * @param length length of the array to hash * @param seed initial seed value * @return 64 bit hash of the given array */ publicstaticlonghash64(finalbyte[] data, int length, int seed){ finallong m = 0xc6a4a7935bd1e995L; finalint r = 47; long h = (seed & 0xffffffffl) ^ (length * m); int length8 = length / 8; for (int i = 0; i < length8; i++) { finalint i8 = i * 8; long k = ((long) data[i8 + 0] & 0xff) + (((long) data[i8 + 1] & 0xff) << 8) + (((long) data[i8 + 2] & 0xff) << 16) + (((long) data[i8 + 3] & 0xff) << 24) + (((long) data[i8 + 4] & 0xff) << 32) + (((long) data[i8 + 5] & 0xff) << 40) + (((long) data[i8 + 6] & 0xff) << 48) + (((long) data[i8 + 7] & 0xff) << 56); k *= m; k ^= k >>> r; k *= m; h ^= k; h *= m; } switch (length % 8) { case7: h ^= (long) (data[(length & ~7) + 6] & 0xff) << 48; case6: h ^= (long) (data[(length & ~7) + 5] & 0xff) << 40; case5: h ^= (long) (data[(length & ~7) + 4] & 0xff) << 32; case4: h ^= (long) (data[(length & ~7) + 3] & 0xff) << 24; case3: h ^= (long) (data[(length & ~7) + 2] & 0xff) << 16; case2: h ^= (long) (data[(length & ~7) + 1] & 0xff) << 8; case1: h ^= (long) (data[length & ~7] & 0xff); h *= m; } ; h ^= h >>> r; h *= m; h ^= h >>> r; return h; }

/** * Generates 64 bit hash from byte array with default seed value. * * @param data byte array to hash * @param length length of the array to hash * @return 64 bit hash of the given string */ publicstaticlonghash64(finalbyte[] data, int length){ return hash64(data, length, 0xe17a1465); }

/** * Generates 64 bit hash from a string. * * @param text string to hash * @return 64 bit hash of the given string */ publicstaticlonghash64(final String text){ finalbyte[] bytes = text.getBytes(); return hash64(bytes, bytes.length); }

/** * Generates 64 bit hash from a substring. * * @param text string to hash * @param from starting index * @param length length of the substring to hash * @return 64 bit hash of the given array */ publicstaticlonghash64(final String text, int from, int length){ return hash64(text.substring(from, from + length)); } }