sp
/
tempest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

slight improvements to bit vector hashmap

tempestpy_adaptions
dehnert 7 years ago
parent
8b557c36a7
commit
95fae73833
4 changed files with 165 additions and 123 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 109
      src/storm/storage/BitVector.cpp
  2. 9
      src/storm/storage/BitVector.h
  3. 120
      src/storm/storage/BitVectorHashMap.cpp
  4. 50
      src/storm/storage/BitVectorHashMap.h

109
src/storm/storage/BitVector.cpp
View File

@ -1018,7 +1018,7 @@ namespace storm {
return seed;
}
inline __attribute__((always_inline)) uint32_t fmix32 (uint32_t h) {
inline __attribute__((always_inline)) uint32_t fmix32(uint32_t h) {
h ^= h >> 16;
h *= 0x85ebca6b;
h ^= h >> 13;
@ -1028,16 +1028,36 @@ namespace storm {
return h;
}
inline __attribute__((always_inline)) uint64_t fmix64(uint64_t k) {
k ^= k >> 33;
k *= 0xff51afd7ed558ccdull;
k ^= k >> 33;
k *= 0xc4ceb9fe1a85ec53ull;
k ^= k >> 33;
return k;
}
inline uint32_t rotl32(uint32_t x, int8_t r) {
return (x << r) | (x >> (32 - r));
}
inline __attribute__((always_inline)) uint32_t getblock32 (uint32_t const* p, int i) {
inline uint64_t rotl64 ( uint64_t x, int8_t r )
{
return (x << r) | (x >> (64 - r));
}
inline __attribute__((always_inline)) uint32_t getblock32(uint32_t const* p, int i) {
return p[i];
}
std::size_t Murmur3_32_BitVectorHash::operator()(storm::storage::BitVector const& bv) const {
const uint8_t * data = reinterpret_cast<uint8_t const*>(bv.buckets);
inline __attribute__((always_inline)) uint32_t getblock64(uint64_t const* p, int i) {
return p[i];
}
template<>
uint32_t Murmur3BitVectorHash<uint32_t>::operator()(storm::storage::BitVector const& bv) const {
uint8_t const* data = reinterpret_cast<uint8_t const*>(bv.buckets);
uint32_t len = bv.bucketCount() * 8;
const int nblocks = bv.bucketCount() * 2;
@ -1074,6 +1094,84 @@ namespace storm {
return h1;
}
template<>
uint64_t Murmur3BitVectorHash<uint64_t>::operator()(storm::storage::BitVector const& bv) const {
uint8_t const* data = reinterpret_cast<uint8_t const*>(bv.buckets);
uint64_t len = bv.bucketCount() * 8;
const int nblocks = bv.bucketCount() / 2;
uint64_t h1 = 0;
uint64_t h2 = 0;
const uint64_t c1 = 0x87c37b91114253d5ull;
const uint64_t c2 = 0x4cf5ad432745937full;
//----------
// body
const uint64_t * blocks = (const uint64_t *)(data);
for (int i = 0; i < nblocks; i++) {
uint64_t k1 = getblock64(blocks,i*2+0);
uint64_t k2 = getblock64(blocks,i*2+1);
k1 *= c1; k1 = rotl64(k1,31); k1 *= c2; h1 ^= k1;
h1 = rotl64(h1,27); h1 += h2; h1 = h1*5+0x52dce729;
k2 *= c2; k2 = rotl64(k2,33); k2 *= c1; h2 ^= k2;
h2 = rotl64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5;
}
//----------
// tail
uint8_t const* tail = reinterpret_cast<uint8_t const*>(data + nblocks*16);
uint64_t k1 = 0;
uint64_t k2 = 0;
switch(len & 15)
{
case 15: k2 ^= ((uint64_t)tail[14]) << 48;
case 14: k2 ^= ((uint64_t)tail[13]) << 40;
case 13: k2 ^= ((uint64_t)tail[12]) << 32;
case 12: k2 ^= ((uint64_t)tail[11]) << 24;
case 11: k2 ^= ((uint64_t)tail[10]) << 16;
case 10: k2 ^= ((uint64_t)tail[ 9]) << 8;
case 9: k2 ^= ((uint64_t)tail[ 8]) << 0;
k2 *= c2; k2 = rotl64(k2,33); k2 *= c1; h2 ^= k2;
case 8: k1 ^= ((uint64_t)tail[ 7]) << 56;
case 7: k1 ^= ((uint64_t)tail[ 6]) << 48;
case 6: k1 ^= ((uint64_t)tail[ 5]) << 40;
case 5: k1 ^= ((uint64_t)tail[ 4]) << 32;
case 4: k1 ^= ((uint64_t)tail[ 3]) << 24;
case 3: k1 ^= ((uint64_t)tail[ 2]) << 16;
case 2: k1 ^= ((uint64_t)tail[ 1]) << 8;
case 1: k1 ^= ((uint64_t)tail[ 0]) << 0;
k1 *= c1; k1 = rotl64(k1,31); k1 *= c2; h1 ^= k1;
};
//----------
// finalization
h1 ^= len; h2 ^= len;
h1 += h2;
h2 += h1;
h1 = fmix64(h1);
h2 = fmix64(h2);
h1 += h2;
h2 += h1;
return h1 ^ h2;
}
// All necessary explicit template instantiations.
template BitVector::BitVector(uint_fast64_t length, std::vector<uint_fast64_t>::iterator begin, std::vector<uint_fast64_t>::iterator end);
template BitVector::BitVector(uint_fast64_t length, std::vector<uint_fast64_t>::const_iterator begin, std::vector<uint_fast64_t>::const_iterator end);
@ -1083,6 +1181,9 @@ namespace storm {
template void BitVector::set(std::vector<uint_fast64_t>::const_iterator begin, std::vector<uint_fast64_t>::const_iterator end, bool value);
template void BitVector::set(boost::container::flat_set<uint_fast64_t>::iterator begin, boost::container::flat_set<uint_fast64_t>::iterator end, bool value);
template void BitVector::set(boost::container::flat_set<uint_fast64_t>::const_iterator begin, boost::container::flat_set<uint_fast64_t>::const_iterator end, bool value);
template struct Murmur3BitVectorHash<uint32_t>;
template struct Murmur3BitVectorHash<uint64_t>;
}
}

9
src/storm/storage/BitVector.h
View File

@ -502,7 +502,9 @@ namespace storm {
friend std::ostream& operator<<(std::ostream& out, BitVector const& bitVector);
friend struct std::hash<storm::storage::BitVector>;
friend struct FNV1aBitVectorHash;
friend struct Murmur3_32_BitVectorHash;
template<typename StateType>
friend struct Murmur3BitVectorHash;
private:
/*!
@ -576,8 +578,9 @@ namespace storm {
std::size_t operator()(storm::storage::BitVector const& bv) const;
};
struct Murmur3_32_BitVectorHash {
std::size_t operator()(storm::storage::BitVector const& bv) const;
template<typename StateType>
struct Murmur3BitVectorHash {
StateType operator()(storm::storage::BitVector const& bv) const;
};
} // namespace storage

120
src/storm/storage/BitVectorHashMap.cpp
View File

@ -69,12 +69,12 @@ namespace storm {
}
template<class ValueType, class Hash>
std::size_t BitVectorHashMap<ValueType, Hash>::size() const {
uint64_t BitVectorHashMap<ValueType, Hash>::size() const {
return numberOfElements;
}
template<class ValueType, class Hash>
std::size_t BitVectorHashMap<ValueType, Hash>::capacity() const {
uint64_t BitVectorHashMap<ValueType, Hash>::capacity() const {
return 1ull << currentSize;
}
@ -88,7 +88,6 @@ namespace storm {
#endif
// Create new containers and swap them with the old ones.
numberOfElements = 0;
storm::storage::BitVector oldBuckets(bucketSize * (1ull << currentSize));
std::swap(oldBuckets, buckets);
storm::storage::BitVector oldOccupied = storm::storage::BitVector(1ull << currentSize);
@ -97,29 +96,9 @@ namespace storm {
std::swap(oldValues, values);
// Now iterate through the elements and reinsert them in the new storage.
bool fail = false;
numberOfElements = 0;
for (auto bucketIndex : oldOccupied) {
fail = !this->insertWithoutIncreasingSize(oldBuckets.get(bucketIndex * bucketSize, bucketSize), oldValues[bucketIndex]);
STORM_LOG_ASSERT(!fail, "Expected to be able to insert all elements.");
}
}
template<class ValueType, class Hash>
bool BitVectorHashMap<ValueType, Hash>::insertWithoutIncreasingSize(storm::storage::BitVector const& key, ValueType const& value) {
std::tuple<bool, std::size_t, bool> flagBucketTuple = this->findBucketToInsert<false>(key);
if (std::get<2>(flagBucketTuple)) {
return false;
}
if (std::get<0>(flagBucketTuple)) {
return true;
} else {
// Insert the new bits into the bucket.
buckets.set(std::get<1>(flagBucketTuple) * bucketSize, key);
occupied.set(std::get<1>(flagBucketTuple));
values[std::get<1>(flagBucketTuple)] = value;
++numberOfElements;
return true;
findOrAddAndGetBucket(oldBuckets.get(bucketIndex * bucketSize, bucketSize), oldValues[bucketIndex]);
}
}
@ -129,59 +108,41 @@ namespace storm {
}
template<class ValueType, class Hash>
void BitVectorHashMap<ValueType, Hash>::setOrAdd(storm::storage::BitVector const& key, ValueType const& value) {
setOrAddAndGetBucket(key, value);
}
std::pair<ValueType, uint64_t> BitVectorHashMap<ValueType, Hash>::findOrAddAndGetBucket(storm::storage::BitVector const& key, ValueType const& value) {
checkIncreaseSize();
template<class ValueType, class Hash>
std::pair<ValueType, std::size_t> BitVectorHashMap<ValueType, Hash>::findOrAddAndGetBucket(storm::storage::BitVector const& key, ValueType const& value) {
// If the load of the map is too high, we increase the size.
if (numberOfElements >= loadFactor * (1ull << currentSize)) {
this->increaseSize();
}
std::tuple<bool, std::size_t, bool> flagBucketTuple = this->findBucketToInsert<true>(key);
STORM_LOG_ASSERT(!std::get<2>(flagBucketTuple), "Failed to find bucket for insertion.");
if (std::get<0>(flagBucketTuple)) {
return std::make_pair(values[std::get<1>(flagBucketTuple)], std::get<1>(flagBucketTuple));
std::pair<bool, uint64_t> flagAndBucket = this->findBucketToInsert(key);
if (flagAndBucket.first) {
return std::make_pair(values[flagAndBucket.second], flagAndBucket.second);
} else {
// Insert the new bits into the bucket.
buckets.set(std::get<1>(flagBucketTuple) * bucketSize, key);
occupied.set(std::get<1>(flagBucketTuple));
values[std::get<1>(flagBucketTuple)] = value;
buckets.set(flagAndBucket.second * bucketSize, key);
occupied.set(flagAndBucket.second);
values[flagAndBucket.second] = value;
++numberOfElements;
return std::make_pair(value, std::get<1>(flagBucketTuple));
return std::make_pair(value, flagAndBucket.second);
}
}
template<class ValueType, class Hash>
std::size_t BitVectorHashMap<ValueType, Hash>::setOrAddAndGetBucket(storm::storage::BitVector const& key, ValueType const& value) {
bool BitVectorHashMap<ValueType, Hash>::checkIncreaseSize() {
// If the load of the map is too high, we increase the size.
if (numberOfElements >= loadFactor * (1ull << currentSize)) {
this->increaseSize();
return true;
}
std::tuple<bool, std::size_t, bool> flagBucketTuple = this->findBucketToInsert<true>(key);
STORM_LOG_ASSERT(!std::get<2>(flagBucketTuple), "Failed to find bucket for insertion.");
if (!std::get<0>(flagBucketTuple)) {
// Insert the new bits into the bucket.
buckets.set(std::get<1>(flagBucketTuple) * bucketSize, key);
occupied.set(std::get<1>(flagBucketTuple));
++numberOfElements;
}
values[std::get<1>(flagBucketTuple)] = value;
return std::get<1>(flagBucketTuple);
return false;
}
template<class ValueType, class Hash>
ValueType BitVectorHashMap<ValueType, Hash>::getValue(storm::storage::BitVector const& key) const {
std::pair<bool, std::size_t> flagBucketPair = this->findBucket(key);
std::pair<bool, uint64_t> flagBucketPair = this->findBucket(key);
STORM_LOG_ASSERT(flagBucketPair.first, "Unknown key.");
return values[flagBucketPair.second];
}
template<class ValueType, class Hash>
ValueType BitVectorHashMap<ValueType, Hash>::getValue(std::size_t bucket) const {
ValueType BitVectorHashMap<ValueType, Hash>::getValue(uint64_t bucket) const {
return values[bucket];
}
@ -201,73 +162,58 @@ namespace storm {
}
template<class ValueType, class Hash>
std::pair<bool, std::size_t> BitVectorHashMap<ValueType, Hash>::findBucket(storm::storage::BitVector const& key) const {
uint64_t BitVectorHashMap<ValueType, Hash>::getCurrentShiftWidth() const {
return (sizeof(decltype(hasher(storm::storage::BitVector()))) * 8 - currentSize);
}
template<class ValueType, class Hash>
std::pair<bool, uint64_t> BitVectorHashMap<ValueType, Hash>::findBucket(storm::storage::BitVector const& key) const {
#ifndef NDEBUG
++numberOfFinds;
#endif
uint_fast64_t initialHash = hasher(key) % (1ull << currentSize);
uint_fast64_t bucket = initialHash;
uint64_t bucket = hasher(key) >> this->getCurrentShiftWidth();
uint_fast64_t i = 0;
while (isBucketOccupied(bucket)) {
++i;
#ifndef NDEBUG
++numberOfFindProbingSteps;
#endif
if (buckets.matches(bucket * bucketSize, key)) {
return std::make_pair(true, bucket);
}
bucket += 1;
++bucket;
if (bucket == (1ull << currentSize)) {
bucket = 0;
}
if (bucket == initialHash) {
return std::make_pair(false, bucket);
}
}
return std::make_pair(false, bucket);
}
template<class ValueType, class Hash>
template<bool increaseStorage>
std::tuple<bool, std::size_t, bool> BitVectorHashMap<ValueType, Hash>::findBucketToInsert(storm::storage::BitVector const& key) {
std::pair<bool, uint64_t> BitVectorHashMap<ValueType, Hash>::findBucketToInsert(storm::storage::BitVector const& key) {
#ifndef NDEBUG
++numberOfInsertions;
#endif
uint_fast64_t initialHash = hasher(key) % (1ull << currentSize);
uint_fast64_t bucket = initialHash;
uint64_t bucket = hasher(key) >> this->getCurrentShiftWidth();
uint64_t i = 0;
while (isBucketOccupied(bucket)) {
++i;
#ifndef NDEBUG
++numberOfInsertionProbingSteps;
#endif
if (buckets.matches(bucket * bucketSize, key)) {
return std::make_tuple(true, bucket, false);
return std::make_pair(true, bucket);
}
bucket += 1;
++bucket;
if (bucket == (1ull << currentSize)) {
bucket = 0;
}
if (bucket == initialHash) {
if (increaseStorage) {
this->increaseSize();
bucket = initialHash = hasher(key) % (1ull << currentSize);
} else {
return std::make_tuple(false, bucket, true);
}
}
}
return std::make_tuple(false, bucket, false);
return std::make_tuple(false, bucket);
}
template<class ValueType, class Hash>
std::pair<storm::storage::BitVector, ValueType> BitVectorHashMap<ValueType, Hash>::getBucketAndValue(std::size_t bucket) const {
std::pair<storm::storage::BitVector, ValueType> BitVectorHashMap<ValueType, Hash>::getBucketAndValue(uint64_t bucket) const {
return std::make_pair(buckets.get(bucket * bucketSize, bucketSize), values[bucket]);
}
@ -278,7 +224,7 @@ namespace storm {
}
}
template class BitVectorHashMap<uint_fast64_t>;
template class BitVectorHashMap<uint64_t>;
template class BitVectorHashMap<uint32_t>;
}
}

50
src/storm/storage/BitVectorHashMap.h
View File

@ -16,7 +16,7 @@ namespace storm {
*/
// template<typename ValueType, typename Hash = std::hash<storm::storage::BitVector>>
// template<typename ValueType, typename Hash = FNV1aBitVectorHash>
template<typename ValueType, typename Hash = Murmur3_32_BitVectorHash>
template<typename ValueType, typename Hash = Murmur3BitVectorHash<ValueType>>
class BitVectorHashMap {
public:
class BitVectorHashMapIterator {
@ -74,15 +74,6 @@ namespace storm {
*/
ValueType findOrAdd(storm::storage::BitVector const& key, ValueType const& value);
/*!
* Sets the given key value pain in the map. If the key is found in the map, the corresponding value is
* overwritten with the given value. Otherwise, the key is inserted with the given value.
*
* @param key The key to search or insert.
* @param value The value to set.
*/
void setOrAdd(storm::storage::BitVector const& key, ValueType const& value);
/*!
* Searches for the given key in the map. If it is found, the mapped-to value is returned. Otherwise, the
* key is inserted with the given value.
@ -93,17 +84,7 @@ namespace storm {
* the provided new value otherwise and whose second component is the index of the bucket into which the key
* was inserted.
*/
std::pair<ValueType, std::size_t> findOrAddAndGetBucket(storm::storage::BitVector const& key, ValueType const& value);
/*!
* Sets the given key value pain in the map. If the key is found in the map, the corresponding value is
* overwritten with the given value. Otherwise, the key is inserted with the given value.
*
* @param key The key to search or insert.
* @param value The value to set.
* @return The index of the bucket into which the key was inserted.
*/
std::size_t setOrAddAndGetBucket(storm::storage::BitVector const& key, ValueType const& value);
std::pair<ValueType, uint64_t> findOrAddAndGetBucket(storm::storage::BitVector const& key, ValueType const& value);
/*!
* Retrieves the key stored in the given bucket (if any) and the value it is mapped to.
@ -111,7 +92,7 @@ namespace storm {
* @param bucket The index of the bucket.
* @return The content and value of the named bucket.
*/
std::pair<storm::storage::BitVector, ValueType> getBucketAndValue(std::size_t bucket) const;
std::pair<storm::storage::BitVector, ValueType> getBucketAndValue(uint64_t bucket) const;
/*!
* Retrieves the value associated with the given key (if any). If the key does not exist, the behaviour is
@ -126,7 +107,7 @@ namespace storm {
*
* @return The value associated with the given bucket (if any).
*/
ValueType getValue(std::size_t bucket) const;
ValueType getValue(uint64_t bucket) const;
/*!
* Checks if the given key is already contained in the map.
@ -155,14 +136,14 @@ namespace storm {
*
* @return The size of the map.
*/
std::size_t size() const;
uint64_t size() const;
/*!
* Retrieves the capacity of the underlying container.
*
* @return The capacity of the underlying container.
*/
std::size_t capacity() const;
uint64_t capacity() const;
/*!
* Performs a remapping of all values stored by applying the given remapping.
@ -187,7 +168,7 @@ namespace storm {
* @return A pair whose first component indicates whether the key is already contained in the map and whose
* second component indicates in which bucket the key is stored.
*/
std::pair<bool, std::size_t> findBucket(storm::storage::BitVector const& key) const;
std::pair<bool, uint64_t> findBucket(storm::storage::BitVector const& key) const;
/*!
* Searches for the bucket into which the given key can be inserted. If no empty bucket can be found, the
@ -200,8 +181,7 @@ namespace storm {
* an error flag indicating that the bucket could not be found (e.g. due to the restriction that the storage
* must not be increased).
*/
template<bool increaseStorage>
std::tuple<bool, std::size_t, bool> findBucketToInsert(storm::storage::BitVector const& key);
std::pair<bool, uint64_t> findBucketToInsert(storm::storage::BitVector const& key);
/*!
* Inserts the given key-value pair without resizing the underlying storage. If that fails, this is
@ -218,6 +198,18 @@ namespace storm {
*/
void increaseSize();
/*!
* Checks whether the size should be increased and does so if necessary.
*
* @return True iff the storage was increased.
*/
bool checkIncreaseSize();
/*!
* Determines the number of bits by which the hash value must be shifted to obtain a value in the legal range.
*/
uint64_t getCurrentShiftWidth() const;
// The load factor determining when the size of the map is increased.
double loadFactor;
@ -237,7 +229,7 @@ namespace storm {
std::vector<ValueType> values;
// The number of elements in this map.
std::size_t numberOfElements;
uint64_t numberOfElements;
// Functor object that are used to perform the actual hashing.
Hash hasher;

Write Preview
Loading…
Cancel
Save