Question

I want to concat a big bitset with a smaller one in a way that wont kill performance. Currently my application spends 20% of cpu time in just the following code:

boost::dynamic_bitset<> encode(const std::vector<char>& data)
{
    boost::dynamic_bitset<> result;

    std::for_each(data.begin(), data.end(), [&](unsigned char symbol)
    {
        for(size_t n = 0; n < codes_[symbol].size(); ++n)
            result.push_back(codes_[symbol][n]); // codes_[symbol][n].size() avarage ~5 bits
    });
    return result;
}

I have read this post which proposes a solution, which unfortunately will not work for me as the size difference between the sizes of destination bitset and the source bitset is very large.

Any ideas?

If this is not possible to do efficiently with boost::dynamic_bitset then I’m open for other suggestions.

Answer 1

I wrote my own bitset class. I appreciate any suggestions for improvements. I will try to look into SSE and see if there is anything useful there.

With my very rough benchmark I got a 11x performance increase while appending 6 bits at a time.

class fast_bitset
{
public:
    typedef unsigned long block_type;
    static const size_t bits_per_block = sizeof(block_type)*8;

    fast_bitset() 
        : is_open_(true)
        , blocks_(1)
        , space_(blocks_.size()*bits_per_block){}

    void append(const fast_bitset& other)
    {
        assert(!other.is_open_);

        for(size_t n = 0; n < other.blocks_.size()-1; ++n)
            append(other.blocks_[n], bits_per_block);
        append(other.blocks_.back() >> other.space_, bits_per_block - other.space_);
    }

    void append(block_type value, size_t n_bits)
    {
        assert(is_open_);
        assert(n_bits < bits_per_block);

        if(space_ < n_bits)
        {
            blocks_.back() = blocks_.back() << space_;
            blocks_.back() = blocks_.back() | (value >> (n_bits - space_));
            blocks_.push_back(value);
            space_ = bits_per_block - (n_bits - space_);
        }
        else
        {
            blocks_.back() = blocks_.back() << n_bits;
            blocks_.back() = blocks_.back() | value;
            space_ -= n_bits;
        }
    }

    void push_back(bool bit)
    {
        append(bit, 1);
    }

    bool operator[](size_t index) const
    {
        assert(!is_open_);

        static const size_t high_bit = 1 << (bits_per_block-1);
        const size_t block_index = index / bits_per_block;
        const size_t bit_index = index % bits_per_block;
        const size_t bit_mask = high_bit >> bit_index;
        return blocks_[block_index] & bit_mask;
    }

    void close()
    {
        blocks_.back() = blocks_.back() << space_;
        is_open_ = false;
    }

    size_t size() const
    {
        return blocks_.size()*bits_per_block-space_;
    }

    const std::vector<block_type>& blocks() const {return blocks_;}

    class reader
    {
    public:
        reader(const fast_bitset& bitset) 
            : bitset_(bitset)
            , index_(0)
            , size_(bitset.size()){}
        bool next_bit(){return bitset_[index_++];}
        bool eof() const{return index_ >= size_;}
    private:
        const fast_bitset& bitset_;
        size_t index_;
        size_t size_;
    };

private:
    bool is_open_;
    std::vector<block_type> blocks_;
    size_t space_;
};