Question

I have a data file generated by hardware. Some data is 4 bit wide and some is 12 bit wide. Matlab is able to process these data using fread(fp,1,’ubit4=>uint16′). I tried to do it in C++, but it seems that there is no simple way. I can read by byte/int/long/long long and then extract out the requested bits. but it seems inefficient dealing with hundreds mega bytes data.

To generalize this problem, the question is how to read bitN integer (N from 1 to 64 for example)? Can anyone recommend a good way to read this kind of data from file in c++?

Answer 1

Thank you all for the contributed answers and they are all very helpful. I am not trying to answer my question and get the credit, but I feel I am obligated to give my feedback on the progress on this question. All credits goes to the above answers.

To achieve the similar function of the matlab fread to read bitN integers, I feel that the template class is not proper, so I came up several functions to deal with <8bit <16bit <32bit and <64bit cases and process them separately.

My idea is: I copy several bytes (from 2 to 8 bytes) to my object and process these bytes and keep unprocessed byte for next processing. Here is my code and testing results (only the <8bit case is implemented):

#include <math.h>
#include <memory.h>
typedef unsigned _int8 _uint8;
typedef unsigned _int16 _uint16;
typedef unsigned _int32 _uint32;
typedef unsigned _int64 _uint64;

class bitbuffer
{
    _uint8 *pbuf;
    _uint8 *pelem; //can be casted to int16/32/64
    _uint32 pbuf_len; //buf length in byte
    _uint32 pelem_len; //element length in byte
    union membuf
    {
        _uint64 buf64;
        _uint32 buf32;
        _uint16 buf16;
        _uint8 buf8[2];
    } tbuf;

    //bookkeeping information
    _uint8 start_bit; //
    _uint32 byte_pos; //current byte position
    _uint32 elem_pos;
public:
    bitbuffer(_uint8 *src,_uint32 src_len,_uint8 *dst,_uint32 dst_len)
    {
        pbuf=src;pelem=dst;
        pbuf_len=src_len;pelem_len=dst_len;
        start_bit=0;byte_pos=0;elem_pos=0;
    } //to define the source and destination
    void set_startbit(_uint8 bit) {start_bit=bit;}
    void set_bytepos(_uint32 pos) {byte_pos=pos;}
    void set_elempos(_uint32 pos) {elem_pos=pos;}
    void reset() {start_bit=0;byte_pos=0;elem_pos=0;} //for restart something from somewhere else
    //OUT getbits(IN a, _uint8 nbits); //get nbits from a using start and byte_pos
    _uint32 get_elem_uint8(_uint32 num_elem,_uint8 nbits) //output limit to 8/16/32/64 only
    {
        _uint32 num_read=0;
        _uint16 mask=pow(2,nbits)-1;//00000111 for example nbit=3 
        while(byte_pos<=pbuf_len-2)
        {
            //memcpy((char*)&tbuf.buf16,pbuf+byte_pos,2); //copy 2 bytes into our buffer, this may introduce redundant copy
            tbuf.buf8[1]=pbuf[byte_pos]; //for little endian machine, swap the bytes
            tbuf.buf8[0]=pbuf[byte_pos+1];
            //now we have start_bits, byte_pos, elem_pos, just finish them all
            while(start_bit<=16-nbits)
            {
                pelem[elem_pos++]=(tbuf.buf16>>(16-start_bit-nbits))&mask;//(tbuf.buf16&(mask<<(16-start_bit))
                start_bit+=nbits; //advance by nbits
                num_read++;
                if(num_read>=num_elem)
                {
                    break;
                }
            }
            //need update the start_bit and byte_pos
            byte_pos+=(start_bit/8);
            start_bit%=8;
            if(num_read>=num_elem)
            {
                break;
            }

        }
        return num_read;
    }
/*  
    _uint32 get_elem_uint16(_uint32 num_elem,_uint8 nbits) //output limit to 8/16/32/64 only
    {
        _uint32 num_read=0;
        _uint32 mask=pow(2,nbits)-1;//00000111 for example nbit=3 
        while(byte_pos<pbuf_len-4)
        {
            memcpy((char*)&tbuf.buf32,pbuf+byte_pos,4); //copy 2 bytes into our buffer, this may introduce redundant copy
            //now we have start_bits, byte_pos, elem_pos, just finish them all
            while(start_bit<=32-nbits)
            {
                pelem[elem_pos++]=(tbuf.buf32>>(32-start_bit-nbits))&mask;//(tbuf.buf16&(mask<<(16-start_bit))
                start_bit+=nbits; //advance by nbits
                num_read++;
                if(num_read>=num_elem)
                {
                    break;
                }
            }
            //need update the start_bit and byte_pos
            start_bit%=8;
            byte_pos+=(start_bit/8);
            if(num_read>=num_elem)
            {
                break;
            }

        }
        return num_read;
    }
    _uint32 get_elem_uint32(_uint32 num_elem,_uint8 nbits) //output limit to 8/16/32/64 only
    {
        _uint32 num_read=0;
        _uint64 mask=pow(2,nbits)-1;//00000111 for example nbit=3 
        while(byte_pos<pbuf_len-8)
        {
            memcpy((char*)&tbuf.buf16,pbuf+byte_pos,8); //copy 2 bytes into our buffer, this may introduce redundant copy
            //now we have start_bits, byte_pos, elem_pos, just finish them all
            while(start_bit<=64-nbits)
            {
                pelem[elem_pos++]=(tbuf.buf64>>(64-start_bit-nbits))&mask;//(tbuf.buf16&(mask<<(16-start_bit))
                start_bit+=nbits; //advance by nbits
                num_read++;
                if(num_read>=num_elem)
                {
                    break;
                }
            }
            //need update the start_bit and byte_pos
            start_bit%=8;
            byte_pos+=(start_bit/8);
            if(num_read>=num_elem)
            {
                break;
            }

        }
        return num_read;
    }

    //not work well for 64 bit!
    _uint64 get_elem_uint64(_uint32 num_elem,_uint8 nbits) //output limit to 8/16/32/64 only
    {
        _uint32 num_read=0;
        _uint64 mask=pow(2,nbits)-1;//00000111 for example nbit=3 
        while(byte_pos<pbuf_len-2)
        {
            memcpy((char*)&tbuf.buf16,pbuf+byte_pos,8); //copy 2 bytes into our buffer, this may introduce redundant copy
            //now we have start_bits, byte_pos, elem_pos, just finish them all
            while(start_bit<=16-nbits)
            {
                pelem[elem_pos++]=(tbuf.buf16>>(16-start_bit-nbits))&mask;//(tbuf.buf16&(mask<<(16-start_bit))
                start_bit+=nbits; //advance by nbits
                num_read++;
                if(num_read>=num_elem)
                {
                    break;
                }
            }
            //need update the start_bit and byte_pos
            start_bit%=8;
            byte_pos+=(start_bit/8);
            if(num_read>=num_elem)
            {
                break;
            }

        }
        return num_read;
    }*/
};

#include <iostream>
using namespace std;

int main()
{
    _uint8 *pbuf=new _uint8[10];
    _uint8 *pelem=new _uint8[80];
    for(int i=0;i<10;i++) pbuf[i]=i*11+11;
    bitbuffer vbit(pbuf,10,pelem,10);

    cout.setf(ios_base::hex,ios_base::basefield);
    cout<<"Bytes: ";
    for(i=0;i<10;i++) cout<<pbuf[i]<<" ";
    cout<<endl;
    cout<<"1 bit: ";
    int num_read=vbit.get_elem_uint8(80,1);
    for(i=0;i<num_read;i++) cout<<(int)pelem[i];
    cout<<endl;
    vbit.reset();
    cout<<"2 bit: ";
    num_read=vbit.get_elem_uint8(40,2);
    for(i=0;i<num_read;i++) cout<<(int)pelem[i]<<" ";
    cout<<endl;
    vbit.reset();
    cout<<"3 bit: ";
    num_read=vbit.get_elem_uint8(26,3);
    for(i=0;i<num_read;i++) cout<<(int)pelem[i]<<' ';
    cout<<endl;
    vbit.reset();
    cout<<"4 bit: ";
    num_read=vbit.get_elem_uint8(20,4);//get 10 bit-12 integers 
    for(i=0;i<num_read;i++) cout<<(int)pelem[i]<<" ";
    cout<<endl;
    vbit.reset();
    cout<<"5 bit: ";
    num_read=vbit.get_elem_uint8(16,5);//get 10 bit-12 integers 
    for(i=0;i<num_read;i++) cout<<(int)pelem[i]<<" ";
    cout<<endl;
    vbit.reset();
    cout<<"6 bit: ";
    num_read=vbit.get_elem_uint8(13,6);//get 10 bit-12 integers 
    for(i=0;i<num_read;i++) cout<<(int)pelem[i]<<" ";
    cout<<endl;
    vbit.reset();
    cout<<"7 bit: ";
    num_read=vbit.get_elem_uint8(11,7);//get 10 bit-12 integers 
    for(i=0;i<num_read;i++) cout<<(int)pelem[i]<<" ";
    cout<<endl;
    vbit.reset();
    cout<<"8 bit: ";
    num_read=vbit.get_elem_uint8(10,8);//get 10 bit-12 integers 
    for(i=0;i<num_read;i++) cout<<(int)pelem[i]<<" ";
    cout<<endl;
    vbit.reset();

    return 0;
}

testing results:

Bytes: b 16 21 2c 37 42 4d 58 63 6e
1 bit: 0000101100010110001000010010110000110111010000100100110101011000011000110
1101110
2 bit: 0 0 2 3 0 1 1 2 0 2 0 1 0 2 3 0 0 3 1 3 1 0 0 2 1 0 3 1 1 1 2 0 1 2 0 3 1
 2 3 2
3 bit: 0 2 6 1 3 0 4 1 1 3 0 3 3 5 0 2 2 3 2 5 4 1 4 3
4 bit: 0 b 1 6 2 1 2 c 3 7 4 2 4 d 5 8 6 3 6 e
5 bit: 1 c b 2 2 b 1 17 8 9 6 15 10 18 1b e
6 bit: 2 31 18 21 b 3 1d 2 13 15 21 23
7 bit: 5 45 44 12 61 5d 4 4d 2c 18 6d
8 bit: b 16 21 2c 37 42 4d 58 63 6e
Press any key to continue