Question

I was looking how to write a multi threaded C++ code for scanning directory and get list of all files underneath. I have written a single threaded code which can do and below the code which can do that.

#include <sys/types.h>
#include <dirent.h>
#include <errno.h>
#include <vector>
#include <string>
#include <iostream>
#include <sys/stat.h> /* for stat() */


using namespace std;
int isDir(string path)
;

/*function... might want it in some class?*/
int getdir (string dir, vector<string> &dirlist, vector<string> &fileList)
{
    DIR *dp;
    struct dirent *dirp, *dirFp ;
    if((dp  = opendir(dir.c_str())) == NULL) {
        cout << "Error(" << errno << ") opening " << dir << endl;
        return errno;
    }

    while ((dirp = readdir(dp)) != NULL) {
        if (strcmp (dirp->d_name, ".") != 0 && strcmp(dirp->d_name, "..") != 0) {
            //dirlist.push_back(string(dirp->d_name));

            string Tmp = dir.c_str()+ string("/") + string(dirp->d_name);
            if(isDir(Tmp)) {
            //if(isDir(string(dir.c_str() + dirp->d_name))) {
                dirlist.push_back(Tmp);
                getdir(Tmp,dirlist,fileList);
            } else {
  //              cout << "Files :"<<dirp->d_name << endl;
                fileList.push_back(string(Tmp));
            }

        }
    }
    closedir(dp);
    return 0;
}

int isDir(string path)
{
    struct stat stat_buf;
    stat( path.c_str(), &stat_buf);
    int is_dir = S_ISDIR( stat_buf.st_mode);
//    cout <<"isDir :Path "<<path.c_str()<<endl;
    return ( is_dir ? 1: 0);
}

int main()
{
    string dir = string("/test1/mfs");
    vector<string> dirlist = vector<string>();
    vector<string> fileList = vector<string>();

    getdir(dir,dirlist,fileList);
#if 0
    for (unsigned int i = 0;i < dirlist.size();i++) {
        cout << "Dir LIst" <<dirlist[i] << endl;
        //string dirF = dir + "/" + dirlist[i];
        //getdir(dirF,fileList);
    }
#endif
    for (unsigned int i = 0; i < fileList.size(); i++)
        cout << "Files :"<<fileList[i]<< endl;
    return 0;
}

Now issue is that it is single threaded and I need to scan say about 8000 directories under which file can be present. So I am not getting how to do so as number of directories can vary as it is decided by N dimension matrix.

Any help in this regard will be great. Thanks in advance.

Answer 1

boost::filesystem has directory_iterator and recursive_directory_iterator, the former will get all the contents of a directory but not recurse sub-directories, the latter will also recurse subdirectories.

With regard to thread-safety, you could lock a mutex then copy the results into a std::vector or two vector instances, one for files and one for directories, in which case you will at least have a local snapshot copy.

To actual “freeze” the file-system at that point to stop any process modifying it is not something you can normally do – well you could try setting the file attributes on it to read-only then change it back later but you will need to have permission to do that first.