Question

I am looking for a way to iterate through a directory containing 100,000’s of files. Using os.listdir is TERRIBLY slow because this function first takes in the path list from the whole specified path.

What are the fastest options?

NOTE: whoever downvoted has never faced this situation for sure.

Answer 1

This other question was referred to in comments as a duplicate:
List files in a folder as a stream to begin process immediately

…But I found the example to be semi not working. Here is the fixed version that works for me:

from ctypes import CDLL, c_int, c_uint8, c_uint16, c_uint32, c_char, c_char_p, Structure, POINTER
from ctypes.util import find_library

import os

class c_dir(Structure):
    pass

class c_dirent(Structure):
    _fields_ = [ 
        ("d_fileno", c_uint32), 
        ("d_reclen", c_uint16),
        ("d_type", c_uint8), 
        ("d_namlen", c_uint8),
        ("d_name", c_char * 4096),
        # proper way of getting platform MAX filename size?
        # ("d_name", c_char * (os.pathconf('.', 'PC_NAME_MAX')+1) ) 
    ]

c_dirent_p = POINTER(c_dirent)
c_dir_p = POINTER(c_dir)

c_lib = CDLL(find_library("c"))
opendir = c_lib.opendir
opendir.argtypes = [c_char_p]
opendir.restype = c_dir_p

# FIXME Should probably use readdir_r here
readdir = c_lib.readdir
readdir.argtypes = [c_dir_p]
readdir.restype = c_dirent_p

closedir = c_lib.closedir
closedir.argtypes = [c_dir_p]
closedir.restype = c_int

def listdir(path):
    """
    A generator to return the names of files in the directory passed in
    """
    dir_p = opendir(".")
    try:
        while True:
            p = readdir(dir_p)
            if not p:
                break
            name = p.contents.d_name
            if name not in (".", ".."):
                yield name
    finally:
        closedir(dir_p)


if __name__ == "__main__":
    for name in listdir("."):
        print name