Question

In a list with some hundred thousand entries, how does one go about comparing each entry with the rest of the list for duplicates?
For example, List fileNames contains both “00012345.pdf” and “12345.pdf” and are considered duplicte. What is the best strategy to flagging this kind of a duplicate?

Thanks

Update: The naming of files is restricted to numbers. They are padded with zeros. Duplicates are where the padding is missing. Thus, “123.pdf” & “000123.pdf” are duplicates.

Answer 1

You probably want to implement your own substring comparer to test equality based on whether a substring is contained within another string.

This isn’t necessarily optimised, but it will work. You could also possibly consider using Parallel Linq if you are using .NET 4.0.

EDIT: Answer updated to reflect refined question after it was edited

void Main()
{
    List<string> stringList = new List<string> { "00012345.pdf","12345.pdf","notaduplicate.jpg","3453456363234.jpg"};

    IEqualityComparer<string> comparer = new NumericFilenameEqualityComparer ();

    var duplicates = stringList.GroupBy (s => s, comparer).Where(grp => grp.Count() > 1);

    // do something with grouped duplicates...

}

// Not safe for null's !
// NB do you own parameter / null checks / string-case options etc !
public class NumericFilenameEqualityComparer : IEqualityComparer<string> {

   private static Regex digitFilenameRegex = new Regex(@"\d+", RegexOptions.Compiled);

   public bool Equals(string left, string right) {

        Match leftDigitsMatch = digitFilenameRegex.Match(left);
        Match rightDigitsMatch = digitFilenameRegex.Match(right);

        long leftValue = leftDigitsMatch.Success ? long.Parse(leftDigitsMatch.Value) : long.MaxValue;
        long rightValue = rightDigitsMatch.Success ? long.Parse(rightDigitsMatch.Value) : long.MaxValue;

        return leftValue == rightValue;
   }

   public int GetHashCode(string value) {
        return base.GetHashCode();
   }

}