Question

there is a dictionary containning many lists, for example,

  list_dic= {
    q1:[1,2,3,4,5]
    q2:[2,3,5]
    q3:[2,5]
    }

and I want to get all common items count for each list, e.g. the common items count for q1 and q2 is 3=(2,3,5)

q1={q2:3, q3:2}
q2={q1:3,q3:2}
q3={q1:2, q2:2}

my code for this task is:

result = {}
for name, source_list in list_dic.items():
    for target_name, target_list in list_dic.items():
        count = 0
        for item in source_list:
            if item in target_list:
                count+=1
    result[name][target_name] = count 

but this algorithm is inefficient, I want to know a better algorithm to do this task

Answer 1

I think this should do it:

import itertools
import collections

q1 = 'q1'
q2 = 'q2'
q3 = 'q3'

dic_list = {
     q1:[1,2,3,4,5],
     q2:[2,3,5],
     q3:[2,5]
     }

#sets are much more efficient for this sort of thing.  Create a dict
#of the same structure as the old one, only with `set` as values 
#instead of `list`
dic_set = {k:set(v) for k,v in dic_list.items()}

new_dic = collections.defaultdict(dict)
for k1,k2 in itertools.combinations(dic_set,2):
     #to get the count, we just need to know the size of the intersection
     #of the 2 sets.
     value = len(dic_set[k1] & dic_set[k2]) 
     new_dic[k1][k2] = value
     new_dic[k2][k1] = value

print (new_dic)

If you’re following the comments, it turns out that combinations is slightly faster than permutations:

import itertools
import collections

q1 = 'q1'
q2 = 'q2'
q3 = 'q3'


dic_list = {
     q1:[1,2,3,4,5],
     q2:[2,3,5],
     q3:[2,5]
     }

dic_set = {k:set(v) for k,v in dic_list.items()}

def combo_solution():
     new_dic = collections.defaultdict(dict)
     for k1,k2 in itertools.combinations(dic_set,2):
          value = len(dic_set[k1] & dic_set[k2])
          new_dic[k1][k2] = value
          new_dic[k1][k2] = value
     return new_dic

def perm_solution():
     new_dic = collections.defaultdict(dict)
     for k1, k2 in itertools.permutations(dic_set,2):
          new_dic[k1][k2] = len(dic_set[k1] & dic_set[k2])
     return new_dic

import timeit
print timeit.timeit('combo_solution()','from __main__ import combo_solution',number=100000)
print timeit.timeit('perm_solution()','from __main__ import perm_solution',number=100000)

with the results:

0.58366894722    #combinations
0.832300901413   #permutations

This is because set.intersection is an O(min(N,M)) operation — Which is cheap, but can add up if you’re doing it twice as many times as you need to.