Question

I built a program to loop through words and get their synonyms from http://www.dicsin.com.br, but this will take ages (literally), because there are 307k words on my testfile.txt, what can I do ? give me advises please, can I make it multi-process or multi-threaded, i don’t know, i’m new to PHP and programming, thank you anyway, btw, this is my full working code:

<?
//Pega palavras do site: www.dicsin.com.br
pegarSinonimos("http://www.dicsin.com.br/content/dicsin_lista.php");

function pegaPalavras()
{
return file('testfile.txt');
}

function pegarSinonimos($url)
{
        $dicionario = pegaPalavras();
        $array_palavras = array();
        $array_palavras2 = array();
        $con = mysql_connect("localhost","root","whatever");
        if (!$con)
         {
          die('Could not connect: ' . mysql_error());
         }
        mysql_select_db("palavras2", $con);
        foreach($dicionario as $palavra)
        {
            $url_final = $url . "?f_pesq=" . $palavra;// . "&pagina=" . $pagina;

            $html = file_get_contents($url_final);

            $dom = new DOMDocument();
            $dom->loadHTML($html);

            $xpath = new DOMXPath($dom);
            $tags = $xpath->query('//div[@class="palavras_encontradas"]/div[@class="box_palavras_encontradas"]');
            foreach ($tags as $tag) 
            {
                $bla = $tag->nodeValue;
                $bla = utf8_decode($bla);
                $bla = str_replace("visualizar palavras", "", $bla);
                $bla = str_replace("(Sinônimo) ", "", $bla);//echo $bla;//array_push($array_palavras,$tag->nodeValue);
                $sql = "CREATE TABLE $palavra(sinonimo varchar(29))";
                mysql_query($sql,$con);
                mysql_query("INSERT INTO $palavra (sinonimo) VALUES ('$bla')");
            }
        }
        mysql_close($con);
}   
?>

Answer 1

Develop a hash table and do a lookup against that. This will achieve O(1) constant time.