In this case its not working good it keep adding the same links to the List.
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using HtmlAgilityPack;
using System.IO;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using System.Net;
using System.Web;
namespace GatherLinks
{
public partial class Form1 : Form
{
int sites = 0;
int y = 0;
string url = @"http://www.google.co.il";
string guys = "http://www.google.com";
public Form1()
{
InitializeComponent();
List<string> a = webCrawler(guys, 2);
}
private void Form1_Load(object sender, EventArgs e)
{
}
private int factorial(int n)
{
if (n == 0) return 1;
else y = n * factorial(n - 1);
richTextBox1.Text = y.ToString();
return y;
}
private List<string> getLinks(HtmlAgilityPack.HtmlDocument document)
{
List<string> mainLinks = new List<string>();
var linkNodes = document.DocumentNode.SelectNodes("//a[@href]");
if (linkNodes != null)
{
foreach (HtmlNode link in linkNodes)
{
var href = link.Attributes["href"].Value;
mainLinks.Add(href);
}
}
return mainLinks;
}
private List<string> webCrawler(string url, int levels)
{
HtmlAgilityPack.HtmlDocument doc;
HtmlWeb hw = new HtmlWeb();
List<string> webSites;// = new List<string>();
List<string> csFiles = new List<string>();
csFiles.Add("temp string to know that something is happening in level = " + levels.ToString());
csFiles.Add("current site name in this level is : " + url);
try
{
doc = hw.Load(url);
webSites = getLinks(doc);
if (levels == 0)
{
return csFiles;
}
else
{
int actual_sites = 0;
for (int i = 0; i < webSites.Count() && i < 20; i++) {
string t = webSites[i];
if ((t.StartsWith("http://") == true) || (t.StartsWith("https://") == true)) // replace this with future FilterJunkLinks function
{
// for (int e = 0; e < csFiles.Count; e++)
// {
if (csFiles.Contains(t))
{
}
else
{
actual_sites++;
csFiles.AddRange(webCrawler(t, levels - 1));
Texts(richTextBox1, "Level Number " + levels + " " + t + Environment.NewLine, Color.Red);
}
// }
}
}
// report to a message box only at high levels..
//if (levels==1)
//MessageBox.Show(actual_sites.ToString());
return csFiles;
}
}
catch
{
return csFiles;
}
}
And the Texts function:
public void Texts(RichTextBox box, string text, Color color)
{
box.SelectionStart = box.TextLength;
box.SelectionLength = 0;
box.SelectionColor = color;
box.AppendText(text);
box.SelectionColor = box.ForeColor;
}
I need two things to do in the webCrawler function:
-
If the url variable can not be resolved then the try and catch should do the job.
-
If the List csFiles contains already the same items do not add them again. For example if in the csFiles there already http://www.google.com then do not add http://www.google.com again so in the end the csFiles List will contain http://www.google.com only once.
Your use of contains is wrong. If I understand your problem.. I will be brave and hazard a guess..
}
Update:
I don’t understand much about recursion. In fact I avoid it but I think that’s another of your problem.
When you recursively call your code you are “forgetting” the previous list containing all the links. So instead of creating a new csFile every time I suggest you pass in a reference to the list in webcrawler. Hope that solves it.
Meaning:
private List<string> webCrawler(string url, int levels,List<string> csFiles)