Home/ Questions/Q 7971437
In Process

The Archive Base Latest Questions

Editorial Team
  • 0
Asked: 2026-06-04T07:38:22+00:00

i’m trying to get the html of a page using this screen scraping in

  • 0

i’m trying to get the html of a page using this screen scraping in c# with httpwebrequest

it works fine when i try to scrap a normal page..but now if i try to get the html of a page that is being loaded on ajax request i get into trouble…here is the two requests it send when i try to get that page..

normal request to bypass login

POST (http)://example/user/login?destination=/events/Sports HTTP/1.1
Host: example
User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:12.0) Gecko/20100101 Firefox/12.0
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language: en-us,en;q=0.5
Accept-Encoding: gzip, deflate
Connection: keep-alive
Referer: (http)://example/user/login?destination=/events/Sports
Cookie: has_js=1; SESSee201d4242c83ea2671330cdceee4623=qdco8gukcm2pk9offdof1uv3a0
Content-Type: application/x-www-form-urlencoded
Content-Length: 121

name=Username&pass=Password&remember_me=1&form_build_id=form11cb87efa605eb9fb384eb9d2a2c686e&form_id=user_login&op=Go

AJAX request to get data

GET (http)://example/views/ajax?name=Sports&view_name=Events&view_display_id=page_1&view_args=Sports&view_path=events%2FSports&view_base_path=events&view_dom_id=1&pager_element=0 HTTP/1.1
Host: example
User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:12.0) Gecko/20100101 Firefox/12.0
Accept: application/json, text/javascript, */*
Accept-Language: en-us,en;q=0.5
Accept-Encoding: gzip, deflate
Connection: keep-alive
X-Requested-With: XMLHttpRequest
Referer: (http)://example//events/Sports
Cookie: has_js=1; SESSee201d4242c83ea2671330cdceee4623=vd36esbpe8065snbfo39ubhmk3
If-Modified-Since: Wed, 23 May 2012 08:13:51 GMT

I have tried writing my code this way but it doesnt’t work..

string sid = String.Empty;
string uri = "http://example/user/login?destination=/events/Sports";
string postData = string.Format("name=UserName&pass=Password&remember_me=1&form_build_id=form-11cb87efa605eb9fb384eb9d2a2c686e&form_id=user_login&op=Go");
byte[] postBytes = Encoding.UTF8.GetBytes(postData);

//web request

HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(uri);
req.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:12.0) Gecko/20100101 Firefox/12.0";

req.KeepAlive = true;

////set the cookie
Cookie cookie = new Cookie();
cookie.Name = "Sports";

cookie.Domain = "SESSee201d4242c83ea2671330cdceee4623";
req.CookieContainer = new CookieContainer();
req.CookieContainer.Add(cookie);

req.Headers.Add("Accept-Encoding", "gzip, deflate");
req.Headers.Add("Accept-Language", "en-us,en;q=0.5");
req.Method = "POST";
req.Host = "example";
req.Referer = "http://example/user/login?destination=/events/Sports";
req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";

req.ContentType = "application/x-www-form-urlencoded";
req.ContentLength = postBytes.Length;

//getting the request stream and posting data
StreamWriter requestwriter = new StreamWriter(req.GetRequestStream(), System.Text.Encoding.ASCII);
requestwriter.Write(postData);
requestwriter.Close();
string url = "http://example/views/ajax?name=Sports&view_name=Events&view_display_id=page_1&view_args=Sports&view_path=events%2FSports&view_base_path=events&view_dom_id=1&pager_element=0";

HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
request.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:12.0) Gecko/20100101 Firefox/12.0";
request.KeepAlive = true;
request.Headers.Add("X-Requested-With", "XMLHttpRequest");
request.Headers.Add("Accept-Encoding", "gzip, deflate");
request.Headers.Add("Accept-Language", "en-us,en;q=0.5");
request.Host = "example";
request.Method = "GET";
request.Referer = "http://example//events/Sports";
request.Accept = "application/json, text/javascript, */*";

request.CookieContainer.Add(cookie);
request.ContentType = "text/javascript; charset=utf-8";

try
{
    HttpWebResponse res = (HttpWebResponse)request.GetResponse();
    StreamReader sr = new StreamReader(res.GetResponseStream());
    sid = sr.ReadToEnd().Trim();
}
catch {}

I get { "status": false, "display": "", "messages": "" } in string sid instead it should give me status true and display some values..

Share

Leave an answer

You must login to add an answer.

Need An Account,

1 Answer

  1. Editorial Team

    You’re creating a new CookieContainer for each request, so assuming your login request returns a cookie containing your session id, your next request won’t have that session id in the cookie container.

    Use the same CookieContainer for both requests, ie:

       //web request

    HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(uri);
    req.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:12.0) Gecko/20100101 Firefox/12.0";

    req.KeepAlive = true;

    CookieContainer cookies = new CookieContainer();
    req.CookieContainer = cookies;

    req.Headers.Add("Accept-Encoding", "gzip, deflate");
    req.Headers.Add("Accept-Language", "en-us,en;q=0.5");
    req.Method = "POST";
    req.Host = "example";
    req.Referer = "http://example/user/login?destination=/events/Sports";
    req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";

    req.ContentType = "application/x-www-form-urlencoded";
      req.ContentLength = postBytes.Length;

    //getting the request stream and posting data
      StreamWriter requestwriter = new StreamWriter(req.GetRequestStream(), System.Text.Encoding.ASCII);
      requestwriter.Write(postData);
      requestwriter.Close();

      var firstResponse = req.GetResponse();
      using(var sr = new StreamReader(firstResponse.GetResponseStream()) {
          sr.ReadToEnd();
      }

      string url = "http://example/views/ajax?name=Sports&view_name=Events&view_display_id=page_1&view_args=Sports&view_path=events%2FSports&view_base_path=events&view_dom_id=1&pager_element=0";

      HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
      request.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:12.0) Gecko/20100101 Firefox/12.0";
      request.KeepAlive = true;
      request.Headers.Add("X-Requested-With", "XMLHttpRequest");
      request.Headers.Add("Accept-Encoding", "gzip, deflate");
      request.Headers.Add("Accept-Language", "en-us,en;q=0.5");
      request.Host = "example";
      request.Method = "GET";
      request.Referer = "http://example//events/Sports";
      request.Accept = "application/json, text/javascript, */*";

      request.CookieContainer = cookies;
      request.ContentType = "text/javascript; charset=utf-8";

    try
    {
        HttpWebResponse res = (HttpWebResponse)request.GetResponse();
        StreamReader sr = new StreamReader(res.GetResponseStream());
        sid = sr.ReadToEnd().Trim();
    • 0