Question

I am doing a project where I need to parse JSON from a URL through HttpClient. My code works fine for JSON object responses with a small amount of data. But when I use the same code to get a response with a huge amount of data (more than 3MB), I have a problem.

Here is my code:

JSONfunctions.java (function for json parsing)

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
//import java.util.HashMap;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
//import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
//import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import android.util.Log;
import android.widget.Toast;

@SuppressWarnings("unused")
public class JSONfunctions {

    public static JSONObject getJSONfromURL(String url){
        InputStream is = null;
        String result = "";
        JSONObject jArray = null;

        //http post
        try{
                HttpClient httpclient = new DefaultHttpClient();
                HttpPost httppost = new HttpPost(url);
                HttpResponse response = httpclient.execute(httppost);
                HttpEntity entity = response.getEntity();
                is = entity.getContent();

        }catch(Exception e){
                Log.e("log_tag", "Error in http connection "+e.toString());
        }

        //convert response to string
        try{
                BufferedReader reader = new BufferedReader(new InputStreamReader(is,"iso-8859-1"),8);
                StringBuilder sb = new StringBuilder();
                String line = null;
                while ((line = reader.readLine()) != null) {
                        sb.append(line + "\n");
                }
                is.close();
                result=sb.toString();

            //    Toast.makeText(getBaseContext, result, Toast.LENGTH_LONG).

        }catch(Exception e){
                Log.e("log_tag", "Error converting result "+e.toString());
        }

        try{

            jArray = new JSONObject(result);            
        }catch(JSONException e){
                Log.e("log_tag", "Error parsing data "+e.toString());
        }

        return jArray;
    }
}

ListJson.java

import java.util.ArrayList;
import java.util.HashMap;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
//import org.w3c.dom.Document;
//import org.w3c.dom.Element;
//import org.w3c.dom.NodeList;



import android.app.ListActivity;
import android.content.Intent;
//import android.content.Intent;
import android.os.Bundle;
import android.util.Log;
import android.view.View;
import android.widget.AdapterView;
import android.widget.AdapterView.OnItemClickListener;
import android.widget.ListAdapter;
import android.widget.ListView;
import android.widget.SimpleAdapter;
import android.widget.Toast;


public class ListJson extends ListActivity {
     public static JSONObject json;
     public static JSONArray data;
    @Override
    public void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.list1);

        ArrayList<HashMap<String, String>> mylist = new ArrayList<HashMap<String, String>>();
     //   String semail = "[e-mail]";
      //  String spassword = "partha123";


      //  Toast.makeText(getApplicationContext(), JSONExample2.strEmail, Toast.LENGTH_LONG).show();
        //Toast.makeText(getApplicationContext(), JSONExample2.strPwd, Toast.LENGTH_LONG).show();

        //    

        json = JSONfunctions.getJSONfromURL("url here");      


            try{

            data = json.getJSONArray("server_list");


            for(int i=0;i<data.length();i++){                       
                HashMap<String, String> map = new HashMap<String, String>();    
                JSONObject e = data.getJSONObject(i);

                map.put("id", String.valueOf(i));
                map.put("name", "" + e.getString("ServUser"));
                map.put("email", "" + e.getString("ServURL"));
                mylist.add(map);
            }       
        }catch(JSONException e)        {
             Log.e("log_tag", "Error parsing data "+e.toString());
        }


//        try {
//          JSONObject newObject=json.getJSONObject("subscription");
//          JSONArray data1 = newObject.getJSONArray("cust_product");
//          
//          Toast.makeText(getApplicationContext(), data1.toString(), Toast.LENGTH_LONG).show();
//         
//          for(int i=0;i<data1.length();i++){                      
//                  HashMap<String, String> map1 = new HashMap<String, String>();   
//                  JSONObject e = data.getJSONObject(i);
//                  
//                  map1.put("id", String.valueOf(i));
//                  map1.put("name", "" + e.getString("ServUser"));
//                  map1.put("email", "" + e.getString("ServURL"));
//                  mylist.add(map1);
//              }       
//          
//      } catch (JSONException e) {
//          // TODO Auto-generated catch block
//          e.printStackTrace();
//      }


         ListAdapter adapter = new SimpleAdapter(this, mylist , R.layout.item_list,
                                  new String[] { "name", "email" }, 
                              new int[] { R.id.item_title, R.id.item_subtitle });
                                  setListAdapter(adapter);


      final ListView lv = getListView();
       lv.setTextFilterEnabled(true);   
        lv.setOnItemClickListener(new OnItemClickListener() {
            public void onItemClick(AdapterView<?> parent, View view, int position, long id) {              
                //@SuppressWarnings("unchecked")
                //HashMap<String, String> o = (HashMap<String, String>) lv.getItemAtPosition(position);                 
            //  Toast.makeText(ListJson.this, "ID '" + o.get("id") + "' was clicked.", Toast.LENGTH_SHORT).show(); 

                Toast.makeText(getApplicationContext(), parent.getItemAtPosition(position).toString(),
                        Toast.LENGTH_LONG).show();

            //   String a= parent.getItemAtPosition(position).toString();

                Intent intent2= new Intent(ListJson.this, ListJson2.class);
                 startActivity(intent2);
            }
        }); 

    }

I am getting an OutOfMemoryException. I changed heap_size to 192MB and ram size to 32MB, but with no luck. How can I fix this?

Answer 1

Big amounts of data of JSON must be cut to smaller pieces. For example you have a 50000 products on your database. Then it’s wise to paginate API requests – get this huge amount of products by 100-500 records on one query. That will solve your problem.

This approach solves one problem more – errors caused by internet and gprs connection loss etc.

If API is yours then you can change this. If not, then this is a big failure of API design and you can send change request.

EDIT:

Did a little reading and found that highly recommended for parsing huge load of JSON data is http://jackson.codehaus.org/ (Jackson Processor). Haven’t tried it, so cannot comment about this library. Also recommend you to save this JSON stream into the file (don’t load it to memory) and then parse it by chunks.