Question

I need to extract XMLs from a huge (around 500mb) file but I have a 32bit JVM which always runs out of heap space.

I have written a program to extract XMLs from this file but to do that the whole file has to be read (I can not go with 100 lines or so per iteration, as I can not make sure the 100th line would be the end of an XML).

SO how do I do it?

My program for extraction:

private static ArrayList<String> extractXml(String xml) {
    String[] newXml = xml.split("\\<\\?");
    ArrayList<String> xmlList = new ArrayList<String>(Arrays.asList(newXml));
    for(int i = 0; i<xmlList.size();i++){
        if(!xmlList.get(i).contains("xml version=\"1.0\" encoding=\"UTF-8\"")){
            xmlList.remove(i);
        }
    }
    int size = xmlList.size();
    if(xml.contains("#"))
    for(int j = 0;j<size;j++){
        xmlList.set(j, "<?"+xmlList.get(j));
        xmlList.set(j,xmlList.get(j).split("\\#")[0]);

    }else
        for(int j = 0;j<size;j++){
            xmlList.set(j, "<?"+xmlList.get(j).trim());
            System.out.println(xmlList.get(j));             
        }



    return xmlList;

}

The XMLs also have a Header (its a JMSStream header. Like a wrapper on the XML) which I have been successfully removing using above logic.

Sample file content:

#---------- #2 : ID:QADC1_HGR1-EMS13112.15DB4FBEA3665328B:4985 ----------#
<MSG_INFO>
    <message type="TextMessage" messageSelector="" receiveTime="2012-09-12T14:37:26.717" jmsServerTimestamp="2012-09-12T14:37:26.775">
        <header JMSMessageID="ID:QADC1_HGR1-EMS13112.15DB4FBEA3665328B:4985" JMSDestination="OPS.FPES.OUTBOUND.FLIGHT.TRACK_A.DISTRIBUTION" JMSDestinationType="Topic" JMSDeliveryMode="2" JMSPriority="4" JMSTimestamp="1347478646775"/>
        <properties>
            ...
        </properties>
    </message>
</MSG_INFO>
BodyLength=31108
<?xml version="1.0" encoding="UTF-8"?>

<ns0:Envelope xmlns:ns0="http://www.wnco.com/opsplatform/flight/flightevent">
    <ns0:Header>
        <ns1:EventHeader xmlns:ns1="http://www.wnco.com/opsplatform/event/header">
            ...
        </ns1:EventHeader>
    </ns0:Header>
    <ns0:Body>
        <ns1:Flight xmlns:ns1="http://www.wnco.com/opsplatform/flight/flight">
           <ns2:ScheduledFlightLeg xmlns:ns2="http://www.wnco.com/opsplatform/flight/flight/scheduledflightleg">
                ...
                <ns3:OperationalFlightLeg xmlns:ns3="http://www.wnco.com/opsplatform/flight/flight/operationalflightleg">
                    ...
                </ns3:OperationalFlightLeg>
            </ns2:ScheduledFlightLeg>
            <ns2:ScheduledFlightLeg xmlns:ns2="http://www.wnco.com/opsplatform/flight/flight/scheduledflightleg">
                ...
                    ...
                </ns3:OperationalFlightLeg>
            </ns2:ScheduledFlightLeg>
            <ns2:ScheduledFlightLeg xmlns:ns2="http://www.wnco.com/opsplatform/flight/flight/scheduledflightleg">
               ...
                    ...
                </ns3:OperationalFlightLeg>
            </ns2:ScheduledFlightLeg>
            <ns2:ScheduledFlightLeg xmlns:ns2="http://www.wnco.com/opsplatform/flight/flight/scheduledflightleg">
                ...
                    ...
                </ns3:OperationalFlightLeg>
            </ns2:ScheduledFlightLeg>
        </ns1:Flight>
    </ns0:Body>
</ns0:Envelope>

#---------- #3 : ID:QADC1_HGR1-EMS13112.15DB4FBEA3665328B:4985 ----------#
        <MSG_INFO>
            <message type="TextMessage" messageSelector="" receiveTime="2012-09-12T14:37:26.717" jmsServerTimestamp="2012-09-12T14:37:26.775">
                <header JMSMessageID="ID:QADC1_HGR1-EMS13112.15DB4FBEA3665328B:4985" JMSDestination="OPS.FPES.OUTBOUND.FLIGHT.TRACK_A.DISTRIBUTION" JMSDestinationType="Topic" JMSDeliveryMode="2" JMSPriority="4" JMSTimestamp="1347478646775"/>
                <properties>
                    ...
                </properties>
            </message>
        </MSG_INFO>
        BodyLength=31108
        <?xml version="1.0" encoding="UTF-8"?>

        <ns0:Envelope xmlns:ns0="http://www.wnco.com/opsplatform/flight/flightevent">
            <ns0:Header>
                <ns1:EventHeader xmlns:ns1="http://www.wnco.com/opsplatform/event/header">
                    ...
                </ns1:EventHeader>
            </ns0:Header>
            <ns0:Body>
                <ns1:Flight xmlns:ns1="http://www.wnco.com/opsplatform/flight/flight">
                   <ns2:ScheduledFlightLeg xmlns:ns2="http://www.wnco.com/opsplatform/flight/flight/scheduledflightleg">
                        ...
                        <ns3:OperationalFlightLeg xmlns:ns3="http://www.wnco.com/opsplatform/flight/flight/operationalflightleg">
                            ...
                        </ns3:OperationalFlightLeg>
                    </ns2:ScheduledFlightLeg>
                    <ns2:ScheduledFlightLeg xmlns:ns2="http://www.wnco.com/opsplatform/flight/flight/scheduledflightleg">
                        ...
                            ...
                        </ns3:OperationalFlightLeg>
                    </ns2:ScheduledFlightLeg>
                    <ns2:ScheduledFlightLeg xmlns:ns2="http://www.wnco.com/opsplatform/flight/flight/scheduledflightleg">
                       ...
                            ...
                        </ns3:OperationalFlightLeg>
                    </ns2:ScheduledFlightLeg>
                    <ns2:ScheduledFlightLeg xmlns:ns2="http://www.wnco.com/opsplatform/flight/flight/scheduledflightleg">
                        ...
                            ...
                        </ns3:OperationalFlightLeg>
                    </ns2:ScheduledFlightLeg>
                </ns1:Flight>
            </ns0:Body>
        </ns0:Envelope>

Answer 1

Using JAVA Scanner to extract the XMLs based on a unique property per XML.

 public static void main(String[] args) {
      String temp = null;
      String searchedData = null;
      File file = new File(fileLocation);
      try{
            Scanner scan = new Scanner (file);
            scan.useDelimiter("xml version");
            while (scan.hasNext()){
                  temp = scan.next();
                  if(temp.contains("<ns1:EventCorrelationID>153721264</ns1:EventCorrelationID>")){
                        searchedData = temp.substring(0, temp.lastIndexOf("</ns0:Envelope>")+15);
                        System.out.println("xml version"+searchedData);
                  }
            }
      }catch(Exception e){
            e.printStackTrace();
      }
}