Here is my dilema, I have a rather large PHP script, that basically reads several XML files and parses them all into my database. It works excellent in my dev environment, however it always causes an apache 500 error when I try and run it on GoDaddy’s servers. I suspect it is because this script uses a lot memory. When I run it in dev, apache’s memory usage increases to almost 250mb. I’ve done everything I can think of to reduce the memory required by the script, and help would be much appreciated!
The file that is read by the getDeals() method has about 50,000 items in it..
<?php
class FMTC_XmlFeed_Model_Test extends CI_Model{
function __construct(){
parent:: __construct();
set_time_limit(0);
//ini_set('memory_limit', '300M');
//ini_set('max_execution_time', 3000);
}
/*the Order of functions for this model is going to be the following
getCategories (V) (˚,,,,˚) (V)
getMerchants (V) (˚,,,,˚) (V)
getTypes (V) (˚,,,,˚) (V)
getCoupons It's zoidberg time on this one! (V) (˚,,,,˚) (V)
-- the get Coupons method also makes calls to the catCoup() and dealsCoup() (V) (˚,,,,˚) (V)
-- changesCheck() checks for changes inbetween runs.. (V) (˚,,,,˚) (V)
*/
function getXML() {
//get the above xml files from FMTC
$catXML = $this->loadXML("private link");
$merchXML = $this->getXMLCurl('private link');
$typeXML = $this->loadXML(private link);
//start initiating the parsin methods, in the above order
$oldDOD = $this->getDod();
$oldManuals = $this->getManuals();
$this->truncateStuffs();
$this->parentCategory($catXML);
$this->childCategory($catXML);
$this->getMerchants($merchXML);
$this->getTypes($typeXML);
$this->getDeals($_SERVER["DOCUMENT_ROOT"].'/xml_fullPull/deals.xml');
$this->buildDOD($oldDOD);
$this->insertManuals($oldManuals);
echo 'done';
}
//curl
function getXMLCurl($link, $filename) {
try {
//$path = 'C:\''.$filename.'.xml';
//$path = realpath($filename.'.xml');
//$path = '/tmp/'.$filename.'gfcfilepulls.xml';
$path = $_SERVER["DOCUMENT_ROOT"].'/xml_fullPull/'.$filename.''.date("d.m.y.H.i.s", time()).'xml';
$url = $link;
$fp = fopen($path, 'w');
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_FILE, $fp);
$fd = fopen($_SERVER["DOCUMENT_ROOT"].'/xml_fullPull/transfer.log', 'a');
curl_setopt($ch, CURLOPT_VERBOSE, TRUE);
curl_setopt($ch, CURLOPT_STDERR, $fd);
$data = curl_exec($ch);
curl_close($ch);
fclose($fp);
//$data = simplexml_load_file($path);
return $path;
unset($fp);
unset($ch);
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
//get return XMLS
function loadXML($link) {
try {
$data = simplexml_load_file($link);
return $data;
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
//################# Generic functions for this model ##############################################################
//format the datetime for MySQL
function getDate($mysqldate) {
$phpdate = new DateTime($mysqldate);
$mysqldate = $phpdate->format('Y-m-d H:i:s');
return $mysqldate;
}
//generic query to return one row
function getVal($table, $column, $value) {
$this->db->where($column, $value);
$this->db->from($table);
$q = $this->db->get();
if ($q->num_rows() == 1) {
$row = $q->row();
return $row;
} else {return false;}
}
//check -- basically see if something exists in the DB
function check($table, $column, $value) {
$this->db->where($column, $value);
$this->db->from($table);
$q = $this->db->get();
if ($q->num_rows() > 0) {
return true;
} else {
return false; }
}
//generic update statement, with infinte number of sets..
function updateDB($table, $pArray, $wArray) {
//pArray is a key=>value array with the update parameters
//wArray is a key=>value array with the where parameters
//table, is obviously, what table we should be updating :)
if ($wArray != null) {
foreach ($wArray as $column=>$value) {
$this->db->where($column, $value);
}}
if ($pArray != null) {
foreach ($pArray as $column=>$value) {
$this->db->set($column, $value);
}}
//run the update
$this->db->update($table);
}
//search - i - size the characters
function searchisize($str) {
$str = preg_replace('/[^a-z\d ]/i', '', $str);
$str = str_replace(' ','',$str);
return $str;
}
//write to errors.text
function errorHandling($message) {
$data = array (
'message'=>addslashes($message),
'time'=>date("d.m.y.H.i.s", time())
);
$this->db->insert('log_feed', $data);
/*$myFile = "errors.txt";
$fh = fopen($myFile, 'a') or die("can't open file");
fwrite($fh, $message, '\n');
fclose($fh);*/
}
//#######################################################################################################################
//truncate stuffs
function truncateStuffs() {
$this->db->query('TRUNCATE TABLE categories_for_coupons');
$this->db->query('TRUNCATE TABLE coupon');
$this->db->query('TRUNCATE TABLE countries_for_coupons');
$this->db->query('TRUNCATE TABLE deal_types_for_coupons');
}
//get the current DOD list
function getDOD() {
try {
$results = $this->db->get('dod');
$dod = array();
if ($results->num_rows() > 0) {
foreach($results->result() as $val) {
$this->db->where('id', $val->id);
$t = $this->db->get('coupon');
if ($t->num_rows() == 1) {
$dod[] = $t->row();
}
}
}
return $dod;
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
//rebuld DOD
function buildDOD($dod) {
try {
$this->db->query('TRUNCATE TABLE dod');
foreach($dod as $val) {
$this->db->where('coupon_id', $val->coupon_id);
$t = $this->db->get('coupon');
if ($t->num_rows() == 1) {
$q = $t->row();
$data = array (
'id'=>'',
'coupon_id'=>$q->id
);
$this->db->insert('dod', $data);
}
}
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
//save the manual coupons
function getManuals() {
try {
$this->db->where('coupon_id', '13377331');
$t = $this->db->get('coupon');
$manuals = array();
if ($t->num_rows() > 0) {
foreach($t->result() as $val) {
$manuals[] = $val;
}
return $manuals;
}
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
//re-insert the manuals now that the other list is done
function insertManuals($manuals) {
try {
foreach ($manuals as $val) {
$data = array (
'coupon_id'=>'13377331',
'merchant_id'=>$val->merchant_id,
'merchant_name'=>$val->merchant_name,
'search_merchant_name'=>$val->search_merchant_name,
'network_id'=>null,
'network_name'=>null,
'program_id'=>null,
'label'=>$val->label,
'restrictions'=>null,
'coupon_code'=>$val->coupon_code,
'start_date'=>$val->start_date,
'end_date'=>$val->end_date,
'link'=>$val->link,
'image'=>null,
'status'=>$val->status,
'last_updated'=>null,
'created'=>null,
'price'=>null,
'discount'=>null,
'percent'=>null,
'is_active'=>$val->is_active,
'is_deleted'=>$val->is_deleted,
'feed'=>'Manual',
'clicks'=>$val->clicks,
'priority'=>$val->priority
);
$this->db->insert('coupon', $data);
}
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
//################################################################################################################################################
//parent categories
function parentCategory($xml) {
try {
foreach($xml->category as $category) {
//stupid addslahses thing....
if ($this->check('parent_category', 'name', addslashes($category->name)) == false) {
//since this is the parent table, skip over all that have children...
if ($category->parent == '') {
//the name is the display name, while the filter is the searchable version
//has children, by default, write a false, and when we parse all the children, if there is a match, update this table from there.. yeah!
$data = array(
'name'=> $category->name == ''? null:addslashes($category->name),
'filter'=> $category->filter == ''? null:addslashes($category->filter),
'hasChildren'=>0
);
$this->db->insert('parent_category', $data);
}
}
}
return true;
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
//child category
function childCategory($xml) {
try {
foreach($xml->category as $category) {
if ($this->check('child_category', 'name', addslashes($category->name)) == false) {
if ($category->parent != '') {
//see if the child already exists in the DB or not, also, addslashes.. this is stupid and I dont get it
$parent_id = $this->getVal('parent_category', 'filter', addslashes($category->parent));
//insert the child into the child table
$data = array(
'name'=>$category->name == ''? null:addslashes($category->name),
'filter'=> $category->filter == ''? null:addslashes($category->filter),
'parent_category_id'=>addslashes($parent_id->id),
'desc'=>null
);
//insert it
$this->db->insert('child_category', $data);
//update the parents table
$wArray = array(
'id'=>$parent_id->id);
$pArray = array(
'hasChildren'=>1);
$this->updateDB('parent_category', $pArray, $wArray);
}
}
}
return true;
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
//get the merchants feed
function getMerchants($path) {
try {
$r = new XMLReader;
$r->open($path);
$r->setParserProperty(2,true);
$doc = new DOMDocument;
$go = false;
$values = null;
$chunk = 0;
while ($r->read()) {
if ($r->name === 'merchant' && $r->nodeType == XMLReader::ELEMENT) {
$merchant = simplexml_import_dom($doc->importNode($r->expand(), true));
if ($this->check('merchant', 'merchant_id', addslashes($merchant->id)) == false) {
$network_id = $this->getVal('network', 'network_code', addslashes($merchant->network));
$search_merchant = $this->searchisize($merchant->name);
if ($chunk <= 200) {
$values .= "('', ";
$values .= $merchant->id == ''? "'', ":"'".addslashes($merchant->id)."', ";
$values .= $merchant->name == ''? "'', ":"'".addslashes($merchant->name)."', ";
$values .= $merchant->name == ''? "'', ":"'".addslashes($search_merchant)."', ";
$values .= $network_id->id == false? "'', ":"'".addslashes($network_id->id)."', ";
$values .= $network_id->name == false? "'', ":"'".addslashes($network_id->name)."', ";
$values .= $merchant->link == ''? "'', ":"'".addslashes($merchant->link)."', ";
$values .= $merchant->homepageurl == ''? "'', ":"'".addslashes($merchant->homepageurl)."', ";
$values .= $merchant->status == ''? "'', ":"'".addslashes($merchant->status)."', ";
$values .= "0, ";
$values .= "0, ";
$values .= "0, ";
$values .= "0, ";
$values .= "''), ";
} else {
$chunk = 0;
$this->bulkInsert($values, 'merchant');
$values = null;
$values .= "('', ";
$values .= $merchant->id == ''? "'', ":"'".addslashes($merchant->id)."', ";
$values .= $merchant->name == ''? "'', ":"'".addslashes($merchant->name)."', ";
$values .= $merchant->name == ''? "'', ":"'".addslashes($search_merchant)."', ";
$values .= $network_id->id == false? "'', ":"'".addslashes($network_id->id)."', ";
$values .= $network_id->name == false? "'', ":"'".addslashes($network_id->name)."', ";
$values .= $merchant->link == ''? "'', ":"'".addslashes($merchant->link)."', ";
$values .= $merchant->homepageurl == ''? "'', ":"'".addslashes($merchant->homepageurl)."', ";
$values .= $merchant->status == ''? "'', ":"'".addslashes($merchant->status)."', ";
$values .= "0, ";
$values .= "0, ";
$values .= "0, ";
$values .= "0, ";
$values .= "''), ";
}
$chunk++;
unset($merchant);
if ($go == true) {
$go = true; } else {$go = true;}
}
}
}
$r->next();
$r->close();
if ($go == true) { $this->bulkInsert($values, 'merchant'); }
return true;
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
//get the dealTypes
function getTypes($xml) {
try {
foreach($xml->type as $type) {
if ($this->check('deal_type', 'filter', addslashes($type->filter)) == false) {
$data = array(
'filter'=> $type->filter == ''? null:addslashes($type->filter),
'name'=> $type->name == ''? null:addslashes($type->name)
);
$this->db->insert('deal_type', $data);
}
}
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
//finally, get teh deals ##############################################################################################
function getDeals($path) {
try {
$r = new XMLReader;
$r->open($path);
$r->setParserProperty(1,true);
$doc = new DOMDocument;
$values = null;
$chunk = 0;
$go = false;
$categories = array();
$dealtypes = array();
$countries = array();
while ($r->read()) {
if ($r->name === 'item' && $r->nodeType == XMLReader::ELEMENT) {
set_time_limit(0);
$coupon = null;
$doc = null;
$active = 0;
$doc = new DOMDocument;
$coupon = simplexml_import_dom($doc->importNode($r->expand(), true));
//get the merchant id and network id from our DB -- when writing nulls.. took 4:47 seconds..
$merchant_id = null;
$network_id = null;
$merchant_id = $this->getVal('merchant', 'merchant_id', addslashes($coupon->merchantid));
$network_id = $this->getVal('network', 'network_code', addslashes($coupon->network));
if ($merchant_id) {
if ($merchant_id->autoload == 0) {
$active = 0;
} else {
$active = 1;
}
} else {
echo $coupon->couponid.'<br />';
}
if ($chunk <= 200) {
$values .= "('',".addslashes($coupon->couponid).", ";
$values .= $merchant_id == false? "'', ": $merchant_id->id.", ";
$values .= $merchant_id == false? "'".addslashes($coupon->merchantname)."', ": "'".addslashes($merchant_id->name)."', ";
$values .= $merchant_id == false? "'".addslashes(str_replace("'","", $coupon->merchantname))."', ": "'".$merchant_id->search_merchant_name."', ";
$values .= $network_id == false? "'', ": $network_id->id.", ";
$values .= $network_id == false? "'', ": "'".$network_id->name."', ";
$values .= $coupon->programid == ''? "'', ":"'".addslashes($coupon->programid)."', ";
$values .= $coupon->label == ''? "'', ":"'".addslashes(str_replace(',',"%%", $coupon->label))."', ";
$values .= $coupon->restrictions == ''? "'', ":"'".addslashes($coupon->restrictions)."', ";
$values .= $coupon->couponcode == ''? "'', ":"'".addslashes($coupon->couponcode)."', ";
$values .= $coupon->startdate == ''? "'', ":"'".$this->getDate($coupon->stardate)."', ";
$values .= $coupon->enddate == ''? "'', ":"'".$this->getDate($coupon->enddate)."', ";
$values .= $coupon->link == ''? "'', ":"'".addslashes($coupon->link)."', ";
$values .= $coupon->image == ''? "'', ":"'".addslashes($coupon->image)."', ";
$values .= $coupon->status == ''? "'', ":"'".addslashes($coupon->status)."', ";
$values .= $coupon->lastupdated == ''? "'', ":"'".$this->getDate($coupon->lastupdated)."', ";
$values .= $coupon->created == ''? "'', ":"'".$this->getDate($coupon->created)."', ";
$values .= $coupon->price == ''? "'', ":"'".addslashes($coupon->price)."', ";
$values .= $coupon->discount == ''? "'', ":"'".addslashes($coupon->discount)."', ";
$values .= $coupon->percent == ''? "'', ":"'".addslashes($coupon->percent)."', ";
//$values .= "0, ";
$values.=$active.", ";
$values .= "0, ";
$values .= "'For Me to Coupon', ";
$values .= "0, ";
$values .= "1), ";
foreach ($coupon->categories as $c) {
foreach($c as $t) {
//$categories .= $t.', ';
$temp = array(
'id'=>$coupon->couponid,
'cat'=>$t);
$categories[] = $temp;
}
}
foreach ($coupon->dealtypes as $d) {
foreach($d as $t) {
$temp = array (
'id'=>$coupon->couponid,
'did'=>$t
);
$dealtypes[] = $temp;
}
}
foreach ($coupon->countries as $c) {
foreach($c as $t) {
$temp = array (
'id'=>$coupon->couponid,
'country'=>$t);
$countries[] = $temp;
}
}
} else {
$chunk = 0;
$this->bulkInsert($values, 'coupon');
$this->catCoup($categories);
$this->dealCoup($dealtypes);
$this->countries($countries);
//reset values
$dealtypes = null;
$countries = null;
$categories = null;
$values = null;
$values .= "('',".addslashes($coupon->couponid).", ";
$values .= $merchant_id == false? "'', ": $merchant_id->id.", ";
$values .= $merchant_id == false? "'".addslashes($coupon->merchantname)."', ": "'".addslashes($merchant_id->name)."', ";
$values .= $merchant_id == false? "'".addslashes(str_replace("'","", $coupon->merchantname))."', ": "'".$merchant_id->search_merchant_name."', ";
$values .= $network_id == false? "'', ": $network_id->id.", ";
$values .= $network_id == false? "'', ": "'".$network_id->name."', ";
$values .= $coupon->programid == ''? "'', ":"'".addslashes($coupon->programid)."', ";
$values .= $coupon->label == ''? "'', ":"'".addslashes(str_replace(',',"%%", $coupon->label))."', ";
$values .= $coupon->restrictions == ''? "'', ":"'".addslashes($coupon->restrictions)."', ";
$values .= $coupon->couponcode == ''? "'', ":"'".addslashes($coupon->couponcode)."', ";
$values .= $coupon->startdate == ''? "'', ":"'".$this->getDate($coupon->stardate)."', ";
$values .= $coupon->enddate == ''? "'', ":"'".$this->getDate($coupon->enddate)."', ";
$values .= $coupon->link == ''? "'', ":"'".addslashes($coupon->link)."', ";
$values .= $coupon->image == ''? "'', ":"'".addslashes($coupon->image)."', ";
$values .= $coupon->status == ''? "'', ":"'".addslashes($coupon->status)."', ";
$values .= $coupon->lastupdated == ''? "'', ":"'".$this->getDate($coupon->lastupdated)."', ";
$values .= $coupon->created == ''? "'', ":"'".$this->getDate($coupon->created)."', ";
$values .= $coupon->price == ''? "'', ":"'".addslashes($coupon->price)."', ";
$values .= $coupon->discount == ''? "'', ":"'".addslashes($coupon->discount)."', ";
$values .= $coupon->percent == ''? "'', ":"'".addslashes($coupon->percent)."', ";
//$values .= "0, ";
$values.=$active.", ";
$values .= "0, ";
$values .= "'For Me to Coupon', ";
$values .= "0, ";
$values .= "1), ";
foreach ($coupon->categories as $c) {
foreach($c as $t) {
//$categories .= $t.', ';
$temp = array(
'id'=>$coupon->couponid,
'cat'=>$t);
$categories[] = $temp;
}
}
foreach ($coupon->dealtypes as $d) {
foreach($d as $t) {
$temp = array (
'id'=>$coupon->couponid,
'did'=>$t
);
$dealtypes[] = $temp;
}
}
foreach ($coupon->countries as $c) {
foreach($c as $t) {
$temp = array (
'id'=>$coupon->couponid,
'country'=>$t);
$countries[] = $temp;
}
}
}
$data = array(
'coupon_id'=>addslashes($coupon->couponid),
'change_type'=>addslashes('new coupon'),
'date'=>addslashes(date('Y-m-d')),
'source'=>'For Me to Coupon'
);
//$this->db->insert('log', $data);
$chunk++;
$go = true;
$coupon = null;
}
}
$r->next('item');
//$r->close();
//echo count($categories);
if ($go == true) {$this->bulkInsert($values, 'coupon');}
return true;
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
function bulkInsert($query, $table) {
try {
$len = strlen($query);
$query = substr($query, 0, ($len -2));
//$query .=";<br /><br /><br /><hr />";
$this->db->query("INSERT INTO ".$table." VALUES ".$query);
//echo $query;
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
// ##############################################################################################
// ##################### all of the below methods are called from teh getDeals() method #########
// ##############################################################################################
//categories for coupons
function catCoup($obj) {
try {
$chunk = 0;
$values = null;
$pid = null;
$cobj = null;
$pc = null;
$go = false;
foreach($obj as $key=>$val) {
//if ($this->check('categories_for_coupons', 'coupon_id', addslashes($val['id'])) == false) {
//first find if this is a parent or child category
if ($this->check('parent_category', 'filter', addslashes($val['cat'])) == true) {
//parent category, so just insert this junk and get the PC id
$pid = $this->getVal('parent_category', 'filter', addslashes($val['cat']));
$pc = 1;
} else {
//this is a child, so get the child id and the parent id .. meh
$cobj = $this->getVal('child_category', 'filter', addslashes($val['cat']));
$pc = 0;
}
//echo $pc.'<br />';
//build the bulk insert string
//echo $chunk."<br />";
if ($chunk <= 499) {
if ($pc == 1) {
$values .= "('', ".$val['id'].", ";
$values .= $pid->id.", ";
$values .= "''), ";
} else {
if ($cobj) {
$values .= "('', ".$val['id'].", ";
$values .= $cobj->parent_category_id.", ";
$values .= $cobj->id."), ";
} else {
$values .= "('', ".$val['id'].", ";
$values .= "'', ";
$values .= "''), ";
}
}
} else {
$this->bulkInsert($values, 'categories_for_coupons');
//echo 'inserted!<br />';
$chunk = 0;
$values = null;
if ($pc == 1) {
$values .= "('', ".$val['id'].", ";
$values .= $pid->id.", ";
$values .= "''), ";
} else {
if ($cobj) {
$values .= "('', ".$val['id'].", ";
$values .= $cobj->parent_category_id.", ";
$values .= $cobj->id."), ";
} else {
$values .= "('', ".$val['id'].", ";
$values .= "'', ";
$values .= "''), ";
}
}
}
$chunk++;
$go = true;
//}
}
if ($go == true) {$this->bulkInsert($values, 'categories_for_coupons');}
$chunk = null;
$values = null;
$pid = null;
$cobj = null;
$pc = null;
$go = null;
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
//dealtypes for coupons
function dealCoup($obj) {
try {
$chunk = 0;
$values = null;
$did = null;
$pc = 0;
$go = false;
foreach($obj as $key=>$val) {
//if ($this->check('deal_types_for_coupons', 'coupon_id', addslashes($val['id'])) == false) {
if ($val['did'] != '') {
//get the dealtypeid
$did = $this->getVal('deal_type', 'filter', addslashes($val['did']));
$pc = 1;
} else {
//this means teh coupon had no deal type, but to keep all the tables in line with each other, i'll write in a null value for the deal type for said coupon
$pc = 0;
}
if ($chunk <= 499) {
if ($pc == 1) {
$values .= "('', ".$val['id'].", ";
$values .= $did->id."), ";
} else {
$values .= "('', ".$val['id'].", ";
$values .= "''), ";
}
} else {
$chunk = 0;
$this->bulkInsert($values, 'deal_types_for_coupons');
$values = null;
if ($pc == 1) {
$values .= "('', ".$val['id'].", ";
$values .= $did->id."), ";
} else {
$values .= "('', ".$val['id'].", ";
$values .= "''), ";
}
}
$chunk++;
$go = true;
//}
}
if ($go == true) {$this->bulkInsert($values, 'deal_types_for_coupons');}
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
//countries -- right now, since we don't have a countries feed, just dump it all to teh countries_for_coupons feed
function countries($obj) {
try {
$go = false;
$chunk = 0;
$values = null;
foreach($obj as $key=>$val) {
//if ($this->check('countries_for_coupons', 'coupon_id', addslashes($val['id'])) == false) {
if ($chunk <= 499) {
$values .= "('', '".$val['id']."', '', ";
$values .= "'".$val['country']."'), ";
} else {
$chunk = 0;
$this->bulkInsert($values, 'countries_for_coupons');
$values = null;
$values .= "('', '".$val['id']."', '', ";
$values .= "'".$val['country']."'), ";
}
$go = true;
//}
}
if ($go == true) {$this->bulkInsert($values, 'countries_for_coupons');}
} catch (Exception $e) {
$this->errorHandling($e->getMessage());
}
}
}
Take a look at a profiling tool like XDebug. http://xdebug.org/ It should show you what’s using memory.