Question

I have a script (pasted below) that pulls in the meta descriptions from a list of UrLS, I then have an export function that will export that data to CSV format.

The only problem is my export function stops when there is a character outside the UTF-8 set for example control characters like –

What is the best way to either remove these characters or replace them so they are friendly? And also how would I put this into my script?

<?php
error_reporting(E_ALL);
//ini_set( "display_errors", 0);
function parseUrl($url){
    //Trim whitespace of the url to ensure proper checking.
    $url = trim($url);
    //Check if a protocol is specified at the beginning of the url. If it's not,   prepend 'http://'.
    if (!preg_match("~^(?:f|ht)tps?://~i", $url)) {
            $url = "http://" . $url;
    }
    //Check if '/' is present at the end of the url. If not, append '/'.
    if (substr($url, -1)!=="/"){
            $url .= "/";
    }
    //Return the processed url.
    return $url;
}
//If the form was submitted
if(isset($_GET['siteurl'])){
    //Put every new line as a new entry in the array
    $urls = explode("\n",trim($_GET["siteurl"]));
    //Iterate through urls
    foreach ($urls as $url) {
            //Parse the url to add 'http://' at the beginning or '/' at the end if not   already there, to avoid errors with the get_meta_tags function
            $url = parseUrl($url);
            //Get the meta data for each url
            $tags = get_meta_tags($url);
            //Check to see if the description tag was present and adjust output    accordingly
            $tags = NULL;
$tags = get_meta_tags($url);
if($tags)
echo "<tr><td>Description($url)</td><td>" .$tags['description']. "</td></tr>";
else 
echo "<tr><td>Description($url)</td><td>No Meta Description</td></tr>";
    }
}
?>

The export code:

// ExportHTMLTable

function ExportHTMLTable(tableId)
{
    this.data=[];
    this.table='';
    this.tableId=tableId;

    this.formId='exportForm';

    this.configuration={url:'../export.php',dataType:'json'};

    this.blockSend=0;
    this.blockSize=100024;

    this.requestNumber=0;

    this.rowLastIndex=0;
    this.cellLastIndex=0;

    this.format='';

    // Export
    this.exportDocument=function() 
    {
        this.reset();
        if(!this.prepareData()) return(false);

        this.send();
    }       

    // Export to CSV
    this.exportToCSV=function() 
    {
        this.format='csv';
        this.exportDocument();
    }

    // Export to XML
    this.exportToXML=function() 
    {
        this.format='xml';
        this.exportDocument();
    }

    // Reset variables
    this.reset=function()
    {
        this.data=[];

        this.blockSend=0;
        this.rowLastIndex=0;
        this.cellLastIndex=0;       
        this.requestNumber=0;
    }

    // Get table
    this.getTable=function()
    {
        this.table=$('#'+this.tableId);
        if(this.table.length!=1) return(false);

        return(true);
    }

    // Get data length
    this.getDataLength=function()
    {
        var sum=0;
        for(var rows in this.data) 
            sum+=this.data[rows].length;

        return(sum);
    }

    // Remove form
    this.removeForm=function()
    {
        var form=$('#'+this.formId);
        if(form.length==1) form.remove();
    }

    // Create field
    this.createField=function(name,value)
    {
        var field=document.createElement('input');

        field.name=name;
        field.value=value;
        field.type='hidden';

        return($(field));
    }

    // Prepare (extract from HTML table) data
    this.prepareData=function()
    {
        if(!this.getTable()) return(false);

        var self=this;
        var r=0,c=0,tr=0,tc=0,length=0;

        // Processing rows
        this.table.children('tbody').children('tr').each(function()
        {
            c=0;
            if(!$.isArray(self.data[r])) self.data[r]=[];

            // Processing cells
            $(this).children('th,td').each(function()
            {       
                length=$(this).attr('colspan')+self.data[r].length;

                for(tc=0;tc<length;tc++)
                {
                    if(self.data[r][tc]) continue;
                    self.data[r][tc]=$(this).text();    
                }

                length=r+$(this).attr('rowspan');
                for(tr=r+1;tr<length;tr++) 
                {
                    if(!$.isArray(self.data[tr])) self.data[tr]=[];
                    self.data[tr][c]=$(this).text();
                }

                c++;
            });

            r++;
        }); 

        return(true);           
    }

    // Send data
    this.send=function()
    {
        var i=0,j=0;

        this.requestNumber++;

        this.removeForm();

        var rowsNumber=this.data.length;
        var form=$(document.createElement('form'));

        form.attr('method','post');
        form.attr('action',this.configuration.url);

        for(i=this.rowLastIndex;i<rowsNumber;i++)
        {   
            var cellsNumber=this.data[i].length;
            for(j=this.cellLastIndex;j<cellsNumber;j++)
            {   
                this.blockSend++;

                var field=this.createField('exportTable['+i+']['+j+']',this.data[i][j]);

                form.append(field);

                if(this.blockSend>=(this.requestNumber*this.blockSize)) break;
            }

            if(this.blockSend>=(this.requestNumber*this.blockSize))   break;
            this.cellLastIndex=0;
        }  

        this.rowLastIndex=i;
        this.cellLastIndex=j==0 ? 0 : j+1;

        form.appendTo('body');

        if(this.requestNumber==1)
            form.append(this.createField('requestFirst',1));

        if(this.blockSend==this.getDataLength()) 
        {       
            form.append(this.createField('requestLast',1));
            form.append(this.createField('format',this.format));
            form.submit().remove();
        }
        else   $.post(this.configuration.url,form.serialize(),$.delegate(this.send,this),'json'); 
    }
}

Answer 1

My goodness, there’s a lot of stuff wrong with your posting..

First: – is not considered a control character. It appears in UTF-8, latin1 and even ASCII.

If there are indeed bytes in your string that are not valid UTF-8, then you must be dealing with some other encoding. It is the most likely that this is latin1, in which case you could use utf8_encode to convert the string.

Don’t blindly do this though, first figure out where your data comes from and if it’s really latin1. Sometimes utf8_encode is kind of applied as a band-aid for anything behaves weird, but that’s not a good way to deal with things. As a rule, it might help to think about it this way:

• Within your application, ensure that every string is always passed around as UTF-8.
• If you’re receiving data from external sources, figure out what character set they use, and convert if needed.

I have no idea how this relates to the code snippet you posted though.