Question

I have a database containing three tables:

1. practices – 8 fields
2. patients – 47 fields
3. exacerbations – 11 fields

The majority of the fields in these tables are recorded in varchar format, other fields include integers, doubles and dates.

I have to transform this data into numerically classified data so that it can be used by a statistician to extrapolate any patterns in the data. To acheive this I will have to convert varchar fields into integers that represent the classification that string belongs to, an example being ‘Severity’ which has the following possible string values:

1. Mild
2. Moderate
3. Severe
4. Very Severe

This field in the patients table has a finite list of string values that can appear, other fields have an endless possibility of string values that cannot be classified until they are encountered by my database (unless I implement some form of intelligent approach).

For the time being I am just trying to construct the best approach to converting each field for all entries in each of the 3 tables to numeric values. The pseudo code I have in my head so far is as follows (it’s not complete):

 function profileDatabase 
   for each table in database 
     for each field that is of type varchar
       select all distinct values and insert into classfication table for that field
     end for
   end for

 function classifyDatabase
   for each table in database 
     for each field that is of type varchar
       // do something efficient to build an insert string to place into new table
     end for
   end for

Can someone suggest the best way of performing this process so that it is efficient giving that there are currently in excess of 100 practices, 15,000 patients and 55,000 exacerbations in the system. I have no need to implement this in PHP, build I would prefer to do so. Any pointers as to how to structure this would be great as I am not sure my approach the best approach.

This process will have to run every month for the next two years as the database grows to have a total of 100,000 patients.

Answer 1

I have managed to build my own solution to this problem which runs in reasonable time. For anyone interested, or anyone who may encounter a similar issue here is the approach I have used:

A PHP script that is run as a cron job by calling php scriptName.php [database-name]. The script builds a classified table for each table name that is within the database (that is not a lookup table for this process). The setting up of each classification creates a new table which mimics the format of the base table but sets all fields to allow NULL values. It then creates blank rows for each of the rows found in the base table. The process then proceeds by analysing each table field by field and updating each row with the correct class for this field.

I am sure I can optimise this function to improve on the current complexity, but for now I shall use this approach until the run-time of the scripts goes outside of an acceptable range.

Script code:

include ("../application.php");

profileDatabase("coco");
classifyDatabase("coco");

function profileDatabase($database) {
    mysql_select_db($database);
    $query = "SHOW TABLES";
    $result = db_query($query);
    $dbProfile = array();
    while ($obj = mysql_fetch_array($result)) {
        if (!preg_match("/_/", $obj[0])) {
            $dbProfile[$obj[0]] = profileTable($obj[0]);
        }
    }
    return $dbProfile;
}

function profileTable($table) {
    $tblProfile = array();
    $query = "DESCRIBE $table";
    $result = db_query($query);
    while ($obj = mysql_fetch_array($result)) {
        $type = substr($obj[1], 0, 7);
//echo $type;
        if (preg_match("/varchar/", $obj[1]) && (!preg_match("/Id/", $obj[0]) && !preg_match("/date/", $obj[0]) && !preg_match("/dob/", $obj[0]))) {
            $x = createLookup($obj[0], $table);
            $arr = array($obj[0], $x);
            $tblProfile[] = $arr;
        }
    }
    return $tblProfile;
}

function getDistinctValues($field, $table) {
    $distinct = array();
    $query = "SELECT DISTINCT $field as 'value', COUNT($field) as 'no' FROM $table GROUP BY $field ORDER BY no DESC";
    $result = db_query($query);
    while ($obj = mysql_fetch_array($result)) {
        $distinct[] = $obj;
    }
    return $distinct;
}

function createLookup($field, $table) {
    $query = "CREATE TABLE IF NOT EXISTS `" . $table . "_" . $field . "`
(
`id` int(5) NOT NULL auto_increment,
`value` varchar(255) NOT NULL,
`no` int(5) NOT NULL,
`map1` int(3) NOT NULL,
`map2` int(3) NOT NULL,
PRIMARY KEY  (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1";
    db_query($query);
    $distinct = getDistinctValues($field, $table);
    $count = count($distinct);
    foreach ($distinct as $val) {
        $val['value'] = addslashes($val['value']);
        $rs = db_query("SELECT id FROM " . $table . "_" . $field . " WHERE value = '" . $val['value'] . "' LIMIT 1");
        if (mysql_num_rows($rs) == 0) {
            $sql = "INSERT INTO " . $table . "_" . $field . " (value,no) VALUES ('" . $val['value'] . "', " . $val['no'] . ")";
        } else {
            $sql = "UPDATE " . $table . "_" . $field . " (value,no) VALUES ('" . $val['value'] . "', " . $val['no'] . ")";
        }
        db_query($sql);
    }
    return $count;
}

function classifyDatabase($database) {
    mysql_select_db($database);
    $query = "SHOW TABLES";
    $result = db_query($query);
    $dbProfile = array();
    while ($obj = mysql_fetch_array($result)) {
        if (!preg_match("/_/", $obj[0])) {
            classifyTable($obj[0]);
            //echo "Classfied $obj[0]\n";
        }
    }
}

function classifyTable($table) {
    $query = "SHOW TABLES";
    $result = db_query($query);
    $dbProfile = array();
    $setup = true;
    while ($obj = mysql_fetch_array($result)) {
        if ($obj[0] == "classify_" . $table)
            $setup = false;
    }
    if ($setup) {
        setupClassifyTable($table);
        //echo "Setup $table\n";
    }

    $query = "DESCRIBE $table";
    $result = db_query($query);
    while ($obj = mysql_fetch_array($result)) {
        if (preg_match("/varchar/", $obj[1]) && (!preg_match("/Id/", $obj[0]) && !preg_match("/date/", $obj[0]) && !preg_match("/dob/", $obj[0]))) {
            $rs = db_query("
        SELECT t.entryId, t.$obj[0], COALESCE(tc.map1,99) as 'group' FROM $table t 
        LEFT JOIN " . $table . "_$obj[0] tc ON t.$obj[0] = tc.value 
        ORDER BY tc.map1 ASC");
            while ($obj2 = mysql_fetch_object($rs)) {
                $sql = "UPDATE classify_$table SET $obj[0] = $obj2->group WHERE entryId = $obj2->entryId";
                db_query($sql);
            }
        } else {
            if ($obj[0] != "entryId") {
                $rs = db_query("
        SELECT t.entryId, t.$obj[0] as 'value' FROM $table t");
                while ($obj2 = mysql_fetch_object($rs)) {
                    $sql = "UPDATE classify_$table SET $obj[0] = '" . addslashes($obj2->value) . "' WHERE entryId = $obj2->entryId";
                    db_query($sql);
                }
            }
        }
    }
}

function setupClassifyTable($table) {
    $tblProfile = array();
    $query = "DESCRIBE $table";
    $result = db_query($query);
    $create = "CREATE TABLE IF NOT EXISTS `classify_$table` (";
    while ($obj = mysql_fetch_array($result)) {
        if (preg_match("/varchar/", $obj[1]) && (!preg_match("/Id/", $obj[0]) && !preg_match("/date/", $obj[0]) && !preg_match("/dob/", $obj[0]))) {
            //echo $obj[1]. " matches<br/>";
            $create .= "$obj[0] int(3) NULL,";
        } else {
            $create .= "$obj[0] $obj[1] NULL,";
        }
    }
    $create .= "PRIMARY KEY(`entryId`)) ENGINE=MyISAM DEFAULT CHARSET=latin1";
    db_query($create);
    $result = mysql_query("SELECT entryId FROM $table");
    while ($obj = mysql_fetch_object($result)) {
        db_query("INSERT IGNORE INTO classify_$table (entryId) VALUES ($obj->entryId)");
    }
}

?>