Question

A query that took under 1 minute when I ran it a couple weeks ago is now taking over 10 minutes with no end in sight.

NEW QUERY (TAKING LONG TIME)

select sds.school_id, 
  detail.year, 
  detail.race,
  ROUND((detail.count / summary.total) * 100 ,2) as percent 
FROM school_data_race_ethnicity_raw as detail
inner join school_data_schools as sds USING (school_id)
inner join (
  select sds2.district_id, year, sum(count) as total
  from school_data_race_ethnicity_raw
  inner join school_data_schools as sds2 USING (school_id)
  group by sds2.district_id, year
  ) as summary on summary.district_id = sds.district_id 
    and summary.year = detail.year

Query:

INSERT INTO school_data_race_ethnicity_schools (school_id, year, race, percent) (
    SELECT school_id,
           year,
           race,
           ROUND((count/(
        SELECT SUM(count) 
          FROM school_data_race_ethnicity_raw as school_data_race_ethnicity_raw_inner 
         WHERE school_id = school_data_race_ethnicity_raw_outer.school_id 
           and year = school_data_race_ethnicity_raw_outer.year)
                        ) * 100,2) as percent
      FROM school_data_race_ethnicity_raw as school_data_race_ethnicity_raw_outer)    

Explain:

mysql> explain SELECT school_id,year,race,ROUND((count/(SELECT SUM(count) 
    -> FROM school_data_race_ethnicity_raw as school_data_race_ethnicity_raw_inner 
    -> WHERE 
    -> school_id = school_data_race_ethnicity_raw_outer.school_id and 
    -> year = school_data_race_ethnicity_raw_outer.year)) * 100,2) as percent
    -> FROM school_data_race_ethnicity_raw as school_data_race_ethnicity_raw_outer;
+----+--------------------+--------------------------------------+------+----------------------------+------+---------+-----------------------------------------------------------------+-------+-------------+
| id | select_type        | table                                | type | possible_keys              | key  | key_len | ref                                                             | rows  | Extra       |
+----+--------------------+--------------------------------------+------+----------------------------+------+---------+-----------------------------------------------------------------+-------+-------------+
|  1 | PRIMARY            | school_data_race_ethnicity_raw_outer | ALL  | NULL                       | NULL | NULL    | NULL                                                            | 84012 |             |
|  2 | DEPENDENT SUBQUERY | school_data_race_ethnicity_raw_inner | ref  | school_id,year,school_id_2 | year | 4       | rocdocs_main_drupal_7.school_data_race_ethnicity_raw_outer.year |  8402 | Using where |
+----+--------------------+--------------------------------------+------+----------------------------+------+---------+-----------------------------------------------------------------+-------+-------------+
2 rows in set (0.00 sec)

Create tables:

mysql> show create table school_data_race_ethnicity_raw;
+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Table                          | Create Table                                                                                                                                                                                                                                                                                                                                                                                                  |
+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| school_data_race_ethnicity_raw | CREATE TABLE `school_data_race_ethnicity_raw` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `school_id` varchar(255) NOT NULL,
  `year` int(11) NOT NULL,
  `race` varchar(255) NOT NULL,
  `count` int(11) NOT NULL,
  PRIMARY KEY (`id`),
  KEY `school_id` (`school_id`,`year`),
  KEY `year` (`year`,`race`),
  KEY `school_id_2` (`school_id`)
) ENGINE=MyISAM AUTO_INCREMENT=84013 DEFAULT CHARSET=latin1 |
+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)

mysql> show create table school_data_race_ethnicity_schools;
+------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Table                              | Create Table                                                                                                                                                                                                                                                                                                                                                                           |
+------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| school_data_race_ethnicity_schools | CREATE TABLE `school_data_race_ethnicity_schools` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `school_id` varchar(255) NOT NULL,
  `year` int(11) NOT NULL,
  `race` varchar(255) NOT NULL,
  `percent` decimal(15,2) NOT NULL,
  PRIMARY KEY (`id`),
  KEY `year` (`year`,`race`),
  KEY `school_id` (`school_id`,`year`)
) ENGINE=MyISAM AUTO_INCREMENT=24961 DEFAULT CHARSET=latin1 |
+------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)


mysql> show processlist;
+------+---------+--------------------+-----------------------+---------+------+--------------+------------------------------------------------------------------------------------------------------+
| Id   | User    | Host               | db                    | Command | Time | State        | Info                                                                                                 |
+------+---------+--------------------+-----------------------+---------+------+--------------+------------------------------------------------------------------------------------------------------+
| 1739 | [REMOVED] |  [REMOVED] | rocdocs_main_drupal_7 | Query   | 1467 | Sending data | INSERT INTO school_data_race_ethnicity_schools (school_id, year, race, percent) (
SELECT school_id,y |
| 1800 | root    | localhost          | rocdocs_main_drupal_7 | Query   |    0 | NULL         | show processlist                                                                                     |
+------+---------+--------------------+-----------------------+---------+------+--------------+------------------------------------------------------------------------------------------------------+
2 rows in set (0.00 sec)

Answer 1

Your SELECT is going to be very slow because of the way you are using a sub-query to calculate the percentages. It’s reading an entire year’s data for each row. If you use a sub-query to select the totals and join to that then it should run much faster.

Off the top of my head, something like this (while not ideal) should be much faster than your existing query:

select detail.school_id, 
  detail.year, 
  detail.race,
  ROUND((detail.count / summary.total) * 100 ,2) as percent 
FROM school_data_race_ethnicity_raw as detail
inner join (
  select school_id, year, sum(count) as total
  from school_data_race_ethnicity_raw
  group by school_id, year
  ) as summary on summary.school_id = detail.school_id 
    and summary.year = detail.year