Question

So I am developing in Net Beans a MySQL/Java program for class that helps us understand execution time and such.

So I created the java program so that a user can enter x amount of Tuples into each of the 6 tables. And so we have to enter data up too 125000 tuples. Well I only got to the problem with 60000 tuples, and when the program is executing it takes around 30 minutes just to insert the 60000 tuples.

Needless to say eventually I will have to enter 125000 tuples into a table and I need to know how to make this insert faster? Any Ideas?

Thank You

Brandon

Edit:

Code

/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package program2;


import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import java.util.Random;


public class Program2 {

/**
 * @param args the command line arguments
 */
public static void main(String[] args) {

            // Variables
            int takesnum=0;
            int instructornum=0;
            int coursenum = 0;
            int studentnum = 0;
            int count = 0;
            int departmentnum =0;
            int counts= 0;
            int minimum=0;
            int x=0;
            int teachesnum=0;
            // Variables

            //Connection to the database
            Connection conn = null;
            String url = "jdbc:mysql://localhost:3306/";
            String dbName = "university2";
            String Driver = "com.mysql.jdbc.Driver";
            // Change the userName & password to what ever your credentials are.
            String userName = "root"; 
            String password = "121089bn";
            //Connection to the database

       try {

            InputStreamReader istream = new InputStreamReader(System.in);
            BufferedReader MyReader = new BufferedReader(istream);

            Class.forName(Driver).newInstance();
            conn = DriverManager.getConnection(url+dbName,userName,password);
            System.out.println("Connected");

            // Ask the user how many tuples in department table.
            System.out.println("How many tuples would you like to create in Department Table?");

            // Takes in as string the number then parse it to an int.
            String dept = MyReader.readLine();
            departmentnum = Integer.parseInt(dept);

// ****************** Department Table ******************//  
            while (count < departmentnum)
            {
                Statement st = conn.createStatement();
                // Counts keeps the counter so the Primary Key is unique.
                st.executeUpdate("Insert into department (dept_name, building, budget) values ('Dept "+counts+"', 'Voigt', '1200')");
                count++;
                counts++;
            }

// ****************** Student Table ******************//                
            count=0;
            counts=0;

            System.out.println("How many tuples would you like to create in Student Table?");
            String student = MyReader.readLine();
            studentnum = Integer.parseInt(student);

            while (count < studentnum)
            {
                int z=0;
                int credit=128;
                Random ran = new Random(); 
                int range = departmentnum - minimum; 
                x =  ran.nextInt(range) + minimum; 

                Random random = new Random(); 
                int totcred = credit - minimum; 
                z =  random.nextInt(totcred) + minimum; 

                Statement st = conn.createStatement();
                st.executeUpdate("Insert into student (id, name, dept_name,tot_cred) select '"+counts+"', 'Student "+counts+"', dept_name, '"+z+"' from department where dept_name='Dept "+x+"'");
                count++;
                counts++;
            }

// ****************** Course Table ******************//                 
            x=0;
            count=0;
            counts=0;   

            System.out.println("How many tuples would you like to create in Course Table?");
            String course = MyReader.readLine();
            coursenum = Integer.parseInt(course);

            while (count < coursenum)
            {
                Random ran = new Random(); 
                int range = departmentnum - minimum; 
                x =  ran.nextInt(range) + minimum; 
                int credit=5;
                int z=0;

                Random random = new Random(); 
                int totcred = credit - minimum; 
                z =  random.nextInt(totcred) + minimum; 

                Statement st = conn.createStatement();
                st.executeUpdate("Insert into course (course_id, title, dept_name,credits) select '"+counts+"', 'Computer Science "+counts+"', dept_name, '"+z+"' from department where dept_name='Dept "+x+"'");
                count++;
                counts++;
            }

// ****************** Instructor Table ******************//                   
            x=0;
            count=0;
            counts=0;   

            System.out.println("How many tuples would you like to create in Instructor Table?");
            String instructor = MyReader.readLine();
            instructornum = Integer.parseInt(instructor);

            while (count < instructornum)
            {
                Random ran = new Random(); 
                int range = departmentnum - minimum; 
                x =  ran.nextInt(range) + minimum; 

                Statement st = conn.createStatement();
                st.executeUpdate("Insert into instructor (id, name, dept_name,salary) select '"+counts+"', 'Instructor "+counts+"', dept_name, '10000' from department where dept_name='Dept "+x+"'");
                count++;
                counts++;

            }

// ****************** Teaches Table ******************//                    
            x=0;
            count=0;
            counts=0;

            System.out.println("How many tuples would you like to create in Teaches Table?");
            String teaches = MyReader.readLine();
            teachesnum = Integer.parseInt(teaches);

            while (count < teachesnum)
            {
                Random ran = new Random(); 
                int range = instructornum - minimum; 
                x =  ran.nextInt(range) + minimum; 

                Random random = new Random(); 
                int courserange = coursenum - minimum; 
                int y =  random.nextInt(courserange) + minimum; 

                int g = 100;
                Random r = new Random();
                int f = g - minimum;
                int h = r.nextInt(f) + minimum;

                int l = 100;
                Random random1 = new Random();
                int j = l - minimum;
                int k = random1.nextInt(j) + minimum;


                Statement st = conn.createStatement();
                st.executeUpdate("Insert into teaches (id, course_id, semester, year) select id, course_id, 'Spr "+h+"', '20"+k+"' from course, instructor where instructor.id='"+x+"' and course.course_id='"+y+"'");
                count++;
                counts++;
            }

// ****************** Takes Table ******************//                    
            x=0;
            count=0;
            counts=0;

            System.out.println("How many tuples would you like to create in Takes Table?");
            String takes = MyReader.readLine();
            takesnum = Integer.parseInt(takes);

            while (count < takesnum)
            {
                Random ran = new Random(); 
                int range = studentnum - minimum; 
                x =  ran.nextInt(range) + minimum; 

                Random random = new Random(); 
                int courserange = coursenum - minimum; 
                int y =  random.nextInt(courserange) + minimum;

                int g = 100;
                Random r = new Random();
                int f = g - minimum;
                int h = r.nextInt(f) + minimum;

                int l = 100;
                Random random1 = new Random();
                int j = l - minimum;
                int k = random1.nextInt(j) + minimum;

                Statement st = conn.createStatement();
                st.executeUpdate("Insert into takes (id, course_id, semester, year, grade) select id, course_id, 'Spr "+h+"', '20"+k+"', 'B' from course, student where student.id='"+x+"' and course.course_id='"+y+"'");
                count++;
                counts++;
            }

            conn.close();
       }

            catch (Exception e) {
            System.err.println("Error");
            System.err.println(e.getMessage());
            }
}
}

Tables

drop database university2;
create database university2;

use university2;

create table department
(dept_name      varchar(20) primary key, 
 building       varchar(15), 
 budget             numeric(12,2)
);

create table student
(ID             int,
 name           varchar(20) not null, 
 dept_name      varchar(20), 
 tot_cred       numeric(10,0),
 primary key (ID),
 foreign key (dept_name) references department(dept_name)

);

create table course
(course_id      int, 
 title          varchar(50), 
 dept_name      varchar(15),
 credits        numeric(2,0),
 primary key (course_id),
 foreign key (dept_name) references department(dept_name)
);



create table instructor
(ID         int, 
 name           varchar(20) not null, 
 dept_name      varchar(20), 
 salary         numeric(8,2),
 primary key (ID),
 foreign key (dept_name) references department(dept_name)
);

create table teaches
(ID         int, 
 course_id      int,
 semester       varchar(6),
 year           numeric(4,0),
 primary key (ID, course_id, semester, year),
 foreign key (course_id) references course(course_id)
    on delete cascade,
 foreign key (ID) references instructor(ID)
    on delete cascade
);


create table takes
(ID         int, 
 course_id      int,
 semester       varchar(6),
 year           numeric(4,0),
 grade              varchar(2),
 primary key (ID, course_id, semester, year),
 foreign key (course_id) references course(course_id)
    on delete cascade,
 foreign key (ID) references student(ID)
    on delete cascade
);

Answer 1

Without having had a too close look at your code, here’s a few things you might want to look at :

1. PreparedStatement will probably help a bit, as the statement doesn’t have to be compiled for every request.
2. Execute statements in batches. Have a look at the addBatch() method on PreparedStatement. Once you’ve added, say, 1000 batches, you can execute them using executeBatch(). If you do this in conjunction with the setting ‘rewriteBatchedStatements‘ on the connection, then the MySQL driver might be able to do some optimisations.
3. Introduce several “insert” workers, which each is responsible for inserting batches of a given size. If you’re only working with a single connection, you need to do one insert at a time. If you work with multiple connections (via multiple threads) you’ll most likely see an increase in speed.