Question

I need to do bulk inserts into SQLite database with NHibernate. The object’s PK is HiLo int32. I tried to use NH stateless session with session.insert, and it works fine. However, I found that using prepared command is faster about 30-40%, so I am trying to utilize it.

Currently I am struggling with assigning id (which is NH HiLo).
I found a solution on stackoverflow (NHibernate HiLo ID Generator. Generating an ID before saving) – see method GenerateIdentifier(), but it queries and updates the database on each call, so it’s not a good option.

It there any way to make it working as it supposed to – when id generator reaches the hivalue, it shold do only 1 roundtrip to the server to get the new low and hi values ?

public static Int64 TestNHSQLiteBulk(bool newDB)
        {
            int ProjectCount = 1000000;

            var factory = Database.CreateSQLiteSessionFactory(newDB);

            Int64 objectCount = 0;

            using (var session = factory.OpenStatelessSession())
            {
                var connection = session.Connection;
                using (var transaction = session.BeginTransaction(System.Data.IsolationLevel.ReadCommitted))
                {

                    // this should be removed when proper HiLo will be implemented.
                    var nextProject = session.CreateCriteria<Project>().SetProjection(Projections.Max<Project>(p => p.ProjectId)).UniqueResult();
                    int startId = (nextProject == null) ? 0 : (int)nextProject + 1;

                    var command = connection.CreateCommand();
                    command.CommandText = "Insert INTO Project (ProjectId, ProjectName) Values(?,?)";

                    var projectIdParameter = command.CreateParameter();
                    projectIdParameter.ParameterName = "ProjectId";
                    projectIdParameter.DbType = System.Data.DbType.Int32;

                    var projectNameParameter = command.CreateParameter();
                    projectNameParameter.ParameterName = "ProjectName";
                    projectNameParameter.DbType = System.Data.DbType.String;

                    command.Parameters.Add(projectIdParameter);
                    command.Parameters.Add(projectNameParameter);
                    command.Prepare();

                    for (int p = startId; p < startId + ProjectCount; p++)
                    {

                        var project = new Project()
                        {
                            ProjectName = "Project " + p
                        };

                        //found on stackoverflow, but is results a roundtrip to server on each call.
                        GenerateIdentifier(project, Database.savedConfig, session);

                        //session.Insert(project);

                        //using prepared command is almost 2x faster!
                        projectIdParameter.Value = project.ProjectId;
                        projectNameParameter.Value = project.ProjectName;
                        command.ExecuteNonQuery();

                        objectCount++;
                    }

                    transaction.Commit();
                }

            }

            return objectCount;
        }

    //this will get the value and update the hi-lo value repository in the datastore
    public static void GenerateIdentifier(object target, NHibernate.Cfg.Configuration conf, IStatelessSession session)
    {
        var targetType = target.GetType();

        var classMapping = conf.GetClassMapping(targetType);
        var impl = session.GetSessionImplementation();

        var newId = classMapping.Identifier.CreateIdentifierGenerator(impl.Factory.Dialect, classMapping.Table.Catalog, classMapping.Table.Schema,
                                                                classMapping.RootClazz).Generate(impl, target);
        classMapping.IdentifierProperty.GetSetter(targetType).Set(target, newId);
    }

Answer 1

you are creating a new HiLogenerator for each entity instead of using the given one. Also the generator normqally initializes itself to the value in the database so it is not nessesary to query for that manually:

public static Int64 TestNHSQLiteBulk(bool newDB)
{
    int ProjectCount = 1000000;

    var factory = Database.CreateSQLiteSessionFactory(newDB);

    var classMapping = (SingleTableEntityPersister)factory.GetClassMetadata(typeof(Project));
    var generator = classMapping.IdentifierGenerator;

    Int64 objectCount = 0;

    using (var session = factory.OpenStatelessSession())
    {
        var connection = session.Connection;
        using (var transaction = session.BeginTransaction(System.Data.IsolationLevel.ReadCommitted))
        {
            var command = connection.CreateCommand();
            command.CommandText = "Insert INTO Project (ProjectId, ProjectName) Values(?,?)";

            var projectIdParameter = command.CreateParameter();
            projectIdParameter.ParameterName = "ProjectId";
            projectIdParameter.DbType = System.Data.DbType.Int32;

            var projectNameParameter = command.CreateParameter();
            projectNameParameter.ParameterName = "ProjectName";
            projectNameParameter.DbType = System.Data.DbType.String;

            command.Parameters.Add(projectIdParameter);
            command.Parameters.Add(projectNameParameter);
            command.Prepare();

            for (int p = 0; p < ProjectCount; p++)
            {
                var projectId = (long)generator.Generate(session.GetSessionImplementation(), null);

                //using prepared command is almost 2x faster!
                projectIdParameter.Value = projectId;
                projectNameParameter.Value = "Project" + projectId;
                command.ExecuteNonQuery();

                objectCount++;
            }

            transaction.Commit();
        }
    }

    return objectCount;
}