Question

I have the following Rake file. Using RoR 2.3.8.

desc "Create shops sitemap"
task(:shops => :environment) do
  sitemap = Sitemap.new
  #add every item
  for i in shop.find(:all, :select => 'id, updated_at', :order => 'updated_at DESC', :limit => 50000)
    sitemap.add_url("http://abc.com/shops/#{i.id}",w3c_date(i.updated_at),'daily','1.0')
  end

  puts "#{sitemap.urls.length} total urls"
  #delete the file
  FileUtils.rm(File.join(RAILS_ROOT, "public/sitemap_shops_1.xml.gz"), :force => true)

  f =File.new(File.join(RAILS_ROOT, "public/sitemap_shops_1.xml"), 'w')

  sitemap.write(f,2)
  f.close

  system("gzip #{File.join(RAILS_ROOT, 'public/sitemap_shops_1.xml')}")
end

The file above searches the first 50,000 records based on last updated, then save in a file numbered 1.

How do I modify the code to have it search the next 50,000, and save the file numbered 2, then next 50,000, save as file numbered 3, etc.?

Thanks.

Answer 1

Instead of find, you can use find_in_batches which will return groups of 1,000 at a time (but you can override this to be 50,000 with the :batch_size option). Throw in a counter variable (since I don’t think find_in_batches has anything like an each_with_index) and you can handle all the files you need.

desc "Create shops sitemap"
task(:shops => :environment) do
  file_name_index = 0
  Shop.find_in_batches(:all, :select => 'id, updated_at', :order => 'updated_at DESC', :batch_size => 50000) do |group_of_50000|
    file_name_index += 1
    sitemap = Sitemap.new
    #add every item
    for i in group_of_50000
      sitemap.add_url("http://abc.com/shops/#{i.id}",w3c_date(i.updated_at),'daily','1.0')
    end

    puts "#{sitemap.urls.length} total urls"
    #delete the file
    FileUtils.rm(File.join(RAILS_ROOT, "public/sitemap_shops_#{file_name_index}.xml.gz"), :force => true)

    f =File.new(File.join(RAILS_ROOT, "public/sitemap_shops_#{file_name_index}.xml"), 'w')

    sitemap.write(f,2)
    f.close

    system("gzip #{File.join(RAILS_ROOT, 'public/sitemap_shops_#{file_name_index}.xml')}")
  end
end