CRAWLER_PATTERN = /archiver|slurp|bot|crawler|jeeves|spider|\.{6}/i

total_lines = File.readlines(ARGV[0]).to_a.length
count = 0
File.readlines(ARGV[0]).each do |line|
  if line =~ CRAWLER_PATTERN
    count += 1
  end
end

print "total lines: #{total_lines}, crawler_pattern: #{count} (#{"%.2f" % (count.to_f/total_lines*100)} %)\n"