http = Net::HTTP.new("hostname", 80)
http.open_timeout = 300
http.read_timeout = 300
pagereq = lambda {
http.request(Net::HTTP::Get.new(page, {"User-Agent" => "Mozilla/5.0"})).body }
some_conditions.to_a.each do |n|
page = "startpage"+n.to_s
pagereq.call.scan(/criteria1/).each do |m|
page = "/"+m.to_s
puts pagereq.call.scan(/criteria2/)
end
end
I use this template to collect links or something else from sites. It produce DNS resolving on each connection and this is not good.
What I interest in. Resolve hostname one time, make connection, make all operations, close connection.
No comments:
Post a Comment