96 lines
3.0 KiB
Ruby
96 lines
3.0 KiB
Ruby
require 'typhoeus/hydra/addable'
|
|
require 'typhoeus/hydra/before'
|
|
require 'typhoeus/hydra/cacheable'
|
|
require 'typhoeus/hydra/block_connection'
|
|
require 'typhoeus/hydra/memoizable'
|
|
require 'typhoeus/hydra/queueable'
|
|
require 'typhoeus/hydra/runnable'
|
|
require 'typhoeus/hydra/stubbable'
|
|
|
|
module Typhoeus
|
|
|
|
# Hydra manages making parallel HTTP requests. This
|
|
# is achieved by using libcurls multi interface:
|
|
# http://curl.haxx.se/libcurl/c/libcurl-multi.html
|
|
# The benefits are that you don't have to worry running
|
|
# the requests by yourself.
|
|
#
|
|
# Hydra will also handle how many requests you can
|
|
# make in parallel. Things will get flakey if you
|
|
# try to make too many requests at the same time.
|
|
# The built in limit is 200. When more requests than
|
|
# that are queued up, hydra will save them for later
|
|
# and start the requests as others are finished. You
|
|
# can raise or lower the concurrency limit through
|
|
# the Hydra constructor.
|
|
#
|
|
# Regarding the asynchronous behavior of the hydra,
|
|
# it is important to know that this is completely hidden
|
|
# from the developer and you are free to apply
|
|
# whatever technique you want to your code. That should not
|
|
# conflict with libcurls internal concurrency mechanism.
|
|
#
|
|
# @example Use the hydra to do multiple requests.
|
|
# hydra = Typhoeus::Hydra.new
|
|
# requests = (0..9).map{ Typhoeus::Request.new("www.example.com") }
|
|
# requests.each{ |request| hydra.queue(request) }
|
|
# hydra.run
|
|
#
|
|
# @note Callbacks are going to delay the request
|
|
# execution.
|
|
class Hydra
|
|
include Hydra::Addable
|
|
include Hydra::Runnable
|
|
include Hydra::Memoizable
|
|
include Hydra::Cacheable
|
|
include Hydra::BlockConnection
|
|
include Hydra::Stubbable
|
|
include Hydra::Before
|
|
include Hydra::Queueable
|
|
|
|
# @example Set max_concurrency.
|
|
# Typhoeus::Hydra.new(max_concurrency: 20)
|
|
attr_accessor :max_concurrency
|
|
|
|
# @api private
|
|
attr_reader :multi
|
|
|
|
class << self
|
|
|
|
# Returns a memoized hydra instance.
|
|
#
|
|
# @example Get a hydra.
|
|
# Typhoeus::Hydra.hydra
|
|
#
|
|
# @return [Typhoeus::Hydra] A new hydra.
|
|
def hydra
|
|
Thread.current[:typhoeus_hydra] ||= new
|
|
end
|
|
end
|
|
|
|
# Create a new hydra. All
|
|
# {http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method Ethon::Multi#initialize}
|
|
# options are also available.
|
|
#
|
|
# @example Create a hydra.
|
|
# Typhoeus::Hydra.new
|
|
#
|
|
# @example Create a hydra with max_concurrency.
|
|
# Typhoeus::Hydra.new(max_concurrency: 20)
|
|
#
|
|
# @param [ Hash ] options The options hash.
|
|
#
|
|
# @option options :max_concurrency [ Integer ] Number
|
|
# of max concurrent connections to create. Default is
|
|
# 200.
|
|
#
|
|
# @see http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method
|
|
# Ethon::Multi#initialize
|
|
def initialize(options = {})
|
|
@options = options
|
|
@max_concurrency = Integer(@options.fetch(:max_concurrency, 200))
|
|
@multi = Ethon::Multi.new(options.reject{|k,_| k==:max_concurrency})
|
|
end
|
|
end
|
|
end
|