This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
require 'typhoeus/hydra/addable'
|
||||
require 'typhoeus/hydra/before'
|
||||
require 'typhoeus/hydra/cacheable'
|
||||
require 'typhoeus/hydra/block_connection'
|
||||
require 'typhoeus/hydra/memoizable'
|
||||
require 'typhoeus/hydra/queueable'
|
||||
require 'typhoeus/hydra/runnable'
|
||||
require 'typhoeus/hydra/stubbable'
|
||||
|
||||
module Typhoeus
|
||||
|
||||
# Hydra manages making parallel HTTP requests. This
|
||||
# is achieved by using libcurls multi interface:
|
||||
# http://curl.haxx.se/libcurl/c/libcurl-multi.html
|
||||
# The benefits are that you don't have to worry running
|
||||
# the requests by yourself.
|
||||
#
|
||||
# Hydra will also handle how many requests you can
|
||||
# make in parallel. Things will get flakey if you
|
||||
# try to make too many requests at the same time.
|
||||
# The built in limit is 200. When more requests than
|
||||
# that are queued up, hydra will save them for later
|
||||
# and start the requests as others are finished. You
|
||||
# can raise or lower the concurrency limit through
|
||||
# the Hydra constructor.
|
||||
#
|
||||
# Regarding the asynchronous behavior of the hydra,
|
||||
# it is important to know that this is completely hidden
|
||||
# from the developer and you are free to apply
|
||||
# whatever technique you want to your code. That should not
|
||||
# conflict with libcurls internal concurrency mechanism.
|
||||
#
|
||||
# @example Use the hydra to do multiple requests.
|
||||
# hydra = Typhoeus::Hydra.new
|
||||
# requests = (0..9).map{ Typhoeus::Request.new("www.example.com") }
|
||||
# requests.each{ |request| hydra.queue(request) }
|
||||
# hydra.run
|
||||
#
|
||||
# @note Callbacks are going to delay the request
|
||||
# execution.
|
||||
class Hydra
|
||||
include Hydra::Addable
|
||||
include Hydra::Runnable
|
||||
include Hydra::Memoizable
|
||||
include Hydra::Cacheable
|
||||
include Hydra::BlockConnection
|
||||
include Hydra::Stubbable
|
||||
include Hydra::Before
|
||||
include Hydra::Queueable
|
||||
|
||||
# @example Set max_concurrency.
|
||||
# Typhoeus::Hydra.new(max_concurrency: 20)
|
||||
attr_accessor :max_concurrency
|
||||
|
||||
# @api private
|
||||
attr_reader :multi
|
||||
|
||||
class << self
|
||||
|
||||
# Returns a memoized hydra instance.
|
||||
#
|
||||
# @example Get a hydra.
|
||||
# Typhoeus::Hydra.hydra
|
||||
#
|
||||
# @return [Typhoeus::Hydra] A new hydra.
|
||||
def hydra
|
||||
Thread.current[:typhoeus_hydra] ||= new
|
||||
end
|
||||
end
|
||||
|
||||
# Create a new hydra. All
|
||||
# {http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method Ethon::Multi#initialize}
|
||||
# options are also available.
|
||||
#
|
||||
# @example Create a hydra.
|
||||
# Typhoeus::Hydra.new
|
||||
#
|
||||
# @example Create a hydra with max_concurrency.
|
||||
# Typhoeus::Hydra.new(max_concurrency: 20)
|
||||
#
|
||||
# @param [ Hash ] options The options hash.
|
||||
#
|
||||
# @option options :max_concurrency [ Integer ] Number
|
||||
# of max concurrent connections to create. Default is
|
||||
# 200.
|
||||
#
|
||||
# @see http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method
|
||||
# Ethon::Multi#initialize
|
||||
def initialize(options = {})
|
||||
@options = options
|
||||
@max_concurrency = Integer(@options.fetch(:max_concurrency, 200))
|
||||
@multi = Ethon::Multi.new(options.reject{|k,_| k==:max_concurrency})
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user