Add bin and edit workflow

bin/gems/bundler-4.0.21/lib/bundler/fetcher/base.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Bundler
+  class Fetcher
+    class Base
+      attr_reader :downloader
+      attr_reader :display_uri
+      attr_reader :remote
+      attr_reader :gem_remote_fetcher
+
+      def initialize(downloader, remote, display_uri, gem_remote_fetcher)
+        raise "Abstract class" if self.class == Base
+        @downloader = downloader
+        @remote = remote
+        @display_uri = display_uri
+        @gem_remote_fetcher = gem_remote_fetcher
+      end
+
+      def remote_uri
+        @remote.uri
+      end
+
+      def fetch_uri
+        @fetch_uri ||= if remote_uri.host == "rubygems.org"
+          uri = remote_uri.dup
+          uri.host = "index.rubygems.org"
+          uri
+        else
+          remote_uri
+        end
+      end
+
+      def available?
+        true
+      end
+
+      def api_fetcher?
+        false
+      end
+
+      def release_resolution_memory!
+      end
+
+      private
+
+      def log_specs(&block)
+        if Bundler.ui.debug?
+          Bundler.ui.debug yield
+        else
+          Bundler.ui.info ".", false
+        end
+      end
+    end
+  end
+end

bin/gems/bundler-4.0.21/lib/bundler/fetcher/compact_index.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../worker"
+
+module Bundler
+  class Fetcher
+    class CompactIndex < Base
+      def self.compact_index_request(method_name)
+        method = instance_method(method_name)
+        undef_method(method_name)
+        define_method(method_name) do |*args, &blk|
+          method.bind_call(self, *args, &blk)
+        rescue NetworkDownError, CompactIndexClient::Updater::MismatchedChecksumError => e
+          raise HTTPError, e.message
+        rescue AuthenticationRequiredError, BadAuthenticationError
+          # Fail since we got a 401 from the server.
+          raise
+        rescue HTTPError => e
+          Bundler.ui.trace(e)
+          nil
+        end
+      end
+
+      def specs(gem_names)
+        specs_for_names(gem_names)
+      end
+      compact_index_request :specs
+
+      def specs_for_names(gem_names)
+        gem_info = []
+        complete_gems = []
+        remaining_gems = gem_names.dup
+
+        until remaining_gems.empty?
+          log_specs { "Looking up gems #{remaining_gems.inspect}" }
+          deps = fetch_gem_infos(remaining_gems).flatten(1)
+          next_gems = deps.flat_map {|d| d[CompactIndexClient::INFO_DEPS].flat_map(&:first) }.uniq
+          deps.each {|dep| gem_info << dep }
+          complete_gems.concat(deps.map(&:first)).uniq!
+          remaining_gems = next_gems - complete_gems
+        end
+        @bundle_worker&.stop
+        @bundle_worker = nil # reset it.  Not sure if necessary
+
+        gem_info
+      end
+
+      def available?
+        unless SharedHelpers.md5_available?
+          Bundler.ui.debug("FIPS mode is enabled, bundler can't use the CompactIndex API")
+          return nil
+        end
+        # Read info file checksums out of /versions, so we can know if gems are up to date
+        compact_index_client.available?
+      rescue CompactIndexClient::Updater::MismatchedChecksumError => e
+        Bundler.ui.debug(e.message)
+        nil
+      end
+      compact_index_request :available?
+
+      def api_fetcher?
+        true
+      end
+
+      # The client holds the parsed checksums of all info files in the
+      # index. Dropping it is always safe because it is rebuilt from the
+      # local cache on demand.
+      def release_resolution_memory!
+        @compact_index_client = nil
+      end
+
+      private
+
+      def compact_index_client
+        @compact_index_client ||=
+          SharedHelpers.filesystem_access(cache_path) do
+            CompactIndexClient.new(cache_path, client_fetcher)
+          end
+      end
+
+      def fetch_gem_infos(names)
+        # Create the client and update the versions file on this thread.
+        # Otherwise the workers race to lazily create the client and update
+        # the versions file concurrently, e.g. when the client was released
+        # after resolution and is being rebuilt for `bundle cache`.
+        compact_index_client.available?
+
+        in_parallel(names) {|name| compact_index_client.info(name) }
+      rescue TooManyRequestsError # rubygems.org is rate limiting us, slow down.
+        @bundle_worker&.stop
+        @bundle_worker = nil # reset it.  Not sure if necessary
+        compact_index_client.reset!
+        names.map {|name| compact_index_client.info(name) }
+      end
+
+      def in_parallel(inputs, &blk)
+        func = lambda {|object, _index| blk.call(object) }
+        worker = bundle_worker(func)
+        inputs.each {|input| worker.enq(input) }
+        inputs.map { worker.deq }
+      end
+
+      def bundle_worker(func = nil)
+        @bundle_worker ||= begin
+          worker_name = "Compact Index (#{display_uri.host})"
+          Bundler::Worker.new(Bundler.settings.processor_count, worker_name, func)
+        end
+        @bundle_worker.tap do |worker|
+          worker.instance_variable_set(:@func, func) if func
+        end
+      end
+
+      def cache_path
+        Bundler.user_cache.join("compact_index", remote.cache_slug)
+      end
+
+      def client_fetcher
+        ClientFetcher.new(self, Bundler.ui)
+      end
+
+      ClientFetcher = Struct.new(:fetcher, :ui) do
+        def call(path, headers)
+          fetcher.downloader.fetch(fetcher.fetch_uri + path, headers)
+        rescue NetworkDownError => e
+          raise unless headers["If-None-Match"]
+          ui.warn "Using the cached data for the new index because of a network error: #{e}"
+          Gem::Net::HTTPNotModified.new(nil, nil, nil)
+        end
+      end
+    end
+  end
+end

bin/gems/bundler-4.0.21/lib/bundler/fetcher/dependency.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require "cgi/escape"
+require "cgi/util" unless defined?(CGI::EscapeExt)
+
+module Bundler
+  class Fetcher
+    class Dependency < Base
+      def available?
+        @available ||= fetch_uri.scheme != "file" && downloader.fetch(dependency_api_uri)
+      rescue NetworkDownError => e
+        raise HTTPError, e.message
+      rescue AuthenticationRequiredError
+        # Fail since we got a 401 from the server.
+        raise
+      rescue HTTPError
+        false
+      end
+
+      def api_fetcher?
+        true
+      end
+
+      def specs(gem_names, full_dependency_list = [], last_spec_list = [])
+        query_list = gem_names.uniq - full_dependency_list
+
+        log_specs { "Query List: #{query_list.inspect}" }
+
+        return last_spec_list if query_list.empty?
+
+        spec_list, deps_list = Bundler::Retry.new("dependency api", FAIL_ERRORS).attempts do
+          dependency_specs(query_list)
+        end
+
+        returned_gems = spec_list.map(&:first).uniq
+        specs(deps_list, full_dependency_list + returned_gems, spec_list + last_spec_list)
+      rescue MarshalError, HTTPError, GemspecError
+        Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
+        Bundler.ui.debug "could not fetch from the dependency API, trying the full index"
+        nil
+      end
+
+      def dependency_specs(gem_names)
+        Bundler.ui.debug "Query Gemcutter Dependency Endpoint API: #{gem_names.join(",")}"
+
+        gem_list = unmarshalled_dep_gems(gem_names)
+        get_formatted_specs_and_deps(gem_list)
+      end
+
+      def unmarshalled_dep_gems(gem_names)
+        gem_list = []
+        gem_names.each_slice(api_request_size) do |names|
+          marshalled_deps = downloader.fetch(dependency_api_uri(names)).body
+          gem_list.concat(Bundler.safe_load_marshal(marshalled_deps))
+        end
+        gem_list
+      end
+
+      def get_formatted_specs_and_deps(gem_list)
+        deps_list = []
+        spec_list = []
+
+        gem_list.each do |s|
+          deps_list.concat(s[:dependencies].map(&:first))
+          deps = s[:dependencies].map {|n, d| [n, d.split(", ")] }
+          spec_list.push([s[:name], s[:number], s[:platform], deps])
+        end
+        [spec_list, deps_list]
+      end
+
+      def dependency_api_uri(gem_names = [])
+        uri = fetch_uri + "api/v1/dependencies"
+        uri.query = "gems=#{CGI.escape(gem_names.sort.join(","))}" if gem_names.any?
+        uri
+      end
+
+      private
+
+      def api_request_size
+        Bundler.settings[:api_request_size]&.to_i || Source::Rubygems::API_REQUEST_SIZE
+      end
+    end
+  end
+end

bin/gems/bundler-4.0.21/lib/bundler/fetcher/downloader.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Bundler
+  class Fetcher
+    class Downloader
+      HTTP_NON_RETRYABLE_ERRORS = [
+        SocketError,
+        Errno::EADDRNOTAVAIL,
+        Errno::ENETDOWN,
+        Errno::ENETUNREACH,
+        Gem::Net::HTTP::Persistent::Error,
+        Errno::EHOSTUNREACH,
+      ].freeze
+
+      HTTP_RETRYABLE_ERRORS = [
+        Gem::Timeout::Error,
+        EOFError,
+        Errno::EINVAL,
+        Errno::ECONNRESET,
+        Errno::ETIMEDOUT,
+        Errno::EAGAIN,
+        Gem::Net::HTTPBadResponse,
+        Gem::Net::HTTPHeaderSyntaxError,
+        Gem::Net::ProtocolError,
+        Zlib::BufError,
+      ].freeze
+
+      attr_reader :connection
+      attr_reader :redirect_limit
+
+      def initialize(connection, redirect_limit)
+        @connection = connection
+        @redirect_limit = redirect_limit
+      end
+
+      def fetch(uri, headers = {}, counter = 0)
+        raise HTTPError, "Too many redirects" if counter >= redirect_limit
+
+        filtered_uri = URICredentialsFilter.credential_filtered_uri(uri)
+
+        response = request(uri, headers)
+        Bundler.ui.debug("HTTP #{response.code} #{response.message} #{filtered_uri}")
+
+        case response
+        when Gem::Net::HTTPSuccess, Gem::Net::HTTPNotModified
+          response
+        when Gem::Net::HTTPRedirection
+          new_uri = Gem::URI.parse(response["location"])
+          # Following a downgrade would put the credentials this request carries
+          # on a plaintext connection, so refuse it like Gem::RemoteFetcher does.
+          if https?(uri) && !https?(new_uri)
+            raise HTTPError, "Redirecting to a non-https URI is not allowed: #{URICredentialsFilter.credential_filtered_uri(new_uri)}"
+          end
+          if new_uri.host == uri.host
+            new_uri.user = uri.user
+            new_uri.password = uri.password
+          end
+          fetch(new_uri, headers, counter + 1)
+        when Gem::Net::HTTPRequestedRangeNotSatisfiable
+          new_headers = headers.dup
+          new_headers.delete("Range")
+          fetch(uri, new_headers)
+        when Gem::Net::HTTPRequestEntityTooLarge
+          raise FallbackError, response.body
+        when Gem::Net::HTTPTooManyRequests
+          raise TooManyRequestsError, response.body
+        when Gem::Net::HTTPUnauthorized
+          raise BadAuthenticationError, uri.host if uri.userinfo
+          raise AuthenticationRequiredError, uri.host
+        when Gem::Net::HTTPForbidden
+          raise AuthenticationForbiddenError, uri.host
+        when Gem::Net::HTTPNotFound
+          raise FallbackError, "Gem::Net::HTTPNotFound: #{filtered_uri}"
+        else
+          message = "Gem::#{response.class.name.gsub(/\AGem::/, "")}"
+          message += ": #{response.body}" unless response.body.empty?
+          raise HTTPError, message
+        end
+      end
+
+      def request(uri, headers)
+        validate_uri_scheme!(uri)
+
+        filtered_uri = URICredentialsFilter.credential_filtered_uri(uri)
+
+        Bundler.ui.debug "HTTP GET #{filtered_uri}"
+        req = Gem::Net::HTTP::Get.new uri.request_uri, headers
+        if uri.user
+          user = CGI.unescape(uri.user)
+          password = uri.password ? CGI.unescape(uri.password) : nil
+          req.basic_auth(user, password)
+        end
+        connection.request(uri, req)
+      rescue OpenSSL::SSL::SSLError
+        raise CertificateFailureError.new(uri)
+      rescue *HTTP_NON_RETRYABLE_ERRORS => e
+        Bundler.ui.trace e
+
+        host = uri.host
+        host_port = "#{host}:#{uri.port}"
+        host = host_port if filtered_uri.to_s.include?(host_port)
+        raise NetworkDownError, "Could not reach host #{host}. Check your network " \
+          "connection and try again."
+      rescue *HTTP_RETRYABLE_ERRORS => e
+        Bundler.ui.trace e
+
+        raise HTTPError, "Network error while fetching #{filtered_uri}" \
+            " (#{e})"
+      end
+
+      private
+
+      def https?(uri)
+        uri.scheme&.casecmp("https")&.zero?
+      end
+
+      def validate_uri_scheme!(uri)
+        return if /\Ahttps?\z/.match?(uri.scheme)
+        raise InvalidOption,
+          "The request uri `#{uri}` has an invalid scheme (`#{uri.scheme}`). " \
+          "Did you mean `http` or `https`?"
+      end
+    end
+  end
+end

bin/gems/bundler-4.0.21/lib/bundler/fetcher/gem_remote_fetcher.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "rubygems/remote_fetcher"
+
+module Bundler
+  class Fetcher
+    class GemRemoteFetcher < Gem::RemoteFetcher
+      def initialize(*)
+        super
+
+        @pool_size = Bundler.settings.installation_parallelization
+        ssl_ca_cert = Bundler.settings[:ssl_ca_cert]
+        @cert_files << ssl_ca_cert if ssl_ca_cert
+      end
+
+      def request(*args)
+        super do |req|
+          req.delete("User-Agent") if headers["User-Agent"]
+          yield req if block_given?
+        end
+      end
+    end
+  end
+end

bin/gems/bundler-4.0.21/lib/bundler/fetcher/index.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Bundler
+  class Fetcher
+    class Index < Base
+      def specs(_gem_names)
+        Bundler.rubygems.fetch_all_remote_specs(remote, gem_remote_fetcher)
+      rescue Gem::RemoteFetcher::FetchError => e
+        case e.message
+        when /certificate verify failed/
+          raise CertificateFailureError.new(display_uri)
+        when /401/
+          raise BadAuthenticationError, remote_uri if remote_uri.userinfo
+          raise AuthenticationRequiredError, remote_uri
+        when /403/
+          raise AuthenticationForbiddenError, remote_uri
+        else
+          raise HTTPError, "Could not fetch specs from #{display_uri} due to underlying error <#{e.message}>"
+        end
+      end
+    end
+  end
+end