From baa4e3e659b462fab1d8a0880cf8c57b2d46cf6f Mon Sep 17 00:00:00 2001 From: Andrew Waage Date: Wed, 18 May 2016 10:13:22 -0700 Subject: [PATCH] created a new fetcher with octokit. functionality should remain same --- Gemfile.lock | 10 +- github_changelog_generator.gemspec | 1 + lib/github_changelog_generator.rb | 1 + .../generator/generator.rb | 3 +- .../octo_fetcher.rb | 293 ++++++++++++++++++ 5 files changed, 305 insertions(+), 3 deletions(-) create mode 100644 lib/github_changelog_generator/octo_fetcher.rb diff --git a/Gemfile.lock b/Gemfile.lock index 3c6158a..2c2ff57 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -4,6 +4,7 @@ PATH github_changelog_generator (1.13.1) github_api (>= 0.14) rainbow (>= 2.1) + octokit (~> 4.0) rake (>= 10.0) GEM @@ -47,9 +48,11 @@ GEM multi_json (~> 1.3) multi_xml (~> 0.5) rack (>= 1.2, < 3) - overcommit (0.34.2) + octokit (4.3.0) + sawyer (~> 0.7.0, >= 0.5.3) childprocess (~> 0.5.8) iniparse (~> 1.4) + overcommit (0.34.2) parser (2.3.1.4) ast (~> 2.2) powerpack (0.1.1) @@ -76,10 +79,13 @@ GEM ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) ruby-progressbar (1.8.1) - simplecov (0.12.0) + sawyer (0.7.0) + addressable (>= 2.3.5, < 2.5) + faraday (~> 0.8, < 0.10) docile (~> 1.1.0) json (>= 1.8, < 3) simplecov-html (~> 0.10.0) + simplecov (0.12.0) simplecov-html (0.10.0) term-ansicolor (1.3.2) tins (~> 1.0) diff --git a/github_changelog_generator.gemspec b/github_changelog_generator.gemspec index d9f2074..d6e1f35 100644 --- a/github_changelog_generator.gemspec +++ b/github_changelog_generator.gemspec @@ -27,4 +27,5 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "rake", ">= 10.0" spec.add_runtime_dependency "github_api", ">= 0.14" spec.add_runtime_dependency "rainbow", ">= 2.1" + spec.add_runtime_dependency("octokit", ["~> 4.0"]) end diff --git a/lib/github_changelog_generator.rb b/lib/github_changelog_generator.rb index c478be5..44b584c 100755 --- a/lib/github_changelog_generator.rb +++ b/lib/github_changelog_generator.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true require "github_api" +require "octokit" require "json" require "benchmark" diff --git a/lib/github_changelog_generator/generator/generator.rb b/lib/github_changelog_generator/generator/generator.rb index 9f7a9bf..8a9003f 100644 --- a/lib/github_changelog_generator/generator/generator.rb +++ b/lib/github_changelog_generator/generator/generator.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true require_relative "../fetcher" +require_relative "../octo_fetcher" require_relative "generator_generation" require_relative "generator_fetcher" require_relative "generator_processor" @@ -21,7 +22,7 @@ module GitHubChangelogGenerator def initialize(options = nil) @options = options || {} @tag_times_hash = {} - @fetcher = GitHubChangelogGenerator::Fetcher.new @options + @fetcher = GitHubChangelogGenerator::OctoFetcher.new @options end def fetch_issues_and_pr diff --git a/lib/github_changelog_generator/octo_fetcher.rb b/lib/github_changelog_generator/octo_fetcher.rb new file mode 100644 index 0000000..3b4b207 --- /dev/null +++ b/lib/github_changelog_generator/octo_fetcher.rb @@ -0,0 +1,293 @@ +module GitHubChangelogGenerator + # A Fetcher responsible for all requests to GitHub and all basic manipulation with related data + # (such as filtering, validating, e.t.c) + # + # Example: + # fetcher = GitHubChangelogGenerator::Fetcher.new options + + class OctoFetcher + PER_PAGE_NUMBER = 100 + MAX_THREAD_NUMBER = 50 + CHANGELOG_GITHUB_TOKEN = "CHANGELOG_GITHUB_TOKEN" + GH_RATE_LIMIT_EXCEEDED_MSG = "Warning: Can't finish operation: GitHub API rate limit exceeded, change log may be " \ + "missing some issues. You can limit the number of issues fetched using the `--max-issues NUM` argument." + NO_TOKEN_PROVIDED = "Warning: No token provided (-t option) and variable $CHANGELOG_GITHUB_TOKEN was not found. " \ + "This script can make only 50 requests to GitHub API per hour without token!" + + def initialize(options = {}) + @options = options || {} + @user = @options[:user] + @project = @options[:project] + # Only issues updated at or after this time are returned. This is a timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ. + @since = @options[:since] # eg. Time.parse("2016-01-01 10:00:00").iso8601 + @github_token = fetch_github_token + + @request_options = { :per_page => PER_PAGE_NUMBER } + @github_options = {} + @github_options[:access_token] = @github_token unless @github_token.nil? + @github_options[:api_endpoint] = @options[:github_endpoint] unless @options[:github_endpoint].nil? + + client_type = @options[:github_endpoint].nil? ? Octokit::Client : Octokit::EnterpriseAdminClient + @client = client_type.new(@github_options) + end + + # Returns GitHub token. First try to use variable, provided by --token option, + # otherwise try to fetch it from CHANGELOG_GITHUB_TOKEN env variable. + # + # @return [String] + def fetch_github_token + env_var = @options[:token] ? @options[:token] : (ENV.fetch CHANGELOG_GITHUB_TOKEN, nil) + + Helper.log.warn NO_TOKEN_PROVIDED.yellow unless env_var + + env_var + end + + # Fetch all tags from repo + # + # @return [Array] array of tags + def get_all_tags + print "Fetching tags...\r" if @options[:verbose] + + check_github_response { github_fetch_tags } + end + + # This is wrapper with rescue block + # + # @return [Object] returns exactly the same, what you put in the block, but wrap it with begin-rescue block + def check_github_response + begin + value = yield + rescue Github::Error::Unauthorized => e + Helper.log.error e.body.red + abort "Error: wrong GitHub token" + rescue Github::Error::Forbidden => e + Helper.log.warn e.body.red + Helper.log.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow + end + value + end + + # @return [String] helper to return Github "user/project" + def user_project + "#{@options[:user]}/#{@options[:project]}" + end + + # Parses a URI and returns a hash of all GET variables + # + # @param [String] uri eg. https://api.github.com/repositories/43914960/tags?page=37&foo=1 + # @return [Hash] of all GET variables. eg. { 'page' => 37, 'foo' => 1 } + def parse_url_for_vars(uri) + URI(uri).query.split("&").inject({}) do |params, get_var| + k,v = get_var.split("=") + params[k] = v + params + end + end + + # Returns the number of pages for a API call + # + # @return [Integer] number of pages for this API call in total + def calculate_pages(client, method, request_options) + # Makes the first API call so that we can call last_response + client.send(method, user_project, @request_options.merge(request_options)) + last_response = client.last_response + + if last_pg = last_response.rels[:last] + parse_url_for_vars(last_pg.href)['page'].to_i + else + 1 + end + end + + # Iterates through all pages until there are no more :next pages to follow + # yields the result per page + # + # @param [Octokit::Client] client + # @param [String] method (eg. 'tags') + # @return [Integer] total number of pages + def iterate_pages(client, method, *args, &block) + if args.size == 1 && args.first.is_a?(Hash) + request_options = args.delete_at(0) + elsif args.size > 1 && args.last.is_a?(Hash) + request_options = args.delete_at(args.length - 1) + end + + args.push(@request_options.merge(request_options)) + # args.push({}.merge(request_options)) + + pages = 1 + client.send(method, user_project, *args) + # client.send(method, 'retentionscience/rsapi', *args) + last_response = client.last_response + + yield last_response.data + + while !(next_one = last_response.rels[:next]).nil? + pages +=1 + + last_response = next_one.get + yield last_response.data + end + + pages + end + + + # Fill input array with tags + # + # @return [Array] array of tags in repo + def github_fetch_tags + tags = [] + page_i = 0 + count_pages = calculate_pages(@client, 'tags', {}) + + iterate_pages(@client, 'tags', {}) do |new_tags| + page_i += PER_PAGE_NUMBER + print_in_same_line("Fetching tags... #{page_i}/#{count_pages * PER_PAGE_NUMBER}") + tags.concat(new_tags) + end + print_empty_line + + if tags.count == 0 + Helper.log.warn "Warning: Can't find any tags in repo.\ +Make sure, that you push tags to remote repo via 'git push --tags'".yellow + else + Helper.log.info "Found #{tags.count} tags" + end + tags + end + + # This method fetch all closed issues and separate them to pull requests and pure issues + # (pull request is kind of issue in term of GitHub) + # + # @return [Tuple] with (issues, pull-requests) + def fetch_closed_issues_and_pr + print "Fetching closed issues...\r" if @options[:verbose] + issues = [] + options = { + :state => "closed", + :filter => "all", + :labels => nil, + :since => @since, + } + + begin + page_i = 0 + count_pages = calculate_pages(@client, 'issues', options) + + iterate_pages(@client, 'issues', options) do |new_issues| + page_i += PER_PAGE_NUMBER + print_in_same_line("Fetching issues... #{page_i}/#{count_pages * PER_PAGE_NUMBER}") + issues.concat(new_issues) + break if @options[:max_issues] && issues.length >= @options[:max_issues] + end + print_empty_line + Helper.log.info "Received issues: #{issues.count}" + + rescue + Helper.log.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow + end + + # separate arrays of issues and pull requests: + issues.partition do |x| + x[:pull_request].nil? + end + end + + # Fetch all pull requests. We need them to detect :merged_at parameter + # + # @return [Array] all pull requests + def fetch_closed_pull_requests + pull_requests = [] + options = { :state => 'closed' } + + if !@options[:release_branch].nil? + options[:base] = @options[:release_branch] + end + + begin + page_i = 0 + count_pages = calculate_pages(@client, 'pull_requests', options) + + iterate_pages(@client, 'pull_requests', options) do |new_pr| + page_i += PER_PAGE_NUMBER + log_string = "Fetching merged dates... #{page_i}/#{count_pages * PER_PAGE_NUMBER}" + print_in_same_line(log_string) + pull_requests.concat(new_pr) + end + print_empty_line + rescue + Helper.log.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow + end + + Helper.log.info "Fetching merged dates: #{pull_requests.count}" + pull_requests + end + + # Print specified line on the same string + # + # @param [String] log_string + def print_in_same_line(log_string) + print log_string + "\r" + end + + # Print long line with spaces on same line to clear prev message + def print_empty_line + print_in_same_line(" ") + end + + # Fetch event for all issues and add them to :events + # + # @param [Array] issues + # @return [Void] + def fetch_events_async(issues) + i = 0 + threads = [] + + issues.each_slice(MAX_THREAD_NUMBER) do |issues_slice| + issues_slice.each do |issue| + threads << Thread.new do + begin + issue[:events] = [] + iterate_pages(@client, 'issue_events', issue['number'], {}) do |new_event| + issue[:events].concat(new_event) + end + rescue + Helper.log.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow + end + print_in_same_line("Fetching events for issues and PR: #{i + 1}/#{issues.count}") + i += 1 + end + end + threads.each(&:join) + threads = [] + end + + # to clear line from prev print + print_empty_line + + Helper.log.info "Fetching events for issues and PR: #{i}" + end + + # Fetch tag time from repo + # + # @param [Hash] tag + # @return [Time] time of specified tag + def fetch_date_of_tag(tag) + begin + commit_data = @client.commit(user_project, tag['commit']['sha']) + commit_data[:commit][:committer][:date] + rescue + Helper.log.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow + end + end + + # Fetch commit for specified event + # + # @return [Hash] + def fetch_commit(event) + @client.commit(user_project, event[:commit_id]) + end + end +end