github-changelog-generator/lib/github_changelog_generator/fetcher.rb

222 lines
7.9 KiB
Ruby
Raw Normal View History

2015-03-26 21:56:47 +00:00
module GitHubChangelogGenerator
2015-04-21 21:00:57 +00:00
# A Fetcher responsible for all requests to GitHub and all basic manipulation with related data
# (such as filtering, validating, e.t.c)
#
# Example:
# fetcher = GitHubChangelogGenerator::Fetcher.new options
2015-03-26 21:56:47 +00:00
class Fetcher
2015-04-21 17:42:33 +00:00
PER_PAGE_NUMBER = 30
MAX_SIMULTANEOUS_REQUESTS = 25
CHANGELOG_GITHUB_TOKEN = "CHANGELOG_GITHUB_TOKEN"
2015-05-18 12:04:15 +00:00
GH_RATE_LIMIT_EXCEEDED_MSG = "Warning: Can't finish operation: GitHub API rate limit exceeded, change log may be " \
"missing some issues. You can limit the number of issues fetched using the `--max-issues NUM` argument."
2015-05-18 12:04:15 +00:00
NO_TOKEN_PROVIDED = "Warning: No token provided (-t option) and variable $CHANGELOG_GITHUB_TOKEN was not found. " \
"This script can make only 50 requests to GitHub API per hour without token!"
2015-04-21 17:42:33 +00:00
def initialize(options = {})
2015-05-25 15:04:32 +00:00
@options = options || {}
@user = @options[:user]
@project = @options[:project]
@github_token = fetch_github_token
@github_options = { per_page: PER_PAGE_NUMBER }
@github_options[:oauth_token] = @github_token unless @github_token.nil?
@github_options[:endpoint] = @options[:github_endpoint] unless @options[:github_endpoint].nil?
@github_options[:site] = @options[:github_endpoint] unless @options[:github_site].nil?
@github = check_github_response { Github.new @github_options }
end
2015-04-21 21:00:57 +00:00
# Returns GitHub token. First try to use variable, provided by --token option,
# otherwise try to fetch it from CHANGELOG_GITHUB_TOKEN env variable.
#
# @return [String]
def fetch_github_token
2015-05-14 12:57:31 +00:00
env_var = @options[:token] ? @options[:token] : (ENV.fetch CHANGELOG_GITHUB_TOKEN, nil)
Helper.log.warn NO_TOKEN_PROVIDED.yellow unless env_var
env_var
end
2015-04-21 17:42:33 +00:00
# Fetch all tags from repo
# @return [Array] array of tags
def get_all_tags
2015-05-22 12:59:29 +00:00
print "Fetching tags...\r" if @options[:verbose]
2015-04-21 17:42:33 +00:00
2015-06-10 11:18:15 +00:00
check_github_response { github_fetch_tags }
2015-05-18 12:04:15 +00:00
end
2015-06-10 11:18:15 +00:00
# This is wrapper with rescue block
# @return [Object] returns exactly the same, what you put in the block, but wrap it with begin-rescue block
2015-05-18 12:04:15 +00:00
def check_github_response
2015-04-21 17:42:33 +00:00
begin
2015-05-18 12:04:15 +00:00
value = yield
2015-05-18 06:26:36 +00:00
rescue Github::Error::Unauthorized => e
2016-08-08 19:37:46 +00:00
Helper.log.warn e.error_messages.map { |m| m[:message] }.join(", ").red
2015-05-18 12:04:15 +00:00
abort "Error: wrong GitHub token"
2015-05-18 06:26:36 +00:00
rescue Github::Error::Forbidden => e
2016-08-08 19:37:46 +00:00
Helper.log.warn e.error_messages.map { |m| m[:message] }.join(", ").red
Helper.log.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow
2015-05-18 06:26:36 +00:00
end
2015-05-18 12:04:15 +00:00
value
2015-05-18 06:26:36 +00:00
end
2015-04-21 17:42:33 +00:00
2015-06-10 11:18:15 +00:00
# Fill input array with tags
# @return [Array] array of tags in repo
def github_fetch_tags
tags = []
response = @github.repos.tags @options[:user], @options[:project]
2015-05-18 06:26:36 +00:00
page_i = 0
count_pages = response.count_pages
response.each_page do |page|
page_i += PER_PAGE_NUMBER
2015-05-19 07:12:53 +00:00
print_in_same_line("Fetching tags... #{page_i}/#{count_pages * PER_PAGE_NUMBER}")
2016-06-06 07:23:38 +00:00
tags.concat(page) unless page.nil?
2015-04-21 17:42:33 +00:00
end
2015-05-19 07:12:53 +00:00
print_empty_line
2015-04-21 17:42:33 +00:00
2015-05-18 06:26:36 +00:00
if tags.count == 0
Helper.log.warn "Warning: Can't find any tags in repo.\
2015-05-18 06:26:36 +00:00
Make sure, that you push tags to remote repo via 'git push --tags'".yellow
2015-05-19 07:12:53 +00:00
else
Helper.log.info "Found #{tags.count} tags"
2015-05-18 06:26:36 +00:00
end
2015-06-10 11:18:15 +00:00
tags
2015-04-21 17:42:33 +00:00
end
# This method fetch all closed issues and separate them to pull requests and pure issues
# (pull request is kind of issue in term of GitHub)
2015-05-19 08:47:56 +00:00
# @return [Tuple] with (issues, pull-requests)
def fetch_closed_issues_and_pr
2015-05-22 12:59:29 +00:00
print "Fetching closed issues...\r" if @options[:verbose]
2015-04-21 17:42:33 +00:00
issues = []
begin
response = @github.issues.list user: @options[:user],
repo: @options[:project],
state: "closed",
2015-04-21 21:00:57 +00:00
filter: "all",
labels: nil
2015-04-21 17:42:33 +00:00
page_i = 0
count_pages = response.count_pages
response.each_page do |page|
page_i += PER_PAGE_NUMBER
2015-05-19 07:12:53 +00:00
print_in_same_line("Fetching issues... #{page_i}/#{count_pages * PER_PAGE_NUMBER}")
2015-04-21 17:42:33 +00:00
issues.concat(page)
break if @options[:max_issues] && issues.length >= @options[:max_issues]
end
2015-05-19 07:12:53 +00:00
print_empty_line
Helper.log.info "Received issues: #{issues.count}"
2015-05-19 07:12:53 +00:00
2015-04-21 17:42:33 +00:00
rescue
Helper.log.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow
2015-04-21 17:42:33 +00:00
end
2015-05-19 08:47:56 +00:00
# separate arrays of issues and pull requests:
2015-05-14 13:49:05 +00:00
issues.partition do |x|
2015-04-21 17:42:33 +00:00
x[:pull_request].nil?
2015-05-14 13:49:05 +00:00
end
2015-04-21 17:42:33 +00:00
end
2015-04-21 20:05:46 +00:00
# Fetch all pull requests. We need them to detect :merged_at parameter
# @return [Array] all pull requests
2015-05-19 08:47:56 +00:00
def fetch_closed_pull_requests
2015-04-21 17:42:33 +00:00
pull_requests = []
begin
2016-02-23 15:18:36 +00:00
response = if @options[:release_branch].nil?
@github.pull_requests.list @options[:user],
@options[:project],
state: "closed"
2016-02-23 15:18:36 +00:00
else
@github.pull_requests.list @options[:user],
@options[:project],
state: "closed",
base: @options[:release_branch]
2016-02-23 15:18:36 +00:00
end
2015-04-21 17:42:33 +00:00
page_i = 0
2015-05-19 07:12:53 +00:00
count_pages = response.count_pages
2015-04-21 17:42:33 +00:00
response.each_page do |page|
page_i += PER_PAGE_NUMBER
2015-05-19 07:12:53 +00:00
log_string = "Fetching merged dates... #{page_i}/#{count_pages * PER_PAGE_NUMBER}"
print_in_same_line(log_string)
2015-04-21 17:42:33 +00:00
pull_requests.concat(page)
end
2015-05-19 07:12:53 +00:00
print_empty_line
2015-04-21 17:42:33 +00:00
rescue
Helper.log.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow
2015-04-21 17:42:33 +00:00
end
Helper.log.info "Fetching merged dates: #{pull_requests.count}"
2015-04-21 20:05:46 +00:00
pull_requests
end
2015-04-21 17:42:33 +00:00
2015-06-10 11:18:15 +00:00
# Print specified line on the same string
# @param [String] log_string
2015-05-19 07:12:53 +00:00
def print_in_same_line(log_string)
print log_string + "\r"
end
2015-06-10 11:18:15 +00:00
# Print long line with spaces on same line to clear prev message
2015-05-19 07:12:53 +00:00
def print_empty_line
print_in_same_line(" ")
end
2015-04-21 20:05:46 +00:00
# Fetch event for all issues and add them to :events
# @param [Array] issues
# @return [Void]
def fetch_events_async(issues)
i = 0
threads = []
issues.each_slice(MAX_SIMULTANEOUS_REQUESTS) do |issues_slice|
2015-05-14 13:49:05 +00:00
issues_slice.each do |issue|
threads << Thread.new do
2015-04-21 20:05:46 +00:00
begin
response = @github.issues.events.list user: @options[:user],
repo: @options[:project],
issue_number: issue["number"]
2015-08-25 11:27:20 +00:00
issue[:events] = []
response.each_page do |page|
issue[:events].concat(page)
end
2015-04-21 20:05:46 +00:00
rescue
Helper.log.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow
2015-04-21 20:05:46 +00:00
end
2015-05-19 07:12:53 +00:00
print_in_same_line("Fetching events for issues and PR: #{i + 1}/#{issues.count}")
2015-04-21 20:05:46 +00:00
i += 1
2015-05-14 13:49:05 +00:00
end
end
2015-04-21 20:05:46 +00:00
threads.each(&:join)
threads = []
2015-05-14 13:49:05 +00:00
end
2015-04-21 17:42:33 +00:00
2015-04-21 20:05:46 +00:00
# to clear line from prev print
2015-05-19 07:12:53 +00:00
print_empty_line
2015-04-21 20:05:46 +00:00
Helper.log.info "Fetching events for issues and PR: #{i}"
2015-04-21 17:42:33 +00:00
end
2015-06-10 13:49:06 +00:00
# Fetch tag time from repo
#
# @param [Hash] tag
2015-04-21 20:05:46 +00:00
# @return [Time] time of specified tag
2015-06-10 13:49:06 +00:00
def fetch_date_of_tag(tag)
2015-04-21 20:05:46 +00:00
begin
commit_data = @github.git_data.commits.get @options[:user],
@options[:project],
tag["commit"]["sha"]
2015-04-21 20:05:46 +00:00
rescue
Helper.log.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow
2015-04-21 20:05:46 +00:00
end
2015-06-10 13:49:06 +00:00
time_string = commit_data["committer"]["date"]
Time.parse(time_string)
2015-04-21 20:05:46 +00:00
end
2015-06-10 11:18:15 +00:00
# Fetch commit for specified event
2015-04-21 20:05:46 +00:00
# @return [Hash]
def fetch_commit(event)
@github.git_data.commits.get @options[:user], @options[:project], event[:commit_id]
2015-04-21 20:05:46 +00:00
end
2015-03-26 21:56:47 +00:00
end
end