github-changelog-generator/lib/github_changelog_generator/fetcher.rb

220 lines
7.3 KiB
Ruby
Raw Normal View History

2015-04-21 21:26:28 +00:00
require "logger"
2015-03-26 21:56:47 +00:00
module GitHubChangelogGenerator
2015-04-21 21:00:57 +00:00
# A Fetcher responsible for all requests to GitHub and all basic manipulation with related data
# (such as filtering, validating, e.t.c)
#
# Example:
# fetcher = GitHubChangelogGenerator::Fetcher.new options
2015-03-26 21:56:47 +00:00
class Fetcher
2015-04-21 17:42:33 +00:00
PER_PAGE_NUMBER = 30
2015-05-14 12:57:31 +00:00
CHANGELOG_GITHUB_TOKEN = "CHANGELOG_GITHUB_TOKEN"
2015-05-18 12:04:15 +00:00
GH_RATE_LIMIT_EXCEEDED_MSG = "Warning: Can't finish operation: GitHub API rate limit exceeded, change log may be " \
"missing some issues. You can limit the number of issues fetched using the `--max-issues NUM` argument."
NO_TOKEN_PROVIDED = "Warning: No token provided (-t option) and variable $CHANGELOG_GITHUB_TOKEN was not found. " \
"This script can make only 50 requests to GitHub API per hour without token!"
2015-04-21 17:42:33 +00:00
def initialize(options = {})
@options = options
2015-04-21 21:26:28 +00:00
@logger = Logger.new(STDOUT)
2015-04-21 21:33:50 +00:00
@logger.formatter = proc do |_severity, _datetime, _progname, msg|
2015-04-21 21:26:28 +00:00
"#{msg}\n"
end
2015-05-18 09:00:08 +00:00
@user = @options[:user]
@project = @options[:project]
@github_token = fetch_github_token
2015-04-21 20:05:46 +00:00
@tag_times_hash = {}
2015-05-18 12:50:10 +00:00
github_options = { per_page: PER_PAGE_NUMBER }
github_options[:oauth_token] = @github_token unless @github_token.nil?
github_options[:endpoint] = options[:github_endpoint] unless options[:github_endpoint].nil?
github_options[:site] = options[:github_endpoint] unless options[:github_site].nil?
2015-05-18 12:50:10 +00:00
@github = check_github_response { Github.new github_options }
end
2015-04-21 21:00:57 +00:00
# Returns GitHub token. First try to use variable, provided by --token option,
# otherwise try to fetch it from CHANGELOG_GITHUB_TOKEN env variable.
#
# @return [String]
def fetch_github_token
2015-05-14 12:57:31 +00:00
env_var = @options[:token] ? @options[:token] : (ENV.fetch CHANGELOG_GITHUB_TOKEN, nil)
unless env_var
2015-05-18 12:04:15 +00:00
@logger.warn NO_TOKEN_PROVIDED.yellow
end
env_var
end
2015-04-21 17:42:33 +00:00
# Fetch all tags from repo
# @return [Array] array of tags
def get_all_tags
if @options[:verbose]
print "Fetching tags...\r"
end
tags = []
2015-05-18 12:04:15 +00:00
check_github_response { github_fetch_tags(tags) }
tags
end
def check_github_response
2015-04-21 17:42:33 +00:00
begin
2015-05-18 12:04:15 +00:00
value = yield
2015-05-18 06:26:36 +00:00
rescue Github::Error::Unauthorized => e
@logger.error e.body.red
2015-05-18 12:04:15 +00:00
abort "Error: wrong GitHub token"
2015-05-18 06:26:36 +00:00
rescue Github::Error::Forbidden => e
@logger.warn e.body.red
2015-05-18 12:04:15 +00:00
@logger.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow
2015-05-18 06:26:36 +00:00
end
2015-05-18 12:04:15 +00:00
value
2015-05-18 06:26:36 +00:00
end
2015-04-21 17:42:33 +00:00
2015-05-18 06:26:36 +00:00
def github_fetch_tags(tags)
response = @github.repos.tags @options[:user], @options[:project]
page_i = 0
count_pages = response.count_pages
response.each_page do |page|
page_i += PER_PAGE_NUMBER
print "Fetching tags... #{page_i}/#{count_pages * PER_PAGE_NUMBER}\r"
tags.concat(page)
2015-04-21 17:42:33 +00:00
end
2015-05-18 06:26:36 +00:00
print " \r"
2015-04-21 17:42:33 +00:00
2015-05-18 06:26:36 +00:00
if tags.count == 0
@logger.warn "Warning: Can't find any tags in repo.\
Make sure, that you push tags to remote repo via 'git push --tags'".yellow
elsif @options[:verbose]
@logger.info "Found #{tags.count} tags"
end
2015-04-21 17:42:33 +00:00
end
# This method fetch all closed issues and separate them to pull requests and pure issues
# (pull request is kind of issue in term of GitHub)
# @return [Tuple] with issues and pull requests
def fetch_issues_and_pull_requests
if @options[:verbose]
print "Fetching closed issues...\r"
end
issues = []
begin
2015-04-21 21:00:57 +00:00
response = @github.issues.list user: @options[:user],
repo: @options[:project],
state: "closed",
filter: "all",
labels: nil
2015-04-21 17:42:33 +00:00
page_i = 0
count_pages = response.count_pages
response.each_page do |page|
page_i += PER_PAGE_NUMBER
print "Fetching issues... #{page_i}/#{count_pages * PER_PAGE_NUMBER}\r"
issues.concat(page)
break if @options[:max_issues] && issues.length >= @options[:max_issues]
end
rescue
2015-04-21 21:26:28 +00:00
@logger.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow
2015-04-21 17:42:33 +00:00
end
print " \r"
if @options[:verbose]
2015-04-21 21:26:28 +00:00
@logger.info "Received issues: #{issues.count}"
2015-04-21 17:42:33 +00:00
end
# remove pull request from issues:
2015-05-14 13:49:05 +00:00
issues.partition do |x|
2015-04-21 17:42:33 +00:00
x[:pull_request].nil?
2015-05-14 13:49:05 +00:00
end
2015-04-21 17:42:33 +00:00
end
2015-04-21 20:05:46 +00:00
# Fetch all pull requests. We need them to detect :merged_at parameter
# @return [Array] all pull requests
def fetch_pull_requests
2015-04-21 17:42:33 +00:00
pull_requests = []
begin
response = @github.pull_requests.list @options[:user], @options[:project], state: "closed"
page_i = 0
response.each_page do |page|
page_i += PER_PAGE_NUMBER
count_pages = response.count_pages
print "Fetching merged dates... #{page_i}/#{count_pages * PER_PAGE_NUMBER}\r"
pull_requests.concat(page)
end
rescue
2015-04-21 21:26:28 +00:00
@logger.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow
2015-04-21 17:42:33 +00:00
end
print " \r"
2015-04-21 20:05:46 +00:00
pull_requests
end
2015-04-21 17:42:33 +00:00
2015-04-21 20:05:46 +00:00
# Fetch event for all issues and add them to :events
# @param [Array] issues
# @return [Void]
def fetch_events_async(issues)
i = 0
max_thread_number = 50
threads = []
2015-05-14 13:49:05 +00:00
issues.each_slice(max_thread_number) do |issues_slice|
issues_slice.each do |issue|
threads << Thread.new do
2015-04-21 20:05:46 +00:00
begin
2015-04-21 21:00:57 +00:00
obj = @github.issues.events.list user: @options[:user],
repo: @options[:project],
issue_number: issue["number"]
2015-04-21 20:05:46 +00:00
rescue
2015-04-21 21:26:28 +00:00
@logger.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow
2015-04-21 20:05:46 +00:00
end
issue[:events] = obj.body
print "Fetching events for issues and PR: #{i + 1}/#{issues.count}\r"
i += 1
2015-05-14 13:49:05 +00:00
end
end
2015-04-21 20:05:46 +00:00
threads.each(&:join)
threads = []
2015-05-14 13:49:05 +00:00
end
2015-04-21 17:42:33 +00:00
2015-04-21 20:05:46 +00:00
# to clear line from prev print
print " \r"
2015-04-21 17:42:33 +00:00
if @options[:verbose]
2015-04-21 21:26:28 +00:00
@logger.info "Fetching events for issues and PR: #{i} Done!"
2015-04-21 17:42:33 +00:00
end
end
2015-04-21 20:05:46 +00:00
# Try to find tag date in local hash.
# Otherwise fFetch tag time and put it to local hash file.
# @param [String] tag_name name of the tag
# @return [Time] time of specified tag
def get_time_of_tag(tag_name)
fail ChangelogGeneratorError, "tag_name is nil".red if tag_name.nil?
if @tag_times_hash[tag_name["name"]]
return @tag_times_hash[tag_name["name"]]
end
begin
2015-04-21 21:00:57 +00:00
github_git_data_commits_get = @github.git_data.commits.get @options[:user],
@options[:project],
tag_name["commit"]["sha"]
2015-04-21 20:05:46 +00:00
rescue
2015-04-21 21:26:28 +00:00
@logger.warn GH_RATE_LIMIT_EXCEEDED_MSG.yellow
2015-04-21 20:05:46 +00:00
end
time_string = github_git_data_commits_get["committer"]["date"]
@tag_times_hash[tag_name["name"]] = Time.parse(time_string)
end
# Fetch commit for specifed event
# @return [Hash]
def fetch_commit(event)
@github.git_data.commits.get @options[:user], @options[:project], event[:commit_id]
end
2015-03-26 21:56:47 +00:00
end
end