2013-10-24 20:25:52 +02:00
|
|
|
class DocsCLI < Thor
|
|
|
|
|
include Thor::Actions
|
|
|
|
|
|
|
|
|
|
def self.to_s
|
|
|
|
|
'Docs'
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def initialize(*args)
|
|
|
|
|
require 'docs'
|
|
|
|
|
trap('INT') { puts; exit! } # hide backtrace on ^C
|
|
|
|
|
super
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
desc 'list', 'List available documentations'
|
2018-11-25 12:21:20 -05:00
|
|
|
option :packaged, type: :boolean
|
2013-10-24 20:25:52 +02:00
|
|
|
def list
|
2018-11-25 12:21:20 -05:00
|
|
|
if options[:packaged]
|
2018-11-25 13:01:09 -05:00
|
|
|
slugs = Dir[File.join(Docs.store_path, '*.tar.gz')].map { |f| File.basename(f, '.tar.gz') }
|
|
|
|
|
names = find_docs_by_slugs(slugs).map do |doc|
|
|
|
|
|
name = if doc.version?
|
|
|
|
|
"#{doc.superclass.to_s.demodulize.underscore}@#{doc.version}"
|
|
|
|
|
else
|
|
|
|
|
doc.to_s.demodulize.underscore
|
|
|
|
|
end
|
|
|
|
|
end
|
2018-11-25 12:21:20 -05:00
|
|
|
else
|
|
|
|
|
names = Docs.all.flat_map do |doc|
|
|
|
|
|
name = doc.to_s.demodulize.underscore
|
|
|
|
|
if doc.versioned?
|
|
|
|
|
doc.versions.map { |_doc| "#{name}@#{_doc.version}" }
|
|
|
|
|
else
|
|
|
|
|
name
|
|
|
|
|
end
|
2016-01-24 10:03:12 -05:00
|
|
|
end
|
2018-11-25 13:01:09 -05:00
|
|
|
end
|
2016-01-24 10:03:12 -05:00
|
|
|
|
2018-11-25 13:01:09 -05:00
|
|
|
output = names.join("\n")
|
2018-11-25 12:21:20 -05:00
|
|
|
|
2018-11-25 13:01:09 -05:00
|
|
|
require 'tty-pager'
|
|
|
|
|
TTY::Pager.new.page(output)
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
|
|
|
|
|
2021-04-30 12:54:40 +02:00
|
|
|
desc 'page (<doc> | <doc@version>) [path] [--verbose] [--debug]', 'Generate a page (no indexing)'
|
2013-10-24 20:25:52 +02:00
|
|
|
option :verbose, type: :boolean
|
|
|
|
|
option :debug, type: :boolean
|
|
|
|
|
def page(name, path = '')
|
|
|
|
|
unless path.empty? || path.start_with?('/')
|
|
|
|
|
return puts 'ERROR: [path] must be an absolute path.'
|
|
|
|
|
end
|
|
|
|
|
|
2017-07-16 11:09:19 -04:00
|
|
|
Docs.install_report :image
|
2013-10-24 20:25:52 +02:00
|
|
|
Docs.install_report :store if options[:verbose]
|
|
|
|
|
if options[:debug]
|
|
|
|
|
GC.disable
|
2018-11-25 16:38:49 -05:00
|
|
|
Docs.install_report :filter, :request, :doc
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
|
|
|
|
|
2021-04-30 12:54:40 +02:00
|
|
|
name, version = name.split(/@|~/)
|
|
|
|
|
if Docs.generate_page(name, version, path)
|
2013-10-24 20:25:52 +02:00
|
|
|
puts 'Done'
|
|
|
|
|
else
|
|
|
|
|
puts "Failed!#{' (try running with --debug for more information)' unless options[:debug]}"
|
|
|
|
|
end
|
2016-01-17 13:30:46 -05:00
|
|
|
rescue Docs::DocNotFound => error
|
|
|
|
|
handle_doc_not_found_error(error)
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
|
|
|
|
|
2021-04-30 12:53:36 +02:00
|
|
|
desc 'generate (<doc> | <doc@version>) [--verbose] [--debug] [--force] [--package]', 'Generate a documentation'
|
2016-03-06 16:15:57 -05:00
|
|
|
option :all, type: :boolean
|
2013-10-24 20:25:52 +02:00
|
|
|
option :verbose, type: :boolean
|
|
|
|
|
option :debug, type: :boolean
|
|
|
|
|
option :force, type: :boolean
|
2015-04-05 14:34:00 -04:00
|
|
|
option :package, type: :boolean
|
2013-10-24 20:25:52 +02:00
|
|
|
def generate(name)
|
2016-09-05 15:21:20 -04:00
|
|
|
Docs.rescue_errors = true
|
2013-10-24 20:25:52 +02:00
|
|
|
Docs.install_report :store if options[:verbose]
|
|
|
|
|
Docs.install_report :scraper if options[:debug]
|
2018-11-18 11:29:50 -05:00
|
|
|
Docs.install_report :progress_bar, :doc, :image, :requester if $stdout.tty?
|
2013-10-24 20:25:52 +02:00
|
|
|
|
2016-05-29 11:33:07 -04:00
|
|
|
require 'unix_utils' if options[:package]
|
|
|
|
|
|
2021-04-30 12:53:36 +02:00
|
|
|
doc = find_doc(name)
|
2016-05-29 11:33:07 -04:00
|
|
|
|
|
|
|
|
if doc < Docs::UrlScraper && !options[:force]
|
2013-10-24 20:25:52 +02:00
|
|
|
puts <<-TEXT.strip_heredoc
|
2016-05-29 11:33:07 -04:00
|
|
|
/!\\ WARNING /!\\
|
|
|
|
|
|
|
|
|
|
Some scrapers send thousands of HTTP requests in a short period of time,
|
|
|
|
|
which can slow down the source site and trouble its maintainers.
|
2013-10-24 20:25:52 +02:00
|
|
|
|
2016-05-29 11:33:07 -04:00
|
|
|
Please scrape responsibly. Don't do it unless you're modifying the code.
|
|
|
|
|
|
|
|
|
|
To download the latest tested version of this documentation, run:
|
|
|
|
|
thor docs:download #{name}\n
|
2013-10-24 20:25:52 +02:00
|
|
|
TEXT
|
|
|
|
|
return unless yes? 'Proceed? (y/n)'
|
|
|
|
|
end
|
|
|
|
|
|
2016-03-06 16:15:57 -05:00
|
|
|
result = if doc.version && options[:all]
|
|
|
|
|
doc.superclass.versions.all? do |_doc|
|
|
|
|
|
puts "==> #{_doc.version}"
|
|
|
|
|
generate_doc(_doc, package: options[:package]).tap { puts "\n" }
|
2015-04-05 14:34:00 -04:00
|
|
|
end
|
2013-10-24 20:25:52 +02:00
|
|
|
else
|
2016-03-06 16:15:57 -05:00
|
|
|
generate_doc(doc, package: options[:package])
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
2016-03-06 16:15:57 -05:00
|
|
|
|
|
|
|
|
generate_manifest if result
|
2016-01-17 13:30:46 -05:00
|
|
|
rescue Docs::DocNotFound => error
|
|
|
|
|
handle_doc_not_found_error(error)
|
2016-09-05 15:21:20 -04:00
|
|
|
ensure
|
|
|
|
|
Docs.rescue_errors = false
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
|
|
|
|
|
|
|
|
|
desc 'manifest', 'Create the manifest'
|
|
|
|
|
def manifest
|
2013-10-28 13:48:34 +01:00
|
|
|
generate_manifest
|
2013-10-24 20:25:52 +02:00
|
|
|
puts 'Done'
|
|
|
|
|
end
|
|
|
|
|
|
2021-04-30 11:38:05 +02:00
|
|
|
desc 'download (<doc> <doc@version>... | --default | --installed | --all)', 'Download documentation packages'
|
2016-01-24 10:51:13 -05:00
|
|
|
option :default, type: :boolean
|
|
|
|
|
option :installed, type: :boolean
|
2013-10-24 20:25:52 +02:00
|
|
|
option :all, type: :boolean
|
|
|
|
|
def download(*names)
|
|
|
|
|
require 'unix_utils'
|
2016-01-24 10:51:13 -05:00
|
|
|
docs = if options[:default]
|
|
|
|
|
Docs.defaults
|
|
|
|
|
elsif options[:installed]
|
|
|
|
|
Docs.installed
|
|
|
|
|
elsif options[:all]
|
|
|
|
|
Docs.all_versions
|
|
|
|
|
else
|
|
|
|
|
find_docs(names)
|
|
|
|
|
end
|
2013-10-24 20:25:52 +02:00
|
|
|
assert_docs(docs)
|
|
|
|
|
download_docs(docs)
|
2013-10-28 13:48:34 +01:00
|
|
|
generate_manifest
|
2013-10-24 20:25:52 +02:00
|
|
|
puts 'Done'
|
|
|
|
|
rescue Docs::DocNotFound => error
|
2016-01-17 13:30:46 -05:00
|
|
|
handle_doc_not_found_error(error)
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
|
|
|
|
|
2021-04-30 11:38:05 +02:00
|
|
|
desc 'package <doc> <doc@version>...', 'Create documentation packages'
|
2013-10-24 20:25:52 +02:00
|
|
|
def package(*names)
|
|
|
|
|
require 'unix_utils'
|
2016-01-24 09:36:12 -05:00
|
|
|
docs = find_docs(names)
|
2013-10-24 20:25:52 +02:00
|
|
|
assert_docs(docs)
|
|
|
|
|
docs.each(&method(:package_doc))
|
|
|
|
|
puts 'Done'
|
|
|
|
|
rescue Docs::DocNotFound => error
|
2016-01-17 13:30:46 -05:00
|
|
|
handle_doc_not_found_error(error)
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
|
|
|
|
|
2013-10-28 15:16:01 +01:00
|
|
|
desc 'clean', 'Delete documentation packages'
|
|
|
|
|
def clean
|
|
|
|
|
File.delete(*Dir[File.join Docs.store_path, '*.tar.gz'])
|
|
|
|
|
puts 'Done'
|
|
|
|
|
end
|
|
|
|
|
|
2016-01-24 11:14:30 -05:00
|
|
|
desc 'upload', '[private]'
|
|
|
|
|
option :dryrun, type: :boolean
|
2016-03-06 17:36:12 -05:00
|
|
|
option :packaged, type: :boolean
|
2016-01-24 11:14:30 -05:00
|
|
|
def upload(*names)
|
2018-11-25 12:55:35 -05:00
|
|
|
if options[:packaged]
|
|
|
|
|
slugs = Dir[File.join(Docs.store_path, '*.tar.gz')].map { |f| File.basename(f, '.tar.gz') }
|
|
|
|
|
docs = find_docs_by_slugs(slugs)
|
|
|
|
|
else
|
|
|
|
|
docs = find_docs(names)
|
|
|
|
|
end
|
|
|
|
|
|
2016-01-24 11:14:30 -05:00
|
|
|
assert_docs(docs)
|
2018-11-25 12:22:08 -05:00
|
|
|
|
2018-12-18 15:47:11 -05:00
|
|
|
# Verify files are present
|
|
|
|
|
docs.each do |doc|
|
2022-09-21 21:00:47 +02:00
|
|
|
unless Dir.exist?(File.join(Docs.store_path, doc.path))
|
2018-12-18 15:47:11 -05:00
|
|
|
puts "ERROR: directory #{File.join(Docs.store_path, doc.path)} not found."
|
|
|
|
|
return
|
|
|
|
|
end
|
|
|
|
|
|
2022-09-19 13:35:47 -07:00
|
|
|
unless File.exist?(File.join(Docs.store_path, "#{doc.path}.tar.gz"))
|
2018-12-18 15:47:11 -05:00
|
|
|
puts "ERROR: package for '#{doc.slug}' documentation not found. Run 'thor docs:package #{doc.slug}' to create it."
|
|
|
|
|
return
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
2018-11-25 12:22:08 -05:00
|
|
|
# Sync files with S3 (used by the web app)
|
|
|
|
|
puts '[S3] Begin syncing.'
|
2016-01-24 11:14:30 -05:00
|
|
|
docs.each do |doc|
|
2018-11-25 12:22:08 -05:00
|
|
|
puts "[S3] Syncing #{doc.path}..."
|
2021-02-16 12:00:13 +01:00
|
|
|
cmd = "aws s3 sync #{File.join(Docs.store_path, doc.path)} s3://devdocs-documents/#{doc.path} --delete --profile devdocs"
|
2016-01-24 11:14:30 -05:00
|
|
|
cmd << ' --dryrun' if options[:dryrun]
|
|
|
|
|
system(cmd)
|
|
|
|
|
end
|
2018-11-25 12:22:08 -05:00
|
|
|
puts '[S3] Done syncing.'
|
|
|
|
|
|
2021-02-16 12:04:35 +01:00
|
|
|
# Upload packages to downloads.devdocs.io (used by the "thor docs:download" command)
|
2021-01-22 18:19:49 +01:00
|
|
|
puts '[S3 bundle] Begin uploading.'
|
|
|
|
|
|
|
|
|
|
docs.each do |doc|
|
|
|
|
|
filename = "#{doc.path}.tar.gz"
|
2021-02-16 17:25:55 +01:00
|
|
|
puts "[S3 bundle] Uploading #{filename}..."
|
2021-02-16 12:06:09 +01:00
|
|
|
cmd = "aws s3 cp #{File.join(Docs.store_path, filename)} s3://devdocs-downloads/#{filename} --profile devdocs"
|
2021-01-22 18:19:49 +01:00
|
|
|
cmd << ' --dryrun' if options[:dryrun]
|
|
|
|
|
system(cmd)
|
2018-11-25 12:22:08 -05:00
|
|
|
end
|
2021-01-22 18:19:49 +01:00
|
|
|
puts '[S3 bundle] Done uploading.'
|
2016-01-24 11:14:30 -05:00
|
|
|
end
|
|
|
|
|
|
2016-04-03 09:54:29 -04:00
|
|
|
desc 'commit', '[private]'
|
|
|
|
|
option :message, type: :string
|
|
|
|
|
option :amend, type: :boolean
|
|
|
|
|
def commit(name)
|
|
|
|
|
doc = Docs.find(name, false)
|
2017-09-10 11:05:26 -04:00
|
|
|
message = options[:message] || "Update #{doc.name} documentation (#{doc.versions.first.release})"
|
2016-04-03 09:54:29 -04:00
|
|
|
amend = " --amend" if options[:amend]
|
2016-04-23 11:44:00 -04:00
|
|
|
system("git add assets/ *#{name}*") && system("git commit -m '#{message}'#{amend}")
|
2016-04-03 09:54:29 -04:00
|
|
|
rescue Docs::DocNotFound => error
|
|
|
|
|
handle_doc_not_found_error(error)
|
|
|
|
|
end
|
|
|
|
|
|
2018-10-21 18:41:35 -04:00
|
|
|
desc 'prepare_deploy', 'Internal task executed before deployment'
|
|
|
|
|
def prepare_deploy
|
|
|
|
|
puts 'Docs -- BEGIN'
|
|
|
|
|
|
|
|
|
|
require 'open-uri'
|
|
|
|
|
require 'thread'
|
|
|
|
|
|
|
|
|
|
docs = Docs.all_versions
|
|
|
|
|
time = Time.now.to_i
|
|
|
|
|
mutex = Mutex.new
|
|
|
|
|
|
|
|
|
|
(1..6).map do
|
|
|
|
|
Thread.new do
|
|
|
|
|
while doc = docs.shift
|
|
|
|
|
dir = File.join(Docs.store_path, doc.path)
|
|
|
|
|
FileUtils.mkpath(dir)
|
|
|
|
|
|
|
|
|
|
['index.json', 'meta.json'].each do |filename|
|
2021-02-16 11:49:32 +01:00
|
|
|
json = "https://documents.devdocs.io/#{doc.path}/#{filename}?#{time}"
|
2020-11-15 22:10:18 +01:00
|
|
|
begin
|
2025-01-21 12:17:23 +01:00
|
|
|
URI.open(json, "Accept-Encoding" => "identity") do |file|
|
2020-11-15 22:10:18 +01:00
|
|
|
mutex.synchronize do
|
|
|
|
|
path = File.join(dir, filename)
|
|
|
|
|
File.write(path, file.read)
|
|
|
|
|
end
|
2018-10-21 18:41:35 -04:00
|
|
|
end
|
2020-11-15 22:10:18 +01:00
|
|
|
rescue => e
|
|
|
|
|
puts "Docs -- Failed to download #{json}!"
|
|
|
|
|
throw e
|
2018-10-21 18:41:35 -04:00
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
puts "Docs -- Downloaded #{doc.slug}"
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
end.map(&:join)
|
|
|
|
|
|
|
|
|
|
puts 'Docs -- Generating manifest...'
|
|
|
|
|
generate_manifest
|
|
|
|
|
|
|
|
|
|
puts 'Docs -- DONE'
|
|
|
|
|
end
|
|
|
|
|
|
2013-10-24 20:25:52 +02:00
|
|
|
private
|
|
|
|
|
|
2021-04-30 12:52:15 +02:00
|
|
|
def find_doc(name)
|
|
|
|
|
name, version = name.split(/@|~/)
|
|
|
|
|
if version == 'all'
|
|
|
|
|
Docs.find(name, false).versions
|
|
|
|
|
else
|
|
|
|
|
Docs.find(name, version)
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
2021-04-30 12:52:15 +02:00
|
|
|
def find_docs(names)
|
|
|
|
|
names.flat_map {|name| find_doc(name)}
|
|
|
|
|
end
|
|
|
|
|
|
2018-11-25 12:55:35 -05:00
|
|
|
def find_docs_by_slugs(slugs)
|
|
|
|
|
slugs.flat_map do |slug|
|
|
|
|
|
slug, version = slug.split(/~/)
|
|
|
|
|
Docs.find_by_slug(slug, version)
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
2013-10-24 20:25:52 +02:00
|
|
|
def assert_docs(docs)
|
|
|
|
|
if docs.empty?
|
|
|
|
|
puts 'ERROR: called with no arguments.'
|
2016-01-24 10:51:13 -05:00
|
|
|
puts 'Run "thor list" for usage patterns.'
|
2013-10-24 20:25:52 +02:00
|
|
|
exit
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
2016-01-17 13:30:46 -05:00
|
|
|
def handle_doc_not_found_error(error)
|
|
|
|
|
puts %(ERROR: #{error}.)
|
|
|
|
|
puts 'Run "thor docs:list" to see the list of docs and versions.'
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
|
|
|
|
|
2016-03-06 16:15:57 -05:00
|
|
|
def generate_doc(doc, package: nil)
|
|
|
|
|
if Docs.generate(doc)
|
|
|
|
|
package_doc(doc) if package
|
|
|
|
|
puts 'Done'
|
|
|
|
|
true
|
|
|
|
|
else
|
|
|
|
|
puts "Failed!#{' (try running with --debug for more information)' unless options[:debug]}"
|
|
|
|
|
false
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
2013-10-24 20:25:52 +02:00
|
|
|
def download_docs(docs)
|
2014-08-02 11:50:32 -04:00
|
|
|
# Don't allow downloaded files to be created as StringIO
|
|
|
|
|
require 'open-uri'
|
|
|
|
|
OpenURI::Buffer.send :remove_const, 'StringMax' if OpenURI::Buffer.const_defined?('StringMax')
|
|
|
|
|
OpenURI::Buffer.const_set 'StringMax', 0
|
|
|
|
|
|
2013-10-24 20:25:52 +02:00
|
|
|
require 'thread'
|
|
|
|
|
length = docs.length
|
2016-01-24 10:51:13 -05:00
|
|
|
mutex = Mutex.new
|
2013-10-24 20:25:52 +02:00
|
|
|
i = 0
|
|
|
|
|
|
|
|
|
|
(1..4).map do
|
|
|
|
|
Thread.new do
|
|
|
|
|
while doc = docs.shift
|
|
|
|
|
status = begin
|
|
|
|
|
download_doc(doc)
|
|
|
|
|
'OK'
|
2014-08-02 11:50:46 -04:00
|
|
|
rescue => e
|
|
|
|
|
"FAILED (#{e.class}: #{e.message})"
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
2016-01-24 10:51:13 -05:00
|
|
|
mutex.synchronize { puts "(#{i += 1}/#{length}) #{doc.name}#{ " #{doc.version}" if doc.version} #{status}" }
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
end.map(&:join)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def download_doc(doc)
|
2016-08-13 11:38:22 -04:00
|
|
|
target_path = File.join(Docs.store_path, doc.path)
|
2021-08-23 22:22:01 +02:00
|
|
|
URI.open "https://downloads.devdocs.io/#{doc.path}.tar.gz" do |file|
|
2016-08-13 11:38:22 -04:00
|
|
|
FileUtils.mkpath(target_path)
|
|
|
|
|
file.close
|
|
|
|
|
tar = UnixUtils.gunzip(file.path)
|
|
|
|
|
dir = UnixUtils.untar(tar)
|
2023-08-29 20:04:29 +02:00
|
|
|
FileUtils.rm(tar)
|
2016-08-13 11:38:22 -04:00
|
|
|
FileUtils.rm_rf(target_path)
|
|
|
|
|
FileUtils.mv(dir, target_path)
|
|
|
|
|
FileUtils.rm(file.path)
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def package_doc(doc)
|
|
|
|
|
path = File.join Docs.store_path, doc.path
|
|
|
|
|
|
|
|
|
|
if File.exist?(path)
|
|
|
|
|
tar = UnixUtils.tar(path)
|
|
|
|
|
gzip = UnixUtils.gzip(tar)
|
|
|
|
|
FileUtils.mv(gzip, "#{path}.tar.gz")
|
|
|
|
|
FileUtils.rm(tar)
|
|
|
|
|
else
|
|
|
|
|
puts %(ERROR: can't find "#{doc.name}" documentation files.)
|
|
|
|
|
end
|
|
|
|
|
end
|
2013-10-28 13:48:34 +01:00
|
|
|
|
|
|
|
|
def generate_manifest
|
|
|
|
|
Docs.generate_manifest
|
|
|
|
|
end
|
2013-10-24 20:25:52 +02:00
|
|
|
end
|
