class Metanorma::Compile

Constants

REQUIREMENT_XPATH
SPLITSECTIONS

Attributes

errors[R]

@return [Array<String>]

processor[R]

@return [Array<String>]

Public Class Methods

new() click to toggle source
# File lib/metanorma/compile.rb, line 17
def initialize
  @registry = Metanorma::Registry.instance
  @errors = []
  @isodoc = IsoDoc::Convert.new({})
  @fontist_installed = false
end

Public Instance Methods

build_collection(xml, presxml, filename, dir, opts = {}) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 22
def build_collection(xml, presxml, filename, dir, opts = {})
  base = File.basename(filename)
  collection_setup(base, dir)
  files = sectionsplit(xml, base, dir)
  collection_manifest(base, files, xml, presxml, dir).render(
    { format: %i(html), output_folder: "#{filename}_collection",
      coverpage: File.join(dir, "cover.html") }.merge(opts),
  )
end
clean_sourcecode(xml) click to toggle source
# File lib/metanorma/compile.rb, line 131
def clean_sourcecode(xml)
  xml.xpath(".//callout | .//annotation | .//xmlns:callout | "\
            ".//xmlns:annotation").each(&:remove)
  xml.xpath(".//br | .//xmlns:br").each { |x| x.replace("\n") }
  HTMLEntities.new.decode(xml.children.to_xml)
end
coll_cover() click to toggle source
# File lib/metanorma/sectionsplit.rb, line 47
    def coll_cover
      <<~COVER
        <html><head/>
            <body>
              <h1>{{ doctitle }}</h1>
              <h2>{{ docnumber }}</h2>
              <nav>{{ labels["navigation"] }}</nav>
            </body>
        </html>
      COVER
    end
collection_manifest(filename, files, origxml, _presxml, dir) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 32
def collection_manifest(filename, files, origxml, _presxml, dir)
  File.open(File.join(dir, "#{filename}.html.yaml"), "w:UTF-8") do |f|
    f.write(collectionyaml(files, origxml))
  end
  Metanorma::Collection.parse File.join(dir, "#{filename}.html.yaml")
end
collection_setup(filename, dir) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 39
def collection_setup(filename, dir)
  FileUtils.mkdir_p "#{filename}_collection" if filename
  FileUtils.mkdir_p dir
  File.open(File.join(dir, "cover.html"), "w:UTF-8") do |f|
    f.write(coll_cover)
  end
end
collectionyaml(files, xml) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 230
def collectionyaml(files, xml)
  ret = {
    directives: ["presentation-xml", "bare-after-first"],
    bibdata: {
      title: {
        type: "title-main", language: @lang,
        content: xml.at(ns("//bibdata/title")).text
      },
      type: "collection",
      docid: {
        type: xml.at(ns("//bibdata/docidentifier/@type")).text,
        id: xml.at(ns("//bibdata/docidentifier")).text,
      },
    },
    manifest: {
      level: "collection", title: "Collection",
      docref: files.sort_by { |f| f[:order] }.each.map do |f|
        { fileref: f[:url], identifier: f[:title] }
      end
    },
  }
  recursive_string_keys(ret).to_yaml
end
compile(filename, options = {}) click to toggle source
# File lib/metanorma/compile.rb, line 24
def compile(filename, options = {})
  require_libraries(options)
  options = options_extract(filename, options)
  validate_options(options)
  @processor = @registry.find_processor(options[:type].to_sym)
  extensions = get_extensions(options) or return nil
  (file, isodoc = process_input(filename, options)) or return nil
  relaton_export(isodoc, options)
  extract(isodoc, options[:extract], options[:extract_type])
  FontistUtils.install_fonts(@processor, options) unless @fontist_installed
  @fontist_installed = true
  process_extensions(extensions, file, isodoc, options)
end
copy_repo_items_biblio(ins, section, xml) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 194
def copy_repo_items_biblio(ins, section, xml)
  xml.xpath(ns("//references/bibitem[docidentifier/@type = 'repository']"))
    .each_with_object([]) do |b, m|
    section.at("//*[@bibitemid = '#{b['id']}']") or next
    ins << b.dup
    m << b["id"]
  end
end
create_sectionfile(xml, out, file, chunk, parentnode) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 91
def create_sectionfile(xml, out, file, chunk, parentnode)
  ins = out.at(ns("//misccontainer")) || out.at(ns("//bibdata"))
  if parentnode
    ins.next = "<#{parentnode}/>"
    ins.next.add_child(chunk.dup)
  else ins.next = chunk.dup
  end
  xref_process(out, xml, @key)
  outname = "#{file}.xml"
  File.open(File.join(@splitdir, outname), "w:UTF-8") { |f| f.write(out) }
  outname
end
emptydoc(xml) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 76
def emptydoc(xml)
  out = xml.dup
  out.xpath(
    ns("//preface | //sections | //annex | //bibliography/clause | "\
       "//bibliography/references[not(@hidden = 'true')] | //indexsect"),
  ).each(&:remove)
  out
end
eref_to_internal_eref(section, xml, key) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 155
def eref_to_internal_eref(section, xml, key)
  eref_to_internal_eref_select(section, xml).each_with_object([]) do |x, m|
    url = xml.at(ns("//bibitem[@id = '#{x}']/url[@type = 'citation']"))
    section.xpath(("//*[@bibitemid = '#{x}']")).each do |e|
      id = eref_to_internal_eref1(e, key, url)
      id and m << id
    end
  end
end
eref_to_internal_eref1(elem, key, url) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 165
def eref_to_internal_eref1(elem, key, url)
  if url
    elem.name = "link"
    elem["target"] = url
    nil
  else
    elem["bibitemid"] = "#{key}_#{elem['bibitemid']}"
    elem << make_anchor(elem["bibitemid"])
    elem["type"] = key
    elem["bibitemid"]
  end
end
eref_to_internal_eref_select(section, xml) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 178
def eref_to_internal_eref_select(section, xml)
  refs = section.xpath(("//*/@bibitemid")).map { |x| x.text } # rubocop:disable Style/SymbolProc
  refs.uniq.reject do |x|
    xml.at(ns("//bibitem[@id = '#{x}'][@type = 'internal']")) ||
      xml.at(ns("//bibitem[@id = '#{x}']"\
                "[docidentifier/@type = 'repository']"))
  end
end
extract(isodoc, dirname, extract_types) click to toggle source
# File lib/metanorma/compile.rb, line 138
def extract(isodoc, dirname, extract_types)
  return unless dirname

  if extract_types.nil? || extract_types.empty?
    extract_types = %i[sourcecode image requirement]
  end
  FileUtils.rm_rf dirname
  FileUtils.mkdir_p dirname
  xml = Nokogiri::XML(isodoc) { |config| config.huge }
  sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
  image_export(xml, dirname) if extract_types.include? :image
  requirement_export(xml, dirname) if extract_types.include? :requirement
end
get_extensions(options) click to toggle source
# File lib/metanorma/compile.rb, line 68
def get_extensions(options)
  options[:extension_keys] ||=
    @processor.output_formats.reduce([]) { |memo, (k, _)| memo << k }
  extensions = options[:extension_keys].reduce([]) do |memo, e|
    if @processor.output_formats[e] then memo << e
    else
      message = "[metanorma] Error: #{e} format is not supported for this standard."
      @errors << message
      Util.log(message, :error)
      memo
    end
  end
  if !extensions.include?(:presentation) && extensions.any? do |e|
    @processor.use_presentation_xml(e)
  end
    extensions << :presentation
  end
  extensions
end
image_export(xml, dirname) click to toggle source
# File lib/metanorma/compile.rb, line 163
def image_export(xml, dirname)
  xml.at("//image | //xmlns:image") or return
  FileUtils.mkdir_p "#{dirname}/image"
  xml.xpath("//image | //xmlns:image").each_with_index do |s, i|
    next unless /^data:image/.match? s["src"]

    %r{^data:image/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ s["src"]
    filename = s["filename"] || sprintf("image-%04d.%s", i, imgtype)
    File.open("#{dirname}/image/#{filename}", "wb") do |f|
      f.write(Base64.strict_decode64(imgdata))
    end
  end
end
insert_indirect_biblio(ins, refs, prefix) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 203
    def insert_indirect_biblio(ins, refs, prefix)
      refs.each do |x|
        ins << <<~BIBENTRY
          <bibitem id="#{x}" type="internal">
          <docidentifier type="repository">#{x.sub(/^#{prefix}_/, "#{prefix}/")}</docidentifier>
          </bibitem>
        BIBENTRY
      end
    end
make_anchor(anchor) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 139
def make_anchor(anchor)
  "<localityStack><locality type='anchor'><referenceFrom>"\
    "#{anchor}</referenceFrom></locality></localityStack>"
end
new_hidden_ref(xmldoc) click to toggle source

from standoc

# File lib/metanorma/sectionsplit.rb, line 188
def new_hidden_ref(xmldoc)
  ins = xmldoc.at("bibliography") or
    xmldoc.root << "<bibliography/>" and ins = xmldoc.at("bibliography")
  ins.add_child("<references hidden='true' normative='false'/>").first
end
ns(xpath) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 18
def ns(xpath)
  @isodoc.ns(xpath)
end
options_extract(filename, options) click to toggle source
# File lib/metanorma/compile.rb, line 52
def options_extract(filename, options)
  content = read_file(filename)
  o = Metanorma::Input::Asciidoc.new.extract_metanorma_options(content)
    .merge(xml_options_extract(content))
  options[:type] ||= o[:type]&.to_sym
  t = @registry.alias(options[:type]) and options[:type] = t
  dir = filename.sub(%r(/[^/]+$), "/")
  options[:relaton] ||= "#{dir}/#{o[:relaton]}" if o[:relaton]
  options[:sourcecode] ||= "#{dir}/#{o[:sourcecode]}" if o[:sourcecode]
  options[:extension_keys] ||= o[:extensions]&.split(/, */)&.map(&:to_sym)
  options[:extension_keys] = nil if options[:extension_keys] == [:all]
  options[:format] ||= :asciidoc
  options[:filename] = filename
  options
end
process_extensions(extensions, file, isodoc, options) click to toggle source

isodoc is Raw Metanorma XML

# File lib/metanorma/compile.rb, line 202
def process_extensions(extensions, file, isodoc, options)
  f = change_output_dir options
  xml_name = f.sub(/\.[^.]+$/, ".xml")
  presentationxml_name = f.sub(/\.[^.]+$/, ".presentation.xml")
  Util.sort_extensions_execution(extensions).each do |ext|
    file_extension = @processor.output_formats[ext]
    outfilename = f.sub(/\.[^.]+$/, ".#{file_extension}")
    isodoc_options = get_isodoc_options(file, options, ext)
    if ext == :rxl
      relaton_export(isodoc, options.merge(relaton: outfilename))
    elsif options[:passthrough_presentation_xml] && ext == :presentation
      FileUtils.cp f, presentationxml_name
    elsif ext == :html && options[:sectionsplit]
      sectionsplit_convert(xml_name, isodoc, outfilename, isodoc_options)
    else
      begin
        if @processor.use_presentation_xml(ext)
          @processor.output(nil, presentationxml_name, outfilename, ext,
                            isodoc_options)
        else
          @processor.output(isodoc, xml_name, outfilename, ext,
                            isodoc_options)
        end
      rescue StandardError => e
        isodoc_error_process(e)
      end
    end
    wrap_html(options, file_extension, outfilename)
  end
end
process_input(filename, options) click to toggle source
# File lib/metanorma/compile.rb, line 88
def process_input(filename, options)
  case extname = File.extname(filename)
  when ".adoc" then process_input_adoc(filename, options)
  when ".xml" then process_input_xml(filename, options)
  else
    Util.log("[metanorma] Error: file extension #{extname} "\
             "is not supported.", :error)
    nil
  end
end
process_input_adoc(filename, options) click to toggle source
# File lib/metanorma/compile.rb, line 99
def process_input_adoc(filename, options)
  Util.log("[metanorma] Processing: AsciiDoc input.", :info)
  file = read_file(filename)
  options[:asciimath] and
    file.sub!(/^(=[^\n]+\n)/, "\\1:mn-keep-asciimath:\n")
  dir = File.dirname(filename)
  dir != "." and
    file.gsub!(/^include::/, "include::#{dir}/")
  [file, @processor.input_to_isodoc(file, filename, options)]
end
process_input_xml(filename, _options) click to toggle source
# File lib/metanorma/compile.rb, line 110
def process_input_xml(filename, _options)
  Util.log("[metanorma] Processing: Metanorma XML input.", :info)
  # TODO NN: this is a hack -- we should provide/bridge the
  # document attributes in Metanorma XML
  ["", read_file(filename)]
end
read_file(filename) click to toggle source
# File lib/metanorma/compile.rb, line 117
def read_file(filename)
  File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
end
recursive_string_keys(hash) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 213
def recursive_string_keys(hash)
  case hash
  when Hash then hash.map { |k, v| [k.to_s, recursive_string_keys(v)] }.to_h
  when Enumerable then hash.map { |v| recursive_string_keys(v) }
  else
    hash
  end
end
relaton_export(isodoc, options) click to toggle source
# File lib/metanorma/compile.rb, line 121
def relaton_export(isodoc, options)
  return unless options[:relaton]

  xml = Nokogiri::XML(isodoc) { |config| config.huge }
  bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
  # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
  # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
  File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
end
require_libraries(options) click to toggle source
# File lib/metanorma/compile.rb, line 38
def require_libraries(options)
  options&.dig(:require)&.each { |r| require r }
end
requirement_export(xml, dirname) click to toggle source
# File lib/metanorma/compile.rb, line 181
def requirement_export(xml, dirname)
  xml.at(REQUIREMENT_XPATH) or return
  FileUtils.mkdir_p "#{dirname}/requirement"
  xml.xpath(REQUIREMENT_XPATH).each_with_index do |s, i|
    filename = s["filename"] || sprintf("%s-%04d.xml", s.name, i)
    File.open("#{dirname}/requirement/#{filename}", "w:UTF-8") do |f|
      f.write s
    end
  end
end
sectionfile(fulldoc, xml, file, chunk, parentnode) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 85
def sectionfile(fulldoc, xml, file, chunk, parentnode)
  fname = create_sectionfile(fulldoc, xml.dup, file, chunk, parentnode)
  { order: chunk["displayorder"].to_i, url: fname,
    title: titlerender(chunk) }
end
sectionsplit(xml, filename, dir) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 65
def sectionsplit(xml, filename, dir)
  @key = xref_preprocess(xml)
  @splitdir = dir
  out = emptydoc(xml)
  SPLITSECTIONS.each_with_object([]) do |n, ret|
    xml.xpath(ns(n[0])).each do |s|
      ret << sectionfile(xml, out, "#{filename}.#{ret.size}", s, n[1])
    end
  end
end
sectionsplit_convert(input_filename, file, output_filename = nil, opts = {}) click to toggle source

assume we pass in Presentation XML, but we want to recover Semantic XML

# File lib/metanorma/sectionsplit.rb, line 6
def sectionsplit_convert(input_filename, file, output_filename = nil,
                         opts = {})
  @isodoc = IsoDoc::Convert.new({})
  input_filename += ".xml" unless input_filename.match?(/\.xml$/)
  File.exist?(input_filename) or
    File.open(input_filename, "w:UTF-8") { |f| f.write(file) }
  presxml = File.read(input_filename, encoding: "utf-8")
  @openmathdelim, @closemathdelim = @isodoc.extract_delims(presxml)
  xml, filename, dir = @isodoc.convert_init(presxml, input_filename, false)
  build_collection(xml, presxml, output_filename || filename, dir, opts)
end
sourcecode_export(xml, dirname) click to toggle source
# File lib/metanorma/compile.rb, line 152
def sourcecode_export(xml, dirname)
  xml.at("//sourcecode | //xmlns:sourcecode") or return
  FileUtils.mkdir_p "#{dirname}/sourcecode"
  xml.xpath("//sourcecode | //xmlns:sourcecode").each_with_index do |s, i|
    filename = s["filename"] || sprintf("sourcecode-%04d.txt", i)
    File.open("#{dirname}/sourcecode/#{filename}", "w:UTF-8") do |f|
      f.write clean_sourcecode(s.dup)
    end
  end
end
svg_preprocess(xml) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 119
def svg_preprocess(xml)
  xml.xpath("//m:svg", "m" => "http://www.w3.org/2000/svg").each do |s|
    m = svgmap_wrap(s)
    s.xpath(".//m:a", "m" => "http://www.w3.org/2000/svg").each do |a|
      next unless /^#/.match? a["href"]

      a["href"] = a["href"].sub(/^#/, "")
      m << "<target href='#{a['href']}'>"\
           "<xref target='#{a['href']}'/></target>"
    end
  end
end
svgmap_wrap(svg) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 132
def svgmap_wrap(svg)
  ret = svg.at("./ancestor::xmlns:svgmap") and return ret
  ret = svg.at("./ancestor::xmlns:figure")
  ret.wrap("<svgmap/>")
  svg.at("./ancestor::xmlns:svgmap")
end
titlerender(section) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 222
def titlerender(section)
  title = section.at(ns("./title")) or return "[Untitled]"
  t = title.dup
  t.xpath(ns(".//tab | .//br")).each { |x| x.replace(" ") }
  t.xpath(ns(".//strong")).each { |x| x.replace(x.children) }
  t.children.to_xml
end
validate_format(options) click to toggle source
# File lib/metanorma/compile_validate.rb, line 17
def validate_format(options)
  unless options[:format] == :asciidoc
    Util.log("[metanorma] Error: Only source file format currently "\
             "supported is 'asciidoc'.", :fatal)
  end
end
validate_options(options) click to toggle source
# File lib/metanorma/compile_validate.rb, line 3
def validate_options(options)
  validate_type(options)
  validate_format(options)
end
validate_type(options) click to toggle source
# File lib/metanorma/compile_validate.rb, line 8
def validate_type(options)
  unless options[:type]
    Util.log("[metanorma] Error: Please specify a standard type: "\
             "#{@registry.supported_backends}.", :fatal)
  end
  stdtype = options[:type].to_sym
  load_flavor(stdtype)
end
wrap_html(options, file_extension, outfilename) click to toggle source
# File lib/metanorma/compile.rb, line 192
def wrap_html(options, file_extension, outfilename)
  if options[:wrapper] && /html$/.match(file_extension)
    outfilename = outfilename.sub(/\.html$/, "")
    FileUtils.mkdir_p outfilename
    FileUtils.mv "#{outfilename}.html", outfilename
    FileUtils.mv "#{outfilename}_images", outfilename, force: true
  end
end
xml_options_extract(file) click to toggle source
# File lib/metanorma/compile.rb, line 42
def xml_options_extract(file)
  xml = Nokogiri::XML(file) { |config| config.huge }
  if xml.root
    @registry.root_tags.each do |k, v|
      return { type: k } if v == xml.root.name
    end
  end
  {}
end
xref_preprocess(xml) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 104
def xref_preprocess(xml)
  svg_preprocess(xml)
  key = (0...8).map { rand(65..90).chr }.join # random string
  xml.root["type"] = key # to force recognition of internal refs
  key
end
xref_process(section, xml, key) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 111
def xref_process(section, xml, key)
  refs = eref_to_internal_eref(section, xml, key)
  refs += xref_to_internal_eref(section, key)
  ins = new_hidden_ref(section)
  copied_refs = copy_repo_items_biblio(ins, section, xml)
  insert_indirect_biblio(ins, refs - copied_refs, key)
end
xref_to_internal_eref(xml, key) click to toggle source
# File lib/metanorma/sectionsplit.rb, line 144
def xref_to_internal_eref(xml, key)
  xml.xpath(ns("//xref")).each_with_object({}) do |x, m|
    x["bibitemid"] = "#{key}_#{x['target']}"
    x << make_anchor(x["target"])
    m[x["bibitemid"]] = true
    x.delete("target")
    x["type"] = key
    x.name = "eref"
  end.keys
end

Private Instance Methods

change_output_dir(options) click to toggle source

@param options [Hash] @return [String]

# File lib/metanorma/compile.rb, line 260
def change_output_dir(options)
  if options[:output_dir]
    File.join options[:output_dir], File.basename(options[:filename])
  else options[:filename]
  end
end
get_isodoc_options(file, options, ext) click to toggle source
# File lib/metanorma/compile.rb, line 244
def get_isodoc_options(file, options, ext)
  isodoc_options = @processor.extract_options(file)
  isodoc_options[:datauriimage] = true if options[:datauriimage]
  isodoc_options[:sourcefilename] = options[:filename]
  %i(bare sectionsplit no_install_fonts).each do |x|
    isodoc_options[x] ||= options[x]
  end
  if ext == :pdf
    floc = FontistUtils.fontist_font_locations(@processor, options) and
      isodoc_options[:mn2pdf] = { font_manifest_file: floc.path }
  end
  isodoc_options
end
isodoc_error_process(err) click to toggle source
# File lib/metanorma/compile.rb, line 235
def isodoc_error_process(err)
  if err.message.include? "Fatal:"
    @errors << err.message
  else
    puts err.message
    puts err.backtrace.join("\n")
  end
end
load_flavor(stdtype) click to toggle source
# File lib/metanorma/compile_validate.rb, line 26
def load_flavor(stdtype)
  flavor = "metanorma-#{stdtype}"
  unless @registry.supported_backends.include? stdtype
    Util.log("[metanorma] Info: Loading `#{flavor}` gem "\
             "for standard type `#{stdtype}`.", :info)
  end
  require_flavor(flavor, stdtype)
  unless @registry.supported_backends.include? stdtype
    Util.log("[metanorma] Error: The `#{flavor}` gem "\
             "still doesn't support `#{stdtype}`. Exiting.", :fatal)
  end
end
require_flavor(flavor, stdtype) click to toggle source
# File lib/metanorma/compile_validate.rb, line 39
def require_flavor(flavor, stdtype)
  require flavor
  Util.log("[metanorma] Info: gem `#{flavor}` loaded.", :info)
rescue Gem::ConflictError
  Util.log("[metanorma] Error: Couldn't resolve dependencies for "\
           "`metanorma-#{stdtype}`, Please add it to your Gemfile "\
           "and run bundle install first", :fatal)
rescue LoadError
  Util.log("[metanorma] Error: loading gem `#{flavor}` "\
           "failed. Exiting.", :fatal)
end