module DataMetaPii

PII support for DataMeta

For command line details either check the new method’s source or the README file, the usage section.

“”

Constants

ALL_FMTS

Collect all the constants from the module ExportFmts, that’s all supported formats

ALL_IMPACTS

All supported impacts collected from the module Impact

ALL_SCOPES

All supported scopes collected from the module Scope

Load the PII Applications Link grammar

And the App Link grammar parser:

ATTRB_DEF_NODE_TYPE

AST Node type

ATTRB_LIST_NODE_TYPE

AST Node type

BASE_RULES

Load base rules from the DataMeta Parsing Commons

CONST_KEY

Constant value key

DECIMAL_CONST_DT

Constant Data type:

GEM_ROOT

Determine the gem root in the local Filesystem

GRAMMAR_ROOT

Advance further to determine the root of the grammars

INDENT

One step for indentation of the output

INT_CONST_DT

Constant Data type:

L

Logger to use for this module

PII_COMMONS

Load PII specific common rules from this very gem’s codebase

REF_KEY

Value Reference Key

REGISTRY

Load the PII Registry grammar

REGISTRY_PARSER

Create all parsers, it’s not expensive. First the Registry (Abstract Defs) grammar parser:

STR_CONST_DT

Constant Data type: string

VERSION

Current version

VO_CLASS_KEY

Value Object Class Key

Public Class Methods

buildAlCst(ast, logString = nil) click to toggle source

Builds the AppLink CST from the given Registry Grammar Parser’s AST

# File lib/dataMetaPii.rb, line 366
def buildAlCst(ast, logString = nil)
    # parse the ad first
    reusables = {}
    log = -> (what) {logString << what if logString}

    log.call("AppLink CST\n")
    if ast.ad&.elements
        log.call("#{INDENT}#{ast.ad.type}:#{ast.ad.a.elements.size}\n")
        ast.ad.a.elements.each { |as| # attrbSection
            raise RuntimeError, %<The attributes set "#{as.pk} is defined more than once"> if reusables.has_key?(as.pk.to_sym)
            keyVals = digAstElsType(ATTRB_DEF_NODE_TYPE, as.a.elements)
            log.call(%<#{INDENT * 2}#{as.pk}:#{keyVals.size}\n>)
            aSect = AttrSect.new(as.pk.to_sym)
            keyVals.each { |kv|
                kvVal = kv.val
                log.call(%<#{INDENT * 3}#{kv.nodeType}:#{kvVal}\\#{kvVal.class}>)
                log.call(" (#{kv.node.key}==#{kv.node.nodeVal})//#{kv.node.type}\\#{kv.node.dataType}") if(kv.nodeType == CONST_KEY)
                log.call("\n")
                # noinspection RubyCaseWithoutElseBlockInspection
                 aSect + case kv.nodeType # else case is caught by the AST parser
                              when CONST_KEY
                                  # noinspection RubyCaseWithoutElseBlockInspection
                                  klass = case kv.node.dataType # else case is caught by the AST parser
                                              when STR_CONST_DT
                                                  AlAttrStr
                                              when DECIMAL_CONST_DT
                                                  AlAttrDec
                                              when INT_CONST_DT
                                                  AlAttrInt
                                          end
                                  klass.new(kv.node.key.to_sym, kv.node.nodeVal)
                              when REF_KEY
                                  AttrRef.new(kvVal)
                              when VO_CLASS_KEY
                                  AlAttrVoClass.new(kvVal)
                          end
            }
            reusables[as.pk.to_sym] = aSect
        }
        log.call(%<#{INDENT * 2}#{reusables}\n>)
    else
        log.call("#{INDENT * 2}No reusables\n")
    end
    apps = {}
    if ast.al&.elements
        log.call("#{INDENT}#{ast.al.type}:#{ast.al.a.elements.size}\n")
        ast.al.a.elements.each { |as| # appLinkApps
            log.call(%<#{INDENT * 3}#{as.ak}:#{as.a.elements.size}\n>)
            appKey = as.ak.to_sym
            raise RuntimeError, %<Application "#{appKey}" defined more than once> if apps.has_key?(appKey)
            attrbs = {}
            as.a.elements.each { |ala| #appLinkAttrbs
                alis = digAstElsType(DataMetaPii::ATTRB_DEF_NODE_TYPE, ala.a.elements)
                log.call(%<#{INDENT * 4}#{ala.pk} (#{ala.type}): #{alis.size}\n>)
                aSect = AttrSect.new(ala.pk.to_sym)
                alis.each { |ali|
                   kvVal = ali.val
                   log.call(%<#{INDENT * 5}#{ali.nodeType}: >)
                    if ali.nodeType == DataMetaPii::CONST_KEY
                        log.call(%<(#{ali.node.dataType}):: #{ali.node.key}=#{ali.node.nodeVal}>)
                    else
                       log.call(%<#{ali.val}>)
                    end
                   # noinspection RubyCaseWithoutElseBlockInspection
                   aSect + case ali.nodeType # else case is caught by the AST parser
                               when CONST_KEY
                                   # noinspection RubyCaseWithoutElseBlockInspection
                                   klass = case ali.node.dataType # else case is caught by the AST parser
                                               when STR_CONST_DT
                                                   AlAttrStr
                                               when DECIMAL_CONST_DT
                                                   AlAttrDec
                                               when INT_CONST_DT
                                                   AlAttrInt
                                           end
                                   klass.new(ali.node.key.to_sym, ali.node.nodeVal)
                               when REF_KEY
                                   AttrRef.new(kvVal)
                               when VO_CLASS_KEY
                                   AlAttrVoClass.new(kvVal)
                           end
                    log.call("\n")
                }
                attrbs[ala.pk.to_sym] = aSect
            }
            log.call(%<#{INDENT}#{attrbs}\n>)
            apps[appKey] = attrbs
        }
    else
        raise ArgumentError, 'No Applink Division'
    end
    AppLink.new(ast.verDef.ver, apps, reusables)
end
buildRegCst(ast) click to toggle source

Builds the Registry CST from the given Registry Grammar Parser’s AST

# File lib/dataMetaPii.rb, line 461
def buildRegCst(ast)

    resultMap = {}

    ast.fields.elements.each { |f|
        fKey = f.pk
        raise ArgumentError, %<The PII field #{fKey} is defined more than once> if resultMap.keys.member?(fKey)
        attrs = {}
        f.attrbLst.attrbs.each { |a|
            raise ArgumentError, %<Attribute "#{a.k}" is defined more than once for #{fKey}> if attrs.keys.member?(a.k)
            attrs[a.k] = a.v
        }
        attrVo = RegKeyVo.new(fKey, attrs)
        resultMap[fKey] = attrVo
    }

    RegVo.new(ast.verDef.ver, resultMap)
end
digAstElsType(type, els, result=[]) click to toggle source

Helper method for the AST traversal to collect the attributes Because of the Treetop AST design, can not just flatten the elements and select of those of the needed type in one call, hence the tree traversal

# File lib/dataMetaPii.rb, line 482
def digAstElsType(type, els, result=[])
    if els.nil? # is it a leaf?
        nil # not a leaf - return nil
    else
        els.each { |e|
            if e.respond_to?(:type) && e.type == type # actual attribute Key/Value?
                result << e # add it
            else
                digAstElsType(type, e.elements, result) # dig deeper into the AST
            end
        }
        result
    end
end
errNamespace(outFmt) click to toggle source
# File lib/dataMetaPii.rb, line 497
def errNamespace(outFmt)
    raise ArgumentError, %<For output format "#{outFmt}", the Namespace is required>
end
genCode(scope, outFmt, outDirName, source, namespace = nil) click to toggle source

API method: generate the PII code

# File lib/dataMetaPii.rb, line 502
    def genCode(scope, outFmt, outDirName, source, namespace = nil)
# noinspection RubyCaseWithoutElseBlockInspection
        codeIndent = ' ' * 2 
        raise ArgumentError, %Q<Unsupported scope definition "#{scope}", supported scopes are: #{
            DataMetaPii::ALL_SCOPES.map(&:to_s).join(', ')}> unless ALL_SCOPES.member?(scope)

        raise ArgumentError, %Q<Unsupported output format definition "#{outFmt}", supported formats are: #{
            DataMetaPii::ALL_FMTS.map(&:to_s).join(', ')}> unless ALL_FMTS.member?(outFmt)

        raise ArgumentError, %<For safety purposes, absolute path names like "#{
            outDirName}" are not supported> if outDirName.start_with?('/')

        raise ArgumentError, %<The output dir "#{outDirName}" is not a directory> unless File.directory?(outDirName)

        # noinspection RubyCaseWithoutElseBlockInspection
        case scope # else case caught up there, on argument validation
            when Scope::ABSTRACT
                reg = DataMetaPii.buildRegCst(DataMetaParse.parse(REGISTRY_PARSER, source))
                L.info(%<PII Registry:
#{reg.to_tree_image(INDENT)}>)
                tmpl = ERB.new(IO.read(File.join(GEM_ROOT, 'tpl', outFmt.to_s, 'master.erb')), $SAFE, '%<>>')
                className = "PiiAbstractDef_#{reg.ver.toVarName}"
                # noinspection RubyCaseWithoutElseBlockInspection
                case outFmt
                    when ExportFmts::JAVA, ExportFmts::SCALA
                        errNamespace(outFmt) unless namespace.is_a?(String) && !namespace.empty?
                        pkgDir = namespace.gsub('.', '/')
                        classDest =File.join(outDirName, pkgDir)
                        FileUtils.mkpath classDest

                        IO.write(File.join(classDest, "#{className}.#{outFmt}"),
                                 tmpl.result(binding).gsub(/\n\n+/, "\n\n"), mode: 'wb') # collapse multiple lines in 2

                    when ExportFmts::JSON
                        IO.write(File.join(outDirName, "#{className}.#{outFmt}"),
                                 tmpl.result(binding).gsub(/\n\n+/, "\n\n"), mode: 'wb') # collapse multiple lines in 2

                    when ExportFmts::PYTHON
                        pkgDir = namespace.gsub('.', '_')
                        classDest =File.join(outDirName, pkgDir)
                        FileUtils.mkpath classDest
                        IO.write(File.join(classDest, "#{className[0].downcase + className[1..-1]}.py"),
                                 tmpl.result(binding).gsub(/\n\n+/, "\n\n"), mode: 'wb') # collapse multiple lines in 2
                        IO.write(File.join(classDest, '__init__.py'), %q<
# see https://docs.python.org/3/library/pkgutil.html
# without this, Python will have trouble finding packages that share some common tree off the root
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)

>, mode: 'wb')
                end

            when DataMetaPii::Scope::APPLICATION
                raise NotImplementedError, 'There is no generic code gen for AppLink, each app/svc should have their own'

        end
    end

Private Instance Methods

buildAlCst(ast, logString = nil) click to toggle source

Builds the AppLink CST from the given Registry Grammar Parser’s AST

# File lib/dataMetaPii.rb, line 366
def buildAlCst(ast, logString = nil)
    # parse the ad first
    reusables = {}
    log = -> (what) {logString << what if logString}

    log.call("AppLink CST\n")
    if ast.ad&.elements
        log.call("#{INDENT}#{ast.ad.type}:#{ast.ad.a.elements.size}\n")
        ast.ad.a.elements.each { |as| # attrbSection
            raise RuntimeError, %<The attributes set "#{as.pk} is defined more than once"> if reusables.has_key?(as.pk.to_sym)
            keyVals = digAstElsType(ATTRB_DEF_NODE_TYPE, as.a.elements)
            log.call(%<#{INDENT * 2}#{as.pk}:#{keyVals.size}\n>)
            aSect = AttrSect.new(as.pk.to_sym)
            keyVals.each { |kv|
                kvVal = kv.val
                log.call(%<#{INDENT * 3}#{kv.nodeType}:#{kvVal}\\#{kvVal.class}>)
                log.call(" (#{kv.node.key}==#{kv.node.nodeVal})//#{kv.node.type}\\#{kv.node.dataType}") if(kv.nodeType == CONST_KEY)
                log.call("\n")
                # noinspection RubyCaseWithoutElseBlockInspection
                 aSect + case kv.nodeType # else case is caught by the AST parser
                              when CONST_KEY
                                  # noinspection RubyCaseWithoutElseBlockInspection
                                  klass = case kv.node.dataType # else case is caught by the AST parser
                                              when STR_CONST_DT
                                                  AlAttrStr
                                              when DECIMAL_CONST_DT
                                                  AlAttrDec
                                              when INT_CONST_DT
                                                  AlAttrInt
                                          end
                                  klass.new(kv.node.key.to_sym, kv.node.nodeVal)
                              when REF_KEY
                                  AttrRef.new(kvVal)
                              when VO_CLASS_KEY
                                  AlAttrVoClass.new(kvVal)
                          end
            }
            reusables[as.pk.to_sym] = aSect
        }
        log.call(%<#{INDENT * 2}#{reusables}\n>)
    else
        log.call("#{INDENT * 2}No reusables\n")
    end
    apps = {}
    if ast.al&.elements
        log.call("#{INDENT}#{ast.al.type}:#{ast.al.a.elements.size}\n")
        ast.al.a.elements.each { |as| # appLinkApps
            log.call(%<#{INDENT * 3}#{as.ak}:#{as.a.elements.size}\n>)
            appKey = as.ak.to_sym
            raise RuntimeError, %<Application "#{appKey}" defined more than once> if apps.has_key?(appKey)
            attrbs = {}
            as.a.elements.each { |ala| #appLinkAttrbs
                alis = digAstElsType(DataMetaPii::ATTRB_DEF_NODE_TYPE, ala.a.elements)
                log.call(%<#{INDENT * 4}#{ala.pk} (#{ala.type}): #{alis.size}\n>)
                aSect = AttrSect.new(ala.pk.to_sym)
                alis.each { |ali|
                   kvVal = ali.val
                   log.call(%<#{INDENT * 5}#{ali.nodeType}: >)
                    if ali.nodeType == DataMetaPii::CONST_KEY
                        log.call(%<(#{ali.node.dataType}):: #{ali.node.key}=#{ali.node.nodeVal}>)
                    else
                       log.call(%<#{ali.val}>)
                    end
                   # noinspection RubyCaseWithoutElseBlockInspection
                   aSect + case ali.nodeType # else case is caught by the AST parser
                               when CONST_KEY
                                   # noinspection RubyCaseWithoutElseBlockInspection
                                   klass = case ali.node.dataType # else case is caught by the AST parser
                                               when STR_CONST_DT
                                                   AlAttrStr
                                               when DECIMAL_CONST_DT
                                                   AlAttrDec
                                               when INT_CONST_DT
                                                   AlAttrInt
                                           end
                                   klass.new(ali.node.key.to_sym, ali.node.nodeVal)
                               when REF_KEY
                                   AttrRef.new(kvVal)
                               when VO_CLASS_KEY
                                   AlAttrVoClass.new(kvVal)
                           end
                    log.call("\n")
                }
                attrbs[ala.pk.to_sym] = aSect
            }
            log.call(%<#{INDENT}#{attrbs}\n>)
            apps[appKey] = attrbs
        }
    else
        raise ArgumentError, 'No Applink Division'
    end
    AppLink.new(ast.verDef.ver, apps, reusables)
end
buildRegCst(ast) click to toggle source

Builds the Registry CST from the given Registry Grammar Parser’s AST

# File lib/dataMetaPii.rb, line 461
def buildRegCst(ast)

    resultMap = {}

    ast.fields.elements.each { |f|
        fKey = f.pk
        raise ArgumentError, %<The PII field #{fKey} is defined more than once> if resultMap.keys.member?(fKey)
        attrs = {}
        f.attrbLst.attrbs.each { |a|
            raise ArgumentError, %<Attribute "#{a.k}" is defined more than once for #{fKey}> if attrs.keys.member?(a.k)
            attrs[a.k] = a.v
        }
        attrVo = RegKeyVo.new(fKey, attrs)
        resultMap[fKey] = attrVo
    }

    RegVo.new(ast.verDef.ver, resultMap)
end
digAstElsType(type, els, result=[]) click to toggle source

Helper method for the AST traversal to collect the attributes Because of the Treetop AST design, can not just flatten the elements and select of those of the needed type in one call, hence the tree traversal

# File lib/dataMetaPii.rb, line 482
def digAstElsType(type, els, result=[])
    if els.nil? # is it a leaf?
        nil # not a leaf - return nil
    else
        els.each { |e|
            if e.respond_to?(:type) && e.type == type # actual attribute Key/Value?
                result << e # add it
            else
                digAstElsType(type, e.elements, result) # dig deeper into the AST
            end
        }
        result
    end
end
errNamespace(outFmt) click to toggle source
# File lib/dataMetaPii.rb, line 497
def errNamespace(outFmt)
    raise ArgumentError, %<For output format "#{outFmt}", the Namespace is required>
end
genCode(scope, outFmt, outDirName, source, namespace = nil) click to toggle source

API method: generate the PII code

# File lib/dataMetaPii.rb, line 502
    def genCode(scope, outFmt, outDirName, source, namespace = nil)
# noinspection RubyCaseWithoutElseBlockInspection
        codeIndent = ' ' * 2 
        raise ArgumentError, %Q<Unsupported scope definition "#{scope}", supported scopes are: #{
            DataMetaPii::ALL_SCOPES.map(&:to_s).join(', ')}> unless ALL_SCOPES.member?(scope)

        raise ArgumentError, %Q<Unsupported output format definition "#{outFmt}", supported formats are: #{
            DataMetaPii::ALL_FMTS.map(&:to_s).join(', ')}> unless ALL_FMTS.member?(outFmt)

        raise ArgumentError, %<For safety purposes, absolute path names like "#{
            outDirName}" are not supported> if outDirName.start_with?('/')

        raise ArgumentError, %<The output dir "#{outDirName}" is not a directory> unless File.directory?(outDirName)

        # noinspection RubyCaseWithoutElseBlockInspection
        case scope # else case caught up there, on argument validation
            when Scope::ABSTRACT
                reg = DataMetaPii.buildRegCst(DataMetaParse.parse(REGISTRY_PARSER, source))
                L.info(%<PII Registry:
#{reg.to_tree_image(INDENT)}>)
                tmpl = ERB.new(IO.read(File.join(GEM_ROOT, 'tpl', outFmt.to_s, 'master.erb')), $SAFE, '%<>>')
                className = "PiiAbstractDef_#{reg.ver.toVarName}"
                # noinspection RubyCaseWithoutElseBlockInspection
                case outFmt
                    when ExportFmts::JAVA, ExportFmts::SCALA
                        errNamespace(outFmt) unless namespace.is_a?(String) && !namespace.empty?
                        pkgDir = namespace.gsub('.', '/')
                        classDest =File.join(outDirName, pkgDir)
                        FileUtils.mkpath classDest

                        IO.write(File.join(classDest, "#{className}.#{outFmt}"),
                                 tmpl.result(binding).gsub(/\n\n+/, "\n\n"), mode: 'wb') # collapse multiple lines in 2

                    when ExportFmts::JSON
                        IO.write(File.join(outDirName, "#{className}.#{outFmt}"),
                                 tmpl.result(binding).gsub(/\n\n+/, "\n\n"), mode: 'wb') # collapse multiple lines in 2

                    when ExportFmts::PYTHON
                        pkgDir = namespace.gsub('.', '_')
                        classDest =File.join(outDirName, pkgDir)
                        FileUtils.mkpath classDest
                        IO.write(File.join(classDest, "#{className[0].downcase + className[1..-1]}.py"),
                                 tmpl.result(binding).gsub(/\n\n+/, "\n\n"), mode: 'wb') # collapse multiple lines in 2
                        IO.write(File.join(classDest, '__init__.py'), %q<
# see https://docs.python.org/3/library/pkgutil.html
# without this, Python will have trouble finding packages that share some common tree off the root
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)

>, mode: 'wb')
                end

            when DataMetaPii::Scope::APPLICATION
                raise NotImplementedError, 'There is no generic code gen for AppLink, each app/svc should have their own'

        end
    end