module DataMetaPii
PII support for DataMeta
For command line details either check the new method’s source or the README file, the usage section.
“”
Constants
- ALL_FMTS
Collect all the constants from the module
ExportFmts
, that’s all supported formats- ALL_IMPACTS
All supported impacts collected from the module
Impact
- ALL_SCOPES
All supported scopes collected from the module
Scope
- APP_LINK
Load the PII Applications Link grammar
- APP_LINK_PARSER
And the App Link grammar parser:
- ATTRB_DEF_NODE_TYPE
AST Node type
- ATTRB_LIST_NODE_TYPE
AST Node type
- BASE_RULES
Load base rules from the DataMeta Parsing Commons
- CONST_KEY
Constant value key
- DECIMAL_CONST_DT
Constant Data type:
- GEM_ROOT
Determine the gem root in the local Filesystem
- GRAMMAR_ROOT
Advance further to determine the root of the grammars
- INDENT
One step for indentation of the output
- INT_CONST_DT
Constant Data type:
- L
Logger to use for this module
- PII_COMMONS
Load PII specific common rules from this very gem’s codebase
- REF_KEY
Value Reference Key
- REGISTRY
Load the PII Registry grammar
- REGISTRY_PARSER
Create all parsers, it’s not expensive. First the Registry (Abstract Defs) grammar parser:
- STR_CONST_DT
Constant Data type: string
- VERSION
Current version
- VO_CLASS_KEY
Value Object Class Key
Public Class Methods
Builds the AppLink
CST from the given Registry Grammar Parser’s AST
# File lib/dataMetaPii.rb, line 366 def buildAlCst(ast, logString = nil) # parse the ad first reusables = {} log = -> (what) {logString << what if logString} log.call("AppLink CST\n") if ast.ad&.elements log.call("#{INDENT}#{ast.ad.type}:#{ast.ad.a.elements.size}\n") ast.ad.a.elements.each { |as| # attrbSection raise RuntimeError, %<The attributes set "#{as.pk} is defined more than once"> if reusables.has_key?(as.pk.to_sym) keyVals = digAstElsType(ATTRB_DEF_NODE_TYPE, as.a.elements) log.call(%<#{INDENT * 2}#{as.pk}:#{keyVals.size}\n>) aSect = AttrSect.new(as.pk.to_sym) keyVals.each { |kv| kvVal = kv.val log.call(%<#{INDENT * 3}#{kv.nodeType}:#{kvVal}\\#{kvVal.class}>) log.call(" (#{kv.node.key}==#{kv.node.nodeVal})//#{kv.node.type}\\#{kv.node.dataType}") if(kv.nodeType == CONST_KEY) log.call("\n") # noinspection RubyCaseWithoutElseBlockInspection aSect + case kv.nodeType # else case is caught by the AST parser when CONST_KEY # noinspection RubyCaseWithoutElseBlockInspection klass = case kv.node.dataType # else case is caught by the AST parser when STR_CONST_DT AlAttrStr when DECIMAL_CONST_DT AlAttrDec when INT_CONST_DT AlAttrInt end klass.new(kv.node.key.to_sym, kv.node.nodeVal) when REF_KEY AttrRef.new(kvVal) when VO_CLASS_KEY AlAttrVoClass.new(kvVal) end } reusables[as.pk.to_sym] = aSect } log.call(%<#{INDENT * 2}#{reusables}\n>) else log.call("#{INDENT * 2}No reusables\n") end apps = {} if ast.al&.elements log.call("#{INDENT}#{ast.al.type}:#{ast.al.a.elements.size}\n") ast.al.a.elements.each { |as| # appLinkApps log.call(%<#{INDENT * 3}#{as.ak}:#{as.a.elements.size}\n>) appKey = as.ak.to_sym raise RuntimeError, %<Application "#{appKey}" defined more than once> if apps.has_key?(appKey) attrbs = {} as.a.elements.each { |ala| #appLinkAttrbs alis = digAstElsType(DataMetaPii::ATTRB_DEF_NODE_TYPE, ala.a.elements) log.call(%<#{INDENT * 4}#{ala.pk} (#{ala.type}): #{alis.size}\n>) aSect = AttrSect.new(ala.pk.to_sym) alis.each { |ali| kvVal = ali.val log.call(%<#{INDENT * 5}#{ali.nodeType}: >) if ali.nodeType == DataMetaPii::CONST_KEY log.call(%<(#{ali.node.dataType}):: #{ali.node.key}=#{ali.node.nodeVal}>) else log.call(%<#{ali.val}>) end # noinspection RubyCaseWithoutElseBlockInspection aSect + case ali.nodeType # else case is caught by the AST parser when CONST_KEY # noinspection RubyCaseWithoutElseBlockInspection klass = case ali.node.dataType # else case is caught by the AST parser when STR_CONST_DT AlAttrStr when DECIMAL_CONST_DT AlAttrDec when INT_CONST_DT AlAttrInt end klass.new(ali.node.key.to_sym, ali.node.nodeVal) when REF_KEY AttrRef.new(kvVal) when VO_CLASS_KEY AlAttrVoClass.new(kvVal) end log.call("\n") } attrbs[ala.pk.to_sym] = aSect } log.call(%<#{INDENT}#{attrbs}\n>) apps[appKey] = attrbs } else raise ArgumentError, 'No Applink Division' end AppLink.new(ast.verDef.ver, apps, reusables) end
Builds the Registry CST from the given Registry Grammar Parser’s AST
# File lib/dataMetaPii.rb, line 461 def buildRegCst(ast) resultMap = {} ast.fields.elements.each { |f| fKey = f.pk raise ArgumentError, %<The PII field #{fKey} is defined more than once> if resultMap.keys.member?(fKey) attrs = {} f.attrbLst.attrbs.each { |a| raise ArgumentError, %<Attribute "#{a.k}" is defined more than once for #{fKey}> if attrs.keys.member?(a.k) attrs[a.k] = a.v } attrVo = RegKeyVo.new(fKey, attrs) resultMap[fKey] = attrVo } RegVo.new(ast.verDef.ver, resultMap) end
Helper method for the AST traversal to collect the attributes Because of the Treetop AST design, can not just flatten the elements and select of those of the needed type in one call, hence the tree traversal
# File lib/dataMetaPii.rb, line 482 def digAstElsType(type, els, result=[]) if els.nil? # is it a leaf? nil # not a leaf - return nil else els.each { |e| if e.respond_to?(:type) && e.type == type # actual attribute Key/Value? result << e # add it else digAstElsType(type, e.elements, result) # dig deeper into the AST end } result end end
# File lib/dataMetaPii.rb, line 497 def errNamespace(outFmt) raise ArgumentError, %<For output format "#{outFmt}", the Namespace is required> end
API method: generate the PII code
# File lib/dataMetaPii.rb, line 502 def genCode(scope, outFmt, outDirName, source, namespace = nil) # noinspection RubyCaseWithoutElseBlockInspection codeIndent = ' ' * 2 raise ArgumentError, %Q<Unsupported scope definition "#{scope}", supported scopes are: #{ DataMetaPii::ALL_SCOPES.map(&:to_s).join(', ')}> unless ALL_SCOPES.member?(scope) raise ArgumentError, %Q<Unsupported output format definition "#{outFmt}", supported formats are: #{ DataMetaPii::ALL_FMTS.map(&:to_s).join(', ')}> unless ALL_FMTS.member?(outFmt) raise ArgumentError, %<For safety purposes, absolute path names like "#{ outDirName}" are not supported> if outDirName.start_with?('/') raise ArgumentError, %<The output dir "#{outDirName}" is not a directory> unless File.directory?(outDirName) # noinspection RubyCaseWithoutElseBlockInspection case scope # else case caught up there, on argument validation when Scope::ABSTRACT reg = DataMetaPii.buildRegCst(DataMetaParse.parse(REGISTRY_PARSER, source)) L.info(%<PII Registry: #{reg.to_tree_image(INDENT)}>) tmpl = ERB.new(IO.read(File.join(GEM_ROOT, 'tpl', outFmt.to_s, 'master.erb')), $SAFE, '%<>>') className = "PiiAbstractDef_#{reg.ver.toVarName}" # noinspection RubyCaseWithoutElseBlockInspection case outFmt when ExportFmts::JAVA, ExportFmts::SCALA errNamespace(outFmt) unless namespace.is_a?(String) && !namespace.empty? pkgDir = namespace.gsub('.', '/') classDest =File.join(outDirName, pkgDir) FileUtils.mkpath classDest IO.write(File.join(classDest, "#{className}.#{outFmt}"), tmpl.result(binding).gsub(/\n\n+/, "\n\n"), mode: 'wb') # collapse multiple lines in 2 when ExportFmts::JSON IO.write(File.join(outDirName, "#{className}.#{outFmt}"), tmpl.result(binding).gsub(/\n\n+/, "\n\n"), mode: 'wb') # collapse multiple lines in 2 when ExportFmts::PYTHON pkgDir = namespace.gsub('.', '_') classDest =File.join(outDirName, pkgDir) FileUtils.mkpath classDest IO.write(File.join(classDest, "#{className[0].downcase + className[1..-1]}.py"), tmpl.result(binding).gsub(/\n\n+/, "\n\n"), mode: 'wb') # collapse multiple lines in 2 IO.write(File.join(classDest, '__init__.py'), %q< # see https://docs.python.org/3/library/pkgutil.html # without this, Python will have trouble finding packages that share some common tree off the root from pkgutil import extend_path __path__ = extend_path(__path__, __name__) >, mode: 'wb') end when DataMetaPii::Scope::APPLICATION raise NotImplementedError, 'There is no generic code gen for AppLink, each app/svc should have their own' end end
Turns the given text into the instance of the AppLink
object.
# File lib/dataMetaPii.rb, line 563 def parseAppLink(source) piiAppLinkParser = PiiAppLinkParser.new ast = DataMetaParse.parse(piiAppLinkParser, source) raise SyntaxError, 'AppLink parse unsuccessful' unless ast if ast.is_a?(DataMetaParse::Err) raise %<#{ast.parser.failure_line} ast.parser.failure_reason}> end DataMetaPii.buildAlCst(ast).resolveRefs end
Private Instance Methods
Builds the AppLink
CST from the given Registry Grammar Parser’s AST
# File lib/dataMetaPii.rb, line 366 def buildAlCst(ast, logString = nil) # parse the ad first reusables = {} log = -> (what) {logString << what if logString} log.call("AppLink CST\n") if ast.ad&.elements log.call("#{INDENT}#{ast.ad.type}:#{ast.ad.a.elements.size}\n") ast.ad.a.elements.each { |as| # attrbSection raise RuntimeError, %<The attributes set "#{as.pk} is defined more than once"> if reusables.has_key?(as.pk.to_sym) keyVals = digAstElsType(ATTRB_DEF_NODE_TYPE, as.a.elements) log.call(%<#{INDENT * 2}#{as.pk}:#{keyVals.size}\n>) aSect = AttrSect.new(as.pk.to_sym) keyVals.each { |kv| kvVal = kv.val log.call(%<#{INDENT * 3}#{kv.nodeType}:#{kvVal}\\#{kvVal.class}>) log.call(" (#{kv.node.key}==#{kv.node.nodeVal})//#{kv.node.type}\\#{kv.node.dataType}") if(kv.nodeType == CONST_KEY) log.call("\n") # noinspection RubyCaseWithoutElseBlockInspection aSect + case kv.nodeType # else case is caught by the AST parser when CONST_KEY # noinspection RubyCaseWithoutElseBlockInspection klass = case kv.node.dataType # else case is caught by the AST parser when STR_CONST_DT AlAttrStr when DECIMAL_CONST_DT AlAttrDec when INT_CONST_DT AlAttrInt end klass.new(kv.node.key.to_sym, kv.node.nodeVal) when REF_KEY AttrRef.new(kvVal) when VO_CLASS_KEY AlAttrVoClass.new(kvVal) end } reusables[as.pk.to_sym] = aSect } log.call(%<#{INDENT * 2}#{reusables}\n>) else log.call("#{INDENT * 2}No reusables\n") end apps = {} if ast.al&.elements log.call("#{INDENT}#{ast.al.type}:#{ast.al.a.elements.size}\n") ast.al.a.elements.each { |as| # appLinkApps log.call(%<#{INDENT * 3}#{as.ak}:#{as.a.elements.size}\n>) appKey = as.ak.to_sym raise RuntimeError, %<Application "#{appKey}" defined more than once> if apps.has_key?(appKey) attrbs = {} as.a.elements.each { |ala| #appLinkAttrbs alis = digAstElsType(DataMetaPii::ATTRB_DEF_NODE_TYPE, ala.a.elements) log.call(%<#{INDENT * 4}#{ala.pk} (#{ala.type}): #{alis.size}\n>) aSect = AttrSect.new(ala.pk.to_sym) alis.each { |ali| kvVal = ali.val log.call(%<#{INDENT * 5}#{ali.nodeType}: >) if ali.nodeType == DataMetaPii::CONST_KEY log.call(%<(#{ali.node.dataType}):: #{ali.node.key}=#{ali.node.nodeVal}>) else log.call(%<#{ali.val}>) end # noinspection RubyCaseWithoutElseBlockInspection aSect + case ali.nodeType # else case is caught by the AST parser when CONST_KEY # noinspection RubyCaseWithoutElseBlockInspection klass = case ali.node.dataType # else case is caught by the AST parser when STR_CONST_DT AlAttrStr when DECIMAL_CONST_DT AlAttrDec when INT_CONST_DT AlAttrInt end klass.new(ali.node.key.to_sym, ali.node.nodeVal) when REF_KEY AttrRef.new(kvVal) when VO_CLASS_KEY AlAttrVoClass.new(kvVal) end log.call("\n") } attrbs[ala.pk.to_sym] = aSect } log.call(%<#{INDENT}#{attrbs}\n>) apps[appKey] = attrbs } else raise ArgumentError, 'No Applink Division' end AppLink.new(ast.verDef.ver, apps, reusables) end
Builds the Registry CST from the given Registry Grammar Parser’s AST
# File lib/dataMetaPii.rb, line 461 def buildRegCst(ast) resultMap = {} ast.fields.elements.each { |f| fKey = f.pk raise ArgumentError, %<The PII field #{fKey} is defined more than once> if resultMap.keys.member?(fKey) attrs = {} f.attrbLst.attrbs.each { |a| raise ArgumentError, %<Attribute "#{a.k}" is defined more than once for #{fKey}> if attrs.keys.member?(a.k) attrs[a.k] = a.v } attrVo = RegKeyVo.new(fKey, attrs) resultMap[fKey] = attrVo } RegVo.new(ast.verDef.ver, resultMap) end
Helper method for the AST traversal to collect the attributes Because of the Treetop AST design, can not just flatten the elements and select of those of the needed type in one call, hence the tree traversal
# File lib/dataMetaPii.rb, line 482 def digAstElsType(type, els, result=[]) if els.nil? # is it a leaf? nil # not a leaf - return nil else els.each { |e| if e.respond_to?(:type) && e.type == type # actual attribute Key/Value? result << e # add it else digAstElsType(type, e.elements, result) # dig deeper into the AST end } result end end
# File lib/dataMetaPii.rb, line 497 def errNamespace(outFmt) raise ArgumentError, %<For output format "#{outFmt}", the Namespace is required> end
API method: generate the PII code
# File lib/dataMetaPii.rb, line 502 def genCode(scope, outFmt, outDirName, source, namespace = nil) # noinspection RubyCaseWithoutElseBlockInspection codeIndent = ' ' * 2 raise ArgumentError, %Q<Unsupported scope definition "#{scope}", supported scopes are: #{ DataMetaPii::ALL_SCOPES.map(&:to_s).join(', ')}> unless ALL_SCOPES.member?(scope) raise ArgumentError, %Q<Unsupported output format definition "#{outFmt}", supported formats are: #{ DataMetaPii::ALL_FMTS.map(&:to_s).join(', ')}> unless ALL_FMTS.member?(outFmt) raise ArgumentError, %<For safety purposes, absolute path names like "#{ outDirName}" are not supported> if outDirName.start_with?('/') raise ArgumentError, %<The output dir "#{outDirName}" is not a directory> unless File.directory?(outDirName) # noinspection RubyCaseWithoutElseBlockInspection case scope # else case caught up there, on argument validation when Scope::ABSTRACT reg = DataMetaPii.buildRegCst(DataMetaParse.parse(REGISTRY_PARSER, source)) L.info(%<PII Registry: #{reg.to_tree_image(INDENT)}>) tmpl = ERB.new(IO.read(File.join(GEM_ROOT, 'tpl', outFmt.to_s, 'master.erb')), $SAFE, '%<>>') className = "PiiAbstractDef_#{reg.ver.toVarName}" # noinspection RubyCaseWithoutElseBlockInspection case outFmt when ExportFmts::JAVA, ExportFmts::SCALA errNamespace(outFmt) unless namespace.is_a?(String) && !namespace.empty? pkgDir = namespace.gsub('.', '/') classDest =File.join(outDirName, pkgDir) FileUtils.mkpath classDest IO.write(File.join(classDest, "#{className}.#{outFmt}"), tmpl.result(binding).gsub(/\n\n+/, "\n\n"), mode: 'wb') # collapse multiple lines in 2 when ExportFmts::JSON IO.write(File.join(outDirName, "#{className}.#{outFmt}"), tmpl.result(binding).gsub(/\n\n+/, "\n\n"), mode: 'wb') # collapse multiple lines in 2 when ExportFmts::PYTHON pkgDir = namespace.gsub('.', '_') classDest =File.join(outDirName, pkgDir) FileUtils.mkpath classDest IO.write(File.join(classDest, "#{className[0].downcase + className[1..-1]}.py"), tmpl.result(binding).gsub(/\n\n+/, "\n\n"), mode: 'wb') # collapse multiple lines in 2 IO.write(File.join(classDest, '__init__.py'), %q< # see https://docs.python.org/3/library/pkgutil.html # without this, Python will have trouble finding packages that share some common tree off the root from pkgutil import extend_path __path__ = extend_path(__path__, __name__) >, mode: 'wb') end when DataMetaPii::Scope::APPLICATION raise NotImplementedError, 'There is no generic code gen for AppLink, each app/svc should have their own' end end
Turns the given text into the instance of the AppLink
object.
# File lib/dataMetaPii.rb, line 563 def parseAppLink(source) piiAppLinkParser = PiiAppLinkParser.new ast = DataMetaParse.parse(piiAppLinkParser, source) raise SyntaxError, 'AppLink parse unsuccessful' unless ast if ast.is_a?(DataMetaParse::Err) raise %<#{ast.parser.failure_line} ast.parser.failure_reason}> end DataMetaPii.buildAlCst(ast).resolveRefs end