001/* 002 * Copyright 2008-2020 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright 2008-2020 Ping Identity Corporation 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020/* 021 * Copyright (C) 2008-2020 Ping Identity Corporation 022 * 023 * This program is free software; you can redistribute it and/or modify 024 * it under the terms of the GNU General Public License (GPLv2 only) 025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 026 * as published by the Free Software Foundation. 027 * 028 * This program is distributed in the hope that it will be useful, 029 * but WITHOUT ANY WARRANTY; without even the implied warranty of 030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 031 * GNU General Public License for more details. 032 * 033 * You should have received a copy of the GNU General Public License 034 * along with this program; if not, see <http://www.gnu.org/licenses>. 035 */ 036package com.unboundid.ldap.sdk.examples; 037 038 039 040import java.io.File; 041import java.io.FileInputStream; 042import java.io.InputStream; 043import java.io.IOException; 044import java.io.OutputStream; 045import java.util.ArrayList; 046import java.util.Iterator; 047import java.util.TreeMap; 048import java.util.LinkedHashMap; 049import java.util.List; 050import java.util.concurrent.atomic.AtomicLong; 051import java.util.zip.GZIPInputStream; 052 053import com.unboundid.ldap.sdk.Entry; 054import com.unboundid.ldap.sdk.LDAPConnection; 055import com.unboundid.ldap.sdk.LDAPException; 056import com.unboundid.ldap.sdk.ResultCode; 057import com.unboundid.ldap.sdk.Version; 058import com.unboundid.ldap.sdk.schema.Schema; 059import com.unboundid.ldap.sdk.schema.EntryValidator; 060import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils; 061import com.unboundid.ldif.DuplicateValueBehavior; 062import com.unboundid.ldif.LDIFException; 063import com.unboundid.ldif.LDIFReader; 064import com.unboundid.ldif.LDIFReaderEntryTranslator; 065import com.unboundid.ldif.LDIFWriter; 066import com.unboundid.util.Debug; 067import com.unboundid.util.LDAPCommandLineTool; 068import com.unboundid.util.StaticUtils; 069import com.unboundid.util.ThreadSafety; 070import com.unboundid.util.ThreadSafetyLevel; 071import com.unboundid.util.args.ArgumentException; 072import com.unboundid.util.args.ArgumentParser; 073import com.unboundid.util.args.BooleanArgument; 074import com.unboundid.util.args.FileArgument; 075import com.unboundid.util.args.IntegerArgument; 076import com.unboundid.util.args.StringArgument; 077 078 079 080/** 081 * This class provides a simple tool that can be used to validate that the 082 * contents of an LDIF file are valid. This includes ensuring that the contents 083 * can be parsed as valid LDIF, and it can also ensure that the LDIF content 084 * conforms to the server schema. It will obtain the schema by connecting to 085 * the server and retrieving the default schema (i.e., the schema which governs 086 * the root DSE). By default, a thorough set of validation will be performed, 087 * but it is possible to disable certain types of validation. 088 * <BR><BR> 089 * Some of the APIs demonstrated by this example include: 090 * <UL> 091 * <LI>Argument Parsing (from the {@code com.unboundid.util.args} 092 * package)</LI> 093 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util} 094 * package)</LI> 095 * <LI>LDIF Processing (from the {@code com.unboundid.ldif} package)</LI> 096 * <LI>Schema Parsing (from the {@code com.unboundid.ldap.sdk.schema} 097 * package)</LI> 098 * </UL> 099 * <BR><BR> 100 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 101 * class (to obtain the information to use to connect to the server to read the 102 * schema), as well as the following additional arguments: 103 * <UL> 104 * <LI>"--schemaDirectory {path}" -- specifies the path to a directory 105 * containing files with schema definitions. If this argument is 106 * provided, then no attempt will be made to communicate with a directory 107 * server.</LI> 108 * <LI>"-f {path}" or "--ldifFile {path}" -- specifies the path to the LDIF 109 * file to be validated.</LI> 110 * <LI>"-c" or "--isCompressed" -- indicates that the LDIF file is 111 * compressed.</LI> 112 * <LI>"-R {path}" or "--rejectFile {path}" -- specifies the path to the file 113 * to be written with information about all entries that failed 114 * validation.</LI> 115 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of 116 * concurrent threads to use when processing the LDIF. If this is not 117 * provided, then a default of one thread will be used.</LI> 118 * <LI>"--ignoreUndefinedObjectClasses" -- indicates that the validation 119 * process should ignore validation failures due to entries that contain 120 * object classes not defined in the server schema.</LI> 121 * <LI>"--ignoreUndefinedAttributes" -- indicates that the validation process 122 * should ignore validation failures due to entries that contain 123 * attributes not defined in the server schema.</LI> 124 * <LI>"--ignoreMalformedDNs" -- indicates that the validation process should 125 * ignore validation failures due to entries with malformed DNs.</LI> 126 * <LI>"--ignoreMissingRDNValues" -- indicates that the validation process 127 * should ignore validation failures due to entries that contain an RDN 128 * attribute value that is not present in the set of entry 129 * attributes.</LI> 130 * <LI>"--ignoreStructuralObjectClasses" -- indicates that the validation 131 * process should ignore validation failures due to entries that either do 132 * not have a structural object class or that have multiple structural 133 * object classes.</LI> 134 * <LI>"--ignoreProhibitedObjectClasses" -- indicates that the validation 135 * process should ignore validation failures due to entries containing 136 * auxiliary classes that are not allowed by a DIT content rule, or 137 * abstract classes that are not subclassed by an auxiliary or structural 138 * class contained in the entry.</LI> 139 * <LI>"--ignoreProhibitedAttributes" -- indicates that the validation process 140 * should ignore validation failures due to entries including attributes 141 * that are not allowed or are explicitly prohibited by a DIT content 142 * rule.</LI> 143 * <LI>"--ignoreMissingAttributes" -- indicates that the validation process 144 * should ignore validation failures due to entries missing required 145 * attributes.</LI> 146 * <LI>"--ignoreSingleValuedAttributes" -- indicates that the validation 147 * process should ignore validation failures due to single-valued 148 * attributes containing multiple values.</LI> 149 * <LI>"--ignoreAttributeSyntax" -- indicates that the validation process 150 * should ignore validation failures due to attribute values which violate 151 * the associated attribute syntax.</LI> 152 * <LI>"--ignoreSyntaxViolationsForAttribute" -- indicates that the validation 153 * process should ignore validation failures due to attribute values which 154 * violate the associated attribute syntax, but only for the specified 155 * attribute types.</LI> 156 * <LI>"--ignoreNameForms" -- indicates that the validation process should 157 * ignore validation failures due to name form violations (in which the 158 * entry's RDN does not comply with the associated name form).</LI> 159 * </UL> 160 */ 161@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 162public final class ValidateLDIF 163 extends LDAPCommandLineTool 164 implements LDIFReaderEntryTranslator 165{ 166 /** 167 * The end-of-line character for this platform. 168 */ 169 private static final String EOL = 170 StaticUtils.getSystemProperty("line.separator", "\n"); 171 172 173 174 // The arguments used by this program. 175 private BooleanArgument ignoreDuplicateValues; 176 private BooleanArgument ignoreUndefinedObjectClasses; 177 private BooleanArgument ignoreUndefinedAttributes; 178 private BooleanArgument ignoreMalformedDNs; 179 private BooleanArgument ignoreMissingRDNValues; 180 private BooleanArgument ignoreMissingSuperiorObjectClasses; 181 private BooleanArgument ignoreStructuralObjectClasses; 182 private BooleanArgument ignoreProhibitedObjectClasses; 183 private BooleanArgument ignoreProhibitedAttributes; 184 private BooleanArgument ignoreMissingAttributes; 185 private BooleanArgument ignoreSingleValuedAttributes; 186 private BooleanArgument ignoreAttributeSyntax; 187 private BooleanArgument ignoreNameForms; 188 private BooleanArgument isCompressed; 189 private FileArgument schemaDirectory; 190 private FileArgument ldifFile; 191 private FileArgument rejectFile; 192 private FileArgument encryptionPassphraseFile; 193 private IntegerArgument numThreads; 194 private StringArgument ignoreSyntaxViolationsForAttribute; 195 196 // The counter used to keep track of the number of entries processed. 197 private final AtomicLong entriesProcessed = new AtomicLong(0L); 198 199 // The counter used to keep track of the number of entries that could not be 200 // parsed as valid entries. 201 private final AtomicLong malformedEntries = new AtomicLong(0L); 202 203 // The entry validator that will be used to validate the entries. 204 private EntryValidator entryValidator; 205 206 // The LDIF writer that will be used to write rejected entries. 207 private LDIFWriter rejectWriter; 208 209 210 211 /** 212 * Parse the provided command line arguments and make the appropriate set of 213 * changes. 214 * 215 * @param args The command line arguments provided to this program. 216 */ 217 public static void main(final String[] args) 218 { 219 final ResultCode resultCode = main(args, System.out, System.err); 220 if (resultCode != ResultCode.SUCCESS) 221 { 222 System.exit(resultCode.intValue()); 223 } 224 } 225 226 227 228 /** 229 * Parse the provided command line arguments and make the appropriate set of 230 * changes. 231 * 232 * @param args The command line arguments provided to this program. 233 * @param outStream The output stream to which standard out should be 234 * written. It may be {@code null} if output should be 235 * suppressed. 236 * @param errStream The output stream to which standard error should be 237 * written. It may be {@code null} if error messages 238 * should be suppressed. 239 * 240 * @return A result code indicating whether the processing was successful. 241 */ 242 public static ResultCode main(final String[] args, 243 final OutputStream outStream, 244 final OutputStream errStream) 245 { 246 final ValidateLDIF validateLDIF = new ValidateLDIF(outStream, errStream); 247 return validateLDIF.runTool(args); 248 } 249 250 251 252 /** 253 * Creates a new instance of this tool. 254 * 255 * @param outStream The output stream to which standard out should be 256 * written. It may be {@code null} if output should be 257 * suppressed. 258 * @param errStream The output stream to which standard error should be 259 * written. It may be {@code null} if error messages 260 * should be suppressed. 261 */ 262 public ValidateLDIF(final OutputStream outStream, 263 final OutputStream errStream) 264 { 265 super(outStream, errStream); 266 } 267 268 269 270 /** 271 * Retrieves the name for this tool. 272 * 273 * @return The name for this tool. 274 */ 275 @Override() 276 public String getToolName() 277 { 278 return "validate-ldif"; 279 } 280 281 282 283 /** 284 * Retrieves the description for this tool. 285 * 286 * @return The description for this tool. 287 */ 288 @Override() 289 public String getToolDescription() 290 { 291 return "Validate the contents of an LDIF file " + 292 "against the server schema."; 293 } 294 295 296 297 /** 298 * Retrieves the version string for this tool. 299 * 300 * @return The version string for this tool. 301 */ 302 @Override() 303 public String getToolVersion() 304 { 305 return Version.NUMERIC_VERSION_STRING; 306 } 307 308 309 310 /** 311 * Indicates whether this tool should provide support for an interactive mode, 312 * in which the tool offers a mode in which the arguments can be provided in 313 * a text-driven menu rather than requiring them to be given on the command 314 * line. If interactive mode is supported, it may be invoked using the 315 * "--interactive" argument. Alternately, if interactive mode is supported 316 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 317 * interactive mode may be invoked by simply launching the tool without any 318 * arguments. 319 * 320 * @return {@code true} if this tool supports interactive mode, or 321 * {@code false} if not. 322 */ 323 @Override() 324 public boolean supportsInteractiveMode() 325 { 326 return true; 327 } 328 329 330 331 /** 332 * Indicates whether this tool defaults to launching in interactive mode if 333 * the tool is invoked without any command-line arguments. This will only be 334 * used if {@link #supportsInteractiveMode()} returns {@code true}. 335 * 336 * @return {@code true} if this tool defaults to using interactive mode if 337 * launched without any command-line arguments, or {@code false} if 338 * not. 339 */ 340 @Override() 341 public boolean defaultsToInteractiveMode() 342 { 343 return true; 344 } 345 346 347 348 /** 349 * Indicates whether this tool should provide arguments for redirecting output 350 * to a file. If this method returns {@code true}, then the tool will offer 351 * an "--outputFile" argument that will specify the path to a file to which 352 * all standard output and standard error content will be written, and it will 353 * also offer a "--teeToStandardOut" argument that can only be used if the 354 * "--outputFile" argument is present and will cause all output to be written 355 * to both the specified output file and to standard output. 356 * 357 * @return {@code true} if this tool should provide arguments for redirecting 358 * output to a file, or {@code false} if not. 359 */ 360 @Override() 361 protected boolean supportsOutputFile() 362 { 363 return true; 364 } 365 366 367 368 /** 369 * Indicates whether this tool should default to interactively prompting for 370 * the bind password if a password is required but no argument was provided 371 * to indicate how to get the password. 372 * 373 * @return {@code true} if this tool should default to interactively 374 * prompting for the bind password, or {@code false} if not. 375 */ 376 @Override() 377 protected boolean defaultToPromptForBindPassword() 378 { 379 return true; 380 } 381 382 383 384 /** 385 * Indicates whether this tool supports the use of a properties file for 386 * specifying default values for arguments that aren't specified on the 387 * command line. 388 * 389 * @return {@code true} if this tool supports the use of a properties file 390 * for specifying default values for arguments that aren't specified 391 * on the command line, or {@code false} if not. 392 */ 393 @Override() 394 public boolean supportsPropertiesFile() 395 { 396 return true; 397 } 398 399 400 401 /** 402 * Indicates whether the LDAP-specific arguments should include alternate 403 * versions of all long identifiers that consist of multiple words so that 404 * they are available in both camelCase and dash-separated versions. 405 * 406 * @return {@code true} if this tool should provide multiple versions of 407 * long identifiers for LDAP-specific arguments, or {@code false} if 408 * not. 409 */ 410 @Override() 411 protected boolean includeAlternateLongIdentifiers() 412 { 413 return true; 414 } 415 416 417 418 /** 419 * Indicates whether this tool should provide a command-line argument that 420 * allows for low-level SSL debugging. If this returns {@code true}, then an 421 * "--enableSSLDebugging}" argument will be added that sets the 422 * "javax.net.debug" system property to "all" before attempting any 423 * communication. 424 * 425 * @return {@code true} if this tool should offer an "--enableSSLDebugging" 426 * argument, or {@code false} if not. 427 */ 428 @Override() 429 protected boolean supportsSSLDebugging() 430 { 431 return true; 432 } 433 434 435 436 /** 437 * Adds the arguments used by this program that aren't already provided by the 438 * generic {@code LDAPCommandLineTool} framework. 439 * 440 * @param parser The argument parser to which the arguments should be added. 441 * 442 * @throws ArgumentException If a problem occurs while adding the arguments. 443 */ 444 @Override() 445 public void addNonLDAPArguments(final ArgumentParser parser) 446 throws ArgumentException 447 { 448 String description = "The path to the LDIF file to process. The tool " + 449 "will automatically attempt to detect whether the file is " + 450 "encrypted or compressed."; 451 ldifFile = new FileArgument('f', "ldifFile", true, 1, "{path}", description, 452 true, true, true, false); 453 ldifFile.addLongIdentifier("ldif-file", true); 454 parser.addArgument(ldifFile); 455 456 457 // Add an argument that makes it possible to read a compressed LDIF file. 458 // Note that this argument is no longer needed for dealing with compressed 459 // files, since the tool will automatically detect whether a file is 460 // compressed. However, the argument is still provided for the purpose of 461 // backward compatibility. 462 description = "Indicates that the specified LDIF file is compressed " + 463 "using gzip compression."; 464 isCompressed = new BooleanArgument('c', "isCompressed", description); 465 isCompressed.addLongIdentifier("is-compressed", true); 466 isCompressed.setHidden(true); 467 parser.addArgument(isCompressed); 468 469 470 // Add an argument that indicates that the tool should read the encryption 471 // passphrase from a file. 472 description = "Indicates that the specified LDIF file is encrypted and " + 473 "that the encryption passphrase is contained in the specified " + 474 "file. If the LDIF data is encrypted and this argument is not " + 475 "provided, then the tool will interactively prompt for the " + 476 "encryption passphrase."; 477 encryptionPassphraseFile = new FileArgument(null, 478 "encryptionPassphraseFile", false, 1, null, description, true, true, 479 true, false); 480 encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file", 481 true); 482 encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true); 483 encryptionPassphraseFile.addLongIdentifier("encryption-password-file", 484 true); 485 parser.addArgument(encryptionPassphraseFile); 486 487 488 description = "The path to the file to which rejected entries should be " + 489 "written."; 490 rejectFile = new FileArgument('R', "rejectFile", false, 1, "{path}", 491 description, false, true, true, false); 492 rejectFile.addLongIdentifier("reject-file", true); 493 parser.addArgument(rejectFile); 494 495 description = "The path to a directory containing one or more LDIF files " + 496 "with the schema information to use. If this is provided, " + 497 "then no LDAP communication will be performed."; 498 schemaDirectory = new FileArgument(null, "schemaDirectory", false, 1, 499 "{path}", description, true, true, false, true); 500 schemaDirectory.addLongIdentifier("schema-directory", true); 501 parser.addArgument(schemaDirectory); 502 503 description = "The number of threads to use when processing the LDIF file."; 504 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}", 505 description, 1, Integer.MAX_VALUE, 1); 506 numThreads.addLongIdentifier("num-threads", true); 507 parser.addArgument(numThreads); 508 509 description = "Ignore validation failures due to entries containing " + 510 "duplicate values for the same attribute."; 511 ignoreDuplicateValues = 512 new BooleanArgument(null, "ignoreDuplicateValues", description); 513 ignoreDuplicateValues.setArgumentGroupName( 514 "Validation Strictness Arguments"); 515 ignoreDuplicateValues.addLongIdentifier("ignore-duplicate-values", true); 516 parser.addArgument(ignoreDuplicateValues); 517 518 description = "Ignore validation failures due to object classes not " + 519 "defined in the schema."; 520 ignoreUndefinedObjectClasses = 521 new BooleanArgument(null, "ignoreUndefinedObjectClasses", description); 522 ignoreUndefinedObjectClasses.setArgumentGroupName( 523 "Validation Strictness Arguments"); 524 ignoreUndefinedObjectClasses.addLongIdentifier( 525 "ignore-undefined-object-classes", true); 526 parser.addArgument(ignoreUndefinedObjectClasses); 527 528 description = "Ignore validation failures due to attributes not defined " + 529 "in the schema."; 530 ignoreUndefinedAttributes = 531 new BooleanArgument(null, "ignoreUndefinedAttributes", description); 532 ignoreUndefinedAttributes.setArgumentGroupName( 533 "Validation Strictness Arguments"); 534 ignoreUndefinedAttributes.addLongIdentifier("ignore-undefined-attributes", 535 true); 536 parser.addArgument(ignoreUndefinedAttributes); 537 538 description = "Ignore validation failures due to entries with malformed " + 539 "DNs."; 540 ignoreMalformedDNs = 541 new BooleanArgument(null, "ignoreMalformedDNs", description); 542 ignoreMalformedDNs.setArgumentGroupName("Validation Strictness Arguments"); 543 ignoreMalformedDNs.addLongIdentifier("ignore-malformed-dns", true); 544 parser.addArgument(ignoreMalformedDNs); 545 546 description = "Ignore validation failures due to entries with RDN " + 547 "attribute values that are missing from the set of entry " + 548 "attributes."; 549 ignoreMissingRDNValues = 550 new BooleanArgument(null, "ignoreMissingRDNValues", description); 551 ignoreMissingRDNValues.setArgumentGroupName( 552 "Validation Strictness Arguments"); 553 ignoreMissingRDNValues.addLongIdentifier("ignore-missing-rdn-values", true); 554 parser.addArgument(ignoreMissingRDNValues); 555 556 description = "Ignore validation failures due to entries without exactly " + 557 "structural object class."; 558 ignoreStructuralObjectClasses = 559 new BooleanArgument(null, "ignoreStructuralObjectClasses", 560 description); 561 ignoreStructuralObjectClasses.setArgumentGroupName( 562 "Validation Strictness Arguments"); 563 ignoreStructuralObjectClasses.addLongIdentifier( 564 "ignore-structural-object-classes", true); 565 parser.addArgument(ignoreStructuralObjectClasses); 566 567 description = "Ignore validation failures due to entries with object " + 568 "classes that are not allowed."; 569 ignoreProhibitedObjectClasses = 570 new BooleanArgument(null, "ignoreProhibitedObjectClasses", 571 description); 572 ignoreProhibitedObjectClasses.setArgumentGroupName( 573 "Validation Strictness Arguments"); 574 ignoreProhibitedObjectClasses.addLongIdentifier( 575 "ignore-prohibited-object-classes", true); 576 parser.addArgument(ignoreProhibitedObjectClasses); 577 578 description = "Ignore validation failures due to entries that are " + 579 "one or more superior object classes."; 580 ignoreMissingSuperiorObjectClasses = 581 new BooleanArgument(null, "ignoreMissingSuperiorObjectClasses", 582 description); 583 ignoreMissingSuperiorObjectClasses.setArgumentGroupName( 584 "Validation Strictness Arguments"); 585 ignoreMissingSuperiorObjectClasses.addLongIdentifier( 586 "ignore-missing-superior-object-classes", true); 587 parser.addArgument(ignoreMissingSuperiorObjectClasses); 588 589 description = "Ignore validation failures due to entries with attributes " + 590 "that are not allowed."; 591 ignoreProhibitedAttributes = 592 new BooleanArgument(null, "ignoreProhibitedAttributes", description); 593 ignoreProhibitedAttributes.setArgumentGroupName( 594 "Validation Strictness Arguments"); 595 ignoreProhibitedAttributes.addLongIdentifier( 596 "ignore-prohibited-attributes", true); 597 parser.addArgument(ignoreProhibitedAttributes); 598 599 description = "Ignore validation failures due to entries missing " + 600 "required attributes."; 601 ignoreMissingAttributes = 602 new BooleanArgument(null, "ignoreMissingAttributes", description); 603 ignoreMissingAttributes.setArgumentGroupName( 604 "Validation Strictness Arguments"); 605 ignoreMissingAttributes.addLongIdentifier("ignore-missing-attributes", 606 true); 607 parser.addArgument(ignoreMissingAttributes); 608 609 description = "Ignore validation failures due to entries with multiple " + 610 "values for single-valued attributes."; 611 ignoreSingleValuedAttributes = 612 new BooleanArgument(null, "ignoreSingleValuedAttributes", description); 613 ignoreSingleValuedAttributes.setArgumentGroupName( 614 "Validation Strictness Arguments"); 615 ignoreSingleValuedAttributes.addLongIdentifier( 616 "ignore-single-valued-attributes", true); 617 parser.addArgument(ignoreSingleValuedAttributes); 618 619 description = "Ignore validation failures due to entries with attribute " + 620 "values that violate their associated syntax. If this is " + 621 "provided, then no attribute syntax violations will be " + 622 "flagged. If this is not provided, then all attribute " + 623 "syntax violations will be flagged except for violations " + 624 "in those attributes excluded by the " + 625 "--ignoreSyntaxViolationsForAttribute argument."; 626 ignoreAttributeSyntax = 627 new BooleanArgument(null, "ignoreAttributeSyntax", description); 628 ignoreAttributeSyntax.setArgumentGroupName( 629 "Validation Strictness Arguments"); 630 ignoreAttributeSyntax.addLongIdentifier("ignore-attribute-syntax", true); 631 parser.addArgument(ignoreAttributeSyntax); 632 633 description = "The name or OID of an attribute for which to ignore " + 634 "validation failures due to violations of the associated " + 635 "attribute syntax. This argument can only be used if the " + 636 "--ignoreAttributeSyntax argument is not provided."; 637 ignoreSyntaxViolationsForAttribute = new StringArgument(null, 638 "ignoreSyntaxViolationsForAttribute", false, 0, "{attr}", description); 639 ignoreSyntaxViolationsForAttribute.setArgumentGroupName( 640 "Validation Strictness Arguments"); 641 ignoreSyntaxViolationsForAttribute.addLongIdentifier( 642 "ignore-syntax-violations-for-attribute", true); 643 parser.addArgument(ignoreSyntaxViolationsForAttribute); 644 645 description = "Ignore validation failures due to entries with RDNs " + 646 "that violate the associated name form definition."; 647 ignoreNameForms = new BooleanArgument(null, "ignoreNameForms", description); 648 ignoreNameForms.setArgumentGroupName("Validation Strictness Arguments"); 649 ignoreNameForms.addLongIdentifier("ignore-name-forms", true); 650 parser.addArgument(ignoreNameForms); 651 652 653 // The ignoreAttributeSyntax and ignoreAttributeSyntaxForAttribute arguments 654 // cannot be used together. 655 parser.addExclusiveArgumentSet(ignoreAttributeSyntax, 656 ignoreSyntaxViolationsForAttribute); 657 } 658 659 660 661 /** 662 * Performs the actual processing for this tool. In this case, it gets a 663 * connection to the directory server and uses it to retrieve the server 664 * schema. It then reads the LDIF file and validates each entry accordingly. 665 * 666 * @return The result code for the processing that was performed. 667 */ 668 @Override() 669 public ResultCode doToolProcessing() 670 { 671 // Get the connection to the directory server and use it to read the schema. 672 final Schema schema; 673 if (schemaDirectory.isPresent()) 674 { 675 final File schemaDir = schemaDirectory.getValue(); 676 677 try 678 { 679 final TreeMap<String,File> fileMap = new TreeMap<>(); 680 for (final File f : schemaDir.listFiles()) 681 { 682 final String name = f.getName(); 683 if (f.isFile() && name.endsWith(".ldif")) 684 { 685 fileMap.put(name, f); 686 } 687 } 688 689 if (fileMap.isEmpty()) 690 { 691 err("No LDIF files found in directory " + 692 schemaDir.getAbsolutePath()); 693 return ResultCode.PARAM_ERROR; 694 } 695 696 final ArrayList<File> fileList = new ArrayList<>(fileMap.values()); 697 schema = Schema.getSchema(fileList); 698 } 699 catch (final Exception e) 700 { 701 Debug.debugException(e); 702 err("Unable to read schema from files in directory " + 703 schemaDir.getAbsolutePath() + ": " + 704 StaticUtils.getExceptionMessage(e)); 705 return ResultCode.LOCAL_ERROR; 706 } 707 } 708 else 709 { 710 try 711 { 712 final LDAPConnection connection = getConnection(); 713 schema = connection.getSchema(); 714 connection.close(); 715 } 716 catch (final LDAPException le) 717 { 718 Debug.debugException(le); 719 err("Unable to connect to the directory server and read the schema: ", 720 le.getMessage()); 721 return le.getResultCode(); 722 } 723 } 724 725 726 // Get the encryption passphrase, if it was provided. 727 String encryptionPassphrase = null; 728 if (encryptionPassphraseFile.isPresent()) 729 { 730 try 731 { 732 encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile( 733 encryptionPassphraseFile.getValue()); 734 } 735 catch (final LDAPException e) 736 { 737 Debug.debugException(e); 738 err(e.getMessage()); 739 return e.getResultCode(); 740 } 741 } 742 743 744 // Create the entry validator and initialize its configuration. 745 entryValidator = new EntryValidator(schema); 746 entryValidator.setCheckAttributeSyntax(!ignoreAttributeSyntax.isPresent()); 747 entryValidator.setCheckMalformedDNs(!ignoreMalformedDNs.isPresent()); 748 entryValidator.setCheckEntryMissingRDNValues( 749 !ignoreMissingRDNValues.isPresent()); 750 entryValidator.setCheckMissingAttributes( 751 !ignoreMissingAttributes.isPresent()); 752 entryValidator.setCheckNameForms(!ignoreNameForms.isPresent()); 753 entryValidator.setCheckProhibitedAttributes( 754 !ignoreProhibitedAttributes.isPresent()); 755 entryValidator.setCheckProhibitedObjectClasses( 756 !ignoreProhibitedObjectClasses.isPresent()); 757 entryValidator.setCheckMissingSuperiorObjectClasses( 758 !ignoreMissingSuperiorObjectClasses.isPresent()); 759 entryValidator.setCheckSingleValuedAttributes( 760 !ignoreSingleValuedAttributes.isPresent()); 761 entryValidator.setCheckStructuralObjectClasses( 762 !ignoreStructuralObjectClasses.isPresent()); 763 entryValidator.setCheckUndefinedAttributes( 764 !ignoreUndefinedAttributes.isPresent()); 765 entryValidator.setCheckUndefinedObjectClasses( 766 !ignoreUndefinedObjectClasses.isPresent()); 767 768 if (ignoreSyntaxViolationsForAttribute.isPresent()) 769 { 770 entryValidator.setIgnoreSyntaxViolationAttributeTypes( 771 ignoreSyntaxViolationsForAttribute.getValues()); 772 } 773 774 775 // Create an LDIF reader that can be used to read through the LDIF file. 776 final LDIFReader ldifReader; 777 rejectWriter = null; 778 try 779 { 780 InputStream inputStream = new FileInputStream(ldifFile.getValue()); 781 782 inputStream = ToolUtils.getPossiblyPassphraseEncryptedInputStream( 783 inputStream, encryptionPassphrase, false, 784 "LDIF file '" + ldifFile.getValue().getPath() + 785 "' is encrypted. Please enter the encryption passphrase:", 786 "ERROR: The provided passphrase was incorrect.", 787 getOut(), getErr()).getFirst(); 788 789 if (isCompressed.isPresent()) 790 { 791 inputStream = new GZIPInputStream(inputStream); 792 } 793 else 794 { 795 inputStream = 796 ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream); 797 } 798 799 ldifReader = new LDIFReader(inputStream, numThreads.getValue(), this); 800 } 801 catch (final Exception e) 802 { 803 Debug.debugException(e); 804 err("Unable to open the LDIF reader: ", 805 StaticUtils.getExceptionMessage(e)); 806 return ResultCode.LOCAL_ERROR; 807 } 808 809 ldifReader.setSchema(schema); 810 if (ignoreDuplicateValues.isPresent()) 811 { 812 ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.STRIP); 813 } 814 else 815 { 816 ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.REJECT); 817 } 818 819 try 820 { 821 // Create an LDIF writer that can be used to write information about 822 // rejected entries. 823 try 824 { 825 if (rejectFile.isPresent()) 826 { 827 rejectWriter = new LDIFWriter(rejectFile.getValue()); 828 } 829 } 830 catch (final Exception e) 831 { 832 Debug.debugException(e); 833 err("Unable to create the reject writer: ", 834 StaticUtils.getExceptionMessage(e)); 835 return ResultCode.LOCAL_ERROR; 836 } 837 838 ResultCode resultCode = ResultCode.SUCCESS; 839 while (true) 840 { 841 try 842 { 843 final Entry e = ldifReader.readEntry(); 844 if (e == null) 845 { 846 // Because we're performing parallel processing and returning null 847 // from the translate method, LDIFReader.readEntry() should never 848 // return a non-null value. However, it can throw an LDIFException 849 // if it encounters an invalid entry, or an IOException if there's 850 // a problem reading from the file, so we should still iterate 851 // through all of the entries to catch and report on those problems. 852 break; 853 } 854 } 855 catch (final LDIFException le) 856 { 857 Debug.debugException(le); 858 malformedEntries.incrementAndGet(); 859 860 if (resultCode == ResultCode.SUCCESS) 861 { 862 resultCode = ResultCode.DECODING_ERROR; 863 } 864 865 if (rejectWriter != null) 866 { 867 try 868 { 869 rejectWriter.writeComment( 870 "Unable to parse an entry read from LDIF:", false, false); 871 if (le.mayContinueReading()) 872 { 873 rejectWriter.writeComment( 874 StaticUtils.getExceptionMessage(le), false, true); 875 } 876 else 877 { 878 rejectWriter.writeComment( 879 StaticUtils.getExceptionMessage(le), false, 880 false); 881 rejectWriter.writeComment("Unable to continue LDIF processing.", 882 false, true); 883 err("Aborting LDIF processing: ", 884 StaticUtils.getExceptionMessage(le)); 885 return ResultCode.LOCAL_ERROR; 886 } 887 } 888 catch (final IOException ioe) 889 { 890 Debug.debugException(ioe); 891 err("Unable to write to the reject file:", 892 StaticUtils.getExceptionMessage(ioe)); 893 err("LDIF parse failure that triggered the rejection: ", 894 StaticUtils.getExceptionMessage(le)); 895 return ResultCode.LOCAL_ERROR; 896 } 897 } 898 } 899 catch (final IOException ioe) 900 { 901 Debug.debugException(ioe); 902 903 if (rejectWriter != null) 904 { 905 try 906 { 907 rejectWriter.writeComment("I/O error reading from LDIF:", false, 908 false); 909 rejectWriter.writeComment(StaticUtils.getExceptionMessage(ioe), 910 false, true); 911 return ResultCode.LOCAL_ERROR; 912 } 913 catch (final Exception ex) 914 { 915 Debug.debugException(ex); 916 err("I/O error reading from LDIF:", 917 StaticUtils.getExceptionMessage(ioe)); 918 return ResultCode.LOCAL_ERROR; 919 } 920 } 921 } 922 } 923 924 if (malformedEntries.get() > 0) 925 { 926 out(malformedEntries.get() + " entries were malformed and could not " + 927 "be read from the LDIF file."); 928 } 929 930 if (entryValidator.getInvalidEntries() > 0) 931 { 932 if (resultCode == ResultCode.SUCCESS) 933 { 934 resultCode = ResultCode.OBJECT_CLASS_VIOLATION; 935 } 936 937 for (final String s : entryValidator.getInvalidEntrySummary(true)) 938 { 939 out(s); 940 } 941 } 942 else 943 { 944 if (malformedEntries.get() == 0) 945 { 946 out("No errors were encountered."); 947 } 948 } 949 950 return resultCode; 951 } 952 finally 953 { 954 try 955 { 956 ldifReader.close(); 957 } 958 catch (final Exception e) 959 { 960 Debug.debugException(e); 961 } 962 963 try 964 { 965 if (rejectWriter != null) 966 { 967 rejectWriter.close(); 968 } 969 } 970 catch (final Exception e) 971 { 972 Debug.debugException(e); 973 } 974 } 975 } 976 977 978 979 /** 980 * Examines the provided entry to determine whether it conforms to the 981 * server schema. 982 * 983 * @param entry The entry to be examined. 984 * @param firstLineNumber The line number of the LDIF source on which the 985 * provided entry begins. 986 * 987 * @return The updated entry. This method will always return {@code null} 988 * because all of the real processing needed for the entry is 989 * performed in this method and the entry isn't needed any more 990 * after this method is done. 991 */ 992 @Override() 993 public Entry translate(final Entry entry, final long firstLineNumber) 994 { 995 final ArrayList<String> invalidReasons = new ArrayList<>(5); 996 if (! entryValidator.entryIsValid(entry, invalidReasons)) 997 { 998 if (rejectWriter != null) 999 { 1000 synchronized (this) 1001 { 1002 try 1003 { 1004 rejectWriter.writeEntry(entry, listToString(invalidReasons)); 1005 } 1006 catch (final IOException ioe) 1007 { 1008 Debug.debugException(ioe); 1009 } 1010 } 1011 } 1012 } 1013 1014 final long numEntries = entriesProcessed.incrementAndGet(); 1015 if ((numEntries % 1000L) == 0L) 1016 { 1017 out("Processed ", numEntries, " entries."); 1018 } 1019 1020 return null; 1021 } 1022 1023 1024 1025 /** 1026 * Converts the provided list of strings into a single string. It will 1027 * contain line breaks after all but the last element. 1028 * 1029 * @param l The list of strings to convert to a single string. 1030 * 1031 * @return The string from the provided list, or {@code null} if the provided 1032 * list is empty or {@code null}. 1033 */ 1034 private static String listToString(final List<String> l) 1035 { 1036 if ((l == null) || (l.isEmpty())) 1037 { 1038 return null; 1039 } 1040 1041 final StringBuilder buffer = new StringBuilder(); 1042 final Iterator<String> iterator = l.iterator(); 1043 while (iterator.hasNext()) 1044 { 1045 buffer.append(iterator.next()); 1046 if (iterator.hasNext()) 1047 { 1048 buffer.append(EOL); 1049 } 1050 } 1051 1052 return buffer.toString(); 1053 } 1054 1055 1056 1057 /** 1058 * {@inheritDoc} 1059 */ 1060 @Override() 1061 public LinkedHashMap<String[],String> getExampleUsages() 1062 { 1063 final LinkedHashMap<String[],String> examples = 1064 new LinkedHashMap<>(StaticUtils.computeMapCapacity(2)); 1065 1066 String[] args = 1067 { 1068 "--hostname", "server.example.com", 1069 "--port", "389", 1070 "--ldifFile", "data.ldif", 1071 "--rejectFile", "rejects.ldif", 1072 "--numThreads", "4" 1073 }; 1074 String description = 1075 "Validate the contents of the 'data.ldif' file using the schema " + 1076 "defined in the specified directory server using four concurrent " + 1077 "threads. All types of validation will be performed, and " + 1078 "information about any errors will be written to the 'rejects.ldif' " + 1079 "file."; 1080 examples.put(args, description); 1081 1082 1083 args = new String[] 1084 { 1085 "--schemaDirectory", "/ds/config/schema", 1086 "--ldifFile", "data.ldif", 1087 "--rejectFile", "rejects.ldif", 1088 "--ignoreStructuralObjectClasses", 1089 "--ignoreAttributeSyntax" 1090 }; 1091 description = 1092 "Validate the contents of the 'data.ldif' file using the schema " + 1093 "defined in LDIF files contained in the /ds/config/schema directory " + 1094 "using a single thread. Any errors resulting from entries that do " + 1095 "not have exactly one structural object class or from values which " + 1096 "violate the syntax for their associated attribute types will be " + 1097 "ignored. Information about any other failures will be written to " + 1098 "the 'rejects.ldif' file."; 1099 examples.put(args, description); 1100 1101 return examples; 1102 } 1103 1104 1105 1106 /** 1107 * @return EntryValidator 1108 * 1109 * Returns the EntryValidator 1110 */ 1111 public EntryValidator getEntryValidator() 1112 { 1113 return entryValidator; 1114 } 1115}