001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.awt.GraphicsEnvironment; 007import java.io.File; 008import java.io.FileNotFoundException; 009import java.io.IOException; 010import java.io.PrintWriter; 011import java.nio.charset.StandardCharsets; 012import java.nio.file.Files; 013import java.nio.file.Path; 014import java.nio.file.Paths; 015import java.util.ArrayList; 016import java.util.Arrays; 017import java.util.Collection; 018import java.util.Collections; 019import java.util.EnumMap; 020import java.util.HashMap; 021import java.util.List; 022import java.util.Map; 023import java.util.SortedMap; 024import java.util.TreeMap; 025import java.util.TreeSet; 026import java.util.function.Predicate; 027import java.util.stream.Collectors; 028 029import javax.swing.JOptionPane; 030 031import org.openstreetmap.josm.Main; 032import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper; 033import org.openstreetmap.josm.data.validation.tests.Addresses; 034import org.openstreetmap.josm.data.validation.tests.ApiCapabilitiesTest; 035import org.openstreetmap.josm.data.validation.tests.BarriersEntrances; 036import org.openstreetmap.josm.data.validation.tests.Coastlines; 037import org.openstreetmap.josm.data.validation.tests.ConditionalKeys; 038import org.openstreetmap.josm.data.validation.tests.CrossingWays; 039import org.openstreetmap.josm.data.validation.tests.DuplicateNode; 040import org.openstreetmap.josm.data.validation.tests.DuplicateRelation; 041import org.openstreetmap.josm.data.validation.tests.DuplicateWay; 042import org.openstreetmap.josm.data.validation.tests.DuplicatedWayNodes; 043import org.openstreetmap.josm.data.validation.tests.Highways; 044import org.openstreetmap.josm.data.validation.tests.InternetTags; 045import org.openstreetmap.josm.data.validation.tests.Lanes; 046import org.openstreetmap.josm.data.validation.tests.LongSegment; 047import org.openstreetmap.josm.data.validation.tests.MapCSSTagChecker; 048import org.openstreetmap.josm.data.validation.tests.MultipolygonTest; 049import org.openstreetmap.josm.data.validation.tests.NameMismatch; 050import org.openstreetmap.josm.data.validation.tests.OpeningHourTest; 051import org.openstreetmap.josm.data.validation.tests.OverlappingWays; 052import org.openstreetmap.josm.data.validation.tests.PowerLines; 053import org.openstreetmap.josm.data.validation.tests.PublicTransportRouteTest; 054import org.openstreetmap.josm.data.validation.tests.RelationChecker; 055import org.openstreetmap.josm.data.validation.tests.SelfIntersectingWay; 056import org.openstreetmap.josm.data.validation.tests.SimilarNamedWays; 057import org.openstreetmap.josm.data.validation.tests.TagChecker; 058import org.openstreetmap.josm.data.validation.tests.TurnrestrictionTest; 059import org.openstreetmap.josm.data.validation.tests.UnclosedWays; 060import org.openstreetmap.josm.data.validation.tests.UnconnectedWays; 061import org.openstreetmap.josm.data.validation.tests.UntaggedNode; 062import org.openstreetmap.josm.data.validation.tests.UntaggedWay; 063import org.openstreetmap.josm.data.validation.tests.WayConnectedToArea; 064import org.openstreetmap.josm.data.validation.tests.WronglyOrderedWays; 065import org.openstreetmap.josm.gui.MainApplication; 066import org.openstreetmap.josm.gui.layer.ValidatorLayer; 067import org.openstreetmap.josm.gui.preferences.projection.ProjectionPreference; 068import org.openstreetmap.josm.spi.preferences.Config; 069import org.openstreetmap.josm.tools.AlphanumComparator; 070import org.openstreetmap.josm.tools.Logging; 071import org.openstreetmap.josm.tools.Utils; 072 073/** 074 * A OSM data validator. 075 * 076 * @author Francisco R. Santos <frsantos@gmail.com> 077 */ 078public final class OsmValidator { 079 080 private OsmValidator() { 081 // Hide default constructor for utilities classes 082 } 083 084 private static volatile ValidatorLayer errorLayer; 085 086 /** Grid detail, multiplier of east,north values for valuable cell sizing */ 087 private static double griddetail; 088 089 private static final Collection<String> ignoredErrors = new TreeSet<>(); 090 091 /** 092 * All registered tests 093 */ 094 private static final Collection<Class<? extends Test>> allTests = new ArrayList<>(); 095 private static final Map<String, Test> allTestsMap = new HashMap<>(); 096 097 /** 098 * All available tests in core 099 */ 100 @SuppressWarnings("unchecked") 101 private static final Class<Test>[] CORE_TEST_CLASSES = new Class[] { 102 /* FIXME - unique error numbers for tests aren't properly unique - ignoring will not work as expected */ 103 DuplicateNode.class, // ID 1 .. 99 104 OverlappingWays.class, // ID 101 .. 199 105 UntaggedNode.class, // ID 201 .. 299 106 UntaggedWay.class, // ID 301 .. 399 107 SelfIntersectingWay.class, // ID 401 .. 499 108 DuplicatedWayNodes.class, // ID 501 .. 599 109 CrossingWays.Ways.class, // ID 601 .. 699 110 CrossingWays.Boundaries.class, // ID 601 .. 699 111 CrossingWays.Barrier.class, // ID 601 .. 699 112 CrossingWays.SelfCrossing.class, // ID 601 .. 699 113 SimilarNamedWays.class, // ID 701 .. 799 114 Coastlines.class, // ID 901 .. 999 115 WronglyOrderedWays.class, // ID 1001 .. 1099 116 UnclosedWays.class, // ID 1101 .. 1199 117 TagChecker.class, // ID 1201 .. 1299 118 UnconnectedWays.UnconnectedHighways.class, // ID 1301 .. 1399 119 UnconnectedWays.UnconnectedRailways.class, // ID 1301 .. 1399 120 UnconnectedWays.UnconnectedWaterways.class, // ID 1301 .. 1399 121 UnconnectedWays.UnconnectedNaturalOrLanduse.class, // ID 1301 .. 1399 122 UnconnectedWays.UnconnectedPower.class, // ID 1301 .. 1399 123 DuplicateWay.class, // ID 1401 .. 1499 124 NameMismatch.class, // ID 1501 .. 1599 125 MultipolygonTest.class, // ID 1601 .. 1699 126 RelationChecker.class, // ID 1701 .. 1799 127 TurnrestrictionTest.class, // ID 1801 .. 1899 128 DuplicateRelation.class, // ID 1901 .. 1999 129 WayConnectedToArea.class, // ID 2301 .. 2399 130 PowerLines.class, // ID 2501 .. 2599 131 Addresses.class, // ID 2601 .. 2699 132 Highways.class, // ID 2701 .. 2799 133 BarriersEntrances.class, // ID 2801 .. 2899 134 OpeningHourTest.class, // 2901 .. 2999 135 MapCSSTagChecker.class, // 3000 .. 3099 136 Lanes.class, // 3100 .. 3199 137 ConditionalKeys.class, // 3200 .. 3299 138 InternetTags.class, // 3300 .. 3399 139 ApiCapabilitiesTest.class, // 3400 .. 3499 140 LongSegment.class, // 3500 .. 3599 141 PublicTransportRouteTest.class, // 3600 .. 3699 142 }; 143 144 /** 145 * Adds a test to the list of available tests 146 * @param testClass The test class 147 */ 148 public static void addTest(Class<? extends Test> testClass) { 149 allTests.add(testClass); 150 try { 151 allTestsMap.put(testClass.getName(), testClass.getConstructor().newInstance()); 152 } catch (ReflectiveOperationException e) { 153 Logging.error(e); 154 } 155 } 156 157 static { 158 for (Class<? extends Test> testClass : CORE_TEST_CLASSES) { 159 addTest(testClass); 160 } 161 } 162 163 /** 164 * Initializes {@code OsmValidator}. 165 */ 166 public static void initialize() { 167 checkValidatorDir(); 168 initializeGridDetail(); 169 loadIgnoredErrors(); //FIXME: load only when needed 170 } 171 172 /** 173 * Returns the validator directory. 174 * 175 * @return The validator directory 176 */ 177 public static String getValidatorDir() { 178 return new File(Config.getDirs().getUserDataDirectory(true), "validator").getAbsolutePath(); 179 } 180 181 /** 182 * Check if validator directory exists (store ignored errors file) 183 */ 184 private static void checkValidatorDir() { 185 File pathDir = new File(getValidatorDir()); 186 if (!pathDir.exists()) { 187 Utils.mkDirs(pathDir); 188 } 189 } 190 191 private static void loadIgnoredErrors() { 192 ignoredErrors.clear(); 193 if (ValidatorPrefHelper.PREF_USE_IGNORE.get()) { 194 Path path = Paths.get(getValidatorDir()).resolve("ignorederrors"); 195 if (path.toFile().exists()) { 196 try { 197 ignoredErrors.addAll(Files.readAllLines(path, StandardCharsets.UTF_8)); 198 } catch (final FileNotFoundException e) { 199 Logging.debug(Logging.getErrorMessage(e)); 200 } catch (final IOException e) { 201 Logging.error(e); 202 } 203 } 204 } 205 } 206 207 /** 208 * Adds an ignored error 209 * @param s The ignore group / sub group name 210 * @see TestError#getIgnoreGroup() 211 * @see TestError#getIgnoreSubGroup() 212 */ 213 public static void addIgnoredError(String s) { 214 ignoredErrors.add(s); 215 } 216 217 /** 218 * Check if a error should be ignored 219 * @param s The ignore group / sub group name 220 * @return <code>true</code> to ignore that error 221 */ 222 public static boolean hasIgnoredError(String s) { 223 return ignoredErrors.contains(s); 224 } 225 226 /** 227 * Saves the names of the ignored errors to a file 228 */ 229 public static void saveIgnoredErrors() { 230 try (PrintWriter out = new PrintWriter(new File(getValidatorDir(), "ignorederrors"), StandardCharsets.UTF_8.name())) { 231 for (String e : ignoredErrors) { 232 out.println(e); 233 } 234 } catch (IOException e) { 235 Logging.error(e); 236 } 237 } 238 239 /** 240 * Initializes error layer. 241 */ 242 public static synchronized void initializeErrorLayer() { 243 if (!ValidatorPrefHelper.PREF_LAYER.get()) 244 return; 245 if (errorLayer == null) { 246 errorLayer = new ValidatorLayer(); 247 MainApplication.getLayerManager().addLayer(errorLayer); 248 } 249 } 250 251 /** 252 * Resets error layer. 253 * @since 11852 254 */ 255 public static synchronized void resetErrorLayer() { 256 errorLayer = null; 257 } 258 259 /** 260 * Gets a map from simple names to all tests. 261 * @return A map of all tests, indexed and sorted by the name of their Java class 262 */ 263 public static SortedMap<String, Test> getAllTestsMap() { 264 applyPrefs(allTestsMap, false); 265 applyPrefs(allTestsMap, true); 266 return new TreeMap<>(allTestsMap); 267 } 268 269 /** 270 * Returns the instance of the given test class. 271 * @param <T> testClass type 272 * @param testClass The class of test to retrieve 273 * @return the instance of the given test class, if any, or {@code null} 274 * @since 6670 275 */ 276 @SuppressWarnings("unchecked") 277 public static <T extends Test> T getTest(Class<T> testClass) { 278 if (testClass == null) { 279 return null; 280 } 281 return (T) allTestsMap.get(testClass.getName()); 282 } 283 284 private static void applyPrefs(Map<String, Test> tests, boolean beforeUpload) { 285 for (String testName : Config.getPref().getList(beforeUpload 286 ? ValidatorPrefHelper.PREF_SKIP_TESTS_BEFORE_UPLOAD : ValidatorPrefHelper.PREF_SKIP_TESTS)) { 287 Test test = tests.get(testName); 288 if (test != null) { 289 if (beforeUpload) { 290 test.testBeforeUpload = false; 291 } else { 292 test.enabled = false; 293 } 294 } 295 } 296 } 297 298 /** 299 * Gets all tests that are possible 300 * @return The tests 301 */ 302 public static Collection<Test> getTests() { 303 return getAllTestsMap().values(); 304 } 305 306 /** 307 * Gets all tests that are run 308 * @param beforeUpload To get the ones that are run before upload 309 * @return The tests 310 */ 311 public static Collection<Test> getEnabledTests(boolean beforeUpload) { 312 Collection<Test> enabledTests = getTests(); 313 for (Test t : new ArrayList<>(enabledTests)) { 314 if (beforeUpload ? t.testBeforeUpload : t.enabled) { 315 continue; 316 } 317 enabledTests.remove(t); 318 } 319 return enabledTests; 320 } 321 322 /** 323 * Gets the list of all available test classes 324 * 325 * @return A collection of the test classes 326 */ 327 public static Collection<Class<? extends Test>> getAllAvailableTestClasses() { 328 return Collections.unmodifiableCollection(allTests); 329 } 330 331 /** 332 * Initialize grid details based on current projection system. Values based on 333 * the original value fixed for EPSG:4326 (10000) using heuristics (that is, test&error 334 * until most bugs were discovered while keeping the processing time reasonable) 335 */ 336 public static void initializeGridDetail() { 337 String code = Main.getProjection().toCode(); 338 if (Arrays.asList(ProjectionPreference.wgs84.allCodes()).contains(code)) { 339 OsmValidator.griddetail = 10_000; 340 } else if (Arrays.asList(ProjectionPreference.mercator.allCodes()).contains(code)) { 341 OsmValidator.griddetail = 0.01; 342 } else if (Arrays.asList(ProjectionPreference.lambert.allCodes()).contains(code)) { 343 OsmValidator.griddetail = 0.1; 344 } else { 345 OsmValidator.griddetail = 1.0; 346 } 347 } 348 349 /** 350 * Returns grid detail, multiplier of east,north values for valuable cell sizing 351 * @return grid detail 352 * @since 11852 353 */ 354 public static double getGridDetail() { 355 return griddetail; 356 } 357 358 private static boolean testsInitialized; 359 360 /** 361 * Initializes all tests if this operations hasn't been performed already. 362 */ 363 public static synchronized void initializeTests() { 364 if (!testsInitialized) { 365 Logging.debug("Initializing validator tests"); 366 final long startTime = System.currentTimeMillis(); 367 initializeTests(getTests()); 368 testsInitialized = true; 369 if (Logging.isDebugEnabled()) { 370 final long elapsedTime = System.currentTimeMillis() - startTime; 371 Logging.debug("Initializing validator tests completed in {0}", Utils.getDurationString(elapsedTime)); 372 } 373 } 374 } 375 376 /** 377 * Initializes all tests 378 * @param allTests The tests to initialize 379 */ 380 public static void initializeTests(Collection<? extends Test> allTests) { 381 for (Test test : allTests) { 382 try { 383 if (test.enabled) { 384 test.initialize(); 385 } 386 } catch (Exception e) { // NOPMD 387 Logging.error(e); 388 if (!GraphicsEnvironment.isHeadless()) { 389 JOptionPane.showMessageDialog(Main.parent, 390 tr("Error initializing test {0}:\n {1}", test.getClass().getSimpleName(), e), 391 tr("Error"), JOptionPane.ERROR_MESSAGE); 392 } 393 } 394 } 395 } 396 397 /** 398 * Groups the given collection of errors by severity, then message, then description. 399 * @param errors list of errors to group 400 * @param filterToUse optional filter 401 * @return collection of errors grouped by severity, then message, then description 402 * @since 12667 403 */ 404 public static Map<Severity, Map<String, Map<String, List<TestError>>>> getErrorsBySeverityMessageDescription( 405 Collection<TestError> errors, Predicate<? super TestError> filterToUse) { 406 return errors.stream().filter(filterToUse).collect( 407 Collectors.groupingBy(TestError::getSeverity, () -> new EnumMap<>(Severity.class), 408 Collectors.groupingBy(TestError::getMessage, () -> new TreeMap<>(AlphanumComparator.getInstance()), 409 Collectors.groupingBy(e -> e.getDescription() == null ? "" : e.getDescription(), 410 () -> new TreeMap<>(AlphanumComparator.getInstance()), 411 Collectors.toList() 412 )))); 413 } 414}