001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.io.InputStream; 008import java.nio.charset.StandardCharsets; 009import java.time.Duration; 010import java.time.LocalDateTime; 011import java.time.Period; 012import java.time.ZoneOffset; 013import java.util.Arrays; 014import java.util.EnumMap; 015import java.util.Locale; 016import java.util.Map; 017import java.util.NoSuchElementException; 018import java.util.Objects; 019import java.util.concurrent.ConcurrentHashMap; 020import java.util.concurrent.TimeUnit; 021import java.util.regex.Matcher; 022import java.util.regex.Pattern; 023 024import javax.xml.stream.XMLStreamConstants; 025import javax.xml.stream.XMLStreamException; 026 027import org.openstreetmap.josm.data.Bounds; 028import org.openstreetmap.josm.data.DataSource; 029import org.openstreetmap.josm.data.coor.LatLon; 030import org.openstreetmap.josm.data.osm.BBox; 031import org.openstreetmap.josm.data.osm.DataSet; 032import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 033import org.openstreetmap.josm.data.osm.PrimitiveId; 034import org.openstreetmap.josm.data.preferences.BooleanProperty; 035import org.openstreetmap.josm.data.preferences.ListProperty; 036import org.openstreetmap.josm.data.preferences.StringProperty; 037import org.openstreetmap.josm.gui.progress.ProgressMonitor; 038import org.openstreetmap.josm.io.NameFinder.SearchResult; 039import org.openstreetmap.josm.tools.HttpClient; 040import org.openstreetmap.josm.tools.Logging; 041import org.openstreetmap.josm.tools.UncheckedParseException; 042import org.openstreetmap.josm.tools.Utils; 043 044/** 045 * Read content from an Overpass server. 046 * 047 * @since 8744 048 */ 049public class OverpassDownloadReader extends BoundingBoxDownloader { 050 051 /** 052 * Property for current Overpass server. 053 * @since 12816 054 */ 055 public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server", 056 "https://overpass-api.de/api/"); 057 /** 058 * Property for list of known Overpass servers. 059 * @since 12816 060 */ 061 public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers", 062 Arrays.asList("https://overpass-api.de/api/", "http://overpass.osm.rambler.ru/cgi/")); 063 /** 064 * Property to determine if Overpass API should be used for multi-fetch download. 065 * @since 12816 066 */ 067 public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false); 068 069 private static final String DATA_PREFIX = "?data="; 070 071 static final class OverpassOsmReader extends OsmReader { 072 @Override 073 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 074 if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) { 075 final String text = parser.getElementText(); 076 if (text.contains("runtime error")) { 077 throw new XMLStreamException(text); 078 } 079 } 080 super.parseUnknown(printWarning); 081 } 082 } 083 084 /** 085 * Possible Overpass API output format, with the {@code [out:<directive>]} statement. 086 * @since 11916 087 */ 088 public enum OverpassOutpoutFormat { 089 /** Default output format: plain OSM XML */ 090 OSM_XML("xml"), 091 /** OSM JSON format (not GeoJson) */ 092 OSM_JSON("json"), 093 /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */ 094 CSV("csv"), 095 /** Custom, see https://overpass-api.de/output_formats.html#custom */ 096 CUSTOM("custom"), 097 /** Popup, see https://overpass-api.de/output_formats.html#popup */ 098 POPUP("popup"), 099 /** PBF, see https://josm.openstreetmap.de/ticket/14653 */ 100 PBF("pbf"); 101 102 private final String directive; 103 104 OverpassOutpoutFormat(String directive) { 105 this.directive = directive; 106 } 107 108 /** 109 * Returns the directive used in {@code [out:<directive>]} statement. 110 * @return the directive used in {@code [out:<directive>]} statement 111 */ 112 public String getDirective() { 113 return directive; 114 } 115 116 /** 117 * Returns the {@code OverpassOutpoutFormat} matching the given directive. 118 * @param directive directive used in {@code [out:<directive>]} statement 119 * @return {@code OverpassOutpoutFormat} matching the given directive 120 * @throws IllegalArgumentException in case of invalid directive 121 */ 122 static OverpassOutpoutFormat from(String directive) { 123 for (OverpassOutpoutFormat oof : values()) { 124 if (oof.directive.equals(directive)) { 125 return oof; 126 } 127 } 128 throw new IllegalArgumentException(directive); 129 } 130 } 131 132 static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL); 133 134 static final Map<OverpassOutpoutFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>(); 135 136 final String overpassServer; 137 final String overpassQuery; 138 139 /** 140 * Constructs a new {@code OverpassDownloadReader}. 141 * 142 * @param downloadArea The area to download 143 * @param overpassServer The Overpass server to use 144 * @param overpassQuery The Overpass query 145 */ 146 public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) { 147 super(downloadArea); 148 setDoAuthenticate(false); 149 this.overpassServer = overpassServer; 150 this.overpassQuery = overpassQuery.trim(); 151 } 152 153 /** 154 * Registers an OSM reader for the given Overpass output format. 155 * @param format Overpass output format 156 * @param readerClass OSM reader class 157 * @return the previous value associated with {@code format}, or {@code null} if there was no mapping 158 */ 159 public static final Class<? extends AbstractReader> registerOverpassOutpoutFormatReader( 160 OverpassOutpoutFormat format, Class<? extends AbstractReader> readerClass) { 161 return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass)); 162 } 163 164 static { 165 registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class); 166 } 167 168 @Override 169 protected String getBaseUrl() { 170 return overpassServer; 171 } 172 173 @Override 174 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) { 175 if (overpassQuery.isEmpty()) 176 return super.getRequestForBbox(lon1, lat1, lon2, lat2); 177 else { 178 final String query = this.overpassQuery 179 .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2)) 180 .replace("{{center}}", center(lon1, lat1, lon2, lat2)); 181 final String expandedOverpassQuery = expandExtendedQueries(query); 182 return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery); 183 } 184 } 185 186 /** 187 * Evaluates some features of overpass turbo extended query syntax. 188 * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries 189 * @param query unexpanded query 190 * @return expanded query 191 */ 192 static String expandExtendedQueries(String query) { 193 final StringBuffer sb = new StringBuffer(); 194 final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query); 195 while (matcher.find()) { 196 try { 197 switch (matcher.group(1)) { 198 case "date": 199 matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now())); 200 break; 201 case "geocodeArea": 202 matcher.appendReplacement(sb, geocodeArea(matcher.group(2))); 203 break; 204 case "geocodeBbox": 205 matcher.appendReplacement(sb, geocodeBbox(matcher.group(2))); 206 break; 207 case "geocodeCoords": 208 matcher.appendReplacement(sb, geocodeCoords(matcher.group(2))); 209 break; 210 case "geocodeId": 211 matcher.appendReplacement(sb, geocodeId(matcher.group(2))); 212 break; 213 default: 214 Logging.warn("Unsupported syntax: " + matcher.group(1)); 215 } 216 } catch (UncheckedParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) { 217 final String msg = tr("Failed to evaluate {0}", matcher.group()); 218 Logging.log(Logging.LEVEL_WARN, msg, ex); 219 matcher.appendReplacement(sb, "// " + msg + "\n"); 220 } 221 } 222 matcher.appendTail(sb); 223 return sb.toString(); 224 } 225 226 static String bbox(double lon1, double lat1, double lon2, double lat2) { 227 return lat1 + "," + lon1 + "," + lat2 + "," + lon2; 228 } 229 230 static String center(double lon1, double lat1, double lon2, double lat2) { 231 LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter(); 232 return c.lat()+ "," + c.lon(); 233 } 234 235 static String date(String humanDuration, LocalDateTime from) { 236 // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes 237 String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "") 238 .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W") 239 .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S"); 240 Matcher matcher = Pattern.compile( 241 "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+ 242 "((?:[0-9]+D)?)" + 243 "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration); 244 boolean javaPer = false; 245 boolean javaDur = false; 246 if (matcher.matches()) { 247 javaPer = matcher.group(1) != null && !matcher.group(1).isEmpty(); 248 javaDur = matcher.group(3) != null && !matcher.group(3).isEmpty(); 249 duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2); 250 if (javaDur) { 251 duration += 'T' + matcher.group(3); 252 } 253 } 254 255 // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely. 256 // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds). 257 Period p = null; 258 Duration d = null; 259 int idx = duration.indexOf('T'); 260 if (javaPer) { 261 p = Period.parse(javaDur ? duration.substring(0, idx) : duration); 262 } 263 if (javaDur) { 264 d = Duration.parse(javaPer ? 'P' + duration.substring(idx, duration.length()) : duration); 265 } else if (!javaPer) { 266 d = Duration.parse(duration); 267 } 268 269 // Now that period and duration are known, compute the correct date/time 270 LocalDateTime dt = from; 271 if (p != null) { 272 dt = dt.minus(p); 273 } 274 if (d != null) { 275 dt = dt.minus(d); 276 } 277 278 // Returns the date/time formatted in ISO 8601 279 return dt.toInstant(ZoneOffset.UTC).toString(); 280 } 281 282 private static SearchResult searchName(String area) throws IOException { 283 return NameFinder.queryNominatim(area).stream().filter( 284 x -> !OsmPrimitiveType.NODE.equals(x.getOsmId().getType())).iterator().next(); 285 } 286 287 static String geocodeArea(String area) throws IOException { 288 // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id 289 final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class); 290 idOffset.put(OsmPrimitiveType.NODE, 0L); 291 idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L); 292 idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L); 293 final PrimitiveId osmId = searchName(area).getOsmId(); 294 return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType())); 295 } 296 297 static String geocodeBbox(String area) throws IOException { 298 Bounds bounds = searchName(area).getBounds(); 299 return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon(); 300 } 301 302 static String geocodeCoords(String area) throws IOException { 303 SearchResult result = searchName(area); 304 return result.getLat() + "," + result.getLon(); 305 } 306 307 static String geocodeId(String area) throws IOException { 308 PrimitiveId osmId = searchName(area).getOsmId(); 309 return String.format("%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId()); 310 } 311 312 @Override 313 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 314 boolean uncompressAccordingToContentDisposition) throws OsmTransferException { 315 try { 316 int index = urlStr.indexOf(DATA_PREFIX); 317 // Make an HTTP POST request instead of a simple GET, allows more complex queries 318 return super.getInputStreamRaw(urlStr.substring(0, index), 319 progressMonitor, reason, uncompressAccordingToContentDisposition, 320 "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8)); 321 } catch (OsmApiException ex) { 322 final String errorIndicator = "Error</strong>: "; 323 if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) { 324 final String errorPlusRest = ex.getMessage().split(errorIndicator)[1]; 325 if (errorPlusRest != null) { 326 ex.setErrorHeader(errorPlusRest.split("</")[0].replaceAll(".*::request_read_and_idx::", "")); 327 } 328 } 329 throw ex; 330 } 331 } 332 333 @Override 334 protected void adaptRequest(HttpClient request) { 335 // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout 336 final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery); 337 final int timeout; 338 if (timeoutMatcher.find()) { 339 timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1))); 340 } else { 341 timeout = (int) TimeUnit.MINUTES.toMillis(3); 342 } 343 request.setConnectTimeout(timeout); 344 request.setReadTimeout(timeout); 345 } 346 347 @Override 348 protected String getTaskName() { 349 return tr("Contacting Server..."); 350 } 351 352 @Override 353 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 354 AbstractReader reader = null; 355 Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery); 356 if (m.matches()) { 357 Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutpoutFormat.from(m.group(1))); 358 if (readerClass != null) { 359 try { 360 reader = readerClass.getDeclaredConstructor().newInstance(); 361 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) { 362 Logging.error(e); 363 } 364 } 365 } 366 if (reader == null) { 367 reader = new OverpassOsmReader(); 368 } 369 return reader.doParseDataSet(source, progressMonitor); 370 } 371 372 @Override 373 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 374 375 DataSet ds = super.parseOsm(progressMonitor); 376 377 // add bounds if necessary (note that Overpass API does not return bounds in the response XML) 378 if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) { 379 if (crosses180th) { 380 Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0); 381 DataSource src = new DataSource(bounds, getBaseUrl()); 382 ds.addDataSource(src); 383 384 bounds = new Bounds(lat1, -180.0, lat2, lon2); 385 src = new DataSource(bounds, getBaseUrl()); 386 ds.addDataSource(src); 387 } else { 388 Bounds bounds = new Bounds(lat1, lon1, lat2, lon2); 389 DataSource src = new DataSource(bounds, getBaseUrl()); 390 ds.addDataSource(src); 391 } 392 } 393 394 return ds; 395 } 396 397 /** 398 * Fixes Overpass API query to make sure it will be accepted by JOSM. 399 * @param query Overpass query to check 400 * @return fixed query 401 * @since 13335 402 */ 403 public static String fixQuery(String query) { 404 return query == null ? query : query 405 .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;") 406 .replaceAll("(?s)\\[out:(json|csv)[^\\]]*\\]", "[out:xml]"); 407 } 408}