001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.actions.downloadtasks; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.net.URL; 008import java.util.ArrayList; 009import java.util.Arrays; 010import java.util.Collection; 011import java.util.Collections; 012import java.util.HashSet; 013import java.util.Set; 014import java.util.concurrent.Future; 015import java.util.regex.Matcher; 016import java.util.regex.Pattern; 017import java.util.stream.Stream; 018 019import org.openstreetmap.josm.data.Bounds; 020import org.openstreetmap.josm.data.DataSource; 021import org.openstreetmap.josm.data.ProjectionBounds; 022import org.openstreetmap.josm.data.ViewportData; 023import org.openstreetmap.josm.data.coor.LatLon; 024import org.openstreetmap.josm.data.osm.DataSet; 025import org.openstreetmap.josm.data.osm.OsmPrimitive; 026import org.openstreetmap.josm.data.osm.Relation; 027import org.openstreetmap.josm.data.osm.Way; 028import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor; 029import org.openstreetmap.josm.gui.MainApplication; 030import org.openstreetmap.josm.gui.MapFrame; 031import org.openstreetmap.josm.gui.PleaseWaitRunnable; 032import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask; 033import org.openstreetmap.josm.gui.layer.OsmDataLayer; 034import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 035import org.openstreetmap.josm.gui.progress.ProgressMonitor; 036import org.openstreetmap.josm.io.BoundingBoxDownloader; 037import org.openstreetmap.josm.io.OsmServerLocationReader; 038import org.openstreetmap.josm.io.OsmServerLocationReader.OsmUrlPattern; 039import org.openstreetmap.josm.io.OsmServerReader; 040import org.openstreetmap.josm.io.OsmTransferCanceledException; 041import org.openstreetmap.josm.io.OsmTransferException; 042import org.openstreetmap.josm.tools.Logging; 043import org.openstreetmap.josm.tools.Utils; 044import org.xml.sax.SAXException; 045 046/** 047 * Open the download dialog and download the data. 048 * Run in the worker thread. 049 */ 050public class DownloadOsmTask extends AbstractDownloadTask<DataSet> { 051 052 protected Bounds currentBounds; 053 protected DownloadTask downloadTask; 054 055 protected String newLayerName; 056 057 /** This allows subclasses to ignore this warning */ 058 protected boolean warnAboutEmptyArea = true; 059 060 @Override 061 public String[] getPatterns() { 062 if (this.getClass() == DownloadOsmTask.class) { 063 return Arrays.stream(OsmUrlPattern.values()).map(OsmUrlPattern::pattern).toArray(String[]::new); 064 } else { 065 return super.getPatterns(); 066 } 067 } 068 069 @Override 070 public String getTitle() { 071 if (this.getClass() == DownloadOsmTask.class) { 072 return tr("Download OSM"); 073 } else { 074 return super.getTitle(); 075 } 076 } 077 078 @Override 079 public Future<?> download(boolean newLayer, Bounds downloadArea, ProgressMonitor progressMonitor) { 080 return download(new BoundingBoxDownloader(downloadArea), newLayer, downloadArea, progressMonitor); 081 } 082 083 /** 084 * Asynchronously launches the download task for a given bounding box. 085 * 086 * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor. 087 * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to 088 * be discarded. 089 * 090 * You can wait for the asynchronous download task to finish by synchronizing on the returned 091 * {@link Future}, but make sure not to freeze up JOSM. Example: 092 * <pre> 093 * Future<?> future = task.download(...); 094 * // DON'T run this on the Swing EDT or JOSM will freeze 095 * future.get(); // waits for the dowload task to complete 096 * </pre> 097 * 098 * The following example uses a pattern which is better suited if a task is launched from 099 * the Swing EDT: 100 * <pre> 101 * final Future<?> future = task.download(...); 102 * Runnable runAfterTask = new Runnable() { 103 * public void run() { 104 * // this is not strictly necessary because of the type of executor service 105 * // Main.worker is initialized with, but it doesn't harm either 106 * // 107 * future.get(); // wait for the download task to complete 108 * doSomethingAfterTheTaskCompleted(); 109 * } 110 * } 111 * MainApplication.worker.submit(runAfterTask); 112 * </pre> 113 * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm}) 114 * @param newLayer true, if the data is to be downloaded into a new layer. If false, the task 115 * selects one of the existing layers as download layer, preferably the active layer. 116 * @param downloadArea the area to download 117 * @param progressMonitor the progressMonitor 118 * @return the future representing the asynchronous task 119 */ 120 public Future<?> download(OsmServerReader reader, boolean newLayer, Bounds downloadArea, ProgressMonitor progressMonitor) { 121 return download(new DownloadTask(newLayer, reader, progressMonitor, zoomAfterDownload), downloadArea); 122 } 123 124 protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) { 125 this.downloadTask = downloadTask; 126 this.currentBounds = new Bounds(downloadArea); 127 // We need submit instead of execute so we can wait for it to finish and get the error 128 // message if necessary. If no one calls getErrorMessage() it just behaves like execute. 129 return MainApplication.worker.submit(downloadTask); 130 } 131 132 /** 133 * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed. 134 * @param url the original URL 135 * @return the modified URL 136 */ 137 protected String modifyUrlBeforeLoad(String url) { 138 return url; 139 } 140 141 /** 142 * Loads a given URL from the OSM Server 143 * @param newLayer True if the data should be saved to a new layer 144 * @param url The URL as String 145 */ 146 @Override 147 public Future<?> loadUrl(boolean newLayer, String url, ProgressMonitor progressMonitor) { 148 String newUrl = modifyUrlBeforeLoad(url); 149 downloadTask = new DownloadTask(newLayer, 150 new OsmServerLocationReader(newUrl), 151 progressMonitor); 152 currentBounds = null; 153 // Extract .osm filename from URL to set the new layer name 154 extractOsmFilename("https?://.*/(.*\\.osm)", newUrl); 155 return MainApplication.worker.submit(downloadTask); 156 } 157 158 protected final void extractOsmFilename(String pattern, String url) { 159 Matcher matcher = Pattern.compile(pattern).matcher(url); 160 newLayerName = matcher.matches() ? matcher.group(1) : null; 161 } 162 163 @Override 164 public void cancel() { 165 if (downloadTask != null) { 166 downloadTask.cancel(); 167 } 168 } 169 170 @Override 171 public boolean isSafeForRemotecontrolRequests() { 172 return true; 173 } 174 175 @Override 176 public ProjectionBounds getDownloadProjectionBounds() { 177 return downloadTask != null ? downloadTask.computeBbox(currentBounds) : null; 178 } 179 180 /** 181 * Superclass of internal download task. 182 * @since 7636 183 */ 184 public abstract static class AbstractInternalTask extends PleaseWaitRunnable { 185 186 protected final boolean newLayer; 187 protected final boolean zoomAfterDownload; 188 protected DataSet dataSet; 189 190 /** 191 * Constructs a new {@code AbstractInternalTask}. 192 * @param newLayer if {@code true}, force download to a new layer 193 * @param title message for the user 194 * @param ignoreException If true, exception will be propagated to calling code. If false then 195 * exception will be thrown directly in EDT. When this runnable is executed using executor framework 196 * then use false unless you read result of task (because exception will get lost if you don't) 197 * @param zoomAfterDownload If true, the map view will zoom to download area after download 198 */ 199 public AbstractInternalTask(boolean newLayer, String title, boolean ignoreException, boolean zoomAfterDownload) { 200 super(title, ignoreException); 201 this.newLayer = newLayer; 202 this.zoomAfterDownload = zoomAfterDownload; 203 } 204 205 /** 206 * Constructs a new {@code AbstractInternalTask}. 207 * @param newLayer if {@code true}, force download to a new layer 208 * @param title message for the user 209 * @param progressMonitor progress monitor 210 * @param ignoreException If true, exception will be propagated to calling code. If false then 211 * exception will be thrown directly in EDT. When this runnable is executed using executor framework 212 * then use false unless you read result of task (because exception will get lost if you don't) 213 * @param zoomAfterDownload If true, the map view will zoom to download area after download 214 */ 215 public AbstractInternalTask(boolean newLayer, String title, ProgressMonitor progressMonitor, boolean ignoreException, 216 boolean zoomAfterDownload) { 217 super(title, progressMonitor, ignoreException); 218 this.newLayer = newLayer; 219 this.zoomAfterDownload = zoomAfterDownload; 220 } 221 222 protected OsmDataLayer getEditLayer() { 223 return MainApplication.getLayerManager().getEditLayer(); 224 } 225 226 /** 227 * Returns the number of modifiable data layers 228 * @return number of modifiable data layers 229 * @deprecated Use {@link #getNumModifiableDataLayers} 230 */ 231 @Deprecated 232 protected int getNumDataLayers() { 233 return (int) getNumModifiableDataLayers(); 234 } 235 236 private static Stream<OsmDataLayer> getModifiableDataLayers() { 237 return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class) 238 .stream().filter(OsmDataLayer::isDownloadable); 239 } 240 241 /** 242 * Returns the number of modifiable data layers 243 * @return number of modifiable data layers 244 * @since 13434 245 */ 246 protected long getNumModifiableDataLayers() { 247 return getModifiableDataLayers().count(); 248 } 249 250 /** 251 * Returns the first modifiable data layer 252 * @return the first modifiable data layer 253 * @since 13434 254 */ 255 protected OsmDataLayer getFirstModifiableDataLayer() { 256 return getModifiableDataLayers().findFirst().orElse(null); 257 } 258 259 protected OsmDataLayer createNewLayer(String layerName) { 260 if (layerName == null || layerName.isEmpty()) { 261 layerName = OsmDataLayer.createNewName(); 262 } 263 return new OsmDataLayer(dataSet, layerName, null); 264 } 265 266 protected OsmDataLayer createNewLayer() { 267 return createNewLayer(null); 268 } 269 270 protected ProjectionBounds computeBbox(Bounds bounds) { 271 BoundingXYVisitor v = new BoundingXYVisitor(); 272 if (bounds != null) { 273 v.visit(bounds); 274 } else { 275 v.computeBoundingBox(dataSet.getNodes()); 276 } 277 return v.getBounds(); 278 } 279 280 protected OsmDataLayer addNewLayerIfRequired(String newLayerName) { 281 long numDataLayers = getNumModifiableDataLayers(); 282 if (newLayer || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) { 283 // the user explicitly wants a new layer, we don't have any layer at all 284 // or it is not clear which layer to merge to 285 final OsmDataLayer layer = createNewLayer(newLayerName); 286 MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload); 287 return layer; 288 } 289 return null; 290 } 291 292 protected void loadData(String newLayerName, Bounds bounds) { 293 OsmDataLayer layer = addNewLayerIfRequired(newLayerName); 294 if (layer == null) { 295 layer = getEditLayer(); 296 if (layer == null || !layer.isDownloadable()) { 297 layer = getFirstModifiableDataLayer(); 298 } 299 Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.data); 300 layer.mergeFrom(dataSet); 301 MapFrame map = MainApplication.getMap(); 302 if (map != null && zoomAfterDownload && bounds != null) { 303 map.mapView.zoomTo(new ViewportData(computeBbox(bounds))); 304 } 305 if (!primitivesToUpdate.isEmpty()) { 306 MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate)); 307 } 308 layer.onPostDownloadFromServer(); 309 } 310 } 311 312 /** 313 * Look for primitives deleted on server (thus absent from downloaded data) 314 * but still present in existing data layer 315 * @param bounds download bounds 316 * @param ds existing data set 317 * @return the primitives to update 318 */ 319 private Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) { 320 if (bounds == null) 321 return Collections.emptySet(); 322 Collection<OsmPrimitive> col = new ArrayList<>(); 323 ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add); 324 if (!col.isEmpty()) { 325 Set<Way> ways = new HashSet<>(); 326 Set<Relation> rels = new HashSet<>(); 327 for (OsmPrimitive n : col) { 328 for (OsmPrimitive ref : n.getReferrers()) { 329 if (ref.isNew()) { 330 continue; 331 } else if (ref instanceof Way) { 332 ways.add((Way) ref); 333 } else if (ref instanceof Relation) { 334 rels.add((Relation) ref); 335 } 336 } 337 } 338 ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add); 339 rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add); 340 } 341 return col; 342 } 343 } 344 345 protected class DownloadTask extends AbstractInternalTask { 346 protected final OsmServerReader reader; 347 348 /** 349 * Constructs a new {@code DownloadTask}. 350 * @param newLayer if {@code true}, force download to a new layer 351 * @param reader OSM data reader 352 * @param progressMonitor progress monitor 353 */ 354 public DownloadTask(boolean newLayer, OsmServerReader reader, ProgressMonitor progressMonitor) { 355 this(newLayer, reader, progressMonitor, true); 356 } 357 358 /** 359 * Constructs a new {@code DownloadTask}. 360 * @param newLayer if {@code true}, force download to a new layer 361 * @param reader OSM data reader 362 * @param progressMonitor progress monitor 363 * @param zoomAfterDownload If true, the map view will zoom to download area after download 364 * @since 8942 365 */ 366 public DownloadTask(boolean newLayer, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) { 367 super(newLayer, tr("Downloading data"), progressMonitor, false, zoomAfterDownload); 368 this.reader = reader; 369 } 370 371 protected DataSet parseDataSet() throws OsmTransferException { 372 return reader.parseOsm(progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false)); 373 } 374 375 @Override 376 public void realRun() throws IOException, SAXException, OsmTransferException { 377 try { 378 if (isCanceled()) 379 return; 380 dataSet = parseDataSet(); 381 } catch (OsmTransferException e) { 382 if (isCanceled()) { 383 Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString())); 384 return; 385 } 386 if (e instanceof OsmTransferCanceledException) { 387 setCanceled(true); 388 return; 389 } else { 390 rememberException(e); 391 } 392 DownloadOsmTask.this.setFailed(true); 393 } 394 } 395 396 @Override 397 protected void finish() { 398 if (isFailed() || isCanceled()) 399 return; 400 if (dataSet == null) 401 return; // user canceled download or error occurred 402 if (dataSet.allPrimitives().isEmpty()) { 403 if (warnAboutEmptyArea) { 404 rememberErrorMessage(tr("No data found in this area.")); 405 } 406 // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work 407 dataSet.addDataSource(new DataSource(currentBounds != null ? currentBounds : 408 new Bounds(LatLon.ZERO), "OpenStreetMap server")); 409 } 410 411 rememberDownloadedData(dataSet); 412 loadData(newLayerName, currentBounds); 413 } 414 415 @Override 416 protected void cancel() { 417 setCanceled(true); 418 if (reader != null) { 419 reader.cancel(); 420 } 421 } 422 } 423 424 @Override 425 public String getConfirmationMessage(URL url) { 426 if (url != null) { 427 String urlString = url.toExternalForm(); 428 if (urlString.matches(OsmUrlPattern.OSM_API_URL.pattern())) { 429 // TODO: proper i18n after stabilization 430 Collection<String> items = new ArrayList<>(); 431 items.add(tr("OSM Server URL:") + ' ' + url.getHost()); 432 items.add(tr("Command")+": "+url.getPath()); 433 if (url.getQuery() != null) { 434 items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", "))); 435 } 436 return Utils.joinAsHtmlUnorderedList(items); 437 } 438 // TODO: other APIs 439 } 440 return null; 441 } 442}