Rework download area support (#908)
Closes #904 * Make all data-related targets like `download*`, `import-osm`, `import-borders`, and `generate-tiles` into `area`-aware -- making it possible for multiple data files to coexist inside the `./data` dir. * Add `make download area=... [url=...]` command to automatically download any kind of area by checking Geofabrik, BBBike, and OSM.fr, optionally from a custom URL. Supports `area=planet` too. * Do not re-download area with `make download-*` if it already exists. * Automatically rename `<area>-latest.osm.pbf` into `<area>.osm.pbf` * If `area=...` parameter is not given to `make`, see if there is exactly one `*.osm.pbf` file, and if so, use `*` as the `area`. * Configure many variables in the .env file, overriding the defaults in tools * If `<area>.osm.pbf` exists, but `<area>.dc-config.pbf` is missing, generate it using `download-osm make-dc` command. Also: * closes #614 * closes #647 * partially addresses #261
This commit is contained in:
Yuri Astrakhan
GitHub
163
Makefile
163
Makefile
@@ -13,9 +13,6 @@ DC_OPTS ?= --rm -u $(shell id -u):$(shell id -g)
|
||||
# If set to a non-empty value, will use postgis-preloaded instead of postgis docker image
|
||||
USE_PRELOADED_IMAGE ?=
|
||||
|
||||
# If set, this file will be imported in the import-osm target
|
||||
PBF_FILE?=
|
||||
|
||||
# Local port to use with postserve
|
||||
PPORT ?= 8090
|
||||
export PPORT
|
||||
@@ -55,6 +52,87 @@ endif
|
||||
OMT_HOST := http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost)
|
||||
|
||||
|
||||
#
|
||||
# Determine area to work on
|
||||
# If $(area) parameter is not set and data/*.osm.pbf finds only one file, use it as $(area).
|
||||
# Otherwise all make targets requiring area param will show an error.
|
||||
# Note: If there are no data files, and user calls make download area=... once,
|
||||
# they will not need to use area= parameter after that because there will be just a single file.
|
||||
#
|
||||
|
||||
# historically we have been using $(area) rather than $(AREA), so make both work
|
||||
area ?= $(AREA)
|
||||
# Ensure the $(AREA) param is set, or try to automatically determine it based on available data files
|
||||
ifeq ($(strip $(area)),)
|
||||
# if $area is not set. set it to the name of the *.osm.pbf file, but only if there is only one
|
||||
data_files := $(wildcard data/*.osm.pbf)
|
||||
ifneq ($(word 2,$(data_files)),)
|
||||
AREA_ERROR := The 'area' parameter (or env var) has not been set, and there are more than one data/*.osm.pbf files. Set area to one of these IDs, or a new one: $(patsubst data/%.osm.pbf,'%',$(data_files))
|
||||
else
|
||||
ifeq ($(word 1,$(data_files)),)
|
||||
AREA_ERROR := The 'area' parameter (or env var) has not been set, and there are no data/*.osm.pbf files
|
||||
else
|
||||
# Keep just the name of the data file, without the .osm.pbf extension
|
||||
area := $(strip $(basename $(basename $(notdir $(data_files)))))
|
||||
# Rename area-latest.osm.pbf to area.osm.pbf
|
||||
# TODO: This if statement could be removed in a few months once everyone is using the file without the `-latest`?
|
||||
ifneq ($(area),$(area:-latest=))
|
||||
$(shell mv "data/$(area).osm.pbf" "data/$(area:-latest=).osm.pbf")
|
||||
area := $(area:-latest=)
|
||||
$(warning ATTENTION: File data/$(area)-latest.osm.pbf was renamed to $(area).osm.pbf.)
|
||||
AREA_INFO := Detected area=$(area) based on the found data/$(area)-latest.osm.pbf (renamed to $(area).osm.pbf). Use 'area' parameter (or env var) to override.
|
||||
else
|
||||
AREA_INFO := Detected area=$(area) based on the found data/ pbf file. Use 'area' parameter (or env var) to override.
|
||||
endif
|
||||
endif
|
||||
endif
|
||||
endif
|
||||
|
||||
# If set, this file will be downloaded in download-osm and imported in the import-osm targets
|
||||
PBF_FILE ?= data/$(area).osm.pbf
|
||||
|
||||
# For download-osm, allow URL parameter to download file from a given URL. Area param must still be provided.
|
||||
ifneq ($(strip $(url)),)
|
||||
DOWNLOAD_AREA := $(url)
|
||||
else
|
||||
DOWNLOAD_AREA := $(area)
|
||||
endif
|
||||
|
||||
# import-borders uses these temp files during border parsing/import
|
||||
export BORDERS_CLEANUP_FILE ?= data/borders/$(area).cleanup.pbf
|
||||
export BORDERS_PBF_FILE ?= data/borders/$(area).filtered.pbf
|
||||
export BORDERS_CSV_FILE ?= data/borders/$(area).lines.csv
|
||||
|
||||
# The file is placed into the $EXPORT_DIR=/export (mapped to ./data)
|
||||
export MBTILES_FILE ?= $(area).mbtiles
|
||||
MBTILES_LOCAL_FILE = data/$(MBTILES_FILE)
|
||||
|
||||
# Location of the dynamically-generated imposm config file
|
||||
export IMPOSM_CONFIG_FILE ?= data/$(area).repl.json
|
||||
|
||||
# download-osm generates this file with metadata about the file
|
||||
AREA_DC_CONFIG_FILE ?= data/$(area).dc-config.yml
|
||||
|
||||
ifeq ($(strip $(area)),)
|
||||
define assert_area_is_given
|
||||
@echo "ERROR: $(AREA_ERROR)"
|
||||
@echo ""
|
||||
@echo " make $@ area=<area-id>"
|
||||
@echo ""
|
||||
@echo "To download an area, use make download <area-id>"
|
||||
@echo "To list downloadable areas, use make list-geofabrik and/or make list-bbbike"
|
||||
@exit 1
|
||||
endef
|
||||
else
|
||||
ifneq ($(strip $(AREA_INFO)),)
|
||||
define assert_area_is_given
|
||||
@echo "$(AREA_INFO)"
|
||||
endef
|
||||
endif
|
||||
endif
|
||||
|
||||
|
||||
|
||||
#
|
||||
# TARGETS
|
||||
#
|
||||
@@ -78,6 +156,8 @@ help:
|
||||
@echo "Hints for developers:"
|
||||
@echo " make # build source code"
|
||||
@echo " make list-geofabrik # list actual geofabrik OSM extracts for download"
|
||||
@echo " make list-bbbike # list actual BBBike OSM extracts for download"
|
||||
@echo " make download area=albania # download OSM data from any source and create config file"
|
||||
@echo " make download-geofabrik area=albania # download OSM data from geofabrik.de and create config file"
|
||||
@echo " make download-osmfr area=asia/qatar # download OSM data from openstreetmap.fr and create config file"
|
||||
@echo " make download-bbbike area=Amsterdam # download OSM data from bbbike.org and create config file"
|
||||
@@ -105,7 +185,7 @@ help:
|
||||
.PHONY: init-dirs
|
||||
init-dirs:
|
||||
@mkdir -p build/sql
|
||||
@mkdir -p data
|
||||
@mkdir -p data/borders
|
||||
@mkdir -p cache
|
||||
|
||||
build/openmaptiles.tm2source/data.yml: init-dirs
|
||||
@@ -165,29 +245,51 @@ list-geofabrik: init-dirs
|
||||
list-bbbike: init-dirs
|
||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list bbbike
|
||||
|
||||
#
|
||||
# download, download-geofabrik, download-osmfr, and download-bbbike are handled here
|
||||
# The --imposm-cfg will fail for some of the sources, but we ignore that error -- only needed for diff mode
|
||||
#
|
||||
OSM_SERVERS := geofabrik osmfr bbbike
|
||||
ALL_DOWNLOADS := $(addprefix download-,$(OSM_SERVERS))
|
||||
OSM_SERVER=$(patsubst download-%,%,$@)
|
||||
ALL_DOWNLOADS := $(addprefix download-,$(OSM_SERVERS)) download
|
||||
OSM_SERVER=$(patsubst download,,$(patsubst download-%,%,$@))
|
||||
.PHONY: $(ALL_DOWNLOADS)
|
||||
$(ALL_DOWNLOADS): init-dirs
|
||||
ifeq ($(strip $(area)),)
|
||||
@echo ""
|
||||
@echo "ERROR: Unable to download an area if area is not given."
|
||||
@echo "Usage:"
|
||||
@echo " make download-$(OSM_SERVER) area=<area-id>"
|
||||
@echo ""
|
||||
$(if $(filter %-geofabrik,$@),@echo "Use make list-geofabrik to get a list of all available areas";echo "")
|
||||
@exit 1
|
||||
else
|
||||
@echo "=============== download-$(OSM_SERVER) ======================="
|
||||
@echo "Download area: $(area)"
|
||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c \
|
||||
'download-osm $(OSM_SERVER) $(area) \
|
||||
@$(assert_area_is_given)
|
||||
ifeq (,$(wildcard $(PBF_FILE)))
|
||||
ifneq ($(strip $(url)),)
|
||||
$(if $(OSM_SERVER),$(error url parameter can only be used with the 'make download area=... url=...'))
|
||||
endif
|
||||
@echo "Downloading $(area) into $(PBF_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
|
||||
@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c ' \
|
||||
download-osm $(OSM_SERVER) $(DOWNLOAD_AREA) \
|
||||
--minzoom $$QUICKSTART_MIN_ZOOM \
|
||||
--maxzoom $$QUICKSTART_MAX_ZOOM \
|
||||
--make-dc /import/docker-compose-config.yml -- -d /import'
|
||||
ls -la ./data/$(notdir $(area))*
|
||||
--make-dc $(AREA_DC_CONFIG_FILE) \
|
||||
--imposm-cfg $(IMPOSM_CONFIG_FILE) \
|
||||
--output $(PBF_FILE) \
|
||||
2>&1 \
|
||||
| tee /tmp/download.out ; \
|
||||
exit_code=$${PIPESTATUS[0]} ; \
|
||||
if [[ "$$exit_code" != "0" ]]; then \
|
||||
if grep -q "Imposm config file cannot be generated from this source" /tmp/download.out; then \
|
||||
echo "WARNING: $(IMPOSM_CONFIG_FILE) could not be generated, but it is only needed to apply updates." ; \
|
||||
else \
|
||||
exit $$exit_code ; \
|
||||
fi ; \
|
||||
fi'
|
||||
@echo ""
|
||||
else
|
||||
ifeq (,$(wildcard $(AREA_DC_CONFIG_FILE)))
|
||||
@echo "Data file $(PBF_FILE) already exists, but the $(AREA_DC_CONFIG_FILE) is not, generating..."
|
||||
@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c ' \
|
||||
download-osm make-dc $(PBF_FILE) \
|
||||
--minzoom $$QUICKSTART_MIN_ZOOM \
|
||||
--maxzoom $$QUICKSTART_MAX_ZOOM \
|
||||
--make-dc $(AREA_DC_CONFIG_FILE) \
|
||||
--id "$(area)"'
|
||||
else
|
||||
@echo "Data files $(PBF_FILE) and $(AREA_DC_CONFIG_FILE) already exists, skipping the download."
|
||||
endif
|
||||
endif
|
||||
|
||||
.PHONY: psql
|
||||
@@ -196,6 +298,7 @@ psql: start-db-nowait
|
||||
|
||||
.PHONY: import-osm
|
||||
import-osm: all start-db-nowait
|
||||
@$(assert_area_is_given)
|
||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-osm $(PBF_FILE)'
|
||||
|
||||
.PHONY: update-osm
|
||||
@@ -212,24 +315,28 @@ import-data: start-db
|
||||
|
||||
.PHONY: import-borders
|
||||
import-borders: start-db-nowait
|
||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-borders $(PBF_FILE)'
|
||||
@$(assert_area_is_given)
|
||||
# If CSV borders file already exists, use it without re-parsing
|
||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \
|
||||
'pgwait && import-borders $$([ -f "$(BORDERS_CSV_FILE)" ] && echo 'load' || echo 'import') $(PBF_FILE)'
|
||||
|
||||
.PHONY: import-sql
|
||||
import-sql: all start-db-nowait
|
||||
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \
|
||||
awk -v s=": WARNING:" '$$0~s{print; print "\n*** WARNING detected, aborting"; exit(1)} 1'
|
||||
|
||||
.PHONY: generate-tiles
|
||||
ifneq ($(wildcard data/docker-compose-config.yml),)
|
||||
DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml
|
||||
ifneq ($(wildcard $(AREA_DC_CONFIG_FILE)),)
|
||||
DC_CONFIG_TILES := -f docker-compose.yml -f $(AREA_DC_CONFIG_FILE)
|
||||
endif
|
||||
.PHONY: generate-tiles
|
||||
generate-tiles: all start-db
|
||||
rm -rf data/tiles.mbtiles
|
||||
echo "Generating tiles ..."; \
|
||||
@$(assert_area_is_given)
|
||||
@echo "Generating tiles into $(MBTILES_LOCAL_FILE) (will delete if already exists)..."
|
||||
@rm -rf "$(MBTILES_LOCAL_FILE)"
|
||||
$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles
|
||||
@echo "Updating generated tile metadata ..."
|
||||
$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) openmaptiles-tools \
|
||||
mbtiles-tools meta-generate ./data/tiles.mbtiles ./openmaptiles.yaml --auto-minmax --show-ranges
|
||||
mbtiles-tools meta-generate "$(MBTILES_LOCAL_FILE)" ./openmaptiles.yaml --auto-minmax --show-ranges
|
||||
|
||||
.PHONY: start-tileserver
|
||||
start-tileserver: init-dirs
|
||||
|
||||
Reference in New Issue
Block a user