Merge remote-tracking branch 'github/master'

# Conflicts: # Makefile
2020-07-01 15:18:59 +02:00
parent 7519c5889f c86f4a557a
commit 566f4c47d5
60 changed files with 5122 additions and 3324 deletions
--- a/262
+++ b/262
@@ -1,44 +1,48 @@
+#
+# First section - common variable initialization
+#
+
 # Ensure that errors don't hide inside pipes
 SHELL         = /bin/bash
 .SHELLFLAGS   = -o pipefail -c

+# Make all .env variables available for make targets
+include .env
+
 # Options to run with docker and docker-compose - ensure the container is destroyed on exit
 # Containers run as the current user rather than root (so that created files are not root-owned)
-DC_OPTS?=--rm -u $(shell id -u):$(shell id -g)
+DC_OPTS ?= --rm -u $(shell id -u):$(shell id -g)

 DC_USER?=openmaptiles
 DC_PASSWORD?=openmaptiles

 # If set to a non-empty value, will use postgis-preloaded instead of postgis docker image
-USE_PRELOADED_IMAGE?=
-
-# If set, this file will be imported in the import-osm target
-PBF_FILE?=
+USE_PRELOADED_IMAGE ?=

 # Local port to use with postserve
-PPORT?=8090
+PPORT ?= 8090
 export PPORT
 # Local port to use with tileserver
-TPORT?=8081
+TPORT ?= 8081
 export TPORT

 # Allow a custom docker-compose project name
 ifeq ($(strip $(DC_PROJECT)),)
-  override DC_PROJECT:=$(notdir $(shell pwd))
-  DOCKER_COMPOSE:= docker-compose
+  DC_PROJECT := $(notdir $(shell pwd))
+  DOCKER_COMPOSE := docker-compose
 else
-  DOCKER_COMPOSE:= docker-compose --project-name $(DC_PROJECT)
+  DOCKER_COMPOSE := docker-compose --project-name $(DC_PROJECT)
 endif

 # Make some operations quieter (e.g. inside the test script)
 ifeq ($(strip $(QUIET)),)
-  QUIET_FLAG:=
+  QUIET_FLAG :=
 else
-  QUIET_FLAG:=--quiet
+  QUIET_FLAG := --quiet
 endif

 # Use `xargs --no-run-if-empty` flag, if supported
-XARGS:=xargs $(shell xargs --no-run-if-empty </dev/null 2>/dev/null && echo --no-run-if-empty)
+XARGS := xargs $(shell xargs --no-run-if-empty </dev/null 2>/dev/null && echo --no-run-if-empty)

 # If running in the test mode, compare files rather than copy them
 TEST_MODE?=no
@@ -50,12 +54,109 @@ else
  GRAPH_PARAMS=./layers
 endif

+# Set OpenMapTiles host
+OMT_HOST := http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost)
+
+# This defines an easy $(newline) value to act as a "\n". Make sure to keep exactly two empty lines after newline.
+define newline
+
+
+endef
+
+
+#
+# Determine area to work on
+# If $(area) parameter is not set and data/*.osm.pbf finds only one file, use it as $(area).
+# Otherwise all make targets requiring area param will show an error.
+# Note: If there are no data files, and user calls  make download area=... once,
+#       they will not need to use area= parameter after that because there will be just a single file.
+#
+
+# historically we have been using $(area) rather than $(AREA), so make both work
+area ?= $(AREA)
+# Ensure the $(AREA) param is set, or try to automatically determine it based on available data files
+ifeq ($(strip $(area)),)
+  # if $area is not set. set it to the name of the *.osm.pbf file, but only if there is only one
+  data_files := $(wildcard data/*.osm.pbf)
+  ifneq ($(word 2,$(data_files)),)
+    AREA_ERROR := The 'area' parameter (or env var) has not been set, and there are more than one data/*.osm.pbf files: $(patsubst data/%.osm.pbf,'%',$(data_files))
+  else
+    ifeq ($(word 1,$(data_files)),)
+      AREA_ERROR := The 'area' parameter (or env var) has not been set, and there are no data/*.osm.pbf files
+    else
+      # Keep just the name of the data file, without the .osm.pbf extension
+      area := $(strip $(basename $(basename $(notdir $(data_files)))))
+      # Rename area-latest.osm.pbf to area.osm.pbf
+      # TODO: This if statement could be removed in a few months once everyone is using the file without the `-latest`?
+      ifneq ($(area),$(area:-latest=))
+        $(shell mv "data/$(area).osm.pbf" "data/$(area:-latest=).osm.pbf")
+        area := $(area:-latest=)
+        $(warning ATTENTION: File data/$(area)-latest.osm.pbf was renamed to $(area).osm.pbf.)
+        AREA_INFO := Detected area=$(area) based on the found data/$(area)-latest.osm.pbf (renamed to $(area).osm.pbf). Use 'area' parameter (or env var) to override.
+      else
+        AREA_INFO := Detected area=$(area) based on the found data/ pbf file. Use 'area' parameter (or env var) to override.
+      endif
+    endif
+  endif
+endif
+
+# If set, this file will be downloaded in download-osm and imported in the import-osm targets
+PBF_FILE ?= data/$(area).osm.pbf
+
+# For download-osm, allow URL parameter to download file from a given URL. Area param must still be provided.
+ifneq ($(strip $(url)),)
+  DOWNLOAD_AREA := $(url)
+else
+  DOWNLOAD_AREA := $(area)
+endif
+
+# import-borders uses these temp files during border parsing/import
+export BORDERS_CLEANUP_FILE ?= data/borders/$(area).cleanup.pbf
+export BORDERS_PBF_FILE ?= data/borders/$(area).filtered.pbf
+export BORDERS_CSV_FILE ?= data/borders/$(area).lines.csv
+
+# The file is placed into the $EXPORT_DIR=/export (mapped to ./data)
+export MBTILES_FILE ?= $(area).mbtiles
+MBTILES_LOCAL_FILE = data/$(MBTILES_FILE)
+
+ifeq ($(strip $(DIFF_MODE)),true)
+  # import-osm implementation requires IMPOSM_CONFIG_FILE to be set to a valid file
+  # For static (no-updates) import, we don't need to override the default value
+  # For the update mode, set location of the dynamically-generated area-based config file
+  export IMPOSM_CONFIG_FILE = data/$(area).repl.json
+endif
+
+# download-osm generates this file with metadata about the file
+AREA_DC_CONFIG_FILE ?= data/$(area).dc-config.yml
+
+ifeq ($(strip $(area)),)
+  define assert_area_is_given
+	@echo ""
+	@echo "ERROR: $(AREA_ERROR)"
+	@echo ""
+	@echo "  make $@ area=<area-id>"
+	@echo ""
+	@echo "To download an area, use   make download <area-id>"
+	@echo "To list downloadable areas, use   make list-geofabrik   and/or   make list-bbbike"
+	@exit 1
+  endef
+else
+  ifneq ($(strip $(AREA_INFO)),)
+    define assert_area_is_given
+	@echo "$(AREA_INFO)"
+    endef
+  endif
+endif
+
+
+
+#
+#  TARGETS
+#
+
 .PHONY: all
 all: init-dirs build/openmaptiles.tm2source/data.yml build/mapping.yaml build-sql

-# Set OpenMapTiles host
-OMT_HOST:=http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost)
-
 .PHONY: help
 help:
 	@echo "=============================================================================="
@@ -72,9 +173,12 @@ help:
 	@echo "Hints for developers:"
 	@echo "  make                                 # build source code"
 	@echo "  make list-geofabrik                  # list actual geofabrik OSM extracts for download"
-	@echo "  make download-geofabrik area=albania # download OSM data from geofabrik,        and create config file"
-	@echo "  make download-osmfr area=asia/qatar  # download OSM data from openstreetmap.fr, and create config file"
-	@echo "  make download-bbike area=Amsterdam   # download OSM data from bbike.org,        and create config file"
+	@echo "  make list-bbbike                     # list actual BBBike OSM extracts for download"
+	@echo "  make download area=albania           # download OSM data from any source       and create config file"
+	@echo "  make download-geofabrik area=albania # download OSM data from geofabrik.de     and create config file"
+	@echo "  make download-osmfr area=asia/qatar  # download OSM data from openstreetmap.fr and create config file"
+	@echo "  make download-bbbike area=Amsterdam  # download OSM data from bbbike.org       and create config file"
+	@echo "  make generate-dc-config              # scan data file and generate tile generation config file with bbox"
 	@echo "  make psql                            # start PostgreSQL console"
 	@echo "  make psql-list-tables                # list all PostgreSQL tables"
 	@echo "  make vacuum-db                       # PostgreSQL: VACUUM ANALYZE"
@@ -98,29 +202,39 @@ help:

 .PHONY: init-dirs
 init-dirs:
-	@mkdir -p build/sql
-	@mkdir -p data
+	@mkdir -p build/sql/parallel
+	@mkdir -p build/openmaptiles.tm2source
+	@mkdir -p data/borders
 	@mkdir -p cache

 build/openmaptiles.tm2source/data.yml: init-dirs
-	mkdir -p build/openmaptiles.tm2source
+ifeq (,$(wildcard build/openmaptiles.tm2source/data.yml))
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="$(DC_USER)" --password="$(DC_PASSWORD)" > $@
+endif

 build/mapping.yaml: init-dirs
+ifeq (,$(wildcard build/mapping.yaml))
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@
+endif

 .PHONY: build-sql
 build-sql: init-dirs
+ifeq (,$(wildcard build/sql/run_last.sql))
+	@mkdir -p build/sql/parallel
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c \
 		'generate-sql openmaptiles.yaml --dir ./build/sql \
-		&& generate-sqltomvt openmaptiles.yaml --key --postgis-ver 2.4.8 --function --fname=getmvt >> "./build/sql/run_last.sql"'
+		&& generate-sqltomvt openmaptiles.yaml \
+							 --key --gzip --postgis-ver 3.0.1 \
+							 --function --fname=getmvt >> ./build/sql/run_last.sql'
+endif

 .PHONY: clean
 clean:
 	rm -rf build

 .PHONY: destroy-db
-destroy-db: override DC_PROJECT:=$(shell echo $(DC_PROJECT) | tr A-Z a-z)
+# TODO:  Use https://stackoverflow.com/a/27852388/177275
+destroy-db: DC_PROJECT := $(shell echo $(DC_PROJECT) | tr A-Z a-z)
 destroy-db:
 	$(DOCKER_COMPOSE) down -v --remove-orphans
 	$(DOCKER_COMPOSE) rm -fv
@@ -152,29 +266,63 @@ stop-db:
 list-geofabrik: init-dirs
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list geofabrik

-OSM_SERVERS:=geofabrik osmfr bbbike
-ALL_DOWNLOADS:=$(addprefix download-,$(OSM_SERVERS))
-OSM_SERVER=$(patsubst download-%,%,$@)
+.PHONY: list-bbbike
+list-bbbike: init-dirs
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list bbbike
+
+#
+# download, download-geofabrik, download-osmfr, and download-bbbike are handled here
+# The --imposm-cfg will fail for some of the sources, but we ignore that error -- only needed for diff mode
+#
+OSM_SERVERS := geofabrik osmfr bbbike
+ALL_DOWNLOADS := $(addprefix download-,$(OSM_SERVERS)) download
+OSM_SERVER=$(patsubst download,,$(patsubst download-%,%,$@))
 .PHONY: $(ALL_DOWNLOADS)
 $(ALL_DOWNLOADS): init-dirs
-ifeq ($(strip $(area)),)
+	@$(assert_area_is_given)
+ifneq ($(strip $(url)),)
+	$(if $(OSM_SERVER),$(error url parameter can only be used with non-specific download target:$(newline)       make download area=$(area) url="$(url)"$(newline)))
+endif
+ifeq (,$(wildcard $(PBF_FILE)))
+ ifeq ($(strip $(DIFF_MODE)),true)
+	@echo "Downloading $(DOWNLOAD_AREA) with replication support into $(PBF_FILE) and $(IMPOSM_CONFIG_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
+	@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \
+				--imposm-cfg "$(IMPOSM_CONFIG_FILE)" \
+				--output "$(PBF_FILE)"
+ else
+	@echo "Downloading $(DOWNLOAD_AREA) into $(PBF_FILE) from $(if $(OSM_SERVER),$(OSM_SERVER),any source)"
+	@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm $(OSM_SERVER) "$(DOWNLOAD_AREA)" \
+				--output "$(PBF_FILE)"
+ endif
 	@echo ""
-	@echo "ERROR: Unable to download an area if area is not given."
-	@echo "Usage:"
-	@echo "  make download-$(OSM_SERVER) area=<area-id>"
-	@echo ""
-	$(if $(filter %-geofabrik,$@),@echo "Use   make list-geofabrik   to get a list of all available areas";echo "")
-	@exit 1
 else
-	@echo "=============== download-$(OSM_SERVER) ======================="
-	@echo "Download area: $(area)"
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c \
-		'download-osm $(OSM_SERVER) $(area) \
-			--minzoom $$QUICKSTART_MIN_ZOOM \
-			--maxzoom $$QUICKSTART_MAX_ZOOM \
-			--make-dc /import/docker-compose-config.yml -- -d /import'
-	ls -la ./data/$(notdir $(area))*
-	@echo ""
+ ifeq ($(strip $(DIFF_MODE)),true)
+  ifeq (,$(wildcard $(IMPOSM_CONFIG_FILE)))
+	$(error \
+		$(newline)   Data files $(PBF_FILE) already exists, but $(IMPOSM_CONFIG_FILE) does not. \
+		$(newline)   You probably downloaded the data file before setting DIFF_MODE=true. \
+		$(newline)   You can delete the data file  $(PBF_FILE) and re-run  make download \
+		$(newline)   to re-download and generate config, or manually create  $(IMPOSM_CONFIG_FILE) \
+		$(newline)   See example    https://github.com/openmaptiles/openmaptiles-tools/blob/v5.2/bin/config/repl_config.json \
+		$(newline))
+  else
+	@echo "Data files $(PBF_FILE) and replication config $(IMPOSM_CONFIG_FILE) already exists, skipping the download."
+  endif
+ else
+	@echo "Data files $(PBF_FILE) already exists, skipping the download."
+ endif
+endif
+
+.PHONY: generate-dc-config
+generate-dc-config:
+	@$(assert_area_is_given)
+ifeq (,$(wildcard $(AREA_DC_CONFIG_FILE)))
+	@$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c ' \
+		download-osm make-dc "$(PBF_FILE)" \
+			--make-dc "$(AREA_DC_CONFIG_FILE)" \
+			--id "$(area)"'
+else
+	@echo "Configuration file $(AREA_DC_CONFIG_FILE) already exists, no need to regenerate."
 endif

 .PHONY: psql
@@ -183,14 +331,17 @@ psql: start-db-nowait

 .PHONY: import-osm
 import-osm: all start-db-nowait
+	@$(assert_area_is_given)
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-osm $(PBF_FILE)'

 .PHONY: update-osm
 update-osm: all start-db-nowait
+	@$(assert_area_is_given)
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-update'

 .PHONY: import-diff
 import-diff: all start-db-nowait
+	@$(assert_area_is_given)
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-diff'

 .PHONY: import-data
@@ -199,27 +350,28 @@ import-data: start-db

 .PHONY: import-borders
 import-borders: start-db-nowait
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-borders'
+	@$(assert_area_is_given)
+	# If CSV borders file already exists, use it without re-parsing
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \
+		'pgwait && import-borders $$([ -f "$(BORDERS_CSV_FILE)" ] && echo load $(BORDERS_CSV_FILE) || echo import $(PBF_FILE))'

 .PHONY: import-sql
 import-sql: all start-db-nowait
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \
 	  awk -v s=": WARNING:" '$$0~s{print; print "\n*** WARNING detected, aborting"; exit(1)} 1'

-.PHONY: show-metadata
-show-metadata: init-dirs
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools mbtiles-tools meta-all ./data/tiles.mbtiles
-
-.PHONY: generate-tiles
-ifneq ($(wildcard data/docker-compose-config.yml),)
-  DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml
+ifneq ($(wildcard $(AREA_DC_CONFIG_FILE)),)
+  DC_CONFIG_TILES := -f docker-compose.yml -f $(AREA_DC_CONFIG_FILE)
 endif
+.PHONY: generate-tiles
 generate-tiles: all start-db
-	rm -rf data/tiles.mbtiles
-	echo "Generating tiles ..."; \
+	@$(assert_area_is_given)
+	@echo "Generating tiles into $(MBTILES_LOCAL_FILE) (will delete if already exists)..."
+	@rm -rf "$(MBTILES_LOCAL_FILE)"
 	$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles
 	@echo "Updating generated tile metadata ..."
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles
+	$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) openmaptiles-tools \
+			mbtiles-tools meta-generate "$(MBTILES_LOCAL_FILE)" ./openmaptiles.yaml --auto-minmax --show-ranges

 .PHONY: start-tileserver
 start-tileserver: init-dirs
@@ -356,7 +508,7 @@ remove-docker-images:
 .PHONY: clean-unnecessary-docker
 clean-unnecessary-docker:
 	@echo "Deleting unnecessary container(s)..."
-	@docker ps -a --filter "status=exited" | $(XARGS) docker rm
+	@docker ps -a -q --filter "status=exited" | $(XARGS) docker rm
 	@echo "Deleting unnecessary image(s)..."
 	@docker images | grep \<none\> | awk -F" " '{print $$3}' | $(XARGS) docker rmi