Use tools v4 with on-the-fly boundary generation (#750)

* Switch OMT to use the new tools v4.0.0 * borders are dynamically generated from the PBF file instead of downloading a prepared CSV file * all tools are executed as current user instead of root, thus files are easier to modify/delete if needed * all data is stored in the local file system instead of docker volumes (Docker currently has a limitation of non-root operation for internal volumes). This also makes it easier to examine and test it. * New `init-dirs` make target creates all the needed dirs - `build, data, cache` * `make clean` deletes the whole `build` dir instead of individual files. * `clean-docker` for backward compatibility deletes `cache` dirs (it used to be a volume) * all `psql` calls are now done with `ON_ERROR_STOP=1` * got rid of `pgclimb-*` targets -- same results can be done with `psql` (`pgclimb-list-views` & `pgclimb-list-tables` renamed to `list-views` and `list-tables`)
2020-03-06 13:15:54 -05:00
parent 036a96fb40
commit 490acf9beb
7 changed files with 114 additions and 109 deletions
--- a/121
+++ b/121
@@ -1,8 +1,6 @@
 # Options to run with docker and docker-compose - ensure the container is destroyed on exit
-DC_OPTS?=--rm
-
-# container runs as the current user rather than root (so that created files are not root-owned)
-DC_USER_OPTS?=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER})
+# Containers run as the current user rather than root (so that created files are not root-owned)
+DC_OPTS?=--rm -u $$(id -u $${USER}):$$(id -g $${USER})

 # If running in the test mode, compare files rather than copy them
 TEST_MODE?=no
@@ -53,30 +51,30 @@ help:
 	@echo "  make help                            # help about available commands"
 	@echo "=============================================================================="

-.PHONY: build
-build:
-	mkdir -p build
+.PHONY: init-dirs
+init-dirs:
+	mkdir -p build && mkdir -p data && mkdir -p cache

-build/openmaptiles.tm2source/data.yml: build
+build/openmaptiles.tm2source/data.yml: init-dirs
 	mkdir -p build/openmaptiles.tm2source
 	docker-compose run $(DC_OPTS) openmaptiles-tools generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > $@

-build/mapping.yaml: build
+build/mapping.yaml: init-dirs
 	docker-compose run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@

-build/tileset.sql: build
-	# FIXME:   switch to openmaptiles-tools after v3.2+ is published
-	docker-compose run $(DC_OPTS) openmaptiles-tools-latest generate-sql openmaptiles.yaml > $@
+build/tileset.sql: init-dirs
+	docker-compose run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > $@

 .PHONY: clean
 clean:
-	rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql
+	rm -rf build

 .PHONY: clean-docker
 clean-docker:
 	docker-compose down -v --remove-orphans
 	docker-compose rm -fv
 	docker volume ls -q | grep openmaptiles  | xargs -r docker volume rm || true
+	rm -rf cache

 .PHONY: db-start
 db-start:
@@ -84,13 +82,17 @@ db-start:
 	@echo "Wait for PostgreSQL to start..."
 	docker-compose run $(DC_OPTS) import-osm  ./pgwait.sh

+.PHONY: db-stop
+db-stop:
+	docker-compose stop postgres
+
 .PHONY: download-geofabrik
-download-geofabrik:
-	@echo ===============  download-geofabrik =======================
-	@echo Download area :   $(area)
-	@echo [[ example: make download-geofabrik  area=albania ]]
-	@echo [[ list areas:  make download-geofabrik-list       ]]
-	docker-compose run $(DC_OPTS) import-osm  ./download-geofabrik.sh $(area)
+download-geofabrik: init-dirs
+	@echo =============== download-geofabrik =======================
+	@echo Download area:   $(area)
+	@echo [[ example: make download-geofabrik area=albania ]]
+	@echo [[ list areas:  make download-geofabrik-list     ]]
+	docker-compose run $(DC_OPTS) import-osm ./download-geofabrik.sh $(area)
 	ls -la ./data/$(area).*
 	@echo "Generated config file: ./data/docker-compose-config.yml"
 	@echo " "
@@ -110,23 +112,37 @@ import-sql: db-start all
 	docker-compose run $(DC_OPTS) openmaptiles-tools import-sql

 .PHONY: import-osmsql
-import-osmsql: db-start all
-	docker-compose run $(DC_OPTS) import-osm
-	docker-compose run $(DC_OPTS) openmaptiles-tools import-sql
+import-osmsql: db-start all import-osm import-sql
+
+.PHONY: import-borders
+import-borders: db-start
+	docker-compose run $(DC_OPTS) openmaptiles-tools import-borders
+
+.PHONY: import-water
+import-water: db-start
+	docker-compose run $(DC_OPTS) import-water
+
+.PHONY: import-natural-earth
+import-natural-earth: db-start
+	docker-compose run $(DC_OPTS) import-natural-earth
+
+.PHONY: import-lakelines
+import-lakelines: db-start
+	docker-compose run $(DC_OPTS) import-lakelines

 .PHONY: generate-tiles
-generate-tiles: db-start all
+generate-tiles: init-dirs db-start all
 	rm -rf data/tiles.mbtiles
 	if [ -f ./data/docker-compose-config.yml ]; then \
-		docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml run $(DC_OPTS) generate-vectortiles; \
+		docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml \
+					   run $(DC_OPTS) generate-vectortiles; \
 	else \
 		docker-compose run $(DC_OPTS) generate-vectortiles; \
 	fi
-	docker-compose run $(DC_OPTS) openmaptiles-tools  generate-metadata ./data/tiles.mbtiles
-	docker-compose run $(DC_OPTS) openmaptiles-tools  chmod 666         ./data/tiles.mbtiles
+	docker-compose run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles

 .PHONY: start-tileserver
-start-tileserver:
+start-tileserver: init-dirs
 	@echo " "
 	@echo "***********************************************************"
 	@echo "* "
@@ -175,9 +191,9 @@ generate-qareports:

 # generate all etl and mapping graphs
 .PHONY: generate-devdoc
-generate-devdoc:
+generate-devdoc: init-dirs
 	mkdir -p ./build/devdoc && \
-	docker-compose run $(DC_USER_OPTS) openmaptiles-tools-latest sh -c \
+	docker-compose run $(DC_OPTS) openmaptiles-tools-latest sh -c \
 			'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \
 			 generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)'

@@ -191,50 +207,51 @@ import-osm-dev:

 # the `download-geofabrik` error message mention `list`, if the area parameter is wrong. so I created a similar make command
 .PHONY: list
-list:
-	docker-compose run $(DC_OPTS) import-osm  ./download-geofabrik-list.sh
+list: download-geofabrik-list

-# same as a `make list`
 .PHONY: download-geofabrik-list
 download-geofabrik-list:
-	docker-compose run $(DC_OPTS) import-osm  ./download-geofabrik-list.sh
+	docker-compose run $(DC_OPTS) import-osm ./download-geofabrik-list.sh

 .PHONY: import-wikidata
 import-wikidata:
-	# FIXME:   switch to openmaptiles-tools after v3.2+ is published
-	docker-compose run $(DC_OPTS) openmaptiles-tools-latest import-wikidata openmaptiles.yaml
-
-.PHONY: psql-list-tables
-psql-list-tables:
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh  -P pager=off  -c "\d+"
+	docker-compose run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml

 .PHONY: psql-pg-stat-reset
 psql-pg-stat-reset:
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh  -P pager=off  -c 'SELECT pg_stat_statements_reset();'
+	docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'SELECT pg_stat_statements_reset();'

 .PHONY: forced-clean-sql
 forced-clean-sql:
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "DROP SCHEMA IF EXISTS public CASCADE ; CREATE SCHEMA IF NOT EXISTS public; "
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent; CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;"
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "GRANT ALL ON SCHEMA public TO public;COMMENT ON SCHEMA public IS 'standard public schema';"
+	docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 \
+		-c "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA IF NOT EXISTS public;" \
+		-c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent;" \
+		-c "CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;" \
+		-c "GRANT ALL ON SCHEMA public TO public; COMMENT ON SCHEMA public IS 'standard public schema';"

-.PHONY: pgclimb-list-views
-pgclimb-list-views:
-	docker-compose run $(DC_OPTS) import-osm ./pgclimb.sh -c "select schemaname,viewname from pg_views where schemaname='public' order by viewname;" csv
+.PHONY: list-views
+list-views:
+	@docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \
+		-c "select schemaname, viewname from pg_views where schemaname='public' order by viewname;"

-.PHONY: pgclimb-list-tables
-pgclimb-list-tables:
-	docker-compose run $(DC_OPTS) import-osm ./pgclimb.sh -c "select schemaname,tablename from pg_tables where schemaname='public' order by tablename;" csv
+.PHONY: list-tables
+list-tables:
+	@docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \
+		-c "select schemaname, tablename from pg_tables where schemaname='public' order by tablename;"
+
+.PHONY: psql-list-tables
+psql-list-tables:
+	docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c "\d+"

 .PHONY: psql-vacuum-analyze
 psql-vacuum-analyze:
 	@echo "Start - postgresql: VACUUM ANALYZE VERBOSE;"
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh  -P pager=off  -c 'VACUUM ANALYZE VERBOSE;'
+	docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'VACUUM ANALYZE VERBOSE;'

 .PHONY: psql-analyze
 psql-analyze:
-	@echo "Start - postgresql: ANALYZE VERBOSE ;"
-	docker-compose run $(DC_OPTS) import-osm ./psql.sh  -P pager=off  -c 'ANALYZE VERBOSE;'
+	@echo "Start - postgresql: ANALYZE VERBOSE;"
+	docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'ANALYZE VERBOSE;'

 .PHONY: list-docker-images
 list-docker-images: