From 7a239b66cfa4b36579abb607d8f60cc7cabfb220 Mon Sep 17 00:00:00 2001 From: Miel Truyen Date: Wed, 16 Jun 2021 21:12:35 +0200 Subject: [PATCH] [WIP] NUTS layer: Municipalities, Provinces, Regions etc as linestrings (with polygons used in intermediate step) representing what is left/right of the border --- .env | 30 +- Makefile | 6 + benelux.sh | 322 ++++++++++++++++++++++ docker-compose.yml | 6 +- europe.sh | 146 ++++++++++ layers/boundary/mapping.yaml | 106 ++++++- layers/boundary/nuts.sql | 182 ++++++++++++ layers/boundary/nuts.yaml | 49 ++++ layers/boundary/nuts.yaml.polygons.backup | 26 ++ openmaptiles.yaml | 33 +-- 10 files changed, 882 insertions(+), 24 deletions(-) create mode 100755 benelux.sh create mode 100755 europe.sh create mode 100644 layers/boundary/nuts.sql create mode 100644 layers/boundary/nuts.yaml create mode 100644 layers/boundary/nuts.yaml.polygons.backup diff --git a/.env b/.env index 3d7fc1b..9411f8f 100644 --- a/.env +++ b/.env @@ -16,11 +16,13 @@ PGPORT=5432 # BBOX may get overwritten by the computed bbox of the specific area: # make generate-bbox-file # By default, the Makefile will use the content of data/$(area).bbox file if it exists. -BBOX=-180.0,-85.0511,180.0,85.0511 +#BBOX=-180.0,-85.0511,180.0,85.0511 +BBOX=4.827919,50.740472,5.798035,51.095291 + # Which zooms to generate in make generate-tiles MIN_ZOOM=0 -MAX_ZOOM=7 +MAX_ZOOM=14 # Use true (case sensitive) to allow data updates DIFF_MODE=false @@ -44,10 +46,28 @@ BORDERS_PBF_FILE=data/borders/filtered.pbf BORDERS_CSV_FILE=data/borders/lines.csv # Number of parallel processes to use when importing sql files -MAX_PARALLEL_PSQL=5 +#MAX_PARALLEL_PSQL=5 +MAX_PARALLEL_PSQL=16 -# Number of parallel threads to use when generating vector map tiles -COPY_CONCURRENCY=10 +# Number of concurrent IO ops to use when generating vector map tiles +# Set to ridiculous high values for SSD/NVME +# Test results: (These were without setting UV_THREADPOOL_SIZE) +# 64: ~100/s peak +# 256: ~200/s peak, stabilizes to a 50% improvement over 64 concurrency +# 1024: ~400/s peak, still at 400/s speed after a few minutes (unlike before, oddly enough) +# 4096: ~250/s peak, stabilizes to 200. But it appears to be choking on something, only updates progress periodically +# 1024 & 24 Threads: ~700/s peak - drops to 214s after 10mins, 178/s after 15mins. EST after 20min: 23hours +# Postgres is mainly idle during all of this (coarse dataset), using only half a core +# To be tested if 1 core is dedicated to postgres and the others to tilelive(UV_THREADPOOL_SIZE) +# will prove to be more efficient on average. Or just UV_THREADPOOL_SIZE==CPU_THREADS. +#COPY_CONCURRENCY=64 +#COPY_CONCURRENCY=256 +#COPY_CONCURRENCY=512 +COPY_CONCURRENCY=1024 +#COPY_CONCURRENCY=4096 + +UV_THREADPOOL_SIZE=16 +#UV_THREADPOOL_SIZE=24 # Variables for generate tiles using PGquery PGHOSTS_LIST= diff --git a/Makefile b/Makefile index 31dc6c8..9e4b8ac 100644 --- a/Makefile +++ b/Makefile @@ -400,6 +400,12 @@ import-sql: all start-db-nowait $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \ awk -v s=": WARNING:" '1{print; fflush()} $$0~s{print "\n*** WARNING detected, aborting"; exit(1)}' +.PHONY: merge-pbf +merge-pbf: + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \ + osmosis --rb data/belgium.osm.pbf --rb data/netherlands.osm.pbf --merge --wb data/merges.osm.pbf + + .PHONY: generate-tiles generate-tiles: all start-db @$(assert_area_is_given) diff --git a/benelux.sh b/benelux.sh new file mode 100755 index 0000000..2613eef --- /dev/null +++ b/benelux.sh @@ -0,0 +1,322 @@ +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + + +########################################### +# This file is a modified quickstart.sh, intended to load benelux region +# + +# If --empty is not given, use preloaded docker image to speed up +if [ $# -gt 0 ] && [[ $1 == --empty ]]; then + export USE_PRELOADED_IMAGE="" + shift +else + export USE_PRELOADED_IMAGE=true +fi + +if [ $# -eq 0 ]; then + # default test area + export area=belgium + echo "No parameter - set area=$area " +else + export area=$1 +fi + +if [ $# -eq 2 ]; then + osm_server=$2 +fi + +## Min versions ... +MIN_COMPOSE_VER=1.7.1 +MIN_DOCKER_VER=1.12.3 +STARTTIME=$(date +%s) +STARTDATE=$(date +"%Y-%m-%dT%H:%M%z") + +log_file=./quickstart.log +rm -f $log_file +echo " " +echo "=====================================================================================" +echo " Docker check & Download images " +echo "-------------------------------------------------------------------------------------" +echo "====> : Please check the Docker and docker-compose version!" +echo " : We are using docker-compose v2 file format! see more at https://docs.docker.com/" +echo " : Minimum required Docker version: $MIN_DOCKER_VER+" +echo " : Minimum required docker-compose version: $MIN_COMPOSE_VER+" +echo " : See the .travis build for the currently supported versions." +echo " : Your docker system:" +docker --version +docker-compose --version + +# based on: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers +function version { echo "$@" | tr -cs '0-9.' '.' | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; } + +COMPOSE_VER=$(docker-compose version --short) +if [ "$(version "$COMPOSE_VER")" -lt "$(version "$MIN_COMPOSE_VER")" ]; then + echo "ERR: Your Docker-compose version is known to have bugs, please update docker-compose!" + exit 1 +fi + +DOCKER_VER="$(docker -v | awk -F '[ ,]+' '{ print $3 }')" +if [ "$(version "$DOCKER_VER")" -lt "$(version "$MIN_DOCKER_VER")" ]; then + echo "ERR: Your Docker version is not compatible. Please Update docker!" + exit 1 +fi + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Pulling or refreshing OpenMapTiles docker images " +make refresh-docker-images + + +##### backup log from here ... +exec &> >(tee -a "$log_file") + +echo " " +echo "=====================================================================================" +echo " Start processing " +echo "-------------------------------------------------------------------------------------" +echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] " +echo " : This will be logged to the $log_file file (for debugging) and to the screen" +echo " : Area : $area " +echo " : Download server : ${osm_server:-unset (automatic)} " +echo " : Preloaded image : $USE_PRELOADED_IMAGE " +echo " : Git version : $(git rev-parse HEAD) " +echo " : Started : $STARTDATE " +echo " : Your bash version: $BASH_VERSION" +echo " : Your OS : $OSTYPE" +docker --version +docker-compose --version + +if [[ "$OSTYPE" == "linux-gnu" ]]; then + echo " " + echo "-------------------------------------------------------------------------------------" + echo " : This is working on x86_64 ; Your kernel is:" + uname -r + uname -m + + KERNEL_CPU_VER=$(uname -m) + if [ "$KERNEL_CPU_VER" != "x86_64" ]; then + echo "ERR: Sorry this is working only on x86_64!" + exit 1 + fi + echo " : --- Memory, CPU info ---- " + mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) + echo "System memory (GB): ${mem}" + grep SwapTotal /proc/meminfo + echo "CPU number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)" + grep Free /proc/meminfo +else + echo " " + echo "Warning : Platforms other than Linux are less tested" + echo " " +fi + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Stopping running services & removing old containers" +make destroy-db + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Existing OpenMapTiles docker images. Will use version $(source .env && echo "$TOOLS_VERSION")" +docker images | grep openmaptiles + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Create directories if they don't exist" +make init-dirs + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Removing old MBTILES if exists ( ./data/${area}.mbtiles ) " +rm -f "./data/${area}.mbtiles" + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Downloading ${area} from ${osm_server:-any source}..." +make "download${osm_server:+-${osm_server}}" + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Remove old generated source files ( ./build/* ) ( if they exist ) " +make clean + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/sql/* )" +echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools " +echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! " +make all + +echo " " +echo "-------------------------------------------------------------------------------------" +if [[ "$USE_PRELOADED_IMAGE" == true ]]; then + echo "====> : Start PostgreSQL service using postgis image preloaded with this data:" + echo " : * Water data from http://osmdata.openstreetmap.de" + echo " : Data license: https://osmdata.openstreetmap.de/info/license.html" + echo " : * Natural Earth from http://www.naturalearthdata.com" + echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use" + echo " : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines" + echo " :" + echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data" + echo " : includes all data from the import-data image" + echo " :" + echo " : Use the --empty flag to start with an empty database:" + echo " : ./quickstart.sh --empty albania " + echo " : If desired, you can manually import data by using these commands:" + echo " : make destroy-db" + echo " : make start-db" + echo " : make import-data" + echo " :" + echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/postgis-preloaded" + echo " : Thank you https://www.postgresql.org ! Thank you http://postgis.org !" + make start-db-preloaded +else + echo "====> : Start PostgreSQL service using empty database and importing all the data:" + echo " : * Water data from http://osmdata.openstreetmap.de" + echo " : Data license: https://osmdata.openstreetmap.de/info/license.html" + echo " : * Natural Earth from http://www.naturalearthdata.com" + echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use" + echo " : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines" + echo " :" + echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data" + echo " : includes all data from the import-data image" + echo " :" + echo " : Thank you https://www.postgresql.org ! Thank you http://postgis.org !" + make start-db + make import-data +fi + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start importing OpenStreetMap data: ${area} -> imposm3[./build/mapping.yaml] -> PostgreSQL" +echo " : Imposm3 documentation: https://imposm.org/docs/imposm3/latest/index.html " +echo " : Thank you Omniscale! " +echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm " +echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) " +echo " : Thank you OpenStreetMap Contributors ! " +make import-osm + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start importing border ${area} data into PostgreSQL using osmborder" +echo " : Source code: https://github.com/pnorman/osmborder" +echo " : Data license: http://www.openstreetmap.org/copyright" +echo " : Thank you Paul Norman" +make import-borders + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start importing Wikidata: Wikidata Query Service -> PostgreSQL" +echo " : The Wikidata license: CC0 - https://www.wikidata.org/wiki/Wikidata:Main_Page " +echo " : Thank you Wikidata Contributors ! " +make import-wikidata + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start SQL postprocessing: ./build/sql/* -> PostgreSQL " +echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/blob/master/bin/import-sql" +# If the output contains a WARNING, stop further processing +# Adapted from https://unix.stackexchange.com/questions/307562 +make import-sql + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Analyze PostgreSQL tables" +make analyze-db + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Testing PostgreSQL tables to match layer definitions metadata" +make test-perf-null + +echo " " +echo "-------------------------------------------------------------------------------------" + +if [[ "$(source .env ; echo "$BBOX")" = "-180.0,-85.0511,180.0,85.0511" ]]; then + if [[ "$area" != "planet" ]]; then + echo "====> : Compute bounding box for tile generation" + make generate-bbox-file ${MIN_ZOOM:+MIN_ZOOM="${MIN_ZOOM}"} ${MAX_ZOOM:+MAX_ZOOM="${MAX_ZOOM}"} + else + echo "====> : Skipping bbox calculation when generating the entire planet" + fi + +else + echo "====> : Bounding box is set in .env file" +fi + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. " +echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml " +echo " : Output MBTiles: ./data/${area}.mbtiles " +echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/generate-vectortiles " +echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox " +echo " : Thank you https://www.mapbox.com !" +echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-tiles " +echo " : " +echo " : You will see a lot of deprecated warning in the log! This is normal! " +echo " : like : Mapnik LOG> ... is deprecated and will be removed in Mapnik 4.x ... " +make generate-tiles + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )" +make stop-db + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Inputs - Outputs md5sum for debugging " +rm -f ./data/quickstart_checklist.chk +{ + find build -type f | sort | xargs md5sum + find data -type f | sort | xargs md5sum +} >> ./data/quickstart_checklist.chk +cat ./data/quickstart_checklist.chk + +ENDTIME=$(date +%s) + +echo " " +echo " " +echo "-------------------------------------------------------------------------------------" +echo "-- S u m m a r y --" +echo "-------------------------------------------------------------------------------------" +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : (disk space) We have created a lot of docker images: " +echo " : Hint: you can remove with: docker rmi IMAGE " +docker images | grep openmaptiles + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : (disk space) We have created the new vectortiles ( ./data/${area}.mbtiles ) " +echo " : Please respect the licenses (OdBL for OSM data) of the sources when distributing the MBTiles file." +echo " : Data directory content:" +ls -la ./data + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "The ./quickstart.sh $area is finished! " +echo "It took $((ENDTIME - STARTTIME)) seconds to complete" +echo "We saved the log file to $log_file (for debugging) You can compare with the travis log !" +echo " " +echo "Start experimenting and check the QUICKSTART.MD file!" +echo " " +echo "* Use make start-maputnik to explore tile generation on request" +echo "* Use make start-tileserver to view pre-generated tiles" +echo " " +echo "Available help commands (make help) " +make help + +echo "-------------------------------------------------------------------------------------" +echo " Acknowledgments " +echo " Generated vector tiles are produced work of OpenStreetMap data. " +echo " Such tiles are reusable under CC-BY license granted by OpenMapTiles team: " +echo " https://github.com/openmaptiles/openmaptiles/#license " +echo " Maps made with these vector tiles must display a visible credit: " +echo " © OpenMapTiles © OpenStreetMap contributors " +echo " " +echo " Thanks to all free, open source software developers and Open Data Contributors! " +echo "-------------------------------------------------------------------------------------" diff --git a/docker-compose.yml b/docker-compose.yml index def4165..d47ef5b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,8 @@ networks: services: postgres: - image: "${POSTGIS_IMAGE:-openmaptiles/postgis}:${TOOLS_VERSION}" + #image: "${POSTGIS_IMAGE:-openmaptiles/postgis}:${TOOLS_VERSION}" + image: harbor.cerxes.net/openmaptiles/postgis:5.3 # Use "command: postgres -c jit=off" for PostgreSQL 11+ because of slow large MVT query processing volumes: - pgdata:/var/lib/postgresql/data @@ -19,7 +20,10 @@ services: - postgres ports: - "5432" + shm_size: 4g env_file: .env-postgres + #command: /docker-entrypoint-initdb.d/01_tune-postgis.sh + #command: /docker-entrypoint-initdb.d/01_tune-postgis.sh && echo 'tuned' && postgres -c 'config_file=/etc/postgresql/postgresql.conf' import-data: image: "openmaptiles/import-data:${TOOLS_VERSION}" diff --git a/europe.sh b/europe.sh new file mode 100755 index 0000000..e984854 --- /dev/null +++ b/europe.sh @@ -0,0 +1,146 @@ +set -e +make refresh-docker-images + +# Show mem info +mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) +echo "System memory (GB): ${mem}" +grep SwapTotal /proc/meminfo +echo "CPU number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)" +grep Free /proc/meminfo + + +make destroy-db +make init-dirs + +# rm -f "./data/${area}.mbtiles" +#echo "====> : Downloading ${area} from ${osm_server:-any source}..." +#make "download${osm_server:+-${osm_server}}" + +make clean +make all + +make start-db +make import-data + +# about 1.25hrs, Presumably Disk IO limited. +# If there is an option to double the amount of processes this might +# also help. Seeing about 50% usage per core +# [May 21 08:52:53] [INFO] Reading OSM data took: 22m9.040318577s +# [May 21 09:42:37] [INFO] Writing OSM data took: 49m44.555518107s +# [May 21 09:45:15] [INFO] Importing OSM data took: 52m21.813870474s +# [May 21 09:45:15] [INFO] [PostGIS] Rotating tables took: 38.69062ms +# [May 21 09:45:15] [INFO] Imposm took: 1h14m30.893125729s +make import-osm area=europe + +make import-borders area=europe + + +make import-wikidata + +# It not doing the materialized view in parallel despite postgres 12?! +# argh +make import-sql + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Analyze PostgreSQL tables" +make analyze-db + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Testing PostgreSQL tables to match layer definitions metadata" +# TODO: this right here shows us whats wrong. settings aren't optimal (see performance tips tabs) +# shared_buffersm work_mem, max_worker_processes, +# TODO, Ideal solution: Autoconfigure according to performance tips (and max_worker_processes == HT Cores ( or 1.5X HT Cores) +# (1.5x HT Cores might make sense as most work is probably memory-limited (latency), not CPU limited) +make test-perf-null + +#echo " " +#echo "-------------------------------------------------------------------------------------" +# +#if [[ "$(source .env ; echo "$BBOX")" = "-180.0,-85.0511,180.0,85.0511" ]]; then +# if [[ "$area" != "planet" ]]; then +# echo "====> : Compute bounding box for tile generation" +# make generate-bbox-file ${MIN_ZOOM:+MIN_ZOOM="${MIN_ZOOM}"} ${MAX_ZOOM:+MAX_ZOOM="${MAX_ZOOM}"} +# else +# echo "====> : Skipping bbox calculation when generating the entire planet" +# fi +# +#else +# echo "====> : Bounding box is set in .env file" +#fi +# +##echo " " +##echo "-------------------------------------------------------------------------------------" +##echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. " +##echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml " +##echo " : Output MBTiles: ./data/${area}.mbtiles " +##echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/generate-vectortiles " +##echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox " +##echo " : Thank you https://www.mapbox.com !" +##echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-tiles " +##echo " : " +##echo " : You will see a lot of deprecated warning in the log! This is normal! " +##echo " : like : Mapnik LOG> ... is deprecated and will be removed in Mapnik 4.x ... " +#make generate-tiles area=europe +# +## +##echo " " +##echo "-------------------------------------------------------------------------------------" +##echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )" +##make stop-db +## +##echo " " +##echo "-------------------------------------------------------------------------------------" +##echo "====> : Inputs - Outputs md5sum for debugging " +##rm -f ./data/quickstart_checklist.chk +##{ +## find build -type f | sort | xargs md5sum +## find data -type f | sort | xargs md5sum +##} >> ./data/quickstart_checklist.chk +##cat ./data/quickstart_checklist.chk +## +##ENDTIME=$(date +%s) +## +##echo " " +##echo " " +##echo "-------------------------------------------------------------------------------------" +##echo "-- S u m m a r y --" +##echo "-------------------------------------------------------------------------------------" +##echo " " +##echo "-------------------------------------------------------------------------------------" +##echo "====> : (disk space) We have created a lot of docker images: " +##echo " : Hint: you can remove with: docker rmi IMAGE " +##docker images | grep openmaptiles +## +##echo " " +##echo "-------------------------------------------------------------------------------------" +##echo "====> : (disk space) We have created the new vectortiles ( ./data/${area}.mbtiles ) " +##echo " : Please respect the licenses (OdBL for OSM data) of the sources when distributing the MBTiles file." +##echo " : Data directory content:" +##ls -la ./data +## +##echo " " +##echo "-------------------------------------------------------------------------------------" +##echo "The ./quickstart.sh $area is finished! " +##echo "It took $((ENDTIME - STARTTIME)) seconds to complete" +##echo "We saved the log file to $log_file (for debugging) You can compare with the travis log !" +##echo " " +##echo "Start experimenting and check the QUICKSTART.MD file!" +##echo " " +##echo "* Use make start-maputnik to explore tile generation on request" +##echo "* Use make start-tileserver to view pre-generated tiles" +##echo " " +##echo "Available help commands (make help) " +##make help +## +##echo "-------------------------------------------------------------------------------------" +##echo " Acknowledgments " +##echo " Generated vector tiles are produced work of OpenStreetMap data. " +##echo " Such tiles are reusable under CC-BY license granted by OpenMapTiles team: " +##echo " https://github.com/openmaptiles/openmaptiles/#license " +##echo " Maps made with these vector tiles must display a visible credit: " +##echo " © OpenMapTiles © OpenStreetMap contributors " +##echo " " +##echo " Thanks to all free, open source software developers and Open Data Contributors! " +##echo "-------------------------------------------------------------------------------------" diff --git a/layers/boundary/mapping.yaml b/layers/boundary/mapping.yaml index 76c0f93..bf8c1f6 100644 --- a/layers/boundary/mapping.yaml +++ b/layers/boundary/mapping.yaml @@ -115,10 +115,112 @@ tables: name: maritime type: bool from_member: true + - name: index + type: member_index + - name: role + type: member_role + - name: type + type: member_type mapping: type: [boundary] filters: require: - #admin_level: ['2'] + #admin_level: ['2'] # this used to be specified, re-enable if bugs show up with country borders admin_level: [__any__] - claimed_by: [__any__] + boundary: ['administrative'] # Filters out boundary administrative_fraction and religious_administration + + +# FOr NUTS in linestring version + administrative_relation: + type: relation + columns: + - name: osm_id + type: id + - key: name + name: name + type: string + - key: boundary + name: boundary + type: string + - key: admin_level + name: admin_level + type: integer + mapping: + boundary: [ 'administrative' ] + filters: + require: + admin_level: [ __any__ ] + + administrative_member: + type: relation_member + columns: + - name: relation_id + type: id + - name: boundary_id + type: id + from_member: true + - key: admin_level + name: admin_level + type: integer + - key: maritime + name: maritime + type: bool + from_member: true + - name: index + type: member_index + - name: role + type: member_role + - name: type + type: member_type + mapping: + type: [boundary] + filters: + require: + admin_level: [__any__] + boundary: ['administrative'] + + administrative_boundary: + type: linestring + columns: + - name: osm_id + type: id + - name: geometry + type: geometry + - key: admin_level + name: admin_level + type: integer + mapping: + boundary: [ 'administrative' ] + filters: + require: + admin_level: [ __any__ ] + + +# not currently used +# # etldoc: imposm3 -> osm_adm_boundary_relation +# adm_boundary_relation: +# type: relation +# columns: +# - name: relation_id +# type: id +# - key: name +# name: name +# type: string +# - key: admin_level +# name: admin_level +# type: integer +# - key: border_type +# name: border_type +# type: string +# - key: default_language +# name: default_language +# type: string +# - key: website +# name: website +# type: string +# mapping: +# type: [boundary] +# filters: +# require: +# boundary: ['administrative'] +# admin_level: [__any__] diff --git a/layers/boundary/nuts.sql b/layers/boundary/nuts.sql new file mode 100644 index 0000000..f8c305b --- /dev/null +++ b/layers/boundary/nuts.sql @@ -0,0 +1,182 @@ +-- This is very crude and not finetuned yet + +-- This statement can be deleted after the border importer image stops creating this object as a table +DO +$$ + BEGIN + DROP TABLE IF EXISTS osm_boundary_polygon CASCADE; + EXCEPTION + WHEN wrong_object_type THEN + END; +$$ LANGUAGE plpgsql; + +-- etldoc: osm_border_linestring -> osm_border_linestring_gen_z13 +-- etldoc: osm_border_linestring_adm -> osm_border_linestring_gen_z13 +DROP MATERIALIZED VIEW IF EXISTS osm_boundary_polygon CASCADE; +CREATE MATERIALIZED VIEW osm_boundary_polygon AS +( + SELECT + r.relation_id, + r.name, + CASE + WHEN r.admin_level = 10 THEN 6 + WHEN r.admin_level = 9 THEN 5 + WHEN r.admin_level = 8 THEN 4 + WHEN r.admin_level = 7 THEN 3 + WHEN r.admin_level = 6 THEN 2 + WHEN r.admin_level = 4 THEN 1 + -- No admin_level =3? + WHEN r.admin_level = 2 THEN 0 + ELSE null + END as nuts_level, + p.geometry + FROM ( + SELECT + relation_id, + ST_BuildArea(ST_Node(ST_Collect(geometry))) as geometry, + min(id) as minid -- Used just to join to one relation, could've also used subquery with LIMIT 1 + FROM osm_border_disp_relation + WHERE (role = 'outer' or role = 'inner') + AND ST_GeometryType(geometry)='ST_LineString' + GROUP BY relation_id + ) as p + LEFT JOIN osm_border_disp_relation as r on p.minid = r.id +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_boundary_polygon_idx ON osm_boundary_polygon USING gist (geometry); + + +-- etldoc: osm_border_linestring -> osm_border_linestring_gen_z13 +-- etldoc: osm_border_linestring_adm -> osm_border_linestring_gen_z13 +DROP MATERIALIZED VIEW IF EXISTS osm_boundary_linestring CASCADE; +CREATE MATERIALIZED VIEW osm_boundary_linestring AS +SELECT + osm_id, + geometry, + nuts_level, + nuts->>'l_nuts_0_name' as l_nuts_0_name, + nuts->>'l_nuts_1_name' as l_nuts_1_name, + nuts->>'l_nuts_2_name' as l_nuts_2_name, + nuts->>'l_nuts_3_name' as l_nuts_3_name, + nuts->>'l_nuts_4_name' as l_nuts_4_name, + nuts->>'l_nuts_5_name' as l_nuts_5_name, + nuts->>'r_nuts_0_name' as r_nuts_0_name, + nuts->>'r_nuts_1_name' as r_nuts_1_name, + nuts->>'r_nuts_2_name' as r_nuts_2_name, + nuts->>'r_nuts_3_name' as r_nuts_3_name, + nuts->>'r_nuts_4_name' as r_nuts_4_name, + nuts->>'r_nuts_5_name' as r_nuts_5_name +-- Shouldnt be needed for the map +-- nuts->'l_nuts_1_id' as l_nuts_1_id, +-- nuts->'l_nuts_2_id' as l_nuts_2_id, +-- nuts->'l_nuts_3_id' as l_nuts_3_id, +-- nuts->'l_nuts_4_id' as l_nuts_4_id, +-- nuts->'l_nuts_5_id' as l_nuts_5_id, +-- nuts->'r_nuts_1_id' as r_nuts_1_id, +-- nuts->'r_nuts_2_id' as r_nuts_2_id, +-- nuts->'r_nuts_3_id' as r_nuts_3_id, +-- nuts->'r_nuts_4_id' as r_nuts_4_id, +-- nuts->'r_nuts_5_id' as r_nuts_5_id +FROM ( + SELECT + osm_id, + geometry, + MIN(nuts_level) as nuts_level, + jsonb_object_agg( + CONCAT(side, '_nuts_', nuts_level, '_name'), name + ) + || jsonb_object_agg( + CONCAT(side, '_nuts_', nuts_level, '_id'), -relation_id + ) as nuts + FROM ( + SELECT + b.*, + m.maritime, + m.index, + m.role, + m.admin_level, + r.nuts_level, + r.name, + r.relation_id, + CASE + WHEN + ST_Within( + ST_OffsetCurve( + (ST_LineSubString(b.geometry, 0.499,0.501)), 10, 'quad_segs=4 join=mitre' + ), + r.geometry + ) + THEN 'r' + WHEN + ST_Within( + ST_OffsetCurve( + (ST_LineSubString(b.geometry, 0.499,0.501)), -10, 'quad_segs=4 join=mitre' + ), + r.geometry + ) + THEN 'l' + ELSE 'unknown' -- TODO: Debug if this ever happens, if so our method isn't fool proof + END as side, + r.geometry as relation + FROM + osm_administrative_boundary as b + INNER JOIN osm_administrative_member as m + ON b.osm_id = m.boundary_id + INNER JOIN osm_boundary_polygon as r + ON m.relation_id = r.relation_id + ) as g + GROUP BY osm_id, geometry + ) as p /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_boundary_linestring_idx ON osm_boundary_linestring USING gist (geometry); + + +-- etldoc: layer_boundary[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label=" layer_boundary | z0 | z1 | z2 | z3 | z4 | z5 | z6 | z7 | z8 | z9 | z10 | z11 | z12| z13+"] +CREATE OR REPLACE FUNCTION layer_nuts(bbox geometry, zoom_level int) + RETURNS TABLE + ( + geometry geometry, + nuts_level int, + l_nuts_0_name text, + l_nuts_1_name text, + l_nuts_2_name text, + l_nuts_3_name text, + l_nuts_4_name text, + l_nuts_5_name text, + r_nuts_0_name text, + r_nuts_1_name text, + r_nuts_2_name text, + r_nuts_3_name text, + r_nuts_4_name text, + r_nuts_5_name text + ) +AS +$$ +SELECT + geometry, + nuts_level, + l_nuts_0_name, + l_nuts_1_name, + l_nuts_2_name, + l_nuts_3_name, + l_nuts_4_name, + l_nuts_5_name, + r_nuts_0_name, + r_nuts_1_name, + r_nuts_2_name, + r_nuts_3_name, + r_nuts_4_name, + r_nuts_5_name +FROM osm_boundary_linestring +WHERE geometry && bbox + AND zoom_level > + ( CASE + WHEN nuts_level=0 THEN 2 + WHEN nuts_level=1 THEN 4 + WHEN nuts_level=2 THEN 6 + WHEN nuts_level=3 THEN 6 + WHEN nuts_level=4 THEN 8 + WHEN nuts_level=5 THEN 10 + END ) +$$ LANGUAGE SQL STABLE + -- STRICT + PARALLEL SAFE; diff --git a/layers/boundary/nuts.yaml b/layers/boundary/nuts.yaml new file mode 100644 index 0000000..9164a24 --- /dev/null +++ b/layers/boundary/nuts.yaml @@ -0,0 +1,49 @@ +layer: + id: "nuts" + description: | + Contains administrative boundaries as linestrings (municipalities, counties, provinces, ...) + Administrative regions are translated to their equivalent NUTS/LAU classification + Fields indicate which NUTS-region is to the left and right of the linestring + fields: + nuts_level: + The mininum NUTS/LAU classification this linestring is part of. + NUTS only goes to 3 thus LAU 1 & 2 are mapped as NUTS 4 & 5. + NUTS 0 = Countries + NUTS 1 = Regions (e.g. Vlaams-brabant) + NUTS 2 = Provinces (e.g. Limburg) + NUTS 3 = Administrative arrondissements (e.g. Antwerpen, best to ignore these) + NUTS 4 = Municipalities (e.g. Lummen) + NUTS 5 = Villages/Suburbs (e.g. Linkhout) + l_nuts_0_name: | + Country on the left side of the linestring + l_nuts_1_name: | + Region on the left side of the linestring + l_nuts_2_name: | + Province on the left side of the linestring + l_nuts_3_name: | + Administrative arrondissement on the left side of the linestring + l_nuts_4_name: | + Municipality on the left side of the linestring + l_nuts_5_name: | + Village/suburb on the left side of the linestring + r_nuts_0_name: | + Country on the right side of the linestring + r_nuts_1_name: | + Region on the right side of the linestring + r_nuts_2_name: | + Province on the right side of the linestring + r_nuts_3_name: | + Administrative arrondissement on the right side of the linestring + r_nuts_4_name: | + Municipality on the right side of the linestring + r_nuts_5_name: | + Village/suburb on the right side of the linestring + buffer_size: 4 + datasource: + geometry_field: geometry + query: (SELECT geometry, nuts_level, l_nuts_0_name, l_nuts_1_name, l_nuts_2_name, l_nuts_3_name, l_nuts_4_name, l_nuts_5_name, r_nuts_0_name, r_nuts_1_name, r_nuts_2_name, r_nuts_3_name, r_nuts_4_name, r_nuts_5_name FROM layer_nuts(!bbox!, z(!scale_denominator!))) AS t +schema: + - ./nuts.sql +datasources: + - type: imposm3 + mapping_file: ./mapping.yaml diff --git a/layers/boundary/nuts.yaml.polygons.backup b/layers/boundary/nuts.yaml.polygons.backup new file mode 100644 index 0000000..29faac5 --- /dev/null +++ b/layers/boundary/nuts.yaml.polygons.backup @@ -0,0 +1,26 @@ +layer: + id: "nuts" + description: | + Contains administrative boundaries as polygons (municipalities, counties, provinces, ...) + Administrative regions are translated to their equivalent NUTS/LAU classification + fields: + nuts_level: + The NUTS/LAU classification this polygon fall in. + NUTS only goes to 3 thus LAU 1 & 2 are mapped as NUTS 4 & 5. + NUTS 0 = Countries + NUTS 1 = Regions (e.g. Vlaams-brabant) + NUTS 2 = Provinces (e.g. Limburg) + NUTS 3 = Administrative arrondissements (e.g. Antwerpen, best to ignore these) + NUTS 4 = Municipalities (e.g. Lummen) + NUTS 5 = Villages/Suburbs (e.g. Linkhout) + name: | + Name of the region + buffer_size: 4 + datasource: + geometry_field: geometry + query: (SELECT geometry, nuts_level, name FROM osm_boundary_polygon) AS t +schema: + - ./nuts.sql +datasources: + - type: imposm3 + mapping_file: ./mapping.yaml diff --git a/openmaptiles.yaml b/openmaptiles.yaml index 094bd8f..74e9c31 100644 --- a/openmaptiles.yaml +++ b/openmaptiles.yaml @@ -1,22 +1,23 @@ tileset: layers: - - layers/water/water.yaml - - layers/waterway/waterway.yaml - - layers/landcover/landcover.yaml - - layers/landuse/landuse.yaml - - layers/mountain_peak/mountain_peak.yaml - - layers/park/park.yaml +# - layers/water/water.yaml +# - layers/waterway/waterway.yaml +# - layers/landcover/landcover.yaml +# - layers/landuse/landuse.yaml +# - layers/mountain_peak/mountain_peak.yaml +# - layers/park/park.yaml - layers/boundary/boundary.yaml - - layers/aeroway/aeroway.yaml - - layers/transportation/transportation.yaml - - layers/building/building.yaml - - layers/water_name/water_name.yaml - - layers/transportation_name/transportation_name.yaml - - layers/place/place.yaml - - layers/housenumber/housenumber.yaml - - layers/poi/poi.yaml - - layers/landmarks/landmark.yaml - - layers/aerodrome_label/aerodrome_label.yaml + - layers/boundary/nuts.yaml +# - layers/aeroway/aeroway.yaml +# - layers/transportation/transportation.yaml +# - layers/building/building.yaml +# - layers/water_name/water_name.yaml +# - layers/transportation_name/transportation_name.yaml +# - layers/place/place.yaml +# - layers/housenumber/housenumber.yaml +# - layers/poi/poi.yaml +# - layers/landmarks/landmark.yaml +# - layers/aerodrome_label/aerodrome_label.yaml name: OpenMapTiles version: 3.12.1 id: openmaptiles