cerxes/openmaptiles
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[WIP] NUTS layer:

Browse Source 
Municipalities, Provinces, Regions etc as linestrings (with polygons used in intermediate step) representing what is left/right of the border
This commit is contained in:
Miel Truyen 2021-06-16 21:12:35 +02:00
parent 2a39e1ebfe
commit 7a239b66cf
10 changed files with 882 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
.env
View File

@ -16,11 +16,13 @@ PGPORT=5432
# BBOX may get overwritten by the computed bbox of the specific area: # BBOX may get overwritten by the computed bbox of the specific area:
# make generate-bbox-file # make generate-bbox-file
# By default, the Makefile will use the content of data/$(area).bbox file if it exists. # By default, the Makefile will use the content of data/$(area).bbox file if it exists.
BBOX=-180.0,-85.0511,180.0,85.0511 #BBOX=-180.0,-85.0511,180.0,85.0511
BBOX=4.827919,50.740472,5.798035,51.095291
# Which zooms to generate in make generate-tiles # Which zooms to generate in make generate-tiles
MIN_ZOOM=0 MIN_ZOOM=0
MAX_ZOOM=7 MAX_ZOOM=14
# Use true (case sensitive) to allow data updates # Use true (case sensitive) to allow data updates
DIFF_MODE=false DIFF_MODE=false
@ -44,10 +46,28 @@ BORDERS_PBF_FILE=data/borders/filtered.pbf
BORDERS_CSV_FILE=data/borders/lines.csv BORDERS_CSV_FILE=data/borders/lines.csv
# Number of parallel processes to use when importing sql files # Number of parallel processes to use when importing sql files
MAX_PARALLEL_PSQL=5 #MAX_PARALLEL_PSQL=5
MAX_PARALLEL_PSQL=16
# Number of parallel threads to use when generating vector map tiles # Number of concurrent IO ops to use when generating vector map tiles
COPY_CONCURRENCY=10 # Set to ridiculous high values for SSD/NVME
# Test results: (These were without setting UV_THREADPOOL_SIZE)
# 64: ~100/s peak
# 256: ~200/s peak, stabilizes to a 50% improvement over 64 concurrency
# 1024: ~400/s peak, still at 400/s speed after a few minutes (unlike before, oddly enough)
# 4096: ~250/s peak, stabilizes to 200. But it appears to be choking on something, only updates progress periodically
# 1024 & 24 Threads: ~700/s peak - drops to 214s after 10mins, 178/s after 15mins. EST after 20min: 23hours
# Postgres is mainly idle during all of this (coarse dataset), using only half a core
# To be tested if 1 core is dedicated to postgres and the others to tilelive(UV_THREADPOOL_SIZE)
# will prove to be more efficient on average. Or just UV_THREADPOOL_SIZE==CPU_THREADS.
#COPY_CONCURRENCY=64
#COPY_CONCURRENCY=256
#COPY_CONCURRENCY=512
COPY_CONCURRENCY=1024
#COPY_CONCURRENCY=4096
UV_THREADPOOL_SIZE=16
#UV_THREADPOOL_SIZE=24
# Variables for generate tiles using PGquery # Variables for generate tiles using PGquery
PGHOSTS_LIST= PGHOSTS_LIST=

6
Makefile
View File

@ -400,6 +400,12 @@ import-sql: all start-db-nowait
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \ $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \
awk -v s=": WARNING:" '1{print; fflush()} $$0~s{print "\n*** WARNING detected, aborting"; exit(1)}' awk -v s=": WARNING:" '1{print; fflush()} $$0~s{print "\n*** WARNING detected, aborting"; exit(1)}'
.PHONY: merge-pbf
merge-pbf:
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \
osmosis --rb data/belgium.osm.pbf --rb data/netherlands.osm.pbf --merge --wb data/merges.osm.pbf
.PHONY: generate-tiles .PHONY: generate-tiles
generate-tiles: all start-db generate-tiles: all start-db
@$(assert_area_is_given) @$(assert_area_is_given)

322
benelux.sh Executable file
View File

@ -0,0 +1,322 @@
#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
###########################################
# This file is a modified quickstart.sh, intended to load benelux region
#
# If --empty is not given, use preloaded docker image to speed up
if [ $# -gt 0 ] && [[ $1 == --empty ]]; then
export USE_PRELOADED_IMAGE=""
shift
else
export USE_PRELOADED_IMAGE=true
fi
if [ $# -eq 0 ]; then
# default test area
export area=belgium
echo "No parameter - set area=$area "
else
export area=$1
fi
if [ $# -eq 2 ]; then
osm_server=$2
fi
## Min versions ...
MIN_COMPOSE_VER=1.7.1
MIN_DOCKER_VER=1.12.3
STARTTIME=$(date +%s)
STARTDATE=$(date +"%Y-%m-%dT%H:%M%z")
log_file=./quickstart.log
rm -f $log_file
echo " "
echo "====================================================================================="
echo " Docker check & Download images "
echo "-------------------------------------------------------------------------------------"
echo "====> : Please check the Docker and docker-compose version!"
echo " : We are using docker-compose v2 file format! see more at https://docs.docker.com/"
echo " : Minimum required Docker version: $MIN_DOCKER_VER+"
echo " : Minimum required docker-compose version: $MIN_COMPOSE_VER+"
echo " : See the .travis build for the currently supported versions."
echo " : Your docker system:"
docker --version
docker-compose --version
# based on: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
function version { echo "$@" | tr -cs '0-9.' '.' | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }
COMPOSE_VER=$(docker-compose version --short)
if [ "$(version "$COMPOSE_VER")" -lt "$(version "$MIN_COMPOSE_VER")" ]; then
echo "ERR: Your Docker-compose version is known to have bugs, please update docker-compose!"
exit 1
fi
DOCKER_VER="$(docker -v | awk -F '[ ,]+' '{ print $3 }')"
if [ "$(version "$DOCKER_VER")" -lt "$(version "$MIN_DOCKER_VER")" ]; then
echo "ERR: Your Docker version is not compatible. Please Update docker!"
exit 1
fi
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Pulling or refreshing OpenMapTiles docker images "
make refresh-docker-images
##### backup log from here ...
exec &> >(tee -a "$log_file")
echo " "
echo "====================================================================================="
echo " Start processing "
echo "-------------------------------------------------------------------------------------"
echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] "
echo " : This will be logged to the $log_file file (for debugging) and to the screen"
echo " : Area : $area "
echo " : Download server : ${osm_server:-unset (automatic)} "
echo " : Preloaded image : $USE_PRELOADED_IMAGE "
echo " : Git version : $(git rev-parse HEAD) "
echo " : Started : $STARTDATE "
echo " : Your bash version: $BASH_VERSION"
echo " : Your OS : $OSTYPE"
docker --version
docker-compose --version
if [[ "$OSTYPE" == "linux-gnu" ]]; then
echo " "
echo "-------------------------------------------------------------------------------------"
echo " : This is working on x86_64 ; Your kernel is:"
uname -r
uname -m
KERNEL_CPU_VER=$(uname -m)
if [ "$KERNEL_CPU_VER" != "x86_64" ]; then
echo "ERR: Sorry this is working only on x86_64!"
exit 1
fi
echo " : --- Memory, CPU info ---- "
mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc )
echo "System memory (GB): ${mem}"
grep SwapTotal /proc/meminfo
echo "CPU number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)"
grep Free /proc/meminfo
else
echo " "
echo "Warning : Platforms other than Linux are less tested"
echo " "
fi
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Stopping running services & removing old containers"
make destroy-db
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Existing OpenMapTiles docker images. Will use version $(source .env && echo "$TOOLS_VERSION")"
docker images | grep openmaptiles
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Create directories if they don't exist"
make init-dirs
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Removing old MBTILES if exists ( ./data/${area}.mbtiles ) "
rm -f "./data/${area}.mbtiles"
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Downloading ${area} from ${osm_server:-any source}..."
make "download${osm_server:+-${osm_server}}"
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Remove old generated source files ( ./build/* ) ( if they exist ) "
make clean
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/sql/* )"
echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools "
echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! "
make all
echo " "
echo "-------------------------------------------------------------------------------------"
if [[ "$USE_PRELOADED_IMAGE" == true ]]; then
echo "====> : Start PostgreSQL service using postgis image preloaded with this data:"
echo " : * Water data from http://osmdata.openstreetmap.de"
echo " : Data license: https://osmdata.openstreetmap.de/info/license.html"
echo " : * Natural Earth from http://www.naturalearthdata.com"
echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use"
echo " : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines"
echo " :"
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data"
echo " : includes all data from the import-data image"
echo " :"
echo " : Use the --empty flag to start with an empty database:"
echo " : ./quickstart.sh --empty albania "
echo " : If desired, you can manually import data by using these commands:"
echo " : make destroy-db"
echo " : make start-db"
echo " : make import-data"
echo " :"
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/postgis-preloaded"
echo " : Thank you https://www.postgresql.org ! Thank you http://postgis.org !"
make start-db-preloaded
else
echo "====> : Start PostgreSQL service using empty database and importing all the data:"
echo " : * Water data from http://osmdata.openstreetmap.de"
echo " : Data license: https://osmdata.openstreetmap.de/info/license.html"
echo " : * Natural Earth from http://www.naturalearthdata.com"
echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use"
echo " : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines"
echo " :"
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data"
echo " : includes all data from the import-data image"
echo " :"
echo " : Thank you https://www.postgresql.org ! Thank you http://postgis.org !"
make start-db
make import-data
fi
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start importing OpenStreetMap data: ${area} -> imposm3[./build/mapping.yaml] -> PostgreSQL"
echo " : Imposm3 documentation: https://imposm.org/docs/imposm3/latest/index.html "
echo " : Thank you Omniscale! "
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm "
echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) "
echo " : Thank you OpenStreetMap Contributors ! "
make import-osm
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start importing border ${area} data into PostgreSQL using osmborder"
echo " : Source code: https://github.com/pnorman/osmborder"
echo " : Data license: http://www.openstreetmap.org/copyright"
echo " : Thank you Paul Norman"
make import-borders
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start importing Wikidata: Wikidata Query Service -> PostgreSQL"
echo " : The Wikidata license: CC0 - https://www.wikidata.org/wiki/Wikidata:Main_Page "
echo " : Thank you Wikidata Contributors ! "
make import-wikidata
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start SQL postprocessing: ./build/sql/* -> PostgreSQL "
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/blob/master/bin/import-sql"
# If the output contains a WARNING, stop further processing
# Adapted from https://unix.stackexchange.com/questions/307562
make import-sql
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Analyze PostgreSQL tables"
make analyze-db
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Testing PostgreSQL tables to match layer definitions metadata"
make test-perf-null
echo " "
echo "-------------------------------------------------------------------------------------"
if [[ "$(source .env ; echo "$BBOX")" = "-180.0,-85.0511,180.0,85.0511" ]]; then
if [[ "$area" != "planet" ]]; then
echo "====> : Compute bounding box for tile generation"
make generate-bbox-file ${MIN_ZOOM:+MIN_ZOOM="${MIN_ZOOM}"} ${MAX_ZOOM:+MAX_ZOOM="${MAX_ZOOM}"}
else
echo "====> : Skipping bbox calculation when generating the entire planet"
fi
else
echo "====> : Bounding box is set in .env file"
fi
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. "
echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml "
echo " : Output MBTiles: ./data/${area}.mbtiles "
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/generate-vectortiles "
echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox "
echo " : Thank you https://www.mapbox.com !"
echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-tiles "
echo " : "
echo " : You will see a lot of deprecated warning in the log! This is normal! "
echo " : like : Mapnik LOG> ... is deprecated and will be removed in Mapnik 4.x ... "
make generate-tiles
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )"
make stop-db
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Inputs - Outputs md5sum for debugging "
rm -f ./data/quickstart_checklist.chk
{
find build -type f | sort | xargs md5sum
find data -type f | sort | xargs md5sum
} >> ./data/quickstart_checklist.chk
cat ./data/quickstart_checklist.chk
ENDTIME=$(date +%s)
echo " "
echo " "
echo "-------------------------------------------------------------------------------------"
echo "-- S u m m a r y --"
echo "-------------------------------------------------------------------------------------"
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : (disk space) We have created a lot of docker images: "
echo " : Hint: you can remove with: docker rmi IMAGE "
docker images | grep openmaptiles
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : (disk space) We have created the new vectortiles ( ./data/${area}.mbtiles ) "
echo " : Please respect the licenses (OdBL for OSM data) of the sources when distributing the MBTiles file."
echo " : Data directory content:"
ls -la ./data
echo " "
echo "-------------------------------------------------------------------------------------"
echo "The ./quickstart.sh $area is finished! "
echo "It took $((ENDTIME - STARTTIME)) seconds to complete"
echo "We saved the log file to $log_file (for debugging) You can compare with the travis log !"
echo " "
echo "Start experimenting and check the QUICKSTART.MD file!"
echo " "
echo "* Use make start-maputnik to explore tile generation on request"
echo "* Use make start-tileserver to view pre-generated tiles"
echo " "
echo "Available help commands (make help) "
make help
echo "-------------------------------------------------------------------------------------"
echo " Acknowledgments "
echo " Generated vector tiles are produced work of OpenStreetMap data. "
echo " Such tiles are reusable under CC-BY license granted by OpenMapTiles team: "
echo " https://github.com/openmaptiles/openmaptiles/#license "
echo " Maps made with these vector tiles must display a visible credit: "
echo " © OpenMapTiles © OpenStreetMap contributors "
echo " "
echo " Thanks to all free, open source software developers and Open Data Contributors! "
echo "-------------------------------------------------------------------------------------"

6
docker-compose.yml
View File

@ -11,7 +11,8 @@ networks:
services: services:
postgres: postgres:
image: "${POSTGIS_IMAGE:-openmaptiles/postgis}:${TOOLS_VERSION}" #image: "${POSTGIS_IMAGE:-openmaptiles/postgis}:${TOOLS_VERSION}"
image: harbor.cerxes.net/openmaptiles/postgis:5.3
# Use "command: postgres -c jit=off" for PostgreSQL 11+ because of slow large MVT query processing # Use "command: postgres -c jit=off" for PostgreSQL 11+ because of slow large MVT query processing
volumes: volumes:
- pgdata:/var/lib/postgresql/data - pgdata:/var/lib/postgresql/data
@ -19,7 +20,10 @@ services:
- postgres - postgres
ports: ports:
- "5432" - "5432"
shm_size: 4g
env_file: .env-postgres env_file: .env-postgres
#command: /docker-entrypoint-initdb.d/01_tune-postgis.sh
#command: /docker-entrypoint-initdb.d/01_tune-postgis.sh && echo 'tuned' && postgres -c 'config_file=/etc/postgresql/postgresql.conf'
import-data: import-data:
image: "openmaptiles/import-data:${TOOLS_VERSION}" image: "openmaptiles/import-data:${TOOLS_VERSION}"

146
europe.sh Executable file
View File

@ -0,0 +1,146 @@
set -e
make refresh-docker-images
# Show mem info
mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc )
echo "System memory (GB): ${mem}"
grep SwapTotal /proc/meminfo
echo "CPU number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)"
grep Free /proc/meminfo
make destroy-db
make init-dirs
# rm -f "./data/${area}.mbtiles"
#echo "====> : Downloading ${area} from ${osm_server:-any source}..."
#make "download${osm_server:+-${osm_server}}"
make clean
make all
make start-db
make import-data
# about 1.25hrs, Presumably Disk IO limited.
# If there is an option to double the amount of processes this might
# also help. Seeing about 50% usage per core
# [May 21 08:52:53] [INFO] Reading OSM data took: 22m9.040318577s
# [May 21 09:42:37] [INFO] Writing OSM data took: 49m44.555518107s
# [May 21 09:45:15] [INFO] Importing OSM data took: 52m21.813870474s
# [May 21 09:45:15] [INFO] [PostGIS] Rotating tables took: 38.69062ms
# [May 21 09:45:15] [INFO] Imposm took: 1h14m30.893125729s
make import-osm area=europe
make import-borders area=europe
make import-wikidata
# It not doing the materialized view in parallel despite postgres 12?!
# argh
make import-sql
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Analyze PostgreSQL tables"
make analyze-db
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Testing PostgreSQL tables to match layer definitions metadata"
# TODO: this right here shows us whats wrong. settings aren't optimal (see performance tips tabs)
# shared_buffersm work_mem, max_worker_processes,
# TODO, Ideal solution: Autoconfigure according to performance tips (and max_worker_processes == HT Cores ( or 1.5X HT Cores)
# (1.5x HT Cores might make sense as most work is probably memory-limited (latency), not CPU limited)
make test-perf-null
#echo " "
#echo "-------------------------------------------------------------------------------------"
#
#if [[ "$(source .env ; echo "$BBOX")" = "-180.0,-85.0511,180.0,85.0511" ]]; then
# if [[ "$area" != "planet" ]]; then
# echo "====> : Compute bounding box for tile generation"
# make generate-bbox-file ${MIN_ZOOM:+MIN_ZOOM="${MIN_ZOOM}"} ${MAX_ZOOM:+MAX_ZOOM="${MAX_ZOOM}"}
# else
# echo "====> : Skipping bbox calculation when generating the entire planet"
# fi
#
#else
# echo "====> : Bounding box is set in .env file"
#fi
#
##echo " "
##echo "-------------------------------------------------------------------------------------"
##echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. "
##echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml "
##echo " : Output MBTiles: ./data/${area}.mbtiles "
##echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/generate-vectortiles "
##echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox "
##echo " : Thank you https://www.mapbox.com !"
##echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-tiles "
##echo " : "
##echo " : You will see a lot of deprecated warning in the log! This is normal! "
##echo " : like : Mapnik LOG> ... is deprecated and will be removed in Mapnik 4.x ... "
#make generate-tiles area=europe
#
##
##echo " "
##echo "-------------------------------------------------------------------------------------"
##echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )"
##make stop-db
##
##echo " "
##echo "-------------------------------------------------------------------------------------"
##echo "====> : Inputs - Outputs md5sum for debugging "
##rm -f ./data/quickstart_checklist.chk
##{
## find build -type f | sort | xargs md5sum
## find data -type f | sort | xargs md5sum
##} >> ./data/quickstart_checklist.chk
##cat ./data/quickstart_checklist.chk
##
##ENDTIME=$(date +%s)
##
##echo " "
##echo " "
##echo "-------------------------------------------------------------------------------------"
##echo "-- S u m m a r y --"
##echo "-------------------------------------------------------------------------------------"
##echo " "
##echo "-------------------------------------------------------------------------------------"
##echo "====> : (disk space) We have created a lot of docker images: "
##echo " : Hint: you can remove with: docker rmi IMAGE "
##docker images | grep openmaptiles
##
##echo " "
##echo "-------------------------------------------------------------------------------------"
##echo "====> : (disk space) We have created the new vectortiles ( ./data/${area}.mbtiles ) "
##echo " : Please respect the licenses (OdBL for OSM data) of the sources when distributing the MBTiles file."
##echo " : Data directory content:"
##ls -la ./data
##
##echo " "
##echo "-------------------------------------------------------------------------------------"
##echo "The ./quickstart.sh $area is finished! "
##echo "It took $((ENDTIME - STARTTIME)) seconds to complete"
##echo "We saved the log file to $log_file (for debugging) You can compare with the travis log !"
##echo " "
##echo "Start experimenting and check the QUICKSTART.MD file!"
##echo " "
##echo "* Use make start-maputnik to explore tile generation on request"
##echo "* Use make start-tileserver to view pre-generated tiles"
##echo " "
##echo "Available help commands (make help) "
##make help
##
##echo "-------------------------------------------------------------------------------------"
##echo " Acknowledgments "
##echo " Generated vector tiles are produced work of OpenStreetMap data. "
##echo " Such tiles are reusable under CC-BY license granted by OpenMapTiles team: "
##echo " https://github.com/openmaptiles/openmaptiles/#license "
##echo " Maps made with these vector tiles must display a visible credit: "
##echo " © OpenMapTiles © OpenStreetMap contributors "
##echo " "
##echo " Thanks to all free, open source software developers and Open Data Contributors! "
##echo "-------------------------------------------------------------------------------------"

106
layers/boundary/mapping.yaml
View File

@ -115,10 +115,112 @@ tables:
name: maritime name: maritime
type: bool type: bool
from_member: true from_member: true
- name: index
type: member_index
- name: role
type: member_role
- name: type
type: member_type
mapping: mapping:
type: [boundary] type: [boundary]
filters: filters:
require: require:
#admin_level: ['2'] #admin_level: ['2'] # this used to be specified, re-enable if bugs show up with country borders
admin_level: [__any__] admin_level: [__any__]
claimed_by: [__any__] boundary: ['administrative'] # Filters out boundary administrative_fraction and religious_administration
# FOr NUTS in linestring version
administrative_relation:
type: relation
columns:
- name: osm_id
type: id
- key: name
name: name
type: string
- key: boundary
name: boundary
type: string
- key: admin_level
name: admin_level
type: integer
mapping:
boundary: [ 'administrative' ]
filters:
require:
admin_level: [ __any__ ]
administrative_member:
type: relation_member
columns:
- name: relation_id
type: id
- name: boundary_id
type: id
from_member: true
- key: admin_level
name: admin_level
type: integer
- key: maritime
name: maritime
type: bool
from_member: true
- name: index
type: member_index
- name: role
type: member_role
- name: type
type: member_type
mapping:
type: [boundary]
filters:
require:
admin_level: [__any__]
boundary: ['administrative']
administrative_boundary:
type: linestring
columns:
- name: osm_id
type: id
- name: geometry
type: geometry
- key: admin_level
name: admin_level
type: integer
mapping:
boundary: [ 'administrative' ]
filters:
require:
admin_level: [ __any__ ]
# not currently used
# # etldoc: imposm3 -> osm_adm_boundary_relation
# adm_boundary_relation:
# type: relation
# columns:
# - name: relation_id
# type: id
# - key: name
# name: name
# type: string
# - key: admin_level
# name: admin_level
# type: integer
# - key: border_type
# name: border_type
# type: string
# - key: default_language
# name: default_language
# type: string
# - key: website
# name: website
# type: string
# mapping:
# type: [boundary]
# filters:
# require:
# boundary: ['administrative']
# admin_level: [__any__]

182
layers/boundary/nuts.sql Normal file
View File

@ -0,0 +1,182 @@
-- This is very crude and not finetuned yet
-- This statement can be deleted after the border importer image stops creating this object as a table
DO
$$
BEGIN
DROP TABLE IF EXISTS osm_boundary_polygon CASCADE;
EXCEPTION
WHEN wrong_object_type THEN
END;
$$ LANGUAGE plpgsql;
-- etldoc: osm_border_linestring -> osm_border_linestring_gen_z13
-- etldoc: osm_border_linestring_adm -> osm_border_linestring_gen_z13
DROP MATERIALIZED VIEW IF EXISTS osm_boundary_polygon CASCADE;
CREATE MATERIALIZED VIEW osm_boundary_polygon AS
(
SELECT
r.relation_id,
r.name,
CASE
WHEN r.admin_level = 10 THEN 6
WHEN r.admin_level = 9 THEN 5
WHEN r.admin_level = 8 THEN 4
WHEN r.admin_level = 7 THEN 3
WHEN r.admin_level = 6 THEN 2
WHEN r.admin_level = 4 THEN 1
-- No admin_level =3?
WHEN r.admin_level = 2 THEN 0
ELSE null
END as nuts_level,
p.geometry
FROM (
SELECT
relation_id,
ST_BuildArea(ST_Node(ST_Collect(geometry))) as geometry,
min(id) as minid -- Used just to join to one relation, could've also used subquery with LIMIT 1
FROM osm_border_disp_relation
WHERE (role = 'outer' or role = 'inner')
AND ST_GeometryType(geometry)='ST_LineString'
GROUP BY relation_id
) as p
LEFT JOIN osm_border_disp_relation as r on p.minid = r.id
) /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
CREATE INDEX IF NOT EXISTS osm_boundary_polygon_idx ON osm_boundary_polygon USING gist (geometry);
-- etldoc: osm_border_linestring -> osm_border_linestring_gen_z13
-- etldoc: osm_border_linestring_adm -> osm_border_linestring_gen_z13
DROP MATERIALIZED VIEW IF EXISTS osm_boundary_linestring CASCADE;
CREATE MATERIALIZED VIEW osm_boundary_linestring AS
SELECT
osm_id,
geometry,
nuts_level,
nuts->>'l_nuts_0_name' as l_nuts_0_name,
nuts->>'l_nuts_1_name' as l_nuts_1_name,
nuts->>'l_nuts_2_name' as l_nuts_2_name,
nuts->>'l_nuts_3_name' as l_nuts_3_name,
nuts->>'l_nuts_4_name' as l_nuts_4_name,
nuts->>'l_nuts_5_name' as l_nuts_5_name,
nuts->>'r_nuts_0_name' as r_nuts_0_name,
nuts->>'r_nuts_1_name' as r_nuts_1_name,
nuts->>'r_nuts_2_name' as r_nuts_2_name,
nuts->>'r_nuts_3_name' as r_nuts_3_name,
nuts->>'r_nuts_4_name' as r_nuts_4_name,
nuts->>'r_nuts_5_name' as r_nuts_5_name
-- Shouldnt be needed for the map
-- nuts->'l_nuts_1_id' as l_nuts_1_id,
-- nuts->'l_nuts_2_id' as l_nuts_2_id,
-- nuts->'l_nuts_3_id' as l_nuts_3_id,
-- nuts->'l_nuts_4_id' as l_nuts_4_id,
-- nuts->'l_nuts_5_id' as l_nuts_5_id,
-- nuts->'r_nuts_1_id' as r_nuts_1_id,
-- nuts->'r_nuts_2_id' as r_nuts_2_id,
-- nuts->'r_nuts_3_id' as r_nuts_3_id,
-- nuts->'r_nuts_4_id' as r_nuts_4_id,
-- nuts->'r_nuts_5_id' as r_nuts_5_id
FROM (
SELECT
osm_id,
geometry,
MIN(nuts_level) as nuts_level,
jsonb_object_agg(
CONCAT(side, '_nuts_', nuts_level, '_name'), name
)
|| jsonb_object_agg(
CONCAT(side, '_nuts_', nuts_level, '_id'), -relation_id
) as nuts
FROM (
SELECT
b.*,
m.maritime,
m.index,
m.role,
m.admin_level,
r.nuts_level,
r.name,
r.relation_id,
CASE
WHEN
ST_Within(
ST_OffsetCurve(
(ST_LineSubString(b.geometry, 0.499,0.501)), 10, 'quad_segs=4 join=mitre'
),
r.geometry
)
THEN 'r'
WHEN
ST_Within(
ST_OffsetCurve(
(ST_LineSubString(b.geometry, 0.499,0.501)), -10, 'quad_segs=4 join=mitre'
),
r.geometry
)
THEN 'l'
ELSE 'unknown' -- TODO: Debug if this ever happens, if so our method isn't fool proof
END as side,
r.geometry as relation
FROM
osm_administrative_boundary as b
INNER JOIN osm_administrative_member as m
ON b.osm_id = m.boundary_id
INNER JOIN osm_boundary_polygon as r
ON m.relation_id = r.relation_id
) as g
GROUP BY osm_id, geometry
) as p /* DELAY_MATERIALIZED_VIEW_CREATION */ ;
CREATE INDEX IF NOT EXISTS osm_boundary_linestring_idx ON osm_boundary_linestring USING gist (geometry);
-- etldoc: layer_boundary[shape=record fillcolor=lightpink, style="rounded,filled",
-- etldoc: label="<sql> layer_boundary |<z0> z0 |<z1> z1 |<z2> z2 | <z3> z3 | <z4> z4 | <z5> z5 | <z6> z6 | <z7> z7 | <z8> z8 | <z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13+"]
CREATE OR REPLACE FUNCTION layer_nuts(bbox geometry, zoom_level int)
RETURNS TABLE
(
geometry geometry,
nuts_level int,
l_nuts_0_name text,
l_nuts_1_name text,
l_nuts_2_name text,
l_nuts_3_name text,
l_nuts_4_name text,
l_nuts_5_name text,
r_nuts_0_name text,
r_nuts_1_name text,
r_nuts_2_name text,
r_nuts_3_name text,
r_nuts_4_name text,
r_nuts_5_name text
)
AS
$$
SELECT
geometry,
nuts_level,
l_nuts_0_name,
l_nuts_1_name,
l_nuts_2_name,
l_nuts_3_name,
l_nuts_4_name,
l_nuts_5_name,
r_nuts_0_name,
r_nuts_1_name,
r_nuts_2_name,
r_nuts_3_name,
r_nuts_4_name,
r_nuts_5_name
FROM osm_boundary_linestring
WHERE geometry && bbox
AND zoom_level >
( CASE
WHEN nuts_level=0 THEN 2
WHEN nuts_level=1 THEN 4
WHEN nuts_level=2 THEN 6
WHEN nuts_level=3 THEN 6
WHEN nuts_level=4 THEN 8
WHEN nuts_level=5 THEN 10
END )
$$ LANGUAGE SQL STABLE
-- STRICT
PARALLEL SAFE;

49
layers/boundary/nuts.yaml Normal file
View File

@ -0,0 +1,49 @@
layer:
id: "nuts"
description: |
Contains administrative boundaries as linestrings (municipalities, counties, provinces, ...)
Administrative regions are translated to their equivalent NUTS/LAU classification
Fields indicate which NUTS-region is to the left and right of the linestring
fields:
nuts_level:
The mininum NUTS/LAU classification this linestring is part of.
NUTS only goes to 3 thus LAU 1 & 2 are mapped as NUTS 4 & 5.
NUTS 0 = Countries
NUTS 1 = Regions (e.g. Vlaams-brabant)
NUTS 2 = Provinces (e.g. Limburg)
NUTS 3 = Administrative arrondissements (e.g. Antwerpen, best to ignore these)
NUTS 4 = Municipalities (e.g. Lummen)
NUTS 5 = Villages/Suburbs (e.g. Linkhout)
l_nuts_0_name: |
Country on the left side of the linestring
l_nuts_1_name: |
Region on the left side of the linestring
l_nuts_2_name: |
Province on the left side of the linestring
l_nuts_3_name: |
Administrative arrondissement on the left side of the linestring
l_nuts_4_name: |
Municipality on the left side of the linestring
l_nuts_5_name: |
Village/suburb on the left side of the linestring
r_nuts_0_name: |
Country on the right side of the linestring
r_nuts_1_name: |
Region on the right side of the linestring
r_nuts_2_name: |
Province on the right side of the linestring
r_nuts_3_name: |
Administrative arrondissement on the right side of the linestring
r_nuts_4_name: |
Municipality on the right side of the linestring
r_nuts_5_name: |
Village/suburb on the right side of the linestring
buffer_size: 4
datasource:
geometry_field: geometry
query: (SELECT geometry, nuts_level, l_nuts_0_name, l_nuts_1_name, l_nuts_2_name, l_nuts_3_name, l_nuts_4_name, l_nuts_5_name, r_nuts_0_name, r_nuts_1_name, r_nuts_2_name, r_nuts_3_name, r_nuts_4_name, r_nuts_5_name FROM layer_nuts(!bbox!, z(!scale_denominator!))) AS t
schema:
- ./nuts.sql
datasources:
- type: imposm3
mapping_file: ./mapping.yaml

26
layers/boundary/nuts.yaml.polygons.backup Normal file
View File

@ -0,0 +1,26 @@
layer:
id: "nuts"
description: |
Contains administrative boundaries as polygons (municipalities, counties, provinces, ...)
Administrative regions are translated to their equivalent NUTS/LAU classification
fields:
nuts_level:
The NUTS/LAU classification this polygon fall in.
NUTS only goes to 3 thus LAU 1 & 2 are mapped as NUTS 4 & 5.
NUTS 0 = Countries
NUTS 1 = Regions (e.g. Vlaams-brabant)
NUTS 2 = Provinces (e.g. Limburg)
NUTS 3 = Administrative arrondissements (e.g. Antwerpen, best to ignore these)
NUTS 4 = Municipalities (e.g. Lummen)
NUTS 5 = Villages/Suburbs (e.g. Linkhout)
name: |
Name of the region
buffer_size: 4
datasource:
geometry_field: geometry
query: (SELECT geometry, nuts_level, name FROM osm_boundary_polygon) AS t
schema:
- ./nuts.sql
datasources:
- type: imposm3
mapping_file: ./mapping.yaml

33
openmaptiles.yaml
View File

@ -1,22 +1,23 @@
tileset: tileset:
layers: layers:
- layers/water/water.yaml # - layers/water/water.yaml
- layers/waterway/waterway.yaml # - layers/waterway/waterway.yaml
- layers/landcover/landcover.yaml # - layers/landcover/landcover.yaml
- layers/landuse/landuse.yaml # - layers/landuse/landuse.yaml
- layers/mountain_peak/mountain_peak.yaml # - layers/mountain_peak/mountain_peak.yaml
- layers/park/park.yaml # - layers/park/park.yaml
- layers/boundary/boundary.yaml - layers/boundary/boundary.yaml
- layers/aeroway/aeroway.yaml - layers/boundary/nuts.yaml
- layers/transportation/transportation.yaml # - layers/aeroway/aeroway.yaml
- layers/building/building.yaml # - layers/transportation/transportation.yaml
- layers/water_name/water_name.yaml # - layers/building/building.yaml
- layers/transportation_name/transportation_name.yaml # - layers/water_name/water_name.yaml
- layers/place/place.yaml # - layers/transportation_name/transportation_name.yaml
- layers/housenumber/housenumber.yaml # - layers/place/place.yaml
- layers/poi/poi.yaml # - layers/housenumber/housenumber.yaml
- layers/landmarks/landmark.yaml # - layers/poi/poi.yaml
- layers/aerodrome_label/aerodrome_label.yaml # - layers/landmarks/landmark.yaml
# - layers/aerodrome_label/aerodrome_label.yaml
name: OpenMapTiles name: OpenMapTiles
version: 3.12.1 version: 3.12.1
id: openmaptiles id: openmaptiles