diff --git a/.env b/.env index 61fca59..b9fe82a 100644 --- a/.env +++ b/.env @@ -6,7 +6,7 @@ POSTGRES_PORT=5432 QUICKSTART_MIN_ZOOM=0 QUICKSTART_MAX_ZOOM=7 DIFF_MODE=false -TOOLS_VERSION=3.1.0 +TOOLS_VERSION=4.0.0 BBOX=-180.0,-85.0511,180.0,85.0511 MIN_ZOOM=0 diff --git a/.github/workflows/omt_ci.yml b/.github/workflows/omt_ci.yml new file mode 100644 index 0000000..a2230d5 --- /dev/null +++ b/.github/workflows/omt_ci.yml @@ -0,0 +1,45 @@ +# MapTiler OpenMapTiles +####################### + +# Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo + +name: OMT_CI + +# Controls when the action will run. Triggers the workflow on push and pull request +# events but only for the master branch +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +# jobs can run parallel +jobs: + # This workflow contains a single job called "build" + build: + # runs on ubuntu (can run on windows and mac os) + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + # Named steps + - name: generate all zooms + run: sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env + + # Runs quickstart + - name: quickstart + env: + area: northamptonshire + run: bash ./quickstart.sh $area + + - name: generate devdoc + run: TEST_MODE=yes make generate-devdoc + + # todo: use artifact to store result of tests + #- uses: actions/upload-artifact@v1 + # with: + # name: quickstart log file + # path: quickstart.log diff --git a/.gitignore b/.gitignore index 9696f56..70f1462 100644 --- a/.gitignore +++ b/.gitignore @@ -11,12 +11,12 @@ quickstart.log # imput / output data data/* -# input wikidata -wikidata/* - # generated source files build/* +# Import cache +cache/* + # any IDE files .idea/ .vscode/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 789527d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -sudo: required - -language: bash - -services: - - docker - -script: - - set -e - - TEST_MODE=yes make generate-devdoc - - ./quickstart.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9e61492..535f3c1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,15 +32,11 @@ Your pull request must: * Use clear commit messages. * Be possible to merge automatically. -When you modify rules of importing data in `mapping.yaml` or `*.sql`, please update also: +When you modify import data rules in `mapping.yaml` or `*.sql`, please update: 1. field description in `[layer].yaml` 2. comments starting with `#etldoc` -3. if needed, generate new `mapping_diagram.png` or `etl_diagram.png` using commands below: -``` -make mapping-graph-[layer] -make etl-graph-[layer] -``` +3. regenerate documentation graphs with `make generate-devdoc` 4. update layer description on https://openmaptiles.org/schema/ (https://github.com/openmaptiles/www.openmaptiles.org/tree/master/layers) 5. check if OMT styles are affected by the PR and if there is a need for style updates diff --git a/Makefile b/Makefile index 1e3b274..729b04c 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,15 @@ # Options to run with docker and docker-compose - ensure the container is destroyed on exit -DC_OPTS?=--rm - -# container runs as the current user rather than root (so that created files are not root-owned) -DC_USER_OPTS?=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER}) +# Containers run as the current user rather than root (so that created files are not root-owned) +DC_OPTS?=--rm -u $$(id -u $${USER}):$$(id -g $${USER}) # If running in the test mode, compare files rather than copy them TEST_MODE?=no ifeq ($(TEST_MODE),yes) - COPY_TO_GIT=diff + # create images in ./build/devdoc and compare them to ./layers + GRAPH_PARAMS=./build/devdoc ./layers else - COPY_TO_GIT=cp + # update graphs in the ./layers dir + GRAPH_PARAMS=./layers endif .PHONY: all @@ -36,9 +36,7 @@ help: @echo " make psql-vacuum-analyze # PostgreSQL: VACUUM ANALYZE" @echo " make psql-analyze # PostgreSQL: ANALYZE" @echo " make generate-qareports # generate reports [./build/qareports]" - @echo " make generate-devdoc # generate devdoc including graphs for all layers [./build/devdoc]" - @echo " make etl-graph # hint for generating a single etl graph" - @echo " make mapping-graph # hint for generating a single mapping graph" + @echo " make generate-devdoc # generate devdoc including graphs for all layers [./layers/...]" @echo " make import-sql-dev # start import-sql /bin/bash terminal" @echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" @echo " make clean-docker # remove docker containers, PG data volume" @@ -53,29 +51,30 @@ help: @echo " make help # help about available commands" @echo "==============================================================================" -.PHONY: build -build: - mkdir -p build +.PHONY: init-dirs +init-dirs: + mkdir -p build && mkdir -p data && mkdir -p cache -build/openmaptiles.tm2source/data.yml: build +build/openmaptiles.tm2source/data.yml: init-dirs mkdir -p build/openmaptiles.tm2source docker-compose run $(DC_OPTS) openmaptiles-tools generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > $@ -build/mapping.yaml: build +build/mapping.yaml: init-dirs docker-compose run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@ -build/tileset.sql: build +build/tileset.sql: init-dirs docker-compose run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > $@ .PHONY: clean clean: - rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql + rm -rf build .PHONY: clean-docker clean-docker: docker-compose down -v --remove-orphans docker-compose rm -fv docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true + rm -rf cache .PHONY: db-start db-start: @@ -83,13 +82,17 @@ db-start: @echo "Wait for PostgreSQL to start..." docker-compose run $(DC_OPTS) import-osm ./pgwait.sh +.PHONY: db-stop +db-stop: + docker-compose stop postgres + .PHONY: download-geofabrik -download-geofabrik: - @echo =============== download-geofabrik ======================= - @echo Download area : $(area) - @echo [[ example: make download-geofabrik area=albania ]] - @echo [[ list areas: make download-geofabrik-list ]] - docker-compose run $(DC_OPTS) import-osm ./download-geofabrik.sh $(area) +download-geofabrik: init-dirs + @echo =============== download-geofabrik ======================= + @echo Download area: $(area) + @echo [[ example: make download-geofabrik area=albania ]] + @echo [[ list areas: make download-geofabrik-list ]] + docker-compose run $(DC_OPTS) import-osm ./download-geofabrik.sh $(area) ls -la ./data/$(area).* @echo "Generated config file: ./data/docker-compose-config.yml" @echo " " @@ -109,23 +112,37 @@ import-sql: db-start all docker-compose run $(DC_OPTS) openmaptiles-tools import-sql .PHONY: import-osmsql -import-osmsql: db-start all - docker-compose run $(DC_OPTS) import-osm - docker-compose run $(DC_OPTS) openmaptiles-tools import-sql +import-osmsql: db-start all import-osm import-sql + +.PHONY: import-borders +import-borders: db-start + docker-compose run $(DC_OPTS) openmaptiles-tools import-borders + +.PHONY: import-water +import-water: db-start + docker-compose run $(DC_OPTS) import-water + +.PHONY: import-natural-earth +import-natural-earth: db-start + docker-compose run $(DC_OPTS) import-natural-earth + +.PHONY: import-lakelines +import-lakelines: db-start + docker-compose run $(DC_OPTS) import-lakelines .PHONY: generate-tiles -generate-tiles: db-start all +generate-tiles: init-dirs db-start all rm -rf data/tiles.mbtiles if [ -f ./data/docker-compose-config.yml ]; then \ - docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml run $(DC_OPTS) generate-vectortiles; \ + docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml \ + run $(DC_OPTS) generate-vectortiles; \ else \ docker-compose run $(DC_OPTS) generate-vectortiles; \ fi - docker-compose run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles - docker-compose run $(DC_OPTS) openmaptiles-tools chmod 666 ./data/tiles.mbtiles + docker-compose run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles .PHONY: start-tileserver -start-tileserver: +start-tileserver: init-dirs @echo " " @echo "***********************************************************" @echo "* " @@ -172,41 +189,13 @@ start-postserve: db-start generate-qareports: ./qa/run.sh -build/devdoc: - mkdir -p ./build/devdoc - - -layers = $(notdir $(wildcard layers/*)) # all layers - -.PHONY: etl-graph -etl-graph: - @echo 'Use' - @echo ' make etl-graph-[layer] to generate etl graph for [layer]' - @echo ' example: make etl-graph-poi' - @echo 'Valid layers: $(layers)' - -# generate etl graph for a certain layer, e.g. etl-graph-building, etl-graph-place -etl-graph-%: layers/% build/devdoc - docker-compose run $(DC_USER_OPTS) openmaptiles-tools generate-etlgraph layers/$*/$*.yaml ./build/devdoc - @$(COPY_TO_GIT) ./build/devdoc/etl_$*.png layers/$*/etl_diagram.png - - -mappingLayers = $(notdir $(patsubst %/mapping.yaml,%, $(wildcard layers/*/mapping.yaml))) # layers with mapping.yaml - -# generate mapping graph for a certain layer, e.g. mapping-graph-building, mapping-graph-place -.PHONY: mapping-graph -mapping-graph: - @echo 'Use' - @echo ' make mapping-graph-[layer] to generate mapping graph for [layer]' - @echo ' example: make mapping-graph-poi' - @echo 'Valid layers: $(mappingLayers)' - -mapping-graph-%: ./layers/%/mapping.yaml build/devdoc - docker-compose run $(DC_USER_OPTS) openmaptiles-tools generate-mapping-graph layers/$*/$*.yaml ./build/devdoc/mapping-diagram-$* - @$(COPY_TO_GIT) ./build/devdoc/mapping-diagram-$*.png layers/$*/mapping_diagram.png - # generate all etl and mapping graphs -generate-devdoc: $(addprefix etl-graph-,$(layers)) $(addprefix mapping-graph-,$(mappingLayers)) +.PHONY: generate-devdoc +generate-devdoc: init-dirs + mkdir -p ./build/devdoc && \ + docker-compose run $(DC_OPTS) openmaptiles-tools-latest sh -c \ + 'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \ + generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)' .PHONY: import-sql-dev import-sql-dev: @@ -218,49 +207,51 @@ import-osm-dev: # the `download-geofabrik` error message mention `list`, if the area parameter is wrong. so I created a similar make command .PHONY: list -list: - docker-compose run $(DC_OPTS) import-osm ./download-geofabrik-list.sh +list: download-geofabrik-list -# same as a `make list` .PHONY: download-geofabrik-list download-geofabrik-list: - docker-compose run $(DC_OPTS) import-osm ./download-geofabrik-list.sh + docker-compose run $(DC_OPTS) import-osm ./download-geofabrik-list.sh -.PHONY: download-wikidata -download-wikidata: - mkdir -p wikidata && docker-compose run $(DC_OPTS) --entrypoint /usr/src/app/download-gz.sh import-wikidata - -.PHONY: psql-list-tables -psql-list-tables: - docker-compose run $(DC_OPTS) import-osm ./psql.sh -P pager=off -c "\d+" +.PHONY: import-wikidata +import-wikidata: + docker-compose run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml .PHONY: psql-pg-stat-reset psql-pg-stat-reset: - docker-compose run $(DC_OPTS) import-osm ./psql.sh -P pager=off -c 'SELECT pg_stat_statements_reset();' + docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'SELECT pg_stat_statements_reset();' .PHONY: forced-clean-sql forced-clean-sql: - docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "DROP SCHEMA IF EXISTS public CASCADE ; CREATE SCHEMA IF NOT EXISTS public; " - docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent; CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;" - docker-compose run $(DC_OPTS) import-osm ./psql.sh -c "GRANT ALL ON SCHEMA public TO public;COMMENT ON SCHEMA public IS 'standard public schema';" + docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 \ + -c "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA IF NOT EXISTS public;" \ + -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent;" \ + -c "CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;" \ + -c "GRANT ALL ON SCHEMA public TO public; COMMENT ON SCHEMA public IS 'standard public schema';" -.PHONY: pgclimb-list-views -pgclimb-list-views: - docker-compose run $(DC_OPTS) import-osm ./pgclimb.sh -c "select schemaname,viewname from pg_views where schemaname='public' order by viewname;" csv +.PHONY: list-views +list-views: + @docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \ + -c "select schemaname, viewname from pg_views where schemaname='public' order by viewname;" -.PHONY: pgclimb-list-tables -pgclimb-list-tables: - docker-compose run $(DC_OPTS) import-osm ./pgclimb.sh -c "select schemaname,tablename from pg_tables where schemaname='public' order by tablename;" csv +.PHONY: list-tables +list-tables: + @docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \ + -c "select schemaname, tablename from pg_tables where schemaname='public' order by tablename;" + +.PHONY: psql-list-tables +psql-list-tables: + docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c "\d+" .PHONY: psql-vacuum-analyze psql-vacuum-analyze: @echo "Start - postgresql: VACUUM ANALYZE VERBOSE;" - docker-compose run $(DC_OPTS) import-osm ./psql.sh -P pager=off -c 'VACUUM ANALYZE VERBOSE;' + docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'VACUUM ANALYZE VERBOSE;' .PHONY: psql-analyze psql-analyze: - @echo "Start - postgresql: ANALYZE VERBOSE ;" - docker-compose run $(DC_OPTS) import-osm ./psql.sh -P pager=off -c 'ANALYZE VERBOSE;' + @echo "Start - postgresql: ANALYZE VERBOSE;" + docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'ANALYZE VERBOSE;' .PHONY: list-docker-images list-docker-images: diff --git a/QUICKSTART.md b/QUICKSTART.md index 90fed7e..dc3e5b1 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -411,7 +411,6 @@ the current output: OpenMapTiles https://github.com/openmaptiles/openmaptiles Hints for testing areas make download-geofabrik-list # list actual geofabrik OSM extracts for download -> <> - make list # list actual geofabrik OSM extracts for download -> <> ./quickstart.sh <> # example: ./quickstart.sh madagascar Hints for designers: @@ -434,8 +433,8 @@ Hints for developers: make docker-unnecessary-clean # clean unnecessary docker image(s) and container(s) make refresh-docker-images # refresh openmaptiles docker images from Docker HUB make remove-docker-images # remove openmaptiles docker images - make pgclimb-list-views # list PostgreSQL public schema views - make pgclimb-list-tables # list PostgreSQL public schema tables + make list-views # list PostgreSQL public schema views + make list-tables # list PostgreSQL public schema tables cat .env # list PG database and MIN_ZOOM and MAX_ZOOM information cat ./quickstart.log # backup of the last ./quickstart.sh make help # help about available commands diff --git a/README.md b/README.md index f1f8c9d..1da2a97 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## OpenMapTiles [![Build Status](https://travis-ci.org/openmaptiles/openmaptiles.svg?branch=master)](https://travis-ci.org/openmaptiles/openmaptiles) +## OpenMapTiles [![Build Status](https://github.com/openmaptiles/openmaptiles/workflows/OMT_CI/badge.svg?branch=master)](https://github.com/openmaptiles/openmaptiles/actions) OpenMapTiles is an extensible and open tile schema based on the OpenStreetMap. This project is used to generate vector tiles for online zoomable maps. OpenMapTiles is about creating a beautiful basemaps with general layers containing topographic information. More information [openmaptiles.org](https://openmaptiles.org/) and [openmaptiles.com](https://openmaptiles.com/). @@ -94,26 +94,15 @@ or use the provided `quickstart.sh` script. Now start up the database container. ```bash -docker-compose up -d postgres +make db-start ``` Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines). ```bash -docker-compose run import-water -docker-compose run import-natural-earth -docker-compose run import-lakelines -docker-compose run import-osmborder -``` - -**[Optional]** -Import latest Wikidata. If an OSM feature has [Key:wikidata](https://wiki.openstreetmap.org/wiki/Key:wikidata), OpenMapTiles check corresponding item in Wikidata and use its [labels](https://www.wikidata.org/wiki/Help:Label) for languages listed in [openmaptiles.yaml](openmaptiles.yaml). So the generated vector tiles includes multi-languages in name field. - -Beware that current [Wikidata dump](https://dumps.wikimedia.org/wikidatawiki/entities/latest-all.json.gz) is more than 55GB, it takes time to download and import it. If you just want to have a quickstart on OpenMapTiles, just skip this step. - -```bash -make download-wikidata -docker-compose run import-wikidata +make import-water +make import-natural-earth +make import-lakelines ``` [Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory. @@ -123,11 +112,26 @@ cd data wget http://download.geofabrik.de/europe/albania-latest.osm.pbf ``` -[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from -`build/mapping.yaml` (which has been created by `make`). +OR ```bash -docker-compose run import-osm +make download-geofabrik area=albania +``` + +[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from +`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definiton. Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool. + +```bash +make import-osm +make import-borders +``` + +Import latest Wikidata. If an OSM feature has [Key:wikidata](https://wiki.openstreetmap.org/wiki/Key:wikidata), OpenMapTiles check corresponding item in Wikidata and use its [labels](https://www.wikidata.org/wiki/Help:Label) for languages listed in [openmaptiles.yaml](openmaptiles.yaml). So the generated vector tiles includes multi-languages in name field. + +This step uses [Wikidata Query Service](https://query.wikidata.org) to download just the Wikidata IDs that already exist in the database. + +```bash +make import-wikidata ``` ### Work on Layers @@ -144,7 +148,7 @@ Now you are ready to **generate the vector tiles**. Using environment variables you can limit the bounding box and zoom levels of what you want to generate (`docker-compose.yml`). ``` -docker-compose run generate-vectortiles +make generate-tiles ``` ## License diff --git a/UPDATE.md b/UPDATE.md index 20e5afe..f107c1a 100644 --- a/UPDATE.md +++ b/UPDATE.md @@ -28,8 +28,12 @@ docker-compose run import-osm-diff ## Generate Changed Tiles -After the import has finished **imposm3** will store a list of tiles in text format in the `diffdir`. -Copy the as `tiles.txt` to the import folder. +After the import has finished **imposm3** will store lists of tiles in text format in subfolders of the `diffdir`, +named for the date(s) on which the import took place (`YYYYMMDD`). +Copy and merge the files to `tiles.txt` in the import folder (`data`), either manually or with the following command, which also removes duplicate tiles so they are only generated once: +``` +cd data && sort ./*/*.tiles | uniq > tiles.txt +``` Now run the command to read the tilelist and write the vector tiles for it to a new MBTiles. diff --git a/docker-compose.yml b/docker-compose.yml index e13d882..41087d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,6 @@ version: "2" volumes: pgdata: - cache: services: postgres: image: "openmaptiles/postgis:${TOOLS_VERSION}" @@ -37,12 +36,7 @@ services: volumes: - ./data:/import - ./build:/mapping - - cache:/cache - import-osmborder: - image: "openmaptiles/import-osmborder:${TOOLS_VERSION}" - env_file: .env - networks: - - postgres_conn + - ./cache:/cache import-osm-diff: image: "openmaptiles/import-osm:${TOOLS_VERSION}" env_file: .env @@ -54,7 +48,7 @@ services: volumes: - ./data:/import - ./build:/mapping - - cache:/cache + - ./cache:/cache update-osm: image: "openmaptiles/import-osm:${TOOLS_VERSION}" env_file: .env @@ -66,26 +60,7 @@ services: volumes: - ./data:/import - ./build:/mapping - - cache:/cache - import-sql: - # This target is obsolete, and was left for backwards compatibility - # Use openmaptiles-tools target instead - image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}" - command: import-sql - env_file: .env - networks: - - postgres_conn - volumes: - - .:/tileset - - ./build:/sql - import-wikidata: - image: "openmaptiles/import-wikidata:${TOOLS_VERSION}" - env_file: .env - command: import-wikidata - networks: - - postgres_conn - volumes: - - ./wikidata:/import + - ./cache:/cache openmaptiles-tools: image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}" env_file: .env @@ -93,6 +68,18 @@ services: - postgres_conn volumes: - .:/tileset + - ./data:/import + - ./build:/sql + openmaptiles-tools-latest: + # This target exists for experimental tools that have not yet been published. + # Do not use this for production. + image: "openmaptiles/openmaptiles-tools:latest" + env_file: .env + networks: + - postgres_conn + volumes: + - .:/tileset + - ./data:/import - ./build:/sql generate-changed-vectortiles: image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}" diff --git a/layers/aerodrome_label/aerodrome_label.yaml b/layers/aerodrome_label/aerodrome_label.yaml index 17b0fbe..0be152d 100644 --- a/layers/aerodrome_label/aerodrome_label.yaml +++ b/layers/aerodrome_label/aerodrome_label.yaml @@ -15,12 +15,23 @@ layer: [`aerodrome`](http://wiki.openstreetmap.org/wiki/Proposed_features/Aerodrome) and `aerodrome:type` tags. values: - - international - - public - - regional - - military - - private - - other + international: + aerodrome: 'international' + aerodrome_type: 'international' + public: + aerodrome: 'public' + aerodrome_type: ['%public%', 'civil'] + regional: + aerodrome: 'regional' + aerodrome_type: 'regional' + military: + aerodrome: 'military' + aerodrome_type: '%military%' + military: 'airfield' + private: + aerodrome: 'private' + aerodrome_type: 'private' + other: iata: 3-character code issued by the IATA. icao: 4-letter code issued by the ICAO. ele: Elevation (`ele`) in meters. diff --git a/layers/aerodrome_label/layer.sql b/layers/aerodrome_label/layer.sql index 8d5adfa..19e9bac 100644 --- a/layers/aerodrome_label/layer.sql +++ b/layers/aerodrome_label/layer.sql @@ -27,27 +27,7 @@ $$ COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de, tags, CASE - WHEN aerodrome = 'international' - OR aerodrome_type = 'international' - THEN 'international' - WHEN - aerodrome = 'public' - OR aerodrome_type LIKE '%public%' - OR aerodrome_type = 'civil' - THEN 'public' - WHEN - aerodrome = 'regional' - OR aerodrome_type = 'regional' - THEN 'regional' - WHEN - aerodrome = 'military' - OR aerodrome_type LIKE '%military%' - OR military = 'airfield' - THEN 'military' - WHEN - aerodrome = 'private' - OR aerodrome_type = 'private' - THEN 'private' + %%FIELD_MAPPING: class %% ELSE 'other' END AS class, NULLIF(iata, '') AS iata, @@ -56,5 +36,6 @@ $$ round(substring(ele from E'^(-?\\d+)(\\D|$)')::int*3.2808399)::int AS ele_ft FROM osm_aerodrome_label_point WHERE geometry && bbox AND zoom_level >= 10; - -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/aerodrome_label/mapping.yaml b/layers/aerodrome_label/mapping.yaml index 46cdb71..ff0f22c 100644 --- a/layers/aerodrome_label/mapping.yaml +++ b/layers/aerodrome_label/mapping.yaml @@ -3,7 +3,7 @@ tables: # etldoc: imposm3 -> osm_aerodrome_label_point aerodrome_label_point: type: geometry - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/aeroway/layer.sql b/layers/aeroway/layer.sql index c182a62..1116c67 100644 --- a/layers/aeroway/layer.sql +++ b/layers/aeroway/layer.sql @@ -40,4 +40,6 @@ RETURNS TABLE(geometry geometry, class text, ref text) AS $$ FROM osm_aeroway_polygon WHERE zoom_level >= 14 ) AS zoom_levels WHERE geometry && bbox; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL IMMUTABLE +PARALLEL SAFE; diff --git a/layers/aeroway/mapping.yaml b/layers/aeroway/mapping.yaml index 7c90897..2e6bf12 100644 --- a/layers/aeroway/mapping.yaml +++ b/layers/aeroway/mapping.yaml @@ -50,7 +50,7 @@ tables: # etldoc: imposm3 -> osm_aeroway_polygon aeroway_polygon: type: polygon - fields: + columns: - *ref - name: osm_id type: id @@ -67,7 +67,7 @@ tables: # etldoc: imposm3 -> osm_aeroway_linestring aeroway_linestring: type: linestring - fields: + columns: - *ref - name: osm_id type: id diff --git a/layers/boundary/boundary.sql b/layers/boundary/boundary.sql index c4eec0e..1b0c318 100644 --- a/layers/boundary/boundary.sql +++ b/layers/boundary/boundary.sql @@ -1,3 +1,114 @@ +-- This statement can be deleted after the border importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_border_linestring_gen1 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_border_linestring -> osm_border_linestring_gen1 +DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen1 CASCADE; +CREATE MATERIALIZED VIEW osm_border_linestring_gen1 AS ( + SELECT ST_Simplify(geometry, 10) AS geometry, osm_id, admin_level, dividing_line, disputed, maritime + FROM osm_border_linestring + WHERE admin_level <= 10 +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_linestring_gen1_idx ON osm_border_linestring_gen1 USING gist (geometry); + +-- This statement can be deleted after the border importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_border_linestring_gen2 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_border_linestring -> osm_border_linestring_gen2 +DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen2 CASCADE; +CREATE MATERIALIZED VIEW osm_border_linestring_gen2 AS ( + SELECT ST_Simplify(geometry, 20) AS geometry, osm_id, admin_level, dividing_line, disputed, maritime + FROM osm_border_linestring + WHERE admin_level <= 10 +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_linestring_gen2_idx ON osm_border_linestring_gen2 USING gist (geometry); + +-- This statement can be deleted after the border importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_border_linestring_gen3 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_border_linestring -> osm_border_linestring_gen3 +DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen3 CASCADE; +CREATE MATERIALIZED VIEW osm_border_linestring_gen3 AS ( + SELECT ST_Simplify(geometry, 40) AS geometry, osm_id, admin_level, dividing_line, disputed, maritime + FROM osm_border_linestring + WHERE admin_level <= 8 +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_linestring_gen3_idx ON osm_border_linestring_gen3 USING gist (geometry); + +-- This statement can be deleted after the border importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_border_linestring_gen4 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_border_linestring -> osm_border_linestring_gen4 +DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen4 CASCADE; +CREATE MATERIALIZED VIEW osm_border_linestring_gen4 AS ( + SELECT ST_Simplify(geometry, 80) AS geometry, osm_id, admin_level, dividing_line, disputed, maritime + FROM osm_border_linestring + WHERE admin_level <= 6 +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_linestring_gen4_idx ON osm_border_linestring_gen4 USING gist (geometry); + +-- This statement can be deleted after the border importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_border_linestring_gen5 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_border_linestring -> osm_border_linestring_gen5 +DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen5 CASCADE; +CREATE MATERIALIZED VIEW osm_border_linestring_gen5 AS ( + SELECT ST_Simplify(geometry, 160) AS geometry, osm_id, admin_level, dividing_line, disputed, maritime + FROM osm_border_linestring + WHERE admin_level <= 6 +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_linestring_gen5_idx ON osm_border_linestring_gen5 USING gist (geometry); + +-- This statement can be deleted after the border importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_border_linestring_gen6 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_border_linestring -> osm_border_linestring_gen6 +DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen6 CASCADE; +CREATE MATERIALIZED VIEW osm_border_linestring_gen6 AS ( + SELECT ST_Simplify(geometry, 300) AS geometry, osm_id, admin_level, dividing_line, disputed, maritime + FROM osm_border_linestring + WHERE admin_level <= 4 +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_linestring_gen6_idx ON osm_border_linestring_gen6 USING gist (geometry); + +-- This statement can be deleted after the border importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_border_linestring_gen7 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_border_linestring -> osm_border_linestring_gen7 +DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen7 CASCADE; +CREATE MATERIALIZED VIEW osm_border_linestring_gen7 AS ( + SELECT ST_Simplify(geometry, 600) AS geometry, osm_id, admin_level, dividing_line, disputed, maritime + FROM osm_border_linestring + WHERE admin_level <= 4 +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_linestring_gen7_idx ON osm_border_linestring_gen7 USING gist (geometry); + +-- This statement can be deleted after the border importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_border_linestring_gen8 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_border_linestring -> osm_border_linestring_gen8 +DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen8 CASCADE; +CREATE MATERIALIZED VIEW osm_border_linestring_gen8 AS ( + SELECT ST_Simplify(geometry, 1200) AS geometry, osm_id, admin_level, dividing_line, disputed, maritime + FROM osm_border_linestring + WHERE admin_level <= 4 +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_linestring_gen8_idx ON osm_border_linestring_gen8 USING gist (geometry); + +-- This statement can be deleted after the border importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_border_linestring_gen9 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_border_linestring -> osm_border_linestring_gen9 +DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen9 CASCADE; +CREATE MATERIALIZED VIEW osm_border_linestring_gen9 AS ( + SELECT ST_Simplify(geometry, 2400) AS geometry, osm_id, admin_level, dividing_line, disputed, maritime + FROM osm_border_linestring + WHERE admin_level <= 4 +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_linestring_gen9_idx ON osm_border_linestring_gen9 USING gist (geometry); + +-- This statement can be deleted after the border importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_border_linestring_gen10 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_border_linestring -> osm_border_linestring_gen10 +DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen10 CASCADE; +CREATE MATERIALIZED VIEW osm_border_linestring_gen10 AS ( + SELECT ST_Simplify(geometry, 4800) AS geometry, osm_id, admin_level, dividing_line, disputed, maritime + FROM osm_border_linestring + WHERE admin_level <= 2 +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_linestring_gen10_idx ON osm_border_linestring_gen10 USING gist (geometry); + + CREATE OR REPLACE FUNCTION edit_name(name VARCHAR) RETURNS TEXT AS $$ SELECT CASE WHEN POSITION(' at ' in name) > 0 @@ -359,4 +470,6 @@ RETURNS TABLE(geometry geometry, admin_level int, disputed int, disputed_name te -- etldoc: boundary_z13 -> layer_boundary:z13 SELECT * FROM boundary_z13 WHERE geometry && bbox AND zoom_level >= 13 ) AS zoom_levels; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL IMMUTABLE +PARALLEL SAFE; diff --git a/layers/boundary/etl_diagram.png b/layers/boundary/etl_diagram.png index c47c5bb..20327fe 100644 Binary files a/layers/boundary/etl_diagram.png and b/layers/boundary/etl_diagram.png differ diff --git a/layers/boundary/mapping.yaml b/layers/boundary/mapping.yaml index 9b3e67b..aacb1b5 100644 --- a/layers/boundary/mapping.yaml +++ b/layers/boundary/mapping.yaml @@ -76,7 +76,7 @@ tables: # etldoc: imposm3 -> osm_border_disp_relation border_disp_relation: type: relation_member - fields: + columns: - name: relation_id type: id - name: osm_id diff --git a/layers/building/building.sql b/layers/building/building.sql index 15c58d0..14a48df 100644 --- a/layers/building/building.sql +++ b/layers/building/building.sql @@ -60,8 +60,8 @@ CREATE OR REPLACE VIEW osm_all_buildings AS ( osm_building_street WHERE role = 'house' AND ST_GeometryType(geometry) = 'ST_Polygon' UNION ALL - -- etldoc: osm_building_multipolygon -> layer_building:z14_ - -- Buildings that are inner/outer + -- etldoc: osm_building_polygon -> layer_building:z14_ + -- Buildings that are from multipolygons SELECT osm_id,geometry, COALESCE(nullif(as_numeric(height),-1),nullif(as_numeric(buildingheight),-1)) as height, COALESCE(nullif(as_numeric(min_height),-1),nullif(as_numeric(buildingmin_height),-1)) as min_height, @@ -71,7 +71,9 @@ CREATE OR REPLACE VIEW osm_all_buildings AS ( nullif(colour, '') AS colour, FALSE as hide_3d FROM - osm_building_polygon obp WHERE EXISTS (SELECT 1 FROM osm_building_multipolygon obm WHERE obp.osm_id = obm.osm_id) + osm_building_polygon obp + WHERE osm_id < 0 + UNION ALL -- etldoc: osm_building_polygon -> layer_building:z14_ -- Standalone buildings @@ -112,7 +114,7 @@ RETURNS TABLE(geometry geometry, osm_id bigint, render_height int, render_min_he WHEN 'sandstone' THEN '#b4a995' -- same as stone WHEN 'clay' THEN '#9d8b75' -- same as mud END) AS colour, - CASE WHEN hide_3d THEN TRUE ELSE NULL::boolean END AS hide_3d + CASE WHEN hide_3d THEN TRUE END AS hide_3d FROM ( -- etldoc: osm_building_polygon_gen1 -> layer_building:z13 SELECT @@ -140,6 +142,8 @@ RETURNS TABLE(geometry geometry, osm_id bigint, render_height int, render_min_he zoom_level >= 14 AND geometry && bbox ) AS zoom_levels ORDER BY render_height ASC, ST_YMin(geometry) DESC; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL IMMUTABLE +PARALLEL SAFE; -- not handled: where a building outline covers building parts diff --git a/layers/building/etl_diagram.png b/layers/building/etl_diagram.png index 8a43f94..bb14c0e 100644 Binary files a/layers/building/etl_diagram.png and b/layers/building/etl_diagram.png differ diff --git a/layers/building/mapping.yaml b/layers/building/mapping.yaml index 118eceb..8c2bf7c 100644 --- a/layers/building/mapping.yaml +++ b/layers/building/mapping.yaml @@ -8,7 +8,7 @@ generalized_tables: tables: # etldoc: imposm3 -> osm_building_polygon building_polygon: - fields: + columns: - name: osm_id type: id - name: geometry @@ -69,7 +69,7 @@ tables: # etldoc: imposm3 -> osm_building_street building_street: - fields: + columns: - name: osm_id type: id - name: geometry @@ -161,7 +161,7 @@ tables: # etldoc: imposm3 -> osm_building_associatedstreet building_associatedstreet: - fields: + columns: - name: osm_id type: id - name: geometry @@ -253,7 +253,7 @@ tables: # etldoc: imposm3 -> osm_building_relation building_relation: - fields: + columns: - name: osm_id type: id - name: geometry @@ -342,76 +342,3 @@ tables: mapping: type: [building] type: relation_member - - # etldoc: imposm3 -> osm_building_multipolygon - building_multipolygon: - fields: - - name: osm_id - type: id - - name: geometry - type: validated_geometry - - name: area - type: area - - name: building - key: building - type: string - from_member: true - - name: material - key: building:material - type: string - - name: colour - key: building:colour - type: string - - name: buildingpart - key: building:part - type: string - from_member: true - - name: buildingheight - key: building:height - type: string - from_member: true - - name: height - key: height - type: string - from_member: true - - name: buildingmin_height - key: building:min_height - type: string - from_member: true - - name: min_height - key: min_height - type: string - from_member: true - - name: buildinglevels - key: building:levels - type: string - from_member: true - - name: levels - key: levels - type: string - from_member: true - - name: buildingmin_level - key: building:min_level - type: string - from_member: true - - name: min_level - key: min_level - type: string - from_member: true - - name: member - type: member_id - - name: index - type: member_index - - name: role - type: member_role - from_member: true - - name: type - type: member_type - mapping: - type: [multipolygon] - filters: - reject: - building: ["no","none","No"] - building:part: ["no","none","No"] - man_made: ["bridge"] - type: relation_member diff --git a/layers/building/mapping_diagram.png b/layers/building/mapping_diagram.png index 8d8341f..9a6ef60 100644 Binary files a/layers/building/mapping_diagram.png and b/layers/building/mapping_diagram.png differ diff --git a/layers/housenumber/layer.sql b/layers/housenumber/layer.sql index eed865f..01f2649 100644 --- a/layers/housenumber/layer.sql +++ b/layers/housenumber/layer.sql @@ -7,4 +7,6 @@ RETURNS TABLE(osm_id bigint, geometry geometry, housenumber text) AS $$ -- etldoc: osm_housenumber_point -> layer_housenumber:z14_ SELECT osm_id, geometry, housenumber FROM osm_housenumber_point WHERE zoom_level >= 14 AND geometry && bbox; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/housenumber/mapping.yaml b/layers/housenumber/mapping.yaml index cf5f255..174335b 100644 --- a/layers/housenumber/mapping.yaml +++ b/layers/housenumber/mapping.yaml @@ -4,7 +4,7 @@ tables: # etldoc: imposm3 -> osm_housenumber_point housenumber_point: type: geometry - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/landcover/landcover.sql b/layers/landcover/landcover.sql index 6d64b9a..3fb62b6 100644 --- a/layers/landcover/landcover.sql +++ b/layers/landcover/landcover.sql @@ -11,18 +11,11 @@ CREATE OR REPLACE FUNCTION landcover_class(subclass VARCHAR) RETURNS TEXT AS $$ SELECT CASE - WHEN subclass IN ('farmland', 'farm', 'orchard', 'vineyard', 'plant_nursery') THEN 'farmland' - WHEN subclass IN ('glacier', 'ice_shelf') THEN 'ice' - WHEN subclass IN ('wood', 'forest') THEN 'wood' - WHEN subclass IN ('bare_rock', 'scree') THEN 'rock' - WHEN subclass IN ('fell', 'grassland', 'heath', 'scrub', 'tundra', 'grass', 'meadow', 'allotments', - 'park', 'village_green', 'recreation_ground', 'garden', 'golf_course') THEN 'grass' - WHEN subclass IN ('wetland', 'bog', 'swamp', 'wet_meadow', 'marsh', 'reedbed', - 'saltern', 'tidalflat', 'saltmarsh', 'mangrove') THEN 'wetland' - WHEN subclass IN ('beach', 'sand', 'dune') THEN 'sand' - ELSE NULL + %%FIELD_MAPPING: class %% END; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; -- etldoc: ne_110m_glaciated_areas -> landcover_z0 CREATE OR REPLACE VIEW landcover_z0 AS ( @@ -137,4 +130,6 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class text, subclass text) AS $$ SELECT * FROM landcover_z14 WHERE zoom_level >= 14 AND geometry && bbox ) AS zoom_levels; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/landcover/landcover.yaml b/layers/landcover/landcover.yaml index 09e4e26..6d51fe4 100644 --- a/layers/landcover/landcover.yaml +++ b/layers/landcover/landcover.yaml @@ -10,12 +10,20 @@ layer: description: | Use the **class** to assign natural colors for **landcover**. values: - - farmland - - ice - - wood - - grass - - wetland - - sand + farmland: + subclass: ['farmland', 'farm', 'orchard', 'vineyard', 'plant_nursery'] + ice: + subclass: ['glacier', 'ice_shelf'] + wood: + subclass: ['wood', 'forest'] + rock: + subclass: ['bare_rock', 'scree'] + grass: + subclass: ['fell', 'grassland', 'heath', 'scrub', 'tundra', 'grass', 'meadow', 'allotments', 'park', 'village_green', 'recreation_ground', 'garden', 'golf_course'] + wetland: + subclass: ['wetland', 'bog', 'swamp', 'wet_meadow', 'marsh', 'reedbed', 'saltern', 'tidalflat', 'saltmarsh', 'mangrove'] + sand: + subclass: ['beach', 'sand', 'dune'] subclass: description: | Use **subclass** to do more precise styling. diff --git a/layers/landcover/mapping.yaml b/layers/landcover/mapping.yaml index 952111c..142f3dd 100644 --- a/layers/landcover/mapping.yaml +++ b/layers/landcover/mapping.yaml @@ -46,7 +46,7 @@ generalized_tables: tables: # etldoc: imposm3 -> osm_landcover_polygon landcover_polygon: - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/landuse/landuse.sql b/layers/landuse/landuse.sql index 252247f..fb3f06a 100644 --- a/layers/landuse/landuse.sql +++ b/layers/landuse/landuse.sql @@ -1,61 +1,61 @@ -- etldoc: ne_50m_urban_areas -> landuse_z4 CREATE OR REPLACE VIEW landuse_z4 AS ( - SELECT NULL::bigint AS osm_id, geometry, 'residential'::text AS landuse, NULL::text AS amenity, NULL::text AS leisure, NULL::text AS tourism, NULL::text AS place, NULL::text AS waterway, scalerank + SELECT NULL::bigint AS osm_id, geometry, 'residential'::text AS landuse, NULL::text AS amenity, NULL::text AS leisure, NULL::text AS tourism, NULL::text AS place, NULL::text AS waterway FROM ne_50m_urban_areas WHERE scalerank <= 2 ); -- etldoc: ne_50m_urban_areas -> landuse_z5 CREATE OR REPLACE VIEW landuse_z5 AS ( - SELECT NULL::bigint AS osm_id, geometry, 'residential'::text AS landuse, NULL::text AS amenity, NULL::text AS leisure, NULL::text AS tourism, NULL::text AS place, NULL::text AS waterway, scalerank + SELECT NULL::bigint AS osm_id, geometry, 'residential'::text AS landuse, NULL::text AS amenity, NULL::text AS leisure, NULL::text AS tourism, NULL::text AS place, NULL::text AS waterway FROM ne_50m_urban_areas ); -- etldoc: osm_landuse_polygon_gen7 -> landuse_z6 CREATE OR REPLACE VIEW landuse_z6 AS ( - SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank + SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway FROM osm_landuse_polygon_gen7 ); -- etldoc: osm_landuse_polygon_gen6 -> landuse_z8 CREATE OR REPLACE VIEW landuse_z8 AS ( - SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank + SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway FROM osm_landuse_polygon_gen6 ); -- etldoc: osm_landuse_polygon_gen5 -> landuse_z9 CREATE OR REPLACE VIEW landuse_z9 AS ( - SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank + SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway FROM osm_landuse_polygon_gen5 ); -- etldoc: osm_landuse_polygon_gen4 -> landuse_z10 CREATE OR REPLACE VIEW landuse_z10 AS ( - SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank + SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway FROM osm_landuse_polygon_gen4 ); -- etldoc: osm_landuse_polygon_gen3 -> landuse_z11 CREATE OR REPLACE VIEW landuse_z11 AS ( - SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank + SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway FROM osm_landuse_polygon_gen3 ); -- etldoc: osm_landuse_polygon_gen2 -> landuse_z12 CREATE OR REPLACE VIEW landuse_z12 AS ( - SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank + SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway FROM osm_landuse_polygon_gen2 ); -- etldoc: osm_landuse_polygon_gen1 -> landuse_z13 CREATE OR REPLACE VIEW landuse_z13 AS ( - SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank + SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway FROM osm_landuse_polygon_gen1 ); -- etldoc: osm_landuse_polygon -> landuse_z14 CREATE OR REPLACE VIEW landuse_z14 AS ( - SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway, NULL::int as scalerank + SELECT osm_id, geometry, landuse, amenity, leisure, tourism, place, waterway FROM osm_landuse_polygon ); @@ -108,4 +108,6 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class text) AS $$ SELECT * FROM landuse_z14 WHERE zoom_level >= 14 ) AS zoom_levels WHERE geometry && bbox; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/landuse/mapping.yaml b/layers/landuse/mapping.yaml index 038313e..b3970f3 100644 --- a/layers/landuse/mapping.yaml +++ b/layers/landuse/mapping.yaml @@ -38,7 +38,7 @@ tables: # etldoc: imposm3 -> osm_landuse_polygon landuse_polygon: type: polygon - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/mountain_peak/layer.sql b/layers/mountain_peak/layer.sql index 6002bd1..2fb8fa9 100644 --- a/layers/mountain_peak/layer.sql +++ b/layers/mountain_peak/layer.sql @@ -3,32 +3,32 @@ -- etldoc: style="rounded,filled", label="layer_mountain_peak | z7+" ] ; CREATE OR REPLACE FUNCTION layer_mountain_peak( - bbox geometry, - zoom_level integer, + bbox geometry, + zoom_level integer, pixel_width numeric) RETURNS TABLE( osm_id bigint, - geometry geometry, - name text, - name_en text, - name_de text, - class text, + geometry geometry, + name text, + name_en text, + name_de text, + class text, tags hstore, - ele int, - ele_ft int, - "rank" int) AS + ele int, + ele_ft int, + "rank" int) AS $$ -- etldoc: osm_peak_point -> layer_mountain_peak:z7_ - SELECT - osm_id, - geometry, - name, - name_en, - name_de, - tags -> 'natural' AS class, + SELECT + osm_id, + geometry, + name, + name_en, + name_de, + tags -> 'natural' AS class, tags, - ele::int, - ele_ft::int, + ele::int, + ele_ft::int, rank::int FROM ( SELECT osm_id, geometry, name, COALESCE(NULLIF(name_en, ''), name) AS name_en, @@ -52,4 +52,6 @@ $$ WHERE zoom_level >= 7 AND (rank <= 5 OR zoom_level >= 14) ORDER BY "rank" ASC; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/mountain_peak/mapping.yaml b/layers/mountain_peak/mapping.yaml index 7b9cc68..84a0146 100644 --- a/layers/mountain_peak/mapping.yaml +++ b/layers/mountain_peak/mapping.yaml @@ -3,7 +3,7 @@ tables: # etldoc: imposm3 -> osm_peak_point peak_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/park/etl_diagram.png b/layers/park/etl_diagram.png index 2ee4242..8c1a744 100644 Binary files a/layers/park/etl_diagram.png and b/layers/park/etl_diagram.png differ diff --git a/layers/park/layer.sql b/layers/park/layer.sql index 2798160..2d3e4f1 100644 --- a/layers/park/layer.sql +++ b/layers/park/layer.sql @@ -15,37 +15,37 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class text, name text, name_en t NULL::int as rank FROM ( -- etldoc: osm_park_polygon_gen8 -> layer_park:z6 - SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title, NULL::int as scalerank + SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title FROM osm_park_polygon_gen8 WHERE zoom_level = 6 AND geometry && bbox UNION ALL -- etldoc: osm_park_polygon_gen7 -> layer_park:z7 - SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title, NULL::int as scalerank + SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title FROM osm_park_polygon_gen7 WHERE zoom_level = 7 AND geometry && bbox UNION ALL -- etldoc: osm_park_polygon_gen6 -> layer_park:z8 - SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title, NULL::int as scalerank + SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title FROM osm_park_polygon_gen6 WHERE zoom_level = 8 AND geometry && bbox UNION ALL -- etldoc: osm_park_polygon_gen5 -> layer_park:z9 - SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title, NULL::int as scalerank + SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title FROM osm_park_polygon_gen5 WHERE zoom_level = 9 AND geometry && bbox UNION ALL -- etldoc: osm_park_polygon_gen4 -> layer_park:z10 - SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title, NULL::int as scalerank + SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title FROM osm_park_polygon_gen4 WHERE zoom_level = 10 AND geometry && bbox UNION ALL -- etldoc: osm_park_polygon_gen3 -> layer_park:z11 - SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title, NULL::int as scalerank + SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title FROM osm_park_polygon_gen3 WHERE zoom_level = 11 AND geometry && bbox UNION ALL -- etldoc: osm_park_polygon_gen2 -> layer_park:z12 - SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title, NULL::int as scalerank + SELECT osm_id, geometry, name, name_en, name_de, tags, leisure, boundary, protection_title FROM osm_park_polygon_gen2 WHERE zoom_level = 12 AND geometry && bbox UNION ALL @@ -130,4 +130,6 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class text, name text, name_en t WHERE zoom_level >= 14 AND geometry_point && bbox ) AS park_point ) AS park_all; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/park/mapping.yaml b/layers/park/mapping.yaml index e4a5ec0..e06cb2c 100644 --- a/layers/park/mapping.yaml +++ b/layers/park/mapping.yaml @@ -52,7 +52,8 @@ tables: # etldoc: imposm3 -> osm_park_polygon park_polygon: type: polygon - fields: + _resolve_wikidata: false + columns: - name: osm_id type: id - name: geometry diff --git a/layers/place/capital.sql b/layers/place/capital.sql index e132530..434d3fb 100644 --- a/layers/place/capital.sql +++ b/layers/place/capital.sql @@ -3,6 +3,7 @@ RETURNS INT AS $$ SELECT CASE WHEN capital IN ('yes', '2') THEN 2 WHEN capital = '4' THEN 4 - ELSE NULL END; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ +LANGUAGE SQL +IMMUTABLE STRICT PARALLEL SAFE; diff --git a/layers/place/city.sql b/layers/place/city.sql index 59b6286..e3ef2d6 100644 --- a/layers/place/city.sql +++ b/layers/place/city.sql @@ -52,4 +52,6 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de OR (zoom_level BETWEEN 11 AND 12 AND (gridrank <= 14 OR "rank" IS NOT NULL)) OR (zoom_level >= 13) ) as city_all; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/place/etl_diagram.png b/layers/place/etl_diagram.png index f63eb6c..41b8c26 100644 Binary files a/layers/place/etl_diagram.png and b/layers/place/etl_diagram.png differ diff --git a/layers/place/island_rank.sql b/layers/place/island_rank.sql index 9d63124..15b8d5a 100644 --- a/layers/place/island_rank.sql +++ b/layers/place/island_rank.sql @@ -6,4 +6,6 @@ CREATE OR REPLACE FUNCTION island_rank(area REAL) RETURNS INT AS $$ WHEN area > 40000000 THEN 3 ELSE 7 END; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ +LANGUAGE SQL +IMMUTABLE STRICT PARALLEL SAFE; diff --git a/layers/place/layer.sql b/layers/place/layer.sql index dd6381f..f4d3d40 100644 --- a/layers/place/layer.sql +++ b/layers/place/layer.sql @@ -96,4 +96,6 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, FROM layer_city(bbox, zoom_level, pixel_width) ORDER BY "rank" ASC ) AS place_all -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/place/mapping.yaml b/layers/place/mapping.yaml index 15d7ef5..29d1f3b 100644 --- a/layers/place/mapping.yaml +++ b/layers/place/mapping.yaml @@ -20,7 +20,7 @@ tables: # etldoc: imposm3 -> osm_continent_point continent_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry @@ -40,7 +40,7 @@ tables: # etldoc: imposm3 -> osm_country_point country_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry @@ -70,7 +70,7 @@ tables: # etldoc: imposm3 -> osm_island_polygon island_polygon: type: polygon - fields: + columns: - name: osm_id type: id - name: geometry @@ -93,7 +93,7 @@ tables: # etldoc: imposm3 -> osm_island_point island_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry @@ -114,7 +114,7 @@ tables: # etldoc: imposm3 -> osm_state_point state_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry @@ -144,7 +144,7 @@ tables: # etldoc: imposm3 -> osm_city_point city_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/poi/class.sql b/layers/poi/class.sql index c991d22..eea2319 100644 --- a/layers/poi/class.sql +++ b/layers/poi/class.sql @@ -25,44 +25,16 @@ RETURNS INT AS $$ WHEN 'bar' THEN 800 ELSE 1000 END; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION poi_class(subclass TEXT, mapping_key TEXT) RETURNS TEXT AS $$ SELECT CASE - WHEN subclass IN ('accessories','antiques','beauty','bed','boutique','camera','carpet','charity','chemist','coffee','computer','convenience','copyshop','cosmetics','garden_centre','doityourself','erotic','electronics','fabric','florist','frozen_food','furniture','video_games','video','general','gift','hardware','hearing_aids','hifi','ice_cream','interior_decoration','jewelry','kiosk','lamps','mall','massage','motorcycle','mobile_phone','newsagent','optician','outdoor','perfumery','perfume','pet','photo','second_hand','shoes','sports','stationery','tailor','tattoo','ticket','tobacco','toys','travel_agency','watches','weapons','wholesale') THEN 'shop' - WHEN subclass IN ('townhall','public_building','courthouse','community_centre') THEN 'town_hall' - WHEN subclass IN ('golf','golf_course','miniature_golf') THEN 'golf' - WHEN subclass IN ('fast_food','food_court') THEN 'fast_food' - WHEN subclass IN ('park','bbq') THEN 'park' - WHEN subclass IN ('bus_stop','bus_station') THEN 'bus' - WHEN (subclass='station' AND mapping_key = 'railway') OR subclass IN ('halt', 'tram_stop', 'subway') THEN 'railway' - WHEN (subclass='station' AND mapping_key = 'aerialway') THEN 'aerialway' - WHEN subclass IN ('subway_entrance','train_station_entrance') THEN 'entrance' - WHEN subclass IN ('camp_site','caravan_site') THEN 'campsite' - WHEN subclass IN ('laundry','dry_cleaning') THEN 'laundry' - WHEN subclass IN ('supermarket','deli','delicatessen','department_store','greengrocer','marketplace') THEN 'grocery' - WHEN subclass IN ('books','library') THEN 'library' - WHEN subclass IN ('university','college') THEN 'college' - WHEN subclass IN ('hotel','motel','bed_and_breakfast','guest_house','hostel','chalet','alpine_hut','dormitory') THEN 'lodging' - WHEN subclass IN ('chocolate','confectionery') THEN 'ice_cream' - WHEN subclass IN ('post_box','post_office') THEN 'post' - WHEN subclass IN ('cafe') THEN 'cafe' - WHEN subclass IN ('school','kindergarten') THEN 'school' - WHEN subclass IN ('alcohol','beverages','wine') THEN 'alcohol_shop' - WHEN subclass IN ('bar','nightclub') THEN 'bar' - WHEN subclass IN ('marina','dock') THEN 'harbor' - WHEN subclass IN ('car','car_repair','taxi') THEN 'car' - WHEN subclass IN ('hospital','nursing_home', 'clinic') THEN 'hospital' - WHEN subclass IN ('grave_yard','cemetery') THEN 'cemetery' - WHEN subclass IN ('attraction','viewpoint') THEN 'attraction' - WHEN subclass IN ('biergarten','pub') THEN 'beer' - WHEN subclass IN ('music','musical_instrument') THEN 'music' - WHEN subclass IN ('american_football','stadium','soccer') THEN 'stadium' - WHEN subclass IN ('art','artwork','gallery','arts_centre') THEN 'art_gallery' - WHEN subclass IN ('bag','clothes') THEN 'clothing_store' - WHEN subclass IN ('swimming_area','swimming') THEN 'swimming' - WHEN subclass IN ('castle','ruins') THEN 'castle' + %%FIELD_MAPPING: class %% ELSE subclass END; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/poi/layer.sql b/layers/poi/layer.sql index c067288..35c80e2 100644 --- a/layers/poi/layer.sql +++ b/layers/poi/layer.sql @@ -21,7 +21,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de agg_stop, NULLIF(layer, 0) AS layer, "level", - CASE WHEN indoor=TRUE THEN 1 ELSE NULL END as indoor, + CASE WHEN indoor=TRUE THEN 1 END as indoor, row_number() OVER ( PARTITION BY LabelGrid(geometry, 100 * pixel_width) ORDER BY CASE WHEN name = '' THEN 2000 ELSE poi_class_rank(poi_class(subclass, mapping_key)) END ASC @@ -70,4 +70,6 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de ) as poi_union ORDER BY "rank" ; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/poi/mapping.yaml b/layers/poi/mapping.yaml index 3fb327b..a50e11d 100644 --- a/layers/poi/mapping.yaml +++ b/layers/poi/mapping.yaml @@ -381,11 +381,11 @@ tables: # etldoc: imposm3 -> osm_poi_point poi_point: type: point - fields: *poi_fields + columns: *poi_fields mapping: *poi_mapping # etldoc: imposm3 -> osm_poi_polygon poi_polygon: type: polygon - fields: *poi_fields + columns: *poi_fields mapping: *poi_mapping diff --git a/layers/poi/poi.yaml b/layers/poi/poi.yaml index bd66285..d678bd6 100644 --- a/layers/poi/poi.yaml +++ b/layers/poi/poi.yaml @@ -9,11 +9,91 @@ layer: name: The OSM [`name`](http://wiki.openstreetmap.org/wiki/Key:name) value of the POI. name_en: English name `name:en` if available, otherwise `name`. name_de: German name `name:de` if available, otherwise `name` or `name:en`. - class: | + class: + description: | More general classes of POIs. If there is no more general `class` for the `subclass` this field will contain the same value as `subclass`. But for example for schools you only need to style the class `school` to filter the subclasses `school` and `kindergarten`. Or use the class `shop` to style all shops. + values: + shop: + subclass: ['accessories', 'antiques', 'beauty', 'bed', 'boutique', 'camera', 'carpet', 'charity', 'chemist', + 'coffee', 'computer', 'convenience', 'copyshop', 'cosmetics', 'garden_centre', 'doityourself', + 'erotic', 'electronics', 'fabric', 'florist', 'frozen_food', 'furniture', 'video_games', 'video', + 'general', 'gift', 'hardware', 'hearing_aids', 'hifi', 'ice_cream', 'interior_decoration', + 'jewelry', 'kiosk', 'lamps', 'mall', 'massage', 'motorcycle', 'mobile_phone', 'newsagent', + 'optician', 'outdoor', 'perfumery', 'perfume', 'pet', 'photo', 'second_hand', 'shoes', 'sports', + 'stationery', 'tailor', 'tattoo', 'ticket', 'tobacco', 'toys', 'travel_agency', 'watches', + 'weapons', 'wholesale'] + town_hall: + subclass: ['townhall', 'public_building', 'courthouse', 'community_centre'] + golf: + subclass: ['golf', 'golf_course', 'miniature_golf'] + fast_food: + subclass: ['fast_food', 'food_court'] + park: + subclass: ['park', 'bbq'] + bus: + subclass: ['bus_stop', 'bus_station'] + railway: + - __AND__: + subclass: 'station' + mapping_key: 'railway' + - subclass: ['halt', 'tram_stop', 'subway'] + aerialway: + __AND__: + subclass: 'station' + mapping_key: 'aerialway' + entrance: + subclass: ['subway_entrance', 'train_station_entrance'] + campsite: + subclass: ['camp_site', 'caravan_site'] + laundry: + subclass: ['laundry', 'dry_cleaning'] + grocery: + subclass: ['supermarket', 'deli', 'delicatessen', 'department_store', 'greengrocer', 'marketplace'] + library: + subclass: ['books', 'library'] + college: + subclass: ['university', 'college'] + lodging: + subclass: ['hotel', 'motel', 'bed_and_breakfast', 'guest_house', 'hostel', 'chalet', 'alpine_hut', 'dormitory'] + ice_cream: + subclass: ['chocolate', 'confectionery'] + post: + subclass: ['post_box', 'post_office'] + cafe: + subclass: ['cafe'] + school: + subclass: ['school', 'kindergarten'] + alcohol_shop: + subclass: ['alcohol', 'beverages', 'wine'] + bar: + subclass: ['bar', 'nightclub'] + harbor: + subclass: ['marina', 'dock'] + car: + subclass: ['car', 'car_repair', 'taxi'] + hospital: + subclass: ['hospital', 'nursing_home', 'clinic'] + cemetery: + subclass: ['grave_yard', 'cemetery'] + attraction: + subclass: ['attraction', 'viewpoint'] + beer: + subclass: ['biergarten', 'pub'] + music: + subclass: ['music', 'musical_instrument'] + stadium: + subclass: ['american_football', 'stadium', 'soccer'] + art_gallery: + subclass: ['art', 'artwork', 'gallery', 'arts_centre'] + clothing_store: + subclass: ['bag', 'clothes'] + swimming: + subclass: ['swimming_area', 'swimming'] + castle: + subclass: ['castle', 'ruins'] subclass: description: | Original value of either the diff --git a/layers/poi/poi_stop_agg.sql b/layers/poi/poi_stop_agg.sql index bffb65d..788bcda 100644 --- a/layers/poi/poi_stop_agg.sql +++ b/layers/poi/poi_stop_agg.sql @@ -3,8 +3,7 @@ CREATE MATERIALIZED VIEW osm_poi_stop_centroid AS ( SELECT uic_ref, count(*) as count, - CASE WHEN count(*) > 2 THEN ST_Centroid(ST_UNION(geometry)) - ELSE NULL END AS centroid + CASE WHEN count(*) > 2 THEN ST_Centroid(ST_UNION(geometry)) END AS centroid FROM osm_poi_point WHERE nullif(uic_ref, '') IS NOT NULL @@ -13,7 +12,7 @@ CREATE MATERIALIZED VIEW osm_poi_stop_centroid AS ( uic_ref HAVING count(*) > 1 -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; DROP MATERIALIZED VIEW IF EXISTS osm_poi_stop_rank CASCADE; CREATE MATERIALIZED VIEW osm_poi_stop_rank AS ( @@ -33,7 +32,7 @@ CREATE MATERIALIZED VIEW osm_poi_stop_rank AS ( WHERE subclass IN ('bus_stop', 'bus_station', 'tram_stop', 'subway') ORDER BY p.uic_ref, rk -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; ALTER TABLE osm_poi_point ADD COLUMN IF NOT EXISTS agg_stop INTEGER DEFAULT NULL; -SELECT update_osm_poi_point_agg(); \ No newline at end of file +SELECT update_osm_poi_point_agg(); diff --git a/layers/poi/update_poi_point.sql b/layers/poi/update_poi_point.sql index c7e1543..dddec1d 100644 --- a/layers/poi/update_poi_point.sql +++ b/layers/poi/update_poi_point.sql @@ -27,7 +27,6 @@ BEGIN SET agg_stop = CASE WHEN p.subclass IN ('bus_stop', 'bus_station', 'tram_stop', 'subway') THEN 1 - ELSE NULL END; UPDATE osm_poi_point p @@ -36,7 +35,6 @@ BEGIN WHEN p.subclass IN ('bus_stop', 'bus_station', 'tram_stop', 'subway') AND r.rk IS NULL OR r.rk = 1 THEN 1 - ELSE NULL END) FROM osm_poi_stop_rank r WHERE p.osm_id = r.osm_id diff --git a/layers/transportation/class.sql b/layers/transportation/class.sql index 187b5f8..43642ba 100644 --- a/layers/transportation/class.sql +++ b/layers/transportation/class.sql @@ -3,36 +3,20 @@ CREATE OR REPLACE FUNCTION brunnel(is_bridge BOOL, is_tunnel BOOL, is_ford BOOL) WHEN is_bridge THEN 'bridge' WHEN is_tunnel THEN 'tunnel' WHEN is_ford THEN 'ford' - ELSE NULL END; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ +LANGUAGE SQL +IMMUTABLE STRICT PARALLEL SAFE; -- The classes for highways are derived from the classes used in ClearTables -- https://github.com/ClearTables/ClearTables/blob/master/transportation.lua CREATE OR REPLACE FUNCTION highway_class(highway TEXT, public_transport TEXT, construction TEXT) RETURNS TEXT AS $$ SELECT CASE - WHEN highway IN ('motorway', 'motorway_link') THEN 'motorway' - WHEN highway IN ('trunk', 'trunk_link') THEN 'trunk' - WHEN highway IN ('primary', 'primary_link') THEN 'primary' - WHEN highway IN ('secondary', 'secondary_link') THEN 'secondary' - WHEN highway IN ('tertiary', 'tertiary_link') THEN 'tertiary' - WHEN highway IN ('unclassified', 'residential', 'living_street', 'road') THEN 'minor' - WHEN highway IN ('pedestrian', 'path', 'footway', 'cycleway', 'steps', 'bridleway', 'corridor') OR public_transport IN ('platform') THEN 'path' - WHEN highway IN ('service', 'track', 'raceway') THEN highway - WHEN highway = 'construction' THEN CASE - WHEN construction IN ('motorway', 'motorway_link') THEN 'motorway_construction' - WHEN construction IN ('trunk', 'trunk_link') THEN 'trunk_construction' - WHEN construction IN ('primary', 'primary_link') THEN 'primary_construction' - WHEN construction IN ('secondary', 'secondary_link') THEN 'secondary_construction' - WHEN construction IN ('tertiary', 'tertiary_link') THEN 'tertiary_construction' - WHEN construction = '' OR construction IN ('unclassified', 'residential', 'living_street', 'road') THEN 'minor_construction' - WHEN construction IN ('pedestrian', 'path', 'footway', 'cycleway', 'steps', 'bridleway', 'corridor') OR public_transport IN ('platform') THEN 'path_construction' - WHEN construction IN ('service', 'track', 'raceway') THEN CONCAT(highway, '_construction') - ELSE NULL - END - ELSE NULL + %%FIELD_MAPPING: class %% END; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; -- The classes for railways are derived from the classes used in ClearTables -- https://github.com/ClearTables/ClearTables/blob/master/transportation.lua @@ -40,18 +24,20 @@ CREATE OR REPLACE FUNCTION railway_class(railway TEXT) RETURNS TEXT AS $$ SELECT CASE WHEN railway IN ('rail', 'narrow_gauge', 'preserved', 'funicular') THEN 'rail' WHEN railway IN ('subway', 'light_rail', 'monorail', 'tram') THEN 'transit' - ELSE NULL END; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ +LANGUAGE SQL +IMMUTABLE STRICT PARALLEL SAFE; -- Limit service to only the most important values to ensure -- we always know the values of service CREATE OR REPLACE FUNCTION service_value(service TEXT) RETURNS TEXT AS $$ SELECT CASE WHEN service IN ('spur', 'yard', 'siding', 'crossover', 'driveway', 'alley', 'parking_aisle') THEN service - ELSE NULL END; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ +LANGUAGE SQL +IMMUTABLE STRICT PARALLEL SAFE; -- Limit surface to only the most important values to ensure -- we always know the values of surface @@ -59,6 +45,7 @@ CREATE OR REPLACE FUNCTION surface_value(surface TEXT) RETURNS TEXT AS $$ SELECT CASE WHEN surface IN ('paved', 'asphalt', 'cobblestone', 'concrete', 'concrete:lanes', 'concrete:plates', 'metal', 'paving_stones', 'sett', 'unhewn_cobblestone', 'wood') THEN 'paved' WHEN surface IN ('unpaved', 'compacted', 'dirt', 'earth', 'fine_gravel', 'grass', 'grass_paver', 'gravel', 'gravel_turf', 'ground', 'ice', 'mud', 'pebblestone', 'salt', 'sand', 'snow', 'woodchips') THEN 'unpaved' - ELSE NULL END; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ +LANGUAGE SQL +IMMUTABLE STRICT PARALLEL SAFE; diff --git a/layers/transportation/layer.sql b/layers/transportation/layer.sql index 561f611..4b8bd88 100644 --- a/layers/transportation/layer.sql +++ b/layers/transportation/layer.sql @@ -1,6 +1,8 @@ CREATE OR REPLACE FUNCTION highway_is_link(highway TEXT) RETURNS BOOLEAN AS $$ SELECT highway LIKE '%_link'; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ +LANGUAGE SQL +IMMUTABLE STRICT PARALLEL SAFE; -- etldoc: layer_transportation[shape=record fillcolor=lightpink, style="rounded,filled", @@ -23,7 +25,6 @@ indoor INT, bicycle TEXT, foot TEXT, horse TEXT, mtb_scale TEXT, surface TEXT) A WHEN (highway IS NOT NULL OR public_transport IS NOT NULL) AND highway_class(highway, public_transport, construction) = 'path' THEN COALESCE(NULLIF(public_transport, ''), highway) - ELSE NULL END AS subclass, -- All links are considered as ramps as well CASE WHEN highway_is_link(highway) OR highway = 'steps' @@ -33,7 +34,7 @@ indoor INT, bicycle TEXT, foot TEXT, horse TEXT, mtb_scale TEXT, surface TEXT) A NULLIF(service, '') AS service, NULLIF(layer, 0) AS layer, "level", - CASE WHEN indoor=TRUE THEN 1 ELSE NULL END as indoor, + CASE WHEN indoor=TRUE THEN 1 END as indoor, NULLIF(bicycle, '') AS bicycle, NULLIF(foot, '') AS foot, NULLIF(horse, '') AS horse, @@ -157,12 +158,8 @@ indoor INT, bicycle TEXT, foot TEXT, horse TEXT, mtb_scale TEXT, surface TEXT) A public_transport, service_value(service) AS service, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, man_made, layer, - CASE WHEN highway IN ('footway', 'steps') THEN "level" - ELSE NULL::int - END AS "level", - CASE WHEN highway IN ('footway', 'steps') THEN indoor - ELSE NULL::boolean - END AS indoor, + CASE WHEN highway IN ('footway', 'steps') THEN "level" END AS "level", + CASE WHEN highway IN ('footway', 'steps') THEN indoor END AS indoor, bicycle, foot, horse, mtb_scale, surface_value(surface) AS "surface", z_order @@ -371,4 +368,6 @@ indoor INT, bicycle TEXT, foot TEXT, horse TEXT, mtb_scale TEXT, surface TEXT) A ) AS zoom_levels WHERE geometry && bbox ORDER BY z_order ASC; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/transportation/mapping.yaml b/layers/transportation/mapping.yaml index 29c0c25..7ad30f9 100644 --- a/layers/transportation/mapping.yaml +++ b/layers/transportation/mapping.yaml @@ -158,7 +158,8 @@ tables: # etldoc: imposm3 -> osm_highway_linestring highway_linestring: type: linestring - fields: + _resolve_wikidata: false + columns: - name: osm_id type: id - name: geometry @@ -231,7 +232,8 @@ tables: # etldoc: imposm3 -> osm_railway_linestring railway_linestring: type: linestring - fields: + _resolve_wikidata: false + columns: - name: osm_id type: id - name: geometry @@ -273,7 +275,8 @@ tables: # etldoc: imposm3 -> osm_aerialway_linestring aerialway_linestring: type: linestring - fields: + _resolve_wikidata: false + columns: - name: osm_id type: id - name: geometry @@ -305,7 +308,8 @@ tables: # etldoc: imposm3 -> osm_shipway_linestring shipway_linestring: type: linestring - fields: + _resolve_wikidata: false + columns: - name: osm_id type: id - name: geometry @@ -336,7 +340,7 @@ tables: # etldoc: imposm3 -> osm_highway_polygon highway_polygon: type: polygon - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/transportation/mapping_diagram.png b/layers/transportation/mapping_diagram.png index eacdce7..959e43b 100644 Binary files a/layers/transportation/mapping_diagram.png and b/layers/transportation/mapping_diagram.png differ diff --git a/layers/transportation/transportation.yaml b/layers/transportation/transportation.yaml index b92f650..700d8b2 100644 --- a/layers/transportation/transportation.yaml +++ b/layers/transportation/transportation.yaml @@ -24,33 +24,69 @@ layer: shipping ways), or [`man_made`](http://wiki.openstreetmap.org/wiki/Key:route). values: - - motorway - - trunk - - primary - - secondary - - tertiary - - minor - - service - - track - - path - - raceway - - motorway_construction - - trunk_construction - - primary_construction - - secondary_construction - - tertiary_construction - - minor_construction - - service_construction - - track_construction - - path_construction - - raceway_construction - - rail - - transit - - cable_car - - gondola - - ferry - - bridge - - pier + motorway: + highway: ['motorway', 'motorway_link'] + trunk: + highway: ['trunk', 'trunk_link'] + primary: + highway: ['primary', 'primary_link'] + secondary: + highway: ['secondary', 'secondary_link'] + tertiary: + highway: ['tertiary', 'tertiary_link'] + minor: + highway: ['unclassified', 'residential', 'living_street', 'road'] + path: + highway: ['pedestrian', 'path', 'footway', 'cycleway', 'steps', 'bridleway', 'corridor'] + public_transport: 'platform' + service: + highway: service + track: + highway: track + raceway: + highway: raceway + motorway_construction: + __AND__: + highway: construction + construction: ['motorway', 'motorway_link'] + trunk_construction: + __AND__: + highway: construction + construction: ['trunk', 'trunk_link'] + primary_construction: + __AND__: + highway: construction + construction: ['primary', 'primary_link'] + secondary_construction: + __AND__: + highway: construction + construction: ['secondary', 'secondary_link'] + tertiary_construction: + __AND__: + highway: construction + construction: ['tertiary', 'tertiary_link'] + minor_construction: + __AND__: + highway: construction + construction: ['', 'unclassified', 'residential', 'living_street', 'road'] + path_construction: + __AND__: + - highway: construction + - __OR__: + construction: ['pedestrian', 'path', 'footway', 'cycleway', 'steps', 'bridleway', 'corridor'] + public_transport: platform + service_construction: + __AND__: + highway: construction + construction: service + track_construction: + __AND__: + highway: construction + construction: track + raceway_construction: + __AND__: + highway: construction + construction: raceway subclass: description: | Distinguish more specific classes of railway and path: diff --git a/layers/transportation/update_transportation_merge.sql b/layers/transportation/update_transportation_merge.sql index a2c1852..4c21900 100644 --- a/layers/transportation/update_transportation_merge.sql +++ b/layers/transportation/update_transportation_merge.sql @@ -41,7 +41,7 @@ CREATE MATERIALIZED VIEW osm_transportation_merge_linestring AS ( AND ST_IsValid(geometry) group by highway, construction ) AS highway_union -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_geometry_idx ON osm_transportation_merge_linestring USING gist(geometry); CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_highway_partial_idx @@ -54,7 +54,7 @@ CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen3 AS ( FROM osm_transportation_merge_linestring WHERE highway IN ('motorway','trunk', 'primary') OR highway = 'construction' AND construction IN ('motorway','trunk', 'primary') -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen3_geometry_idx ON osm_transportation_merge_linestring_gen3 USING gist(geometry); CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen3_highway_partial_idx @@ -67,7 +67,7 @@ CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen4 AS ( FROM osm_transportation_merge_linestring_gen3 WHERE (highway IN ('motorway','trunk', 'primary') OR highway = 'construction' AND construction IN ('motorway','trunk', 'primary')) AND ST_Length(geometry) > 50 -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen4_geometry_idx ON osm_transportation_merge_linestring_gen4 USING gist(geometry); CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen4_highway_partial_idx @@ -80,7 +80,7 @@ CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen5 AS ( FROM osm_transportation_merge_linestring_gen4 WHERE (highway IN ('motorway','trunk') OR highway = 'construction' AND construction IN ('motorway','trunk')) AND ST_Length(geometry) > 100 -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen5_geometry_idx ON osm_transportation_merge_linestring_gen5 USING gist(geometry); CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen5_highway_partial_idx @@ -92,7 +92,7 @@ CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen6 AS ( SELECT ST_Simplify(geometry, 1000) AS geometry, osm_id, highway, construction, z_order FROM osm_transportation_merge_linestring_gen5 WHERE (highway IN ('motorway','trunk') OR highway = 'construction' AND construction IN ('motorway','trunk')) AND ST_Length(geometry) > 500 -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen6_geometry_idx ON osm_transportation_merge_linestring_gen6 USING gist(geometry); CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen6_highway_partial_idx @@ -104,7 +104,7 @@ CREATE MATERIALIZED VIEW osm_transportation_merge_linestring_gen7 AS ( SELECT ST_Simplify(geometry, 2000) AS geometry, osm_id, highway, construction, z_order FROM osm_transportation_merge_linestring_gen6 WHERE (highway = 'motorway' OR highway = 'construction' AND construction = 'motorway') AND ST_Length(geometry) > 1000 -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_transportation_merge_linestring_gen7_geometry_idx ON osm_transportation_merge_linestring_gen7 USING gist(geometry); diff --git a/layers/transportation_name/layer.sql b/layers/transportation_name/layer.sql index 9f108d6..ffc3a3d 100644 --- a/layers/transportation_name/layer.sql +++ b/layers/transportation_name/layer.sql @@ -23,11 +23,10 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, CASE WHEN highway IS NOT NULL AND highway_class(highway, '', construction) = 'path' THEN highway - ELSE NULL END AS subclass, NULLIF(layer, 0) AS layer, "level", - CASE WHEN indoor=TRUE THEN 1 ELSE NULL END as indoor + CASE WHEN indoor=TRUE THEN 1 END as indoor FROM ( -- etldoc: osm_transportation_name_linestring_gen4 -> layer_transportation_name:z6 @@ -127,4 +126,6 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, ) AS zoom_levels WHERE geometry && bbox ORDER BY z_order ASC; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/transportation_name/mapping_diagram.png b/layers/transportation_name/mapping_diagram.png index eacdce7..959e43b 100644 Binary files a/layers/transportation_name/mapping_diagram.png and b/layers/transportation_name/mapping_diagram.png differ diff --git a/layers/transportation_name/update_route_member.sql b/layers/transportation_name/update_route_member.sql index 60716a3..921b451 100644 --- a/layers/transportation_name/update_route_member.sql +++ b/layers/transportation_name/update_route_member.sql @@ -56,7 +56,6 @@ BEGIN THEN 'ca-transcanada'::route_network_type WHEN network = 'omt-gb-motorway' THEN 'gb-motorway'::route_network_type WHEN network = 'omt-gb-trunk' THEN 'gb-trunk'::route_network_type - ELSE NULL END ; diff --git a/layers/transportation_name/update_transportation_name.sql b/layers/transportation_name/update_transportation_name.sql index 58a521f..236e9ac 100644 --- a/layers/transportation_name/update_transportation_name.sql +++ b/layers/transportation_name/update_transportation_name.sql @@ -25,21 +25,15 @@ CREATE MATERIALIZED VIEW osm_transportation_name_network AS ( end as ref, hl.highway, hl.construction, - CASE WHEN highway IN ('footway', 'steps') THEN layer - ELSE NULL::int - END AS layer, - CASE WHEN highway IN ('footway', 'steps') THEN "level" - ELSE NULL::int - END AS "level", - CASE WHEN highway IN ('footway', 'steps') THEN indoor - ELSE NULL::boolean - END AS indoor, + CASE WHEN highway IN ('footway', 'steps') THEN layer END AS layer, + CASE WHEN highway IN ('footway', 'steps') THEN "level" END AS "level", + CASE WHEN highway IN ('footway', 'steps') THEN indoor END AS indoor, ROW_NUMBER() OVER(PARTITION BY hl.osm_id ORDER BY rm.network_type) AS "rank", hl.z_order FROM osm_highway_linestring hl left join osm_route_member rm on (rm.member = hl.osm_id) -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_transportation_name_network_geometry_idx ON osm_transportation_name_network USING gist(geometry); @@ -82,7 +76,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring AS ( AND NULLIF(highway, '') IS NOT NULL group by name, name_en, name_de, ref, highway, construction, "level", layer, indoor, network_type ) AS highway_union -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_geometry_idx ON osm_transportation_name_linestring USING gist(geometry); CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_highway_partial_idx @@ -94,7 +88,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen1 AS ( SELECT ST_Simplify(geometry, 50) AS geometry, osm_id, name, name_en, name_de, tags, ref, highway, construction, network, z_order FROM osm_transportation_name_linestring WHERE (highway IN ('motorway','trunk') OR highway = 'construction' AND construction IN ('motorway','trunk')) AND ST_Length(geometry) > 8000 -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen1_geometry_idx ON osm_transportation_name_linestring_gen1 USING gist(geometry); CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen1_highway_partial_idx @@ -106,7 +100,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen2 AS ( SELECT ST_Simplify(geometry, 120) AS geometry, osm_id, name, name_en, name_de, tags, ref, highway, construction, network, z_order FROM osm_transportation_name_linestring_gen1 WHERE (highway IN ('motorway','trunk') OR highway = 'construction' AND construction IN ('motorway','trunk')) AND ST_Length(geometry) > 14000 -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen2_geometry_idx ON osm_transportation_name_linestring_gen2 USING gist(geometry); CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen2_highway_partial_idx @@ -118,7 +112,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen3 AS ( SELECT ST_Simplify(geometry, 200) AS geometry, osm_id, name, name_en, name_de, tags, ref, highway, construction, network, z_order FROM osm_transportation_name_linestring_gen2 WHERE (highway = 'motorway' OR highway = 'construction' AND construction = 'motorway') AND ST_Length(geometry) > 20000 -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_geometry_idx ON osm_transportation_name_linestring_gen3 USING gist(geometry); CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_highway_partial_idx @@ -130,7 +124,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen4 AS ( SELECT ST_Simplify(geometry, 500) AS geometry, osm_id, name, name_en, name_de, tags, ref, highway, construction, network, z_order FROM osm_transportation_name_linestring_gen3 WHERE (highway = 'motorway' OR highway = 'construction' AND construction = 'motorway') AND ST_Length(geometry) > 20000 -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen4_geometry_idx ON osm_transportation_name_linestring_gen4 USING gist(geometry); -- Handle updates diff --git a/layers/water/etl_diagram.png b/layers/water/etl_diagram.png index 031bf9d..ffcc395 100644 Binary files a/layers/water/etl_diagram.png and b/layers/water/etl_diagram.png differ diff --git a/layers/water/mapping.yaml b/layers/water/mapping.yaml index 68c5bfc..2bca61c 100644 --- a/layers/water/mapping.yaml +++ b/layers/water/mapping.yaml @@ -49,7 +49,7 @@ tables: # etldoc: imposm3 -> osm_water_polygon water_polygon: - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/water/water.sql b/layers/water/water.sql index a7ca1f6..a478428 100644 --- a/layers/water/water.sql +++ b/layers/water/water.sql @@ -1,20 +1,66 @@ +-- This statement can be deleted after the water importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_ocean_polygon_gen1 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_ocean_polygon -> osm_ocean_polygon_gen1 +DROP MATERIALIZED VIEW IF EXISTS osm_ocean_polygon_gen1 CASCADE; +CREATE MATERIALIZED VIEW osm_ocean_polygon_gen1 AS ( + SELECT ST_Simplify(geometry, 20) AS geometry + FROM osm_ocean_polygon +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_ocean_polygon_gen1_idx ON osm_ocean_polygon_gen1 USING gist (geometry); + + +-- This statement can be deleted after the water importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_ocean_polygon_gen2 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_ocean_polygon -> osm_ocean_polygon_gen2 +DROP MATERIALIZED VIEW IF EXISTS osm_ocean_polygon_gen2 CASCADE; +CREATE MATERIALIZED VIEW osm_ocean_polygon_gen2 AS ( + SELECT ST_Simplify(geometry, 40) AS geometry + FROM osm_ocean_polygon +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_ocean_polygon_gen2_idx ON osm_ocean_polygon_gen2 USING gist (geometry); + + +-- This statement can be deleted after the water importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_ocean_polygon_gen3 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_ocean_polygon -> osm_ocean_polygon_gen3 +DROP MATERIALIZED VIEW IF EXISTS osm_ocean_polygon_gen3 CASCADE; +CREATE MATERIALIZED VIEW osm_ocean_polygon_gen3 AS ( + SELECT ST_Simplify(geometry, 80) AS geometry + FROM osm_ocean_polygon +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_ocean_polygon_gen3_idx ON osm_ocean_polygon_gen3 USING gist (geometry); + + +-- This statement can be deleted after the water importer image stops creating this object as a table +DO $$ BEGIN DROP TABLE IF EXISTS osm_ocean_polygon_gen4 CASCADE; EXCEPTION WHEN wrong_object_type THEN END; $$ language 'plpgsql'; +-- etldoc: osm_ocean_polygon -> osm_ocean_polygon_gen4 +DROP MATERIALIZED VIEW IF EXISTS osm_ocean_polygon_gen4 CASCADE; +CREATE MATERIALIZED VIEW osm_ocean_polygon_gen4 AS ( + SELECT ST_Simplify(geometry, 160) AS geometry + FROM osm_ocean_polygon +) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_ocean_polygon_gen4_idx ON osm_ocean_polygon_gen4 USING gist (geometry); + + + CREATE OR REPLACE FUNCTION water_class(waterway TEXT) RETURNS TEXT AS $$ SELECT CASE - WHEN waterway='' THEN 'lake' - WHEN waterway='lake' THEN 'lake' - WHEN waterway='dock' THEN 'dock' + %%FIELD_MAPPING: class %% ELSE 'river' END; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION waterway_brunnel(is_bridge BOOL, is_tunnel BOOL) RETURNS TEXT AS $$ SELECT CASE WHEN is_bridge THEN 'bridge' WHEN is_tunnel THEN 'tunnel' - ELSE NULL END; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ +LANGUAGE SQL +IMMUTABLE STRICT PARALLEL SAFE; @@ -333,4 +379,6 @@ RETURNS TABLE(geometry geometry, class text, brunnel text, intermittent int) AS SELECT * FROM water_z14 WHERE zoom_level >= 14 ) AS zoom_levels WHERE geometry && bbox; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/water/water.yaml b/layers/water/water.yaml index 2e77b7b..24ecabe 100644 --- a/layers/water/water.yaml +++ b/layers/water/water.yaml @@ -14,9 +14,12 @@ layer: All water polygons from [OpenStreetMapData](http://osmdata.openstreetmap.de/) have the class `ocean`. Water bodies are classified as `lake` or `river` for water bodies with the [`waterway`](http://wiki.openstreetmap.org/wiki/Key:waterway) tag. values: - - ocean - - lake - - river + lake: + waterway: ['', 'lake'] + dock: + waterway: 'dock' + river: + ocean: intermittent: description: | Mark with `1` if it is an [intermittent](http://wiki.openstreetmap.org/wiki/Key:intermittent) water polygon. diff --git a/layers/water_name/etl_diagram.png b/layers/water_name/etl_diagram.png index b0e4a40..8a2ae1c 100644 Binary files a/layers/water_name/etl_diagram.png and b/layers/water_name/etl_diagram.png differ diff --git a/layers/water_name/layer.sql b/layers/water_name/layer.sql index e0b60cd..0aab292 100644 --- a/layers/water_name/layer.sql +++ b/layers/water_name/layer.sql @@ -54,4 +54,6 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, name_de OR (zoom_level >= "rank" AND "rank" IS NOT NULL) OR (zoom_level >= 8) ); -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/layers/water_name/mapping.yaml b/layers/water_name/mapping.yaml index 7d27ac4..558ad61 100644 --- a/layers/water_name/mapping.yaml +++ b/layers/water_name/mapping.yaml @@ -2,7 +2,7 @@ tables: # etldoc: imposm3 -> osm_marine_point marine_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/water_name/mapping_diagram.png b/layers/water_name/mapping_diagram.png index 2dce3fc..b1e881f 100644 Binary files a/layers/water_name/mapping_diagram.png and b/layers/water_name/mapping_diagram.png differ diff --git a/layers/water_name/update_water_lakeline.sql b/layers/water_name/update_water_lakeline.sql index 3136d8e..566d9b7 100644 --- a/layers/water_name/update_water_lakeline.sql +++ b/layers/water_name/update_water_lakeline.sql @@ -15,7 +15,7 @@ CREATE MATERIALIZED VIEW osm_water_lakeline AS ( FROM osm_water_polygon AS wp INNER JOIN lake_centerline ll ON wp.osm_id = ll.osm_id WHERE wp.name <> '' AND ST_IsValid(wp.geometry) -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_water_lakeline_geometry_idx ON osm_water_lakeline USING gist(geometry); -- Handle updates diff --git a/layers/water_name/update_water_point.sql b/layers/water_name/update_water_point.sql index 66bff8f..26176a7 100644 --- a/layers/water_name/update_water_point.sql +++ b/layers/water_name/update_water_point.sql @@ -15,7 +15,7 @@ CREATE MATERIALIZED VIEW osm_water_point AS ( FROM osm_water_polygon AS wp LEFT JOIN lake_centerline ll ON wp.osm_id = ll.osm_id WHERE ll.osm_id IS NULL AND wp.name <> '' -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_water_point_geometry_idx ON osm_water_point USING gist (geometry); -- Handle updates diff --git a/layers/waterway/mapping.yaml b/layers/waterway/mapping.yaml index 2255b16..cda6d05 100644 --- a/layers/waterway/mapping.yaml +++ b/layers/waterway/mapping.yaml @@ -28,7 +28,7 @@ tables: # etldoc: imposm3 -> osm_waterway_linestring waterway_linestring: type: linestring - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/waterway/update_important_waterway.sql b/layers/waterway/update_important_waterway.sql index 1c6f2c5..6e84fc1 100644 --- a/layers/waterway/update_important_waterway.sql +++ b/layers/waterway/update_important_waterway.sql @@ -30,7 +30,7 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring AS ( WHERE name <> '' AND waterway = 'river' AND ST_IsValid(geometry) GROUP BY name, name_en, name_de, slice_language_tags(tags) ) AS waterway_union -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_geometry_idx ON osm_important_waterway_linestring USING gist(geometry); -- etldoc: osm_important_waterway_linestring -> osm_important_waterway_linestring_gen1 @@ -38,7 +38,7 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen1 AS ( SELECT ST_Simplify(geometry, 60) AS geometry, name, name_en, name_de, tags FROM osm_important_waterway_linestring WHERE ST_Length(geometry) > 1000 -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen1_geometry_idx ON osm_important_waterway_linestring_gen1 USING gist(geometry); -- etldoc: osm_important_waterway_linestring_gen1 -> osm_important_waterway_linestring_gen2 @@ -46,7 +46,7 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen2 AS ( SELECT ST_Simplify(geometry, 100) AS geometry, name, name_en, name_de, tags FROM osm_important_waterway_linestring_gen1 WHERE ST_Length(geometry) > 4000 -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen2_geometry_idx ON osm_important_waterway_linestring_gen2 USING gist(geometry); -- etldoc: osm_important_waterway_linestring_gen2 -> osm_important_waterway_linestring_gen3 @@ -54,7 +54,7 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen3 AS ( SELECT ST_Simplify(geometry, 200) AS geometry, name, name_en, name_de, tags FROM osm_important_waterway_linestring_gen2 WHERE ST_Length(geometry) > 8000 -); +) /* DELAY_MATERIALIZED_VIEW_CREATION */; CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen3_geometry_idx ON osm_important_waterway_linestring_gen3 USING gist(geometry); -- Handle updates diff --git a/layers/waterway/waterway.sql b/layers/waterway/waterway.sql index 64838f6..35c277f 100644 --- a/layers/waterway/waterway.sql +++ b/layers/waterway/waterway.sql @@ -2,9 +2,10 @@ CREATE OR REPLACE FUNCTION waterway_brunnel(is_bridge BOOL, is_tunnel BOOL) RETU SELECT CASE WHEN is_bridge THEN 'bridge' WHEN is_tunnel THEN 'tunnel' - ELSE NULL END; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ +LANGUAGE SQL +IMMUTABLE STRICT PARALLEL SAFE; -- etldoc: ne_110m_rivers_lake_centerlines -> waterway_z3 CREATE OR REPLACE VIEW waterway_z3 AS ( @@ -106,4 +107,6 @@ RETURNS TABLE(geometry geometry, class text, name text, name_en text, name_de te SELECT * FROM waterway_z14 WHERE zoom_level >= 14 ) AS zoom_levels WHERE geometry && bbox; -$$ LANGUAGE SQL IMMUTABLE; +$$ +LANGUAGE SQL +IMMUTABLE PARALLEL SAFE; diff --git a/openmaptiles.yaml b/openmaptiles.yaml index 542574c..a573571 100644 --- a/openmaptiles.yaml +++ b/openmaptiles.yaml @@ -28,7 +28,7 @@ tileset: pixel_scale: 256 languages: - am # Amharic - - ar # Arabic + - ar # Arabic - az # Azerbaijani, Latin - be # Belarusian - bg # Bulgarian diff --git a/quickstart.sh b/quickstart.sh index f0bedfe..7970fa6 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -39,7 +39,7 @@ githash=$( git rev-parse HEAD ) # Options to run with docker and docker-compose - ensure the container is destroyed on exit, # as well as pass any other common parameters. # In the future this should use -u $(id -u "$USER"):$(id -g "$USER") instead of running docker as root. -DC_OPTS="--rm" +DC_OPTS="--rm -u $(id -u "$USER"):$(id -g "$USER")" log_file=./quickstart.log rm -f $log_file @@ -57,7 +57,7 @@ docker --version docker-compose --version # based on: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers -function version { echo "$@" | tr -cs '0-9.' '.' | gawk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; } +function version { echo "$@" | tr -cs '0-9.' '.' | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; } COMPOSE_VER=$(docker-compose version --short) if [ "$(version "$COMPOSE_VER")" -lt "$(version "$MIN_COMPOSE_VER")" ]; then @@ -130,11 +130,8 @@ docker images | grep openmaptiles echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : Making directories - if they don't exist ( ./build ./data ./pgdata ) " -mkdir -p pgdata -mkdir -p build -mkdir -p data -mkdir -p wikidata +echo "====> : Create directories if they don't exist" +make init-dirs echo " " echo "-------------------------------------------------------------------------------------" @@ -186,7 +183,7 @@ echo "-------------------------------------------------------------------------- echo "====> : Start PostgreSQL service ; create PostgreSQL data volume " echo " : Source code: https://github.com/openmaptiles/postgis " echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" -docker-compose up -d postgres +make db-start echo " " echo "-------------------------------------------------------------------------------------" @@ -201,15 +198,7 @@ echo "====> : Start importing water data from http://osmdata.openstreetmap.de/ i echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-water " echo " : Data license: https://osmdata.openstreetmap.de/info/license.html " echo " : Thank you: https://osmdata.openstreetmap.de/info/ " -docker-compose run $DC_OPTS import-water - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Start importing border data from http://openstreetmap.org into PostgreSQL " -echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osmborder" -echo " : Data license: http://www.openstreetmap.org/copyright" -echo " : Thank you: https://github.com/pnorman/osmborder " -docker-compose run $DC_OPTS import-osmborder +make import-water echo " " echo "-------------------------------------------------------------------------------------" @@ -217,7 +206,7 @@ echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-natural-earth " echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use " echo " : Thank you: Natural Earth Contributors! " -docker-compose run $DC_OPTS import-natural-earth +make import-natural-earth echo " " echo "-------------------------------------------------------------------------------------" @@ -225,7 +214,7 @@ echo "====> : Start importing OpenStreetMap Lakelines data " echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-lakelines " echo " : https://github.com/lukasmartinelli/osm-lakelines " echo " : Data license: .. " -docker-compose run $DC_OPTS import-lakelines +make import-lakelines echo " " echo "-------------------------------------------------------------------------------------" @@ -235,23 +224,23 @@ echo " : Thank you Omniscale! " echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm " echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) " echo " : Thank you OpenStreetMap Contributors ! " -docker-compose run $DC_OPTS import-osm +make import-osm echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : Start importing Wikidata: ./wikidata/latest-all.json.gz -> PostgreSQL" -echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-wikidata " -echo " : The Wikidata license: https://www.wikidata.org/wiki/Wikidata:Database_download/en#License " -echo " : Thank you Wikidata Contributors ! " -docker-compose run $DC_OPTS import-wikidata +echo "====> : Start importing border data from ./data/${testdata} into PostgreSQL using osmborder" +echo " : Source code: https://github.com/pnorman/osmborder" +echo " : Data license: http://www.openstreetmap.org/copyright" +echo " : Thank you: Paul Norman" +make import-borders echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Start SQL postprocessing: ./build/tileset.sql -> PostgreSQL " -echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-sql " +echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/blob/master/bin/import-sql" # If the output contains a WARNING, stop further processing # Adapted from https://unix.stackexchange.com/questions/307562 -docker-compose run $DC_OPTS openmaptiles-tools import-sql | \ +make import-sql | \ awk -v s=": WARNING:" '$0~s{print; print "\n*** WARNING detected, aborting"; exit(1)} 1' echo " " @@ -259,6 +248,13 @@ echo "-------------------------------------------------------------------------- echo "====> : Analyze PostgreSQL tables" make psql-analyze +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start importing Wikidata: Wikidata Query Service -> PostgreSQL" +echo " : The Wikidata license: CC0 - https://www.wikidata.org/wiki/Wikidata:Main_Page " +echo " : Thank you Wikidata Contributors ! " +make import-wikidata + echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Testing PostgreSQL tables to match layer definitions metadata" @@ -276,19 +272,12 @@ echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-til echo " : " echo " : You will see a lot of deprecated warning in the log! This is normal! " echo " : like : Mapnik LOG> ... is deprecated and will be removed in Mapnik 4.x ... " - -docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml run $DC_OPTS generate-vectortiles - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Add special metadata to mbtiles! " -docker-compose run $DC_OPTS openmaptiles-tools generate-metadata ./data/tiles.mbtiles -docker-compose run $DC_OPTS openmaptiles-tools chmod 666 ./data/tiles.mbtiles +make generate-tiles echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )" -docker-compose stop postgres +make db-stop echo " " echo "-------------------------------------------------------------------------------------" @@ -322,13 +311,6 @@ echo "====> : (disk space) We have created a lot of docker images: " echo " : Hint: you can remove with: docker rmi IMAGE " docker images | grep openmaptiles - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:" -echo " : Hint: you can remove with : docker volume rm openmaptiles_pgdata " -docker volume ls -q | grep openmaptiles - echo " " echo "-------------------------------------------------------------------------------------" echo "====> : (disk space) We have created the new vectortiles ( ./data/tiles.mbtiles ) "