diff --git a/.env b/.env
index aa66d0c..70d706a 100644
--- a/.env
+++ b/.env
@@ -1,12 +1,16 @@
+# This file defines default environment variables for all images
+
+TOOLS_VERSION=4.1.0
+
POSTGRES_DB=openmaptiles
POSTGRES_USER=openmaptiles
POSTGRES_PASSWORD=openmaptiles
POSTGRES_HOST=postgres
POSTGRES_PORT=5432
+
QUICKSTART_MIN_ZOOM=0
QUICKSTART_MAX_ZOOM=7
DIFF_MODE=false
-TOOLS_VERSION=4.1.0
BBOX=-180.0,-85.0511,180.0,85.0511
MIN_ZOOM=0
diff --git a/.github/workflows/omt_ci.yml b/.github/workflows/omt_ci.yml
deleted file mode 100644
index a2230d5..0000000
--- a/.github/workflows/omt_ci.yml
+++ /dev/null
@@ -1,45 +0,0 @@
-# MapTiler OpenMapTiles
-#######################
-
-# Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo
-
-name: OMT_CI
-
-# Controls when the action will run. Triggers the workflow on push and pull request
-# events but only for the master branch
-on:
- push:
- branches: [ master ]
- pull_request:
- branches: [ master ]
-
-# jobs can run parallel
-jobs:
- # This workflow contains a single job called "build"
- build:
- # runs on ubuntu (can run on windows and mac os)
- runs-on: ubuntu-latest
-
- # Steps represent a sequence of tasks that will be executed as part of the job
- steps:
- # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- - uses: actions/checkout@v2
-
- # Named steps
- - name: generate all zooms
- run: sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env
-
- # Runs quickstart
- - name: quickstart
- env:
- area: northamptonshire
- run: bash ./quickstart.sh $area
-
- - name: generate devdoc
- run: TEST_MODE=yes make generate-devdoc
-
- # todo: use artifact to store result of tests
- #- uses: actions/upload-artifact@v1
- # with:
- # name: quickstart log file
- # path: quickstart.log
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 0000000..886ec47
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,200 @@
+# Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo
+
+name: OpenMapTiles CI
+
+on:
+ push:
+ branches: [ master ]
+ pull_request:
+
+jobs:
+
+ integrity_test:
+ name: Run integrity test
+ runs-on: ubuntu-latest
+ steps:
+
+ - name: Checkout the changes
+ uses: actions/checkout@v2
+
+ - name: Run quickstart for a small area
+ env:
+ area: monaco
+ run: |
+ # For now, change the quickstart values directly in the .env file
+ # TODO: We should probably use env vars instead
+ sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env
+ ./quickstart.sh $area
+
+ - name: Save quickstart.log
+ uses: actions/upload-artifact@v1
+ with:
+ name: quickstart.log
+ path: quickstart.log
+
+ - name: Test etldoc images
+ run: |
+ export TEST_MODE=yes
+ make generate-devdoc
+
+ performance:
+ name: Evaluate performance
+ runs-on: ubuntu-latest
+ # Even though we technically don't need to wait for integrity test to finish,
+ # there is no point to run long perf test until we know the code is OK
+ needs: integrity_test
+ env:
+ # Smaller tests (runs everything in about 30 minutes)
+ TEST_PERF_PARAMS: "--minzoom 0 --maxzoom 14 --test equatorial-guinea --test liechtenstein"
+ TEST_DATA_URL: "https://drive.google.com/uc?export=download&id=12vw07f9W0MiAHIqMztRiIMwahJfqTi21"
+
+ ## Large test data -- we should switch to it after everything is working ok
+ # TEST_PERF_PARAMS: "--minzoom 0 --maxzoom 14 --test hungary --test isle-of-man"
+ # TEST_DATA_URL: "https://drive.google.com/uc?export=download&id=1kw7XPDPd1Rc-Zi2XxGLTXdinUSq-S4pT"
+ steps:
+ - name: Cache test data download
+ id: cache-testdata
+ uses: actions/cache@v1
+ with:
+ path: ci_cache
+ key: "${{ env.TEST_DATA_URL }}"
+
+ - name: Download test data on cache miss
+ if: steps.cache-testdata.outputs.cache-hit != 'true'
+ run: |
+ echo "Data file does not exist, downloading $TEST_DATA_URL"
+ mkdir -p ci_cache
+ curl --silent --show-error --location --output ci_cache/perf-test-areas-latest.osm.pbf "$TEST_DATA_URL"
+
+ - name: Get code
+ uses: actions/checkout@v2
+ with:
+ # Fetch the last two commits in case this is a PR,
+ # and we need to profile the base branch first
+ fetch-depth: 2
+ path: code
+
+ - name: Compute git revision hash to cache
+ id: calc
+ run: |
+ # If this is a pull request, we should cache the parent (base) revision
+ # Otherwise cache the current one
+ cd code
+ REV_HASH=$(git log -1 --format="%H")
+ if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+ # Take the first parent of the grafted commit (cannot use HEAD^1 with shallow clones)
+ REV_HASH=$(git cat-file -p $REV_HASH | awk 'NR > 1 {if(/^parent/){print $2; exit}}')
+ fi
+ echo "::set-output name=hash::$REV_HASH"
+
+ - name: Set up caching for the performance results
+ uses: actions/cache@v1
+ with:
+ path: perf_cache
+ key: "${{ steps.calc.outputs.hash }}-${{ env.TEST_DATA_URL }}"
+
+ - name: Load test data into DB and run performance test
+ id: main
+ env:
+ CACHE_SHA: "${{ steps.calc.outputs.hash }}"
+ run: |
+ create_db() {
+ make clean
+ make init-dirs
+ cp ../ci_cache/perf-test-areas-latest.osm.pbf data/perf-test-areas-latest.osm.pbf
+ make db-destroy
+ make all
+ make db-start
+ time make import-data
+ time make import-osm
+ time make import-borders
+ time make import-wikidata
+ time make import-sql
+ }
+
+ mkdir -p perf_cache
+ mkdir -p artifacts
+ cd code
+
+ CURRENT_SHA=$(git log -1 --format="%H")
+
+ if [ ! -f ../perf_cache/results.json ]; then
+ echo "We do not have cached performance results, create them..."
+ if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+ git reset --hard ${CURRENT_SHA}^1
+ fi
+
+ create_db
+
+ # Use latest tools version because these specific tests do not yet exist in the 4.1 tools version
+ # Custom TOOLS_VERSION can be removed once OMT master is migrated to the next tools version
+ TOOLS_VERSION=latest docker-compose run --rm openmaptiles-tools \
+ test-perf openmaptiles.yaml $TEST_PERF_PARAMS \
+ --record /tileset/results.json
+ mv results.json ../perf_cache
+
+ if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+ # For pull requests, restore to the PR version before continuing
+ git reset --hard ${CURRENT_SHA}
+ fi
+ else
+ echo "Found cached performance results"
+ fi
+
+ if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+ cp ../perf_cache/results.json ../artifacts/base-results.json
+ else
+ cp ../perf_cache/results.json ../artifacts/results.json
+ fi
+
+ if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+ echo "Comparing pull request results with the base..."
+
+ create_db
+
+ # Use latest tools version because these specific tests do not yet exist in the 4.1 tools version
+ # Custom TOOLS_VERSION can be removed once OMT master is migrated to the next tools version
+ cp ../perf_cache/results.json .
+ OUTPUT="$(TOOLS_VERSION=latest docker-compose run --rm openmaptiles-tools \
+ test-perf openmaptiles.yaml $TEST_PERF_PARAMS \
+ --compare /tileset/results.json --record /tileset/pr-results.json)"
+ rm results.json
+ mv pr-results.json ../artifacts/
+
+ # Convert multiline output into a single long string.
+ # See https://github.community/t5/GitHub-Actions/set-output-Truncates-Multiline-Strings/td-p/37870
+ OUTPUT="${OUTPUT//'%'/'%25'}"
+ OUTPUT="${OUTPUT//$'\n'/'%0A'}"
+ OUTPUT="${OUTPUT//$'\r'/'%0D'}"
+
+ # Split into two parts -- before and after the ===== SUMMARY =====
+ echo "::set-output name=summary::${OUTPUT##*========}"
+ echo "::set-output name=details::${OUTPUT%%========*}"
+ fi
+
+ - name: Save artifacts
+ uses: actions/upload-artifact@v1
+ with:
+ name: performance_results
+ path: artifacts
+
+ - name: Post a comment on Pull Request
+ if: "github.event_name == 'pull_request'"
+ uses: marocchino/sticky-pull-request-comment@v1
+ timeout-minutes: 1
+ continue-on-error: true
+ with:
+ message: |-
+ ```
+ ${{ steps.main.outputs.summary }}
+ ```
+
+
+ expand for details...
+
+ ```
+ ${{ steps.main.outputs.details }}
+ ```
+
+
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/Makefile b/Makefile
index 775f670..19acae2 100644
--- a/Makefile
+++ b/Makefile
@@ -24,7 +24,7 @@ else
endif
.PHONY: all
-all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql
+all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build-sql
# Set OpenMapTiles host
OMT_HOST:=http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost)
@@ -38,8 +38,9 @@ help:
@echo " ./quickstart.sh <> # example: ./quickstart.sh madagascar "
@echo " "
@echo "Hints for designers:"
- @echo " make start-postserve # start Postserver + Maputnik Editor [ see $(OMT_HOST):8088 ] "
- @echo " make start-tileserver # start maptiler/tileserver-gl [ see $(OMT_HOST):8080 ] "
+ @echo " make maputnik-start # start Maputnik Editor + dynamic tile server [ see $(OMT_HOST):8088 ]"
+ @echo " make postserve-start # start dynamic tile server [ see $(OMT_HOST):8090 ]"
+ @echo " make tileserver-start # start maptiler/tileserver-gl [ see $(OMT_HOST):8080 ]"
@echo " "
@echo "Hints for developers:"
@echo " make # build source code"
@@ -53,10 +54,12 @@ help:
@echo " make psql-analyze # PostgreSQL: ANALYZE"
@echo " make generate-qareports # generate reports [./build/qareports]"
@echo " make generate-devdoc # generate devdoc including graphs for all layers [./layers/...]"
+ @echo " make tools-dev # start openmaptiles-tools /bin/bash terminal"
+ @echo " make db-destroy # remove docker containers and PostgreSQL data volume"
+ @echo " make db-start # start PostgreSQL, creating it if it doesn't exist"
+ @echo " make db-stop # stop PostgreSQL database without destroying the data"
@echo " make import-sql-dev # start import-sql /bin/bash terminal"
@echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)"
- @echo " make clean-docker # remove docker containers, PG data volume"
- @echo " make forced-clean-sql # drop all PostgreSQL tables for clean environment"
@echo " make docker-unnecessary-clean # clean unnecessary docker image(s) and container(s)"
@echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB"
@echo " make remove-docker-images # remove openmaptiles docker images"
@@ -69,7 +72,9 @@ help:
.PHONY: init-dirs
init-dirs:
- mkdir -p build && mkdir -p data && mkdir -p cache
+ @mkdir -p build
+ @mkdir -p data
+ @mkdir -p cache
build/openmaptiles.tm2source/data.yml: init-dirs
mkdir -p build/openmaptiles.tm2source
@@ -78,15 +83,16 @@ build/openmaptiles.tm2source/data.yml: init-dirs
build/mapping.yaml: init-dirs
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@
-build/tileset.sql: init-dirs
- $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > $@
+.PHONY: build-sql
+build-sql: init-dirs
+ $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > build/tileset.sql
.PHONY: clean
clean:
rm -rf build
-.PHONY: clean-docker
-clean-docker:
+.PHONY: db-destroy
+db-destroy:
$(DOCKER_COMPOSE) down -v --remove-orphans
$(DOCKER_COMPOSE) rm -fv
docker volume ls -q -f "name=^$${DC_PROJECT,,*}_" | $(XARGS) docker volume rm
@@ -94,14 +100,19 @@ clean-docker:
.PHONY: db-start
db-start:
- $(DOCKER_COMPOSE) up -d postgres
+ $(DOCKER_COMPOSE) up --no-recreate -d postgres
@echo "Wait for PostgreSQL to start..."
$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./pgwait.sh
.PHONY: db-stop
db-stop:
+ @echo "Stopping PostgreSQL..."
$(DOCKER_COMPOSE) stop postgres
+.PHONY: list-geofabrik
+list-geofabrik:
+ $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list geofabrik
+
OSM_SERVERS:=geofabrik osmfr bbbike
ALL_DOWNLOADS:=$(addprefix download-,$(OSM_SERVERS))
OSM_SERVER=$(patsubst download-%,%,$@)
@@ -135,28 +146,20 @@ psql: db-start
import-osm: db-start all
$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm
-.PHONY: import-sql
-import-sql: db-start all
- $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql
-
.PHONY: import-osmsql
import-osmsql: db-start all import-osm import-sql
+.PHONY: import-data
+import-data: db-start
+ $(DOCKER_COMPOSE) run $(DC_OPTS) import-data
+
.PHONY: import-borders
import-borders: db-start
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders
-.PHONY: import-water
-import-water: db-start
- $(DOCKER_COMPOSE) run $(DC_OPTS) import-water
-
-.PHONY: import-natural-earth
-import-natural-earth: db-start
- $(DOCKER_COMPOSE) run $(DC_OPTS) import-natural-earth
-
-.PHONY: import-lakelines
-import-lakelines: db-start
- $(DOCKER_COMPOSE) run $(DC_OPTS) import-lakelines
+.PHONY: import-sql
+import-sql: db-start all
+ $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql
.PHONY: generate-tiles
ifneq ($(wildcard data/docker-compose-config.yml),)
@@ -164,11 +167,13 @@ ifneq ($(wildcard data/docker-compose-config.yml),)
endif
generate-tiles: init-dirs db-start all
rm -rf data/tiles.mbtiles
+ echo "Generating tiles ..."; \
$(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles
+ @echo "Updating generated tile metadata ..."
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles
-.PHONY: start-tileserver
-start-tileserver: init-dirs
+.PHONY: tileserver-start
+tileserver-start: init-dirs
@echo " "
@echo "***********************************************************"
@echo "* "
@@ -188,29 +193,42 @@ start-tileserver: init-dirs
@echo " "
docker run $(DC_OPTS) -it --name tileserver-gl -v $$(pwd)/data:/data -p 8080:8080 maptiler/tileserver-gl --port 8080
-.PHONY: start-postserve
-start-postserve: db-start
+.PHONY: postserve-start
+postserve-start: db-start
@echo " "
@echo "***********************************************************"
@echo "* "
@echo "* Bring up postserve at $(OMT_HOST):8090"
+ @echo "* --> can view it locally (use make maputnik-start)"
+ @echo "* --> or can use https://maputnik.github.io/editor"
+ @echo "* "
+ @echo "* set data source / TileJSON URL to http://$(OMT_HOST):8090"
@echo "* "
@echo "***********************************************************"
@echo " "
$(DOCKER_COMPOSE) up -d postserve
- docker pull maputnik/editor
+
+.PHONY: postserve-stop
+postserve-stop:
+ $(DOCKER_COMPOSE) stop postserve
+
+.PHONY: maputnik-start
+maputnik-start: maputnik-stop postserve-start
@echo " "
@echo "***********************************************************"
@echo "* "
@echo "* Start maputnik/editor "
@echo "* ---> go to http://$(OMT_HOST):8088 "
- @echo "* ---> set 'data source' to http://$(OMT_HOST):8090"
+ @echo "* ---> set data source / TileJSON URL to http://$(OMT_HOST):8090"
@echo "* "
@echo "***********************************************************"
@echo " "
- -docker rm -f maputnik_editor
docker run $(DC_OPTS) --name maputnik_editor -d -p 8088:8888 maputnik/editor
+.PHONY: maputnik-stop
+maputnik-stop:
+ -docker rm -f maputnik_editor
+
.PHONY: generate-qareports
generate-qareports:
./qa/run.sh
@@ -219,12 +237,12 @@ generate-qareports:
.PHONY: generate-devdoc
generate-devdoc: init-dirs
mkdir -p ./build/devdoc && \
- $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools-latest sh -c \
+ $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \
'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \
generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)'
-.PHONY: import-sql-dev
-import-sql-dev:
+.PHONY: tools-dev
+tools-dev:
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash
.PHONY: import-osm-dev
@@ -233,7 +251,7 @@ import-osm-dev:
.PHONY: import-wikidata
import-wikidata:
- $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml
+ $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata --cache /cache/wikidata-cache.json openmaptiles.yaml
.PHONY: psql-pg-stat-reset
psql-pg-stat-reset:
@@ -277,7 +295,13 @@ list-docker-images:
.PHONY: refresh-docker-images
refresh-docker-images:
+ifneq ($(strip $(NO_REFRESH)),)
+ @echo "Skipping docker image refresh"
+else
+ @echo ""
+ @echo "Refreshing docker images... Use NO_REFRESH=1 to skip."
$(DOCKER_COMPOSE) pull --ignore-pull-failures
+endif
.PHONY: remove-docker-images
remove-docker-images:
diff --git a/README.md b/README.md
index 1da2a97..5e34b48 100644
--- a/README.md
+++ b/README.md
@@ -100,9 +100,7 @@ make db-start
Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines).
```bash
-make import-water
-make import-natural-earth
-make import-lakelines
+make import-data
```
[Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory.
@@ -119,7 +117,7 @@ make download-geofabrik area=albania
```
[Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from
-`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definiton. Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool.
+`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definition. Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool.
```bash
make import-osm
diff --git a/docker-compose.yml b/docker-compose.yml
index 41087d4..75c479f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,9 +1,17 @@
-version: "2"
+# This version must match the MAKE_DC_VERSION value below
+version: "2.3"
+
volumes:
pgdata:
+
+networks:
+ postgres_conn:
+ driver: bridge
+
services:
+
postgres:
- image: "openmaptiles/postgis:${TOOLS_VERSION}"
+ image: "${POSTGIS_IMAGE:-openmaptiles/postgis}:${TOOLS_VERSION}"
volumes:
- pgdata:/var/lib/postgresql/data
networks:
@@ -11,21 +19,13 @@ services:
ports:
- "5432"
env_file: .env
- import-natural-earth:
- image: "openmaptiles/import-natural-earth:${TOOLS_VERSION}"
- env_file: .env
- networks:
- - postgres_conn
- import-water:
- image: "openmaptiles/import-water:${TOOLS_VERSION}"
- env_file: .env
- networks:
- - postgres_conn
- import-lakelines:
- image: "openmaptiles/import-lakelines:${TOOLS_VERSION}"
+
+ import-data:
+ image: "openmaptiles/import-data:${TOOLS_VERSION}"
env_file: .env
networks:
- postgres_conn
+
import-osm:
image: "openmaptiles/import-osm:${TOOLS_VERSION}"
env_file: .env
@@ -37,6 +37,7 @@ services:
- ./data:/import
- ./build:/mapping
- ./cache:/cache
+
import-osm-diff:
image: "openmaptiles/import-osm:${TOOLS_VERSION}"
env_file: .env
@@ -49,6 +50,7 @@ services:
- ./data:/import
- ./build:/mapping
- ./cache:/cache
+
update-osm:
image: "openmaptiles/import-osm:${TOOLS_VERSION}"
env_file: .env
@@ -61,26 +63,25 @@ services:
- ./data:/import
- ./build:/mapping
- ./cache:/cache
+
openmaptiles-tools:
image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
env_file: .env
+ environment:
+ # Must match the version of this file (first line)
+ # download-osm will use it when generating a composer file
+ MAKE_DC_VERSION: "2.3"
+ # Allow DIFF_MODE to be overwritten from shell
+ DIFF_MODE: ${DIFF_MODE}
networks:
- postgres_conn
volumes:
- .:/tileset
- ./data:/import
- ./build:/sql
- openmaptiles-tools-latest:
- # This target exists for experimental tools that have not yet been published.
- # Do not use this for production.
- image: "openmaptiles/openmaptiles-tools:latest"
- env_file: .env
- networks:
- - postgres_conn
- volumes:
- - .:/tileset
- - ./data:/import
- - ./build:/sql
+ - ./build:/mapping
+ - ./cache:/cache
+
generate-changed-vectortiles:
image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
command: ./export-list.sh
@@ -90,6 +91,7 @@ services:
networks:
- postgres_conn
env_file: .env
+
generate-vectortiles:
image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
volumes:
@@ -102,6 +104,7 @@ services:
BBOX: ${BBOX}
MIN_ZOOM: ${MIN_ZOOM}
MAX_ZOOM: ${MAX_ZOOM}
+
postserve:
image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
command: postserve openmaptiles.yaml --verbose
@@ -112,7 +115,3 @@ services:
- "8090:8090"
volumes:
- .:/tileset
-
-networks:
- postgres_conn:
- driver: bridge
diff --git a/layers/building/building.sql b/layers/building/building.sql
index 95f729c..81f39fd 100644
--- a/layers/building/building.sql
+++ b/layers/building/building.sql
@@ -72,7 +72,8 @@ CREATE OR REPLACE VIEW osm_all_buildings AS (
FALSE as hide_3d
FROM
osm_building_polygon obp
- WHERE osm_id < 0
+ -- OSM mulipolygons once imported can give unique postgis polygons with holes, or multi parts polygons
+ WHERE osm_id < 0 AND ST_GeometryType(geometry) IN ('ST_Polygon', 'ST_MultiPolygon')
UNION ALL
-- etldoc: osm_building_polygon -> layer_building:z14_
@@ -88,7 +89,8 @@ CREATE OR REPLACE VIEW osm_all_buildings AS (
FROM
osm_building_polygon obp
LEFT JOIN osm_building_relation obr ON (obr.member = obp.osm_id)
- WHERE obp.osm_id >= 0
+ -- Only check for ST_Polygon as we exclude buildings from relations keeping only positive ids
+ WHERE obp.osm_id >= 0 AND ST_GeometryType(obp.geometry) = 'ST_Polygon'
);
CREATE OR REPLACE FUNCTION layer_building(bbox geometry, zoom_level int)
diff --git a/layers/poi/poi.yaml b/layers/poi/poi.yaml
index d678bd6..ed49fea 100644
--- a/layers/poi/poi.yaml
+++ b/layers/poi/poi.yaml
@@ -146,10 +146,10 @@ layer:
query: (SELECT osm_id, geometry, name, name_en, name_de, {name_languages}, class, subclass, agg_stop, layer, level, indoor, rank FROM layer_poi(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t
schema:
- ./public_transport_stop_type.sql
- - ./update_poi_polygon.sql
- - ./update_poi_point.sql
- ./class.sql
- ./poi_stop_agg.sql
+ - ./update_poi_polygon.sql
+ - ./update_poi_point.sql
- ./layer.sql
datasources:
- type: imposm3
diff --git a/layers/poi/poi_stop_agg.sql b/layers/poi/poi_stop_agg.sql
index 788bcda..4e4f40a 100644
--- a/layers/poi/poi_stop_agg.sql
+++ b/layers/poi/poi_stop_agg.sql
@@ -33,6 +33,3 @@ CREATE MATERIALIZED VIEW osm_poi_stop_rank AS (
subclass IN ('bus_stop', 'bus_station', 'tram_stop', 'subway')
ORDER BY p.uic_ref, rk
) /* DELAY_MATERIALIZED_VIEW_CREATION */;
-
-ALTER TABLE osm_poi_point ADD COLUMN IF NOT EXISTS agg_stop INTEGER DEFAULT NULL;
-SELECT update_osm_poi_point_agg();
diff --git a/layers/poi/update_poi_point.sql b/layers/poi/update_poi_point.sql
index dddec1d..bd6aeb4 100644
--- a/layers/poi/update_poi_point.sql
+++ b/layers/poi/update_poi_point.sql
@@ -43,6 +43,9 @@ BEGIN
END;
$$ LANGUAGE plpgsql;
+ALTER TABLE osm_poi_point ADD COLUMN IF NOT EXISTS agg_stop INTEGER DEFAULT NULL;
+SELECT update_osm_poi_point_agg();
+
-- Handle updates
CREATE SCHEMA IF NOT EXISTS poi_point;
diff --git a/quickstart.sh b/quickstart.sh
index 4af930b..0108343 100755
--- a/quickstart.sh
+++ b/quickstart.sh
@@ -105,8 +105,8 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc )
echo "system memory (GB): ${mem}"
grep SwapTotal /proc/meminfo
- echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1)"
- cat /proc/meminfo | grep Free
+ echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)"
+ grep Free /proc/meminfo
else
echo " "
echo "Warning : Platforms other than Linux are less tested"
@@ -116,11 +116,11 @@ fi
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Stopping running services & removing old containers"
-make clean-docker
+make db-destroy
echo " "
echo "-------------------------------------------------------------------------------------"
-echo "====> : Checking OpenMapTiles docker images "
+echo "====> : Existing OpenMapTiles docker images. Will use version $(source .env && echo "$TOOLS_VERSION")"
docker images | grep openmaptiles
echo " "
@@ -162,7 +162,7 @@ echo "--------------------------------------------------------------------------
echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )"
echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools "
echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! "
-make
+make all
echo " "
echo "-------------------------------------------------------------------------------------"
@@ -178,29 +178,18 @@ echo "====> : Drop and Recreate PostgreSQL public schema "
# This adds an extra safety belt if the user modifies the docker volume settings
make forced-clean-sql
-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Start importing water data from http://osmdata.openstreetmap.de/ into PostgreSQL "
-echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-water "
-echo " : Data license: https://osmdata.openstreetmap.de/info/license.html "
-echo " : Thank you: https://osmdata.openstreetmap.de/info/ "
-make import-water
-
-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL "
-echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-natural-earth "
-echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use "
-echo " : Thank you: Natural Earth Contributors! "
-make import-natural-earth
-
-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Start importing OpenStreetMap Lakelines data "
-echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-lakelines "
-echo " : https://github.com/lukasmartinelli/osm-lakelines "
-echo " : Data license: .. "
-make import-lakelines
+echo "====> : Importing all the data:"
+echo " : * Water data from http://osmdata.openstreetmap.de"
+echo " : Data license: https://osmdata.openstreetmap.de/info/license.html"
+echo " : * Natural Earth from http://www.naturalearthdata.com"
+echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use"
+echo " : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines"
+echo " :"
+echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data"
+echo " : includes all data from the import-data image"
+echo " :"
+echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !"
+make import-data
echo " "
echo "-------------------------------------------------------------------------------------"
@@ -269,16 +258,13 @@ echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Inputs - Outputs md5sum for debugging "
rm -f ./data/quickstart_checklist.chk
-md5sum build/mapping.yaml >> ./data/quickstart_checklist.chk
-md5sum build/tileset.sql >> ./data/quickstart_checklist.chk
-md5sum build/openmaptiles.tm2source/data.yml >> ./data/quickstart_checklist.chk
-md5sum "./data/${testdata}" >> ./data/quickstart_checklist.chk
-md5sum ./data/tiles.mbtiles >> ./data/quickstart_checklist.chk
-md5sum ./data/docker-compose-config.yml >> ./data/quickstart_checklist.chk
+{
+ find build -type f | sort | xargs md5sum ;
+ find data -type f | sort | xargs md5sum ;
+} >> ./data/quickstart_checklist.chk
cat ./data/quickstart_checklist.chk
ENDTIME=$(date +%s)
-ENDDATE=$(date +"%Y-%m-%dT%H:%M%z")
if stat --help >/dev/null 2>&1; then
MODDATE=$(stat -c %y "./data/${testdata}" )
else
@@ -312,8 +298,8 @@ echo "We saved the log file to $log_file ( for debugging ) You can compare with
echo " "
echo "Start experimenting! And check the QUICKSTART.MD file!"
echo " "
-echo "* Use make start-postserve to explore tile generation on request"
-echo "* Use make start-tileserver to view pre-generated tiles"
+echo "* Use make maputnik-start to explore tile generation on request"
+echo "* Use make tileserver-start to view pre-generated tiles"
echo " "
echo "Available help commands (make help) "
make help