From 4028b6b49c03bed6b5f1bdc9524a784686fd2676 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Tue, 15 Nov 2016 23:02:28 +0100 Subject: [PATCH 01/39] improved ./quickstart.sh ( more comments, debug infos, more safe ) --- .gitignore | 9 ++++ Makefile | 3 ++ quickstart.sh | 135 ++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 138 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index e5d43ba..b7b1da6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,12 @@ data.xml *.thumb.png *.swp *.pbf + +# quickstart log +quickstart.log + +# imput / output data +data/* + +# generated sourece files +build/* \ No newline at end of file diff --git a/Makefile b/Makefile index ca51ae4..c6c3626 100644 --- a/Makefile +++ b/Makefile @@ -53,3 +53,6 @@ layers/housenumber/README.md: clean: rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql && rm -f layers/**/README.md&& rm -f layers/**/*.png + +clean_build: + rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql diff --git a/quickstart.sh b/quickstart.sh index ffcfa8a..c247f6a 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -3,44 +3,161 @@ set -o errexit set -o pipefail set -o nounset -#Check installed versions -docker --version +## +## OpenMapTiles quickstart.sh +## + +STARTTIME=$(date +%s) +STARTDATE=$(date -Iminutes) +githash=$( git rev-parse HEAD ) + +log_file=quickstart.log +rm -f $log_file +exec &> >(tee -a "$log_file") + +echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] " +echo " : This will be logged to the $log_file file ( for debugging) and to the screen" +echo " : git version: $githash / started: $STARTDATE " +echo " : Your system is:" +lsb_release -a + +echo "====> : Please check the docker and docker-compose version!" +echo " : We are using docker-compose V2 file format ! see more: https://docs.docker.com/ " +echo " : (theoretically;not tested) minumum Docker version is 1.10.0+. " +echo " : (theoretically;not tested) minimum Docker-compose version is 1.6.0+." +echo " : See the .travis testfile for the current supported versions " +echo " : Your docker systems is:" +docker --version docker-compose --version -#Remove +echo "====> : Checking OpenMapTiles docker images " +docker images | grep openmaptiles + +echo "====> : Stop running services & Removing old containers " docker-compose down docker-compose rm -fv -echo "Remove old volume" + +echo "====> : For a clean start, We are removing old postgresql data volume ( if exists )" docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true -echo "Make directories " +echo "====> : Making directories - if not exists (./build ./data ) " mkdir -p build mkdir -p data +echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) " +rm -f ./data/*.mbtiles + testdata=zurich_switzerland.osm.pbf +testdataurl=https://s3.amazonaws.com/metro-extracts.mapzen.com/$testdata if [ ! -f ./data/${testdata} ]; then - echo "Download $testdata " + echo "====> : The testdata downloading $testdata " rm -f ./data/* - wget https://s3.amazonaws.com/metro-extracts.mapzen.com/zurich_switzerland.osm.pbf -P ./data + wget $testdataurl -P ./data +else + echo "====> : The testdata ./data/$testdata exists, we don't download! " fi +echo " " +echo "====> : Clean old generated source files ( ./build/* ) ( if exists ) " +docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make clean_build +echo " " +echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )" +echo " : the tool source code: https://github.com/openmaptiles/openmaptiles-tools " +echo " : but we generate code from this directory informations! " docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make + +echo " " +echo "====> : Start PostgreSQL service ; create PostgreSQL data volume " +echo " : source code: https://github.com/openmaptiles/postgis " +echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" docker-compose up -d postgres sleep 30 +echo " " +echo "====> : Start Importing water data from http://openstreetmapdata.com into PostgreSQL " +echo " : source code: https://github.com/openmaptiles/import-water " +echo " : Data license: http://openstreetmapdata.com/info/license " +echo " : Thank you: http://openstreetmapdata.com/info/supporting " docker-compose run --rm import-water + +echo " " +echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL " +echo " : source code: https://github.com/openmaptiles/import-natural-earth " +echo " : term-of-use: http://www.naturalearthdata.com/about/terms-of-use " +echo " : Thank you: Natural Earth Contributors! " docker-compose run --rm import-natural-earth + +echo " " +echo "====> : Start importing OpenStreetMap Lakelines data " +echo " : Source code: https://github.com/openmaptiles/import-lakelines " +echo " : https://github.com/lukasmartinelli/osm-lakelines " +echo " : Data license: .. " docker-compose run --rm import-lakelines + +echo " " +echo "====> : Start importing OpenStreetMap data: ./data/${testdata} -> imposm3[./build/mapping.yaml] -> PostgreSQL" +echo " : Imposm3 documentation: https://imposm.org/docs/imposm3/latest/index.html " +echo " : Thank you Omniscale! " +echo " : source code: https://github.com/openmaptiles/import-osm " +echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) " +echo " : Thank you OpenStreetMap Contributors ! " docker-compose run --rm import-osm + +echo " " +echo "====> : Start SQL postprocessing: ./build/tileset.sql -> PostgreSQL " +echo " : source code: https://github.com/openmaptiles/import-sql " docker-compose run --rm import-sql +echo " " +echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. " +echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml " +echo " : output MBTiles: ./data/tiles.mbtiles " +echo " : source code: https://github.com/openmaptiles/generate-vectortiles " +echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox " +echo " : Thank you https://www.mapbox.com !" +echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-tiles " docker-compose -f docker-compose.yml -f docker-compose-test-override.yml run --rm generate-vectortiles +echo " " +echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )" docker-compose stop postgres -echo "The vectortiles created from $testdata " + +ENDTIME=$(date +%s) +ENDDATE=$(date -Iminutes) +MODDATE=$(stat -c %y ./data/${testdata} ) + +echo " " +echo " " +echo "------------------------------------------------------------" +echo "-- Summary --" +echo "------------------------------------------------------------" +echo " " +echo " " +echo "====> : (disk space) We have created a lot of docker images: " +echo " : hint: you can remove with: docker rmi IMAGE " +docker images | grep openmaptiles + +echo " " +echo "====> : (disk space) We have created this new docker volumes for PostgreSQL data:" +echo " : hint: you can remove with : docker volume rm openmaptiles_pgdata " +docker volume ls -q | grep openmaptiles + +echo " " +echo "====> : (disk space) We have created the new vectortiles ( ./data/tiles.mbtiles ) " +echo " : The OpenMapTiles MBTILES license: ..... " +echo " : We created from $testdata ( file moddate: $MODDATE ) " +echo " : Size: " ls ./data/*.mbtiles -la -echo "Hello ... start experimenting - see docs ! " + +echo " " +echo "============================================================" +echo "The quickstart.sh is finished! " +echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete" +echo "We saved the log file to $log_file ( for debugging ) " +echo " " +echo "Start experimenting -> see the documentations! " +echo "============================================================" From 47569fd09bcf10470250929bc5f2669dfc86aeb7 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Tue, 15 Nov 2016 17:54:53 -0500 Subject: [PATCH 02/39] Fix minor typos --- quickstart.sh | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/quickstart.sh b/quickstart.sh index c247f6a..fd4aa6e 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -16,14 +16,14 @@ rm -f $log_file exec &> >(tee -a "$log_file") echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] " -echo " : This will be logged to the $log_file file ( for debugging) and to the screen" +echo " : This will be logged to the $log_file file ( for debugging ) and to the screen" echo " : git version: $githash / started: $STARTDATE " echo " : Your system is:" lsb_release -a echo "====> : Please check the docker and docker-compose version!" -echo " : We are using docker-compose V2 file format ! see more: https://docs.docker.com/ " -echo " : (theoretically;not tested) minumum Docker version is 1.10.0+. " +echo " : We are using docker-compose V2 file format ! see more: https://docs.docker.com/" +echo " : (theoretically;not tested) minumum Docker version is 1.10.0+." echo " : (theoretically;not tested) minimum Docker-compose version is 1.6.0+." echo " : See the .travis testfile for the current supported versions " echo " : Your docker systems is:" @@ -33,14 +33,14 @@ docker-compose --version echo "====> : Checking OpenMapTiles docker images " docker images | grep openmaptiles -echo "====> : Stop running services & Removing old containers " +echo "====> : Stopping running services & removing old containers " docker-compose down docker-compose rm -fv -echo "====> : For a clean start, We are removing old postgresql data volume ( if exists )" +echo "====> : For a clean start, we are removing old postgresql data volume ( if it exists )" docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true -echo "====> : Making directories - if not exists (./build ./data ) " +echo "====> : Making directories - if they don't exist ( ./build ./data ) " mkdir -p build mkdir -p data @@ -50,41 +50,41 @@ rm -f ./data/*.mbtiles testdata=zurich_switzerland.osm.pbf testdataurl=https://s3.amazonaws.com/metro-extracts.mapzen.com/$testdata if [ ! -f ./data/${testdata} ]; then - echo "====> : The testdata downloading $testdata " + echo "====> : Downloading testdata $testdata " rm -f ./data/* wget $testdataurl -P ./data else - echo "====> : The testdata ./data/$testdata exists, we don't download! " + echo "====> : The testdata ./data/$testdata exists, we don't need to download! " fi echo " " -echo "====> : Clean old generated source files ( ./build/* ) ( if exists ) " +echo "====> : Remove old generated source files ( ./build/* ) ( if they exist ) " docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make clean_build echo " " echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )" -echo " : the tool source code: https://github.com/openmaptiles/openmaptiles-tools " -echo " : but we generate code from this directory informations! " +echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools " +echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! " docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make echo " " echo "====> : Start PostgreSQL service ; create PostgreSQL data volume " -echo " : source code: https://github.com/openmaptiles/postgis " +echo " : Source code: https://github.com/openmaptiles/postgis " echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" docker-compose up -d postgres sleep 30 echo " " -echo "====> : Start Importing water data from http://openstreetmapdata.com into PostgreSQL " -echo " : source code: https://github.com/openmaptiles/import-water " +echo "====> : Start importing water data from http://openstreetmapdata.com into PostgreSQL " +echo " : Source code: https://github.com/openmaptiles/import-water " echo " : Data license: http://openstreetmapdata.com/info/license " echo " : Thank you: http://openstreetmapdata.com/info/supporting " docker-compose run --rm import-water echo " " echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL " -echo " : source code: https://github.com/openmaptiles/import-natural-earth " -echo " : term-of-use: http://www.naturalearthdata.com/about/terms-of-use " +echo " : Source code: https://github.com/openmaptiles/import-natural-earth " +echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use " echo " : Thank you: Natural Earth Contributors! " docker-compose run --rm import-natural-earth @@ -106,14 +106,14 @@ docker-compose run --rm import-osm echo " " echo "====> : Start SQL postprocessing: ./build/tileset.sql -> PostgreSQL " -echo " : source code: https://github.com/openmaptiles/import-sql " +echo " : Source code: https://github.com/openmaptiles/import-sql " docker-compose run --rm import-sql echo " " echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. " echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml " -echo " : output MBTiles: ./data/tiles.mbtiles " -echo " : source code: https://github.com/openmaptiles/generate-vectortiles " +echo " : Output MBTiles: ./data/tiles.mbtiles " +echo " : Source code: https://github.com/openmaptiles/generate-vectortiles " echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox " echo " : Thank you https://www.mapbox.com !" echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-tiles " @@ -135,12 +135,12 @@ echo "------------------------------------------------------------" echo " " echo " " echo "====> : (disk space) We have created a lot of docker images: " -echo " : hint: you can remove with: docker rmi IMAGE " +echo " : Hint: you can remove with: docker rmi IMAGE " docker images | grep openmaptiles echo " " -echo "====> : (disk space) We have created this new docker volumes for PostgreSQL data:" -echo " : hint: you can remove with : docker volume rm openmaptiles_pgdata " +echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:" +echo " : Hint: you can remove with : docker volume rm openmaptiles_pgdata " docker volume ls -q | grep openmaptiles echo " " @@ -156,7 +156,7 @@ echo "The quickstart.sh is finished! " echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete" echo "We saved the log file to $log_file ( for debugging ) " echo " " -echo "Start experimenting -> see the documentations! " +echo "Start experimenting -> see the documentation! " echo "============================================================" From fb09bf7703297b1a475dda4156aa66848fec8fbd Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 16 Nov 2016 01:45:44 +0100 Subject: [PATCH 03/39] add kernel and bash information to the logs --- quickstart.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/quickstart.sh b/quickstart.sh index fd4aa6e..aed9b7a 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -4,7 +4,7 @@ set -o pipefail set -o nounset ## -## OpenMapTiles quickstart.sh +## OpenMapTiles quickstart.sh for x86_64 linux ## STARTTIME=$(date +%s) @@ -15,12 +15,19 @@ log_file=quickstart.log rm -f $log_file exec &> >(tee -a "$log_file") +echo " " echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] " echo " : This will be logged to the $log_file file ( for debugging ) and to the screen" -echo " : git version: $githash / started: $STARTDATE " +echo " : Git version: $githash / Started: $STARTDATE " +echo " : Your bash version: $BASH_VERSION" echo " : Your system is:" lsb_release -a +echo " " +echo " : This is working on x86_64 ; Your kernel is:" +uname -a + +echo " " echo "====> : Please check the docker and docker-compose version!" echo " : We are using docker-compose V2 file format ! see more: https://docs.docker.com/" echo " : (theoretically;not tested) minumum Docker version is 1.10.0+." @@ -30,20 +37,25 @@ echo " : Your docker systems is:" docker --version docker-compose --version +echo " " echo "====> : Checking OpenMapTiles docker images " docker images | grep openmaptiles +echo " " echo "====> : Stopping running services & removing old containers " docker-compose down docker-compose rm -fv +echo " " echo "====> : For a clean start, we are removing old postgresql data volume ( if it exists )" docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true +echo " " echo "====> : Making directories - if they don't exist ( ./build ./data ) " mkdir -p build mkdir -p data +echo " " echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) " rm -f ./data/*.mbtiles @@ -99,7 +111,7 @@ echo " " echo "====> : Start importing OpenStreetMap data: ./data/${testdata} -> imposm3[./build/mapping.yaml] -> PostgreSQL" echo " : Imposm3 documentation: https://imposm.org/docs/imposm3/latest/index.html " echo " : Thank you Omniscale! " -echo " : source code: https://github.com/openmaptiles/import-osm " +echo " : Source code: https://github.com/openmaptiles/import-osm " echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) " echo " : Thank you OpenStreetMap Contributors ! " docker-compose run --rm import-osm From 2ba6e4940d02e01db68dd322a4715864f6d4f14b Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 16 Nov 2016 01:47:15 +0100 Subject: [PATCH 04/39] fix typo --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b7b1da6..b67820f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,5 @@ quickstart.log # imput / output data data/* -# generated sourece files +# generated source files build/* \ No newline at end of file From e8a7305ea5c067371af74b508180eaa86fe1f463 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 16 Nov 2016 01:54:27 +0100 Subject: [PATCH 05/39] add: md5sum info for debugging --- quickstart.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/quickstart.sh b/quickstart.sh index aed9b7a..e546ddd 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -145,6 +145,15 @@ echo "------------------------------------------------------------" echo "-- Summary --" echo "------------------------------------------------------------" echo " " +echo "====> : Inputs - Outputs md5sum for debugging " +rm -f quickstart_checklist.chk +md5sum build/mapping.yaml >> quickstart_checklist.chk +md5sum build/tileset.sql >> quickstart_checklist.chk +md5sum build/testmaptiles.tm2source/data.yml >> quickstart_checklist.chk +md5sum ./data/${testdata} >> quickstart_checklist.chk +md5sum ./data/tiles.mbtiles >> quickstart_checklist.chk +cat quickstart_checklist.chk + echo " " echo "====> : (disk space) We have created a lot of docker images: " echo " : Hint: you can remove with: docker rmi IMAGE " @@ -166,7 +175,7 @@ echo " " echo "============================================================" echo "The quickstart.sh is finished! " echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete" -echo "We saved the log file to $log_file ( for debugging ) " +echo "We saved the log file to $log_file ( for debugging ) you can compare with the travis log !" echo " " echo "Start experimenting -> see the documentation! " echo "============================================================" From 851c5a40ffd28c61b58932b6b6f601e2a470da47 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 16 Nov 2016 01:56:57 +0100 Subject: [PATCH 06/39] add quickstart_checklist.chk to .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b67820f..ee38ce1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,9 @@ data.xml *.swp *.pbf -# quickstart log +# quickstart log chk quickstart.log +quickstart_checklist.chk # imput / output data data/* From af423d4d221807e86190602399d186545d3637f5 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 16 Nov 2016 02:16:18 +0100 Subject: [PATCH 07/39] fix md5sum chklist error --- quickstart.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/quickstart.sh b/quickstart.sh index e546ddd..0ebe82f 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -135,6 +135,16 @@ echo " " echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )" docker-compose stop postgres +echo " " +echo "====> : Inputs - Outputs md5sum for debugging " +rm -f quickstart_checklist.chk +md5sum build/mapping.yaml >> quickstart_checklist.chk +md5sum build/tileset.sql >> quickstart_checklist.chk +md5sum build/openmaptiles.tm2source/data.yml >> quickstart_checklist.chk +md5sum ./data/${testdata} >> quickstart_checklist.chk +md5sum ./data/tiles.mbtiles >> quickstart_checklist.chk +cat quickstart_checklist.chk + ENDTIME=$(date +%s) ENDDATE=$(date -Iminutes) MODDATE=$(stat -c %y ./data/${testdata} ) @@ -144,16 +154,6 @@ echo " " echo "------------------------------------------------------------" echo "-- Summary --" echo "------------------------------------------------------------" -echo " " -echo "====> : Inputs - Outputs md5sum for debugging " -rm -f quickstart_checklist.chk -md5sum build/mapping.yaml >> quickstart_checklist.chk -md5sum build/tileset.sql >> quickstart_checklist.chk -md5sum build/testmaptiles.tm2source/data.yml >> quickstart_checklist.chk -md5sum ./data/${testdata} >> quickstart_checklist.chk -md5sum ./data/tiles.mbtiles >> quickstart_checklist.chk -cat quickstart_checklist.chk - echo " " echo "====> : (disk space) We have created a lot of docker images: " echo " : Hint: you can remove with: docker rmi IMAGE " @@ -175,7 +175,7 @@ echo " " echo "============================================================" echo "The quickstart.sh is finished! " echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete" -echo "We saved the log file to $log_file ( for debugging ) you can compare with the travis log !" +echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !" echo " " echo "Start experimenting -> see the documentation! " echo "============================================================" From 79a16709b755fe094401deeb03f88d5a07b12711 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 16 Nov 2016 02:29:44 +0100 Subject: [PATCH 08/39] add more formatting + minimalize kernel info --- quickstart.sh | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/quickstart.sh b/quickstart.sh index 0ebe82f..d44b2dd 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -16,7 +16,8 @@ rm -f $log_file exec &> >(tee -a "$log_file") echo " " -echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] " +echo "-------------------------------------------------------------------------------------" +echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] " echo " : This will be logged to the $log_file file ( for debugging ) and to the screen" echo " : Git version: $githash / Started: $STARTDATE " echo " : Your bash version: $BASH_VERSION" @@ -24,10 +25,13 @@ echo " : Your system is:" lsb_release -a echo " " +echo "-------------------------------------------------------------------------------------" echo " : This is working on x86_64 ; Your kernel is:" -uname -a +uname -r +uname -m echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Please check the docker and docker-compose version!" echo " : We are using docker-compose V2 file format ! see more: https://docs.docker.com/" echo " : (theoretically;not tested) minumum Docker version is 1.10.0+." @@ -38,48 +42,60 @@ docker --version docker-compose --version echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Checking OpenMapTiles docker images " docker images | grep openmaptiles echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Stopping running services & removing old containers " docker-compose down docker-compose rm -fv echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : For a clean start, we are removing old postgresql data volume ( if it exists )" docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Making directories - if they don't exist ( ./build ./data ) " mkdir -p build mkdir -p data echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) " rm -f ./data/*.mbtiles testdata=zurich_switzerland.osm.pbf testdataurl=https://s3.amazonaws.com/metro-extracts.mapzen.com/$testdata if [ ! -f ./data/${testdata} ]; then + echo " " + echo "-------------------------------------------------------------------------------------" echo "====> : Downloading testdata $testdata " rm -f ./data/* wget $testdataurl -P ./data else + echo " " + echo "-------------------------------------------------------------------------------------" echo "====> : The testdata ./data/$testdata exists, we don't need to download! " fi echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Remove old generated source files ( ./build/* ) ( if they exist ) " docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make clean_build echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )" echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools " echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! " docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Start PostgreSQL service ; create PostgreSQL data volume " echo " : Source code: https://github.com/openmaptiles/postgis " echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" @@ -87,6 +103,7 @@ docker-compose up -d postgres sleep 30 echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Start importing water data from http://openstreetmapdata.com into PostgreSQL " echo " : Source code: https://github.com/openmaptiles/import-water " echo " : Data license: http://openstreetmapdata.com/info/license " @@ -94,6 +111,7 @@ echo " : Thank you: http://openstreetmapdata.com/info/supporting " docker-compose run --rm import-water echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL " echo " : Source code: https://github.com/openmaptiles/import-natural-earth " echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use " @@ -101,6 +119,7 @@ echo " : Thank you: Natural Earth Contributors! " docker-compose run --rm import-natural-earth echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Start importing OpenStreetMap Lakelines data " echo " : Source code: https://github.com/openmaptiles/import-lakelines " echo " : https://github.com/lukasmartinelli/osm-lakelines " @@ -108,6 +127,7 @@ echo " : Data license: .. " docker-compose run --rm import-lakelines echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Start importing OpenStreetMap data: ./data/${testdata} -> imposm3[./build/mapping.yaml] -> PostgreSQL" echo " : Imposm3 documentation: https://imposm.org/docs/imposm3/latest/index.html " echo " : Thank you Omniscale! " @@ -117,11 +137,13 @@ echo " : Thank you OpenStreetMap Contributors ! " docker-compose run --rm import-osm echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Start SQL postprocessing: ./build/tileset.sql -> PostgreSQL " echo " : Source code: https://github.com/openmaptiles/import-sql " docker-compose run --rm import-sql echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. " echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml " echo " : Output MBTiles: ./data/tiles.mbtiles " @@ -132,10 +154,12 @@ echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-til docker-compose -f docker-compose.yml -f docker-compose-test-override.yml run --rm generate-vectortiles echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )" docker-compose stop postgres echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : Inputs - Outputs md5sum for debugging " rm -f quickstart_checklist.chk md5sum build/mapping.yaml >> quickstart_checklist.chk @@ -151,20 +175,23 @@ MODDATE=$(stat -c %y ./data/${testdata} ) echo " " echo " " -echo "------------------------------------------------------------" -echo "-- Summary --" -echo "------------------------------------------------------------" +echo "-------------------------------------------------------------------------------------" +echo "-- S u m m a r y --" +echo "-------------------------------------------------------------------------------------" echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : (disk space) We have created a lot of docker images: " echo " : Hint: you can remove with: docker rmi IMAGE " docker images | grep openmaptiles echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:" echo " : Hint: you can remove with : docker volume rm openmaptiles_pgdata " docker volume ls -q | grep openmaptiles echo " " +echo "-------------------------------------------------------------------------------------" echo "====> : (disk space) We have created the new vectortiles ( ./data/tiles.mbtiles ) " echo " : The OpenMapTiles MBTILES license: ..... " echo " : We created from $testdata ( file moddate: $MODDATE ) " @@ -172,13 +199,11 @@ echo " : Size: " ls ./data/*.mbtiles -la echo " " -echo "============================================================" +echo "-------------------------------------------------------------------------------------" echo "The quickstart.sh is finished! " echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete" echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !" echo " " echo "Start experimenting -> see the documentation! " -echo "============================================================" - - +echo "-------------------------------------------------------------------------------------" From 4f16d8a27d215a1b4ae3ce44a541eaca151d37b7 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 16 Nov 2016 04:45:03 +0100 Subject: [PATCH 09/39] add Memory, CPU info --- quickstart.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/quickstart.sh b/quickstart.sh index d44b2dd..57c3293 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -30,6 +30,13 @@ echo " : This is working on x86_64 ; Your kernel is:" uname -r uname -m +echo " : --- Memory, CPU info ---- " +mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) +echo "system memory (GB): ${mem} " +grep SwapTotal /proc/meminfo +echo cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1) +cat /proc/meminfo | grep Free + echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Please check the docker and docker-compose version!" From 0690750fe1b37112f43b1b9fa5fb921670f10b57 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 16 Nov 2016 10:20:08 +0100 Subject: [PATCH 10/39] add info for the user about the Mapnik logs --- quickstart.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/quickstart.sh b/quickstart.sh index 57c3293..081affb 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -158,6 +158,10 @@ echo " : Source code: https://github.com/openmaptiles/generate-vectortiles echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox " echo " : Thank you https://www.mapbox.com !" echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-tiles " +echo " : " +echo " : You will see a lot of deprecated warning in the log! This is normal! " +echo " : like : Mapnik LOG> ... is deprecated and will be removed in Mapnik 4.x ... " + docker-compose -f docker-compose.yml -f docker-compose-test-override.yml run --rm generate-vectortiles echo " " From 2957fb6048169cb76aaa4a79f5a35185a519a647 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 27 Nov 2016 11:31:43 +0100 Subject: [PATCH 11/39] using geofabrik-download for quickstart --- Makefile | 34 ++++++++++++++++++++++++++++++++ docker-compose-test-override.yml | 8 -------- quickstart.sh | 31 +++++++++++++++++++++++++---- 3 files changed, 61 insertions(+), 12 deletions(-) delete mode 100644 docker-compose-test-override.yml diff --git a/Makefile b/Makefile index c6c3626..8911620 100644 --- a/Makefile +++ b/Makefile @@ -56,3 +56,37 @@ clean: clean_build: rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql + +psql: + docker-compose run --rm import-osm /usr/src/app/psql.sh + +import-sql-dev: + docker-compose run --rm import-sql /bin/bash + +import-osm-dev: + docker-compose run --rm import-osm /bin/bash + +download-geofabrik: + @echo Download area : $(area) + docker-compose run --rm import-osm ./download-geofabrik.sh $(area) + ls ./data/* + +download-geofabrik-list: + docker-compose run --rm import-osm ./download-geofabrik-list.sh + +etlgraph: + generate-etlgraph layers/boundary/boundary.yaml + generate-etlgraph layers/highway/highway.yaml + generate-etlgraph layers/housenumber/housenumber.yaml + generate-etlgraph layers/landuse/landuse.yaml + generate-etlgraph layers/poi/poi.yaml + generate-etlgraph layers/water/water.yaml + generate-etlgraph layers/waterway/waterway.yaml + generate-etlgraph layers/building/building.yaml + generate-etlgraph layers/highway_name/highway_name.yaml + generate-etlgraph layers/landcover/landcover.yaml + generate-etlgraph layers/place/place.yaml + generate-etlgraph layers/railway/railway.yaml + generate-etlgraph layers/water_name/water_name.yaml + + \ No newline at end of file diff --git a/docker-compose-test-override.yml b/docker-compose-test-override.yml deleted file mode 100644 index 0ae9e4a..0000000 --- a/docker-compose-test-override.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: "2" - -services: - generate-vectortiles: - environment: - BBOX: "8.25,46.97,9.58,47.52" - MIN_ZOOM: "0" - MAX_ZOOM: "7" diff --git a/quickstart.sh b/quickstart.sh index 081affb..1393219 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -3,6 +3,15 @@ set -o errexit set -o pipefail set -o nounset +if [ $# -eq 0 ]; then + echo "No parameter - set area=albania " + osm_area=albania +else + osm_area=$1 +fi + +testdata=${osm_area}.osm.pbf + ## ## OpenMapTiles quickstart.sh for x86_64 linux ## @@ -75,20 +84,30 @@ echo "-------------------------------------------------------------------------- echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) " rm -f ./data/*.mbtiles -testdata=zurich_switzerland.osm.pbf -testdataurl=https://s3.amazonaws.com/metro-extracts.mapzen.com/$testdata + if [ ! -f ./data/${testdata} ]; then echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Downloading testdata $testdata " rm -f ./data/* - wget $testdataurl -P ./data + #wget $testdataurl -P ./data + docker-compose run --rm import-osm ./download-geofabrik.sh ${osm_area} else echo " " echo "-------------------------------------------------------------------------------------" echo "====> : The testdata ./data/$testdata exists, we don't need to download! " fi + +if [ ! -f ./data/${testdata} ]; then + echo " " + echo "Missing ./data/$testdata , Download error? " + exit 404 +fi + + + + echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Remove old generated source files ( ./build/* ) ( if they exist ) " @@ -162,7 +181,7 @@ echo " : " echo " : You will see a lot of deprecated warning in the log! This is normal! " echo " : like : Mapnik LOG> ... is deprecated and will be removed in Mapnik 4.x ... " -docker-compose -f docker-compose.yml -f docker-compose-test-override.yml run --rm generate-vectortiles +docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml run --rm generate-vectortiles echo " " echo "-------------------------------------------------------------------------------------" @@ -209,6 +228,10 @@ echo " : We created from $testdata ( file moddate: $MODDATE ) " echo " : Size: " ls ./data/*.mbtiles -la +echo " " +echo "-------------------------------------------------------------------------------------" +echo " Acknowledgments " +echo " Thanks to all free, open source software developers and Open Data Contributors ! " echo " " echo "-------------------------------------------------------------------------------------" echo "The quickstart.sh is finished! " From 1d5bf2f1a04b5decb8b2f3a50371516193797568 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 27 Nov 2016 19:01:41 +0100 Subject: [PATCH 12/39] set docker=1.10 docker-compose=1.7.1 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8bb8d4d..aa9894c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,9 +13,9 @@ before_install: - sudo apt-get clean - sudo apt-get update - sudo apt-key update - - sudo apt-get -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine=1.12.0-0~precise + - sudo apt-get -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine=1.10.0-0~precise - sudo rm -f /usr/local/bin/docker-compose - - curl -L https://github.com/docker/compose/releases/download/1.8.0/docker-compose-`uname -s`-`uname -m` > docker-compose + - curl -L https://github.com/docker/compose/releases/download/1.7.1/docker-compose-`uname -s`-`uname -m` > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - sudo service docker restart From e79ff9628031f8d76f1b9e7c80e12ea11313088a Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 27 Nov 2016 20:01:01 +0100 Subject: [PATCH 13/39] check minimal versions, add QUICKSTART.md --- QUICKSTART.md | 247 ++++++++++++++++++++++++++++++++++++++++++++++++++ quickstart.sh | 35 ++++++- 2 files changed, 280 insertions(+), 2 deletions(-) create mode 100644 QUICKSTART.md diff --git a/QUICKSTART.md b/QUICKSTART.md new file mode 100644 index 0000000..b78532f --- /dev/null +++ b/QUICKSTART.md @@ -0,0 +1,247 @@ + + +## Quickstart + +### Req: +* linux x86_64 +* docker >=1.10 +* docker-compose >=1.7.1 +* git +* free disk space >= ~15Gb ( more is better! ) + + +### First experiment - with albania ( small extracts! ) + +```bash +git clone https://github.com/openmaptiles/openmaptiles.git +cd openmaptiles +./quickstart.sh +``` + +if you have problems check the ./quickstart.log! + + + +### Check other extracts + +IF the previous step is working, +THAN you can test other available quickstart extracts ( based on geofabrik extracts) ! + * We are using https://github.com/julien-noblet/download-geofabrik tool + +This is generating mbtiles for your area : [ MIN_ZOOM: "0" - MAX_ZOOM: "7" ] + +```bash +./quickstart.sh africa +./quickstart.sh alabama +./quickstart.sh alaska +./quickstart.sh albania +./quickstart.sh alberta +./quickstart.sh alps +./quickstart.sh alsace +./quickstart.sh andorra +./quickstart.sh antarctica +./quickstart.sh aquitaine +./quickstart.sh argentina +./quickstart.sh arizona +./quickstart.sh arkansas +./quickstart.sh asia +./quickstart.sh australia +./quickstart.sh austria +./quickstart.sh auvergne +./quickstart.sh azerbaijan +./quickstart.sh azores +./quickstart.sh bangladesh +./quickstart.sh bayern +./quickstart.sh belarus +./quickstart.sh belgium +./quickstart.sh belize +./quickstart.sh berlin +./quickstart.sh bolivia +./quickstart.sh botswana +./quickstart.sh bourgogne +./quickstart.sh brandenburg +./quickstart.sh brazil +./quickstart.sh bremen +./quickstart.sh bretagne +./quickstart.sh buckinghamshire +./quickstart.sh bulgaria +./quickstart.sh california +./quickstart.sh cambridgeshire +./quickstart.sh cameroon +./quickstart.sh canada +./quickstart.sh centre +./quickstart.sh cheshire +./quickstart.sh chile +./quickstart.sh china +./quickstart.sh colombia +./quickstart.sh colorado +./quickstart.sh connecticut +./quickstart.sh cornwall +./quickstart.sh corse +./quickstart.sh croatia +./quickstart.sh cuba +./quickstart.sh cumbria +./quickstart.sh cyprus +./quickstart.sh dach +./quickstart.sh delaware +./quickstart.sh denmark +./quickstart.sh derbyshire +./quickstart.sh devon +./quickstart.sh dorset +./quickstart.sh ecuador +./quickstart.sh egypt +./quickstart.sh england +./quickstart.sh essex +./quickstart.sh estonia +./quickstart.sh ethiopia +./quickstart.sh europe +./quickstart.sh fiji +./quickstart.sh finland +./quickstart.sh florida +./quickstart.sh france +./quickstart.sh germany +./quickstart.sh gloucestershire +./quickstart.sh greece +./quickstart.sh greenland +./quickstart.sh guadeloupe +./quickstart.sh guatemala +./quickstart.sh guinea +./quickstart.sh guyane +./quickstart.sh hamburg +./quickstart.sh hampshire +./quickstart.sh hawaii +./quickstart.sh herefordshire +./quickstart.sh hertfordshire +./quickstart.sh hessen +./quickstart.sh hungary +./quickstart.sh iceland +./quickstart.sh idaho +./quickstart.sh illinois +./quickstart.sh india +./quickstart.sh indiana +./quickstart.sh indonesia +./quickstart.sh iowa +./quickstart.sh irak +./quickstart.sh iran +./quickstart.sh italy +./quickstart.sh japan +./quickstart.sh jordan +./quickstart.sh kansas +./quickstart.sh kazakhstan +./quickstart.sh kent +./quickstart.sh kentucky +./quickstart.sh kenya +./quickstart.sh kosovo +./quickstart.sh kyrgyzstan +./quickstart.sh lancashire +./quickstart.sh latvia +./quickstart.sh lebanon +./quickstart.sh leicestershire +./quickstart.sh lesotho +./quickstart.sh liberia +./quickstart.sh libya +./quickstart.sh liechtenstein +./quickstart.sh limousin +./quickstart.sh lithuania +./quickstart.sh lorraine +./quickstart.sh louisiana +./quickstart.sh luxembourg +./quickstart.sh macedonia +./quickstart.sh madagascar +./quickstart.sh maine +./quickstart.sh malta +./quickstart.sh manitoba +./quickstart.sh martinique +./quickstart.sh maryland +./quickstart.sh massachusetts +./quickstart.sh mayotte +./quickstart.sh mexico +./quickstart.sh michigan +./quickstart.sh minnesota +./quickstart.sh mississippi +./quickstart.sh missouri +./quickstart.sh mittelfranken +./quickstart.sh moldova +./quickstart.sh monaco +./quickstart.sh mongolia +./quickstart.sh montana +./quickstart.sh montenegro +./quickstart.sh morocco +./quickstart.sh nebraska +./quickstart.sh nepal +./quickstart.sh netherlands +./quickstart.sh nevada +./quickstart.sh niederbayern +./quickstart.sh niedersachsen +./quickstart.sh nigeria +./quickstart.sh norfolk +./quickstart.sh norway +./quickstart.sh nottinghamshire +./quickstart.sh nunavut +./quickstart.sh oberbayern +./quickstart.sh oberfranken +./quickstart.sh oberpfalz +./quickstart.sh ohio +./quickstart.sh oklahoma +./quickstart.sh ontario +./quickstart.sh oregon +./quickstart.sh oxfordshire +./quickstart.sh pakistan +./quickstart.sh paraguay +./quickstart.sh pennsylvania +./quickstart.sh peru +./quickstart.sh philippines +./quickstart.sh picardie +./quickstart.sh poland +./quickstart.sh portugal +./quickstart.sh quebec +./quickstart.sh reunion +./quickstart.sh romania +./quickstart.sh saarland +./quickstart.sh sachsen +./quickstart.sh saskatchewan +./quickstart.sh schwaben +./quickstart.sh scotland +./quickstart.sh serbia +./quickstart.sh shropshire +./quickstart.sh slovakia +./quickstart.sh slovenia +./quickstart.sh somalia +./quickstart.sh somerset +./quickstart.sh spain +./quickstart.sh staffordshire +./quickstart.sh suffolk +./quickstart.sh surrey +./quickstart.sh sweden +./quickstart.sh switzerland +./quickstart.sh syria +./quickstart.sh taiwan +./quickstart.sh tajikistan +./quickstart.sh tanzania +./quickstart.sh tennessee +./quickstart.sh texas +./quickstart.sh thailand +./quickstart.sh thueringen +./quickstart.sh turkey +./quickstart.sh turkmenistan +./quickstart.sh ukraine +./quickstart.sh unterfranken +./quickstart.sh uruguay +./quickstart.sh us +./quickstart.sh utah +./quickstart.sh uzbekistan +./quickstart.sh vermont +./quickstart.sh vietnam +./quickstart.sh virginia +./quickstart.sh wales +./quickstart.sh washington +./quickstart.sh wiltshire +./quickstart.sh wisconsin +./quickstart.sh wyoming +./quickstart.sh yukon +``` + +### Change MAX_ZOOM ... + +TODO ... + diff --git a/quickstart.sh b/quickstart.sh index 1393219..19e83c4 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -3,6 +3,12 @@ set -o errexit set -o pipefail set -o nounset +# +# ./quickstart.sh +# ./quickstart.sh +# +# + if [ $# -eq 0 ]; then echo "No parameter - set area=albania " osm_area=albania @@ -12,9 +18,13 @@ fi testdata=${osm_area}.osm.pbf + ## ## OpenMapTiles quickstart.sh for x86_64 linux ## +MIN_COMPOSE_VER=1.7.1 +MIN_DOCKER_VER=1.10.0 + STARTTIME=$(date +%s) STARTDATE=$(date -Iminutes) @@ -39,6 +49,13 @@ echo " : This is working on x86_64 ; Your kernel is:" uname -r uname -m +KERNEL_CPU_VER=$(uname -m) +if [ "$KERNEL_CPU_VER" != "x86_64" ]; then + echo "ERR: Sorry this is working only on x86_64!" + exit 1 +fi + + echo " : --- Memory, CPU info ---- " mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) echo "system memory (GB): ${mem} " @@ -50,13 +67,26 @@ echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Please check the docker and docker-compose version!" echo " : We are using docker-compose V2 file format ! see more: https://docs.docker.com/" -echo " : (theoretically;not tested) minumum Docker version is 1.10.0+." -echo " : (theoretically;not tested) minimum Docker-compose version is 1.6.0+." +echo " : (theoretically;not tested) minumum Docker version is $MIN_DOCKER_VER+." +echo " : (theoretically;not tested) minimum Docker-compose version is $MIN_COMPOSE_VER+." echo " : See the .travis testfile for the current supported versions " echo " : Your docker systems is:" docker --version docker-compose --version +COMPOSE_VER=$(docker-compose version --short) +if [ $COMPOSE_VER "<" $MIN_COMPOSE_VER ]; then + echo "ERR: Your Docker-compose version is Known to have bugs , Please Update docker-compose!" + exit 1 +fi + +DOCKER_VER="$(docker -v | awk -F '[ ,]+' '{ print $3 }')" +if [ $DOCKER_VER "<" $MIN_DOCKER_VER ]; then + echo "ERR: Your Docker version is not compatible , Please Update docker!" + exit 1 +fi + + echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Checking OpenMapTiles docker images " @@ -66,6 +96,7 @@ echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Stopping running services & removing old containers " docker-compose down +docker-compose kill docker-compose rm -fv echo " " From a4e8b4ac546f0b15a7fbdba48fef198f44697d01 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 27 Nov 2016 23:32:03 +0100 Subject: [PATCH 14/39] add QUICKSTART_MIN_ZOOM and QUICKSTART_MAX_ZOOM environment variables --- .env | 2 ++ QUICKSTART.md | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 6a65572..ed3cf2a 100644 --- a/.env +++ b/.env @@ -3,3 +3,5 @@ POSTGRES_USER=openmaptiles POSTGRES_PASSWORD=openmaptiles POSTGRES_HOST=postgres POSTGRES_PORT=5432 +QUICKSTART_MIN_ZOOM=0 +QUICKSTART_MAX_ZOOM=7 diff --git a/QUICKSTART.md b/QUICKSTART.md index b78532f..84557fc 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -1,6 +1,6 @@ -## Quickstart +## Quickstart - for small extracts ### Req: * linux x86_64 @@ -241,7 +241,11 @@ This is generating mbtiles for your area : [ MIN_ZOOM: "0" - MAX_ZOOM: "7" ] ./quickstart.sh yukon ``` -### Change MAX_ZOOM ... +### Change MIN_ZOOM and MAX_ZOOM -TODO ... +modify the settings in the `.env` file +* QUICKSTART_MIN_ZOOM=0 +* QUICKSTART_MAX_ZOOM=7 +and re-start `./quickstart.sh ` +* the new config file re-generating to here ./data/docker-compose-config.yml From ceb66e638c272ef3ebe4ae3e77c12460a52eb7bc Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Mon, 28 Nov 2016 16:44:30 +0100 Subject: [PATCH 15/39] add more documentation, small fixes --- Makefile | 3 +++ quickstart.sh | 47 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 8911620..67dfea5 100644 --- a/Makefile +++ b/Makefile @@ -74,6 +74,9 @@ download-geofabrik: download-geofabrik-list: docker-compose run --rm import-osm ./download-geofabrik-list.sh +start-mapbox-studio: + docker-compose up mapbox-studio + etlgraph: generate-etlgraph layers/boundary/boundary.yaml generate-etlgraph layers/highway/highway.yaml diff --git a/quickstart.sh b/quickstart.sh index 19e83c4..2a4216f 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -3,10 +3,18 @@ set -o errexit set -o pipefail set -o nounset -# -# ./quickstart.sh -# ./quickstart.sh -# +# Example calls ... +# ./quickstart.sh +# ./quickstart.sh africa +# ./quickstart.sh alabama +# ./quickstart.sh alaska +# ./quickstart.sh albania +# ./quickstart.sh alberta +# ./quickstart.sh alps +# .... +# +# to list areas : make download-geofabrik-list +# see more QUICKSTART.md # if [ $# -eq 0 ]; then @@ -228,6 +236,7 @@ md5sum build/tileset.sql >> quickstart_checklist.chk md5sum build/openmaptiles.tm2source/data.yml >> quickstart_checklist.chk md5sum ./data/${testdata} >> quickstart_checklist.chk md5sum ./data/tiles.mbtiles >> quickstart_checklist.chk +md5sum ./data/docker-compose-config.yml >> quickstart_checklist.chk cat quickstart_checklist.chk ENDTIME=$(date +%s) @@ -261,14 +270,30 @@ ls ./data/*.mbtiles -la echo " " echo "-------------------------------------------------------------------------------------" -echo " Acknowledgments " -echo " Thanks to all free, open source software developers and Open Data Contributors ! " -echo " " -echo "-------------------------------------------------------------------------------------" -echo "The quickstart.sh is finished! " +echo "The ./quickstart.sh $osm_area is finished! " echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete" echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !" echo " " -echo "Start experimenting -> see the documentation! " +echo "Start experimenting ! And check the QUICKSTART.MD file !" +echo " " +echo "Hints for testing other areas" +echo " make download-geofabrik-list # list actual geofabrik OSM extracts for download -> <> " +echo " ./quickstart.sh <> # example: ./quickstart.sh madagascar " +echo " " +echo "Hints for designers:" +echo " .... # start Maputnik " +echo " .... # start Tileserver-gl-light" +echo " make start-mapbox-studio # start Mapbox Studio " +echo " " +echo "Hints for developers:" +echo " make download-geofabrik area=albania # download OSM data from geofabrik area=albania" +echo " make psql # start PostgreSQL console " +echo " make import-sql-dev # start import-sql /bin/bash terminal " +echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" +echo " .... # start lukasmartinelli/postgis-editor" +echo " cat .env # list PG database and MIN_ZOOM and MAX_ZOOM informations" +echo " cat quickstart.log # backup ./quickstart.log for Technical Support :) " +echo "-------------------------------------------------------------------------------------" +echo " Acknowledgments " +echo " Thanks to all free, open source software developers and Open Data Contributors ! " echo "-------------------------------------------------------------------------------------" - From 355f4baf577750c67da01df9a50a312a7714d618 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Mon, 28 Nov 2016 23:15:47 +0100 Subject: [PATCH 16/39] more improvments --- Makefile | 49 +++++++++++++++++++- quickstart.sh | 123 +++++++++++++++++++++++++++++--------------------- 2 files changed, 118 insertions(+), 54 deletions(-) diff --git a/Makefile b/Makefile index 67dfea5..8f552c3 100644 --- a/Makefile +++ b/Makefile @@ -57,9 +57,55 @@ clean: clean_build: rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql +clean-docker: + docker-compose down -v --remove-orphans + docker-compose rm -fv + docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true + +list-docker-images: + docker images | grep openmaptiles + +refresh-docker-images: + docker pull openmaptiles/generate-vectortiles + docker pull openmaptiles/import-lakelines + docker pull openmaptiles/import-natural-earth + docker pull openmaptiles/import-osm + docker pull openmaptiles/import-sql + docker pull openmaptiles/import-water + docker pull openmaptiles/openmaptiles-tools + docker pull openmaptiles/postgis + docker pull osm2vectortiles/mapbox-studio + +remove-docker-images: + docker rmi openmaptiles/generate-vectortiles + docker rmi openmaptiles/import-lakelines + docker rmi openmaptiles/import-natural-earth + docker rmi openmaptiles/import-osm + docker rmi openmaptiles/import-sql + docker rmi openmaptiles/import-water + docker rmi openmaptiles/openmaptiles-tools + docker rmi openmaptiles/postgis + docker rmi osm2vectortiles/mapbox-studio + psql: docker-compose run --rm import-osm /usr/src/app/psql.sh +psql-list-tables: + docker-compose run --rm import-osm /usr/src/app/psql.sh -P pager=off -c "\d+" + +psql-pg-stat-reset: + docker-compose run --rm import-osm /usr/src/app/psql.sh -P pager=off -c 'SELECT pg_stat_statements_reset();' + +forced-clean-sql: + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "DROP SCHEMA IF EXISTS public CASCADE" + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE SCHEMA IF NOT EXISTS public" + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION hstore" + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION postgis" + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION pg_stat_statements" + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "GRANT ALL ON SCHEMA public TO postgres;" + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "GRANT ALL ON SCHEMA public TO public;" + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "COMMENT ON SCHEMA public IS 'standard public schema';" + import-sql-dev: docker-compose run --rm import-sql /bin/bash @@ -91,5 +137,4 @@ etlgraph: generate-etlgraph layers/place/place.yaml generate-etlgraph layers/railway/railway.yaml generate-etlgraph layers/water_name/water_name.yaml - - \ No newline at end of file + diff --git a/quickstart.sh b/quickstart.sh index 2a4216f..69ed956 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -3,42 +3,41 @@ set -o errexit set -o pipefail set -o nounset + +########################################### +# OpenMapTiles quickstart.sh for x86_64 linux +# # Example calls ... -# ./quickstart.sh -# ./quickstart.sh africa -# ./quickstart.sh alabama -# ./quickstart.sh alaska -# ./quickstart.sh albania -# ./quickstart.sh alberta -# ./quickstart.sh alps +# ./quickstart.sh +# ./quickstart.sh africa +# ./quickstart.sh alabama +# ./quickstart.sh alaska +# ./quickstart.sh albania +# ./quickstart.sh alberta +# ./quickstart.sh alps # .... -# +# # to list areas : make download-geofabrik-list # see more QUICKSTART.md # if [ $# -eq 0 ]; then - echo "No parameter - set area=albania " - osm_area=albania -else + osm_area=albania # default test country + echo "No parameter - set area=$osm_area " +else osm_area=$1 fi - testdata=${osm_area}.osm.pbf - -## -## OpenMapTiles quickstart.sh for x86_64 linux -## +## Min versions ... MIN_COMPOSE_VER=1.7.1 MIN_DOCKER_VER=1.10.0 - STARTTIME=$(date +%s) STARTDATE=$(date -Iminutes) githash=$( git rev-parse HEAD ) -log_file=quickstart.log +log_file=./quickstart.log rm -f $log_file exec &> >(tee -a "$log_file") @@ -94,6 +93,10 @@ if [ $DOCKER_VER "<" $MIN_DOCKER_VER ]; then exit 1 fi +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Pulling or Refreshing OpenMapTiles docker images " +#make refresh-docker-images echo " " echo "-------------------------------------------------------------------------------------" @@ -103,18 +106,12 @@ docker images | grep openmaptiles echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Stopping running services & removing old containers " -docker-compose down -docker-compose kill -docker-compose rm -fv +make clean-docker echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : For a clean start, we are removing old postgresql data volume ( if it exists )" -docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Making directories - if they don't exist ( ./build ./data ) " +echo "====> : Making directories - if they don't exist ( ./build ./data ./pgdata ) " +mkdir -p pgdata mkdir -p build mkdir -p data @@ -123,6 +120,12 @@ echo "-------------------------------------------------------------------------- echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) " rm -f ./data/*.mbtiles +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Removing pgdata " +# rm -rf ./pgdata/* + + if [ ! -f ./data/${testdata} ]; then echo " " @@ -131,22 +134,28 @@ if [ ! -f ./data/${testdata} ]; then rm -f ./data/* #wget $testdataurl -P ./data docker-compose run --rm import-osm ./download-geofabrik.sh ${osm_area} + echo " " + echo "-------------------------------------------------------------------------------------" + echo "====> : Osm metadata : $testdata " + cat ./data/osmstat.txt + echo " " + echo "-------------------------------------------------------------------------------------" + echo "====> : Generated docker-compose config " + cat ./data/docker-compose-config.yml else echo " " echo "-------------------------------------------------------------------------------------" - echo "====> : The testdata ./data/$testdata exists, we don't need to download! " + echo "====> : The testdata ./data/$testdata exists, we don't need to download! " fi if [ ! -f ./data/${testdata} ]; then echo " " - echo "Missing ./data/$testdata , Download error? " + echo "Missing ./data/$testdata , Download or Parameter error? " exit 404 fi - - echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Remove old generated source files ( ./build/* ) ( if they exist ) " @@ -165,7 +174,9 @@ echo "====> : Start PostgreSQL service ; create PostgreSQL data volume " echo " : Source code: https://github.com/openmaptiles/postgis " echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" docker-compose up -d postgres -sleep 30 + +# Drop all PostgreSQL tables ... +make forced-clean-sql echo " " echo "-------------------------------------------------------------------------------------" @@ -198,7 +209,7 @@ echo " : Imposm3 documentation: https://imposm.org/docs/imposm3/latest/inde echo " : Thank you Omniscale! " echo " : Source code: https://github.com/openmaptiles/import-osm " echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) " -echo " : Thank you OpenStreetMap Contributors ! " +echo " : Thank you OpenStreetMap Contributors ! " docker-compose run --rm import-osm echo " " @@ -230,16 +241,17 @@ docker-compose stop postgres echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Inputs - Outputs md5sum for debugging " -rm -f quickstart_checklist.chk -md5sum build/mapping.yaml >> quickstart_checklist.chk -md5sum build/tileset.sql >> quickstart_checklist.chk -md5sum build/openmaptiles.tm2source/data.yml >> quickstart_checklist.chk -md5sum ./data/${testdata} >> quickstart_checklist.chk -md5sum ./data/tiles.mbtiles >> quickstart_checklist.chk -md5sum ./data/docker-compose-config.yml >> quickstart_checklist.chk -cat quickstart_checklist.chk +rm -f ./data/quickstart_checklist.chk +md5sum build/mapping.yaml >> ./data/quickstart_checklist.chk +md5sum build/tileset.sql >> ./data/quickstart_checklist.chk +md5sum build/openmaptiles.tm2source/data.yml >> ./data/quickstart_checklist.chk +md5sum ./data/${testdata} >> ./data/quickstart_checklist.chk +md5sum ./data/tiles.mbtiles >> ./data/quickstart_checklist.chk +md5sum ./data/docker-compose-config.yml >> ./data/quickstart_checklist.chk +md5sum ./data/osmstat.txt >> ./data/quickstart_checklist.chk +cat ./data/quickstart_checklist.chk -ENDTIME=$(date +%s) +ENDTIME=$(date +%s) ENDDATE=$(date -Iminutes) MODDATE=$(stat -c %y ./data/${testdata} ) @@ -254,11 +266,12 @@ echo "====> : (disk space) We have created a lot of docker images: " echo " : Hint: you can remove with: docker rmi IMAGE " docker images | grep openmaptiles -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:" -echo " : Hint: you can remove with : docker volume rm openmaptiles_pgdata " -docker volume ls -q | grep openmaptiles +# +# echo " " +# echo "-------------------------------------------------------------------------------------" +# echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:" +# echo " : Hint: you can remove with : docker volume rm openmaptiles_pgdata " +# docker volume ls -q | grep openmaptiles echo " " echo "-------------------------------------------------------------------------------------" @@ -273,7 +286,7 @@ echo "-------------------------------------------------------------------------- echo "The ./quickstart.sh $osm_area is finished! " echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete" echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !" -echo " " +echo " " echo "Start experimenting ! And check the QUICKSTART.MD file !" echo " " echo "Hints for testing other areas" @@ -283,17 +296,23 @@ echo " " echo "Hints for designers:" echo " .... # start Maputnik " echo " .... # start Tileserver-gl-light" -echo " make start-mapbox-studio # start Mapbox Studio " +echo " make start-mapbox-studio # start Mapbox Studio" echo " " echo "Hints for developers:" echo " make download-geofabrik area=albania # download OSM data from geofabrik area=albania" echo " make psql # start PostgreSQL console " +echo " make psql-list-tables # list all PostgreSQL tables " echo " make import-sql-dev # start import-sql /bin/bash terminal " echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" -echo " .... # start lukasmartinelli/postgis-editor" +echo " make clean-docker # remove docker containers, PG data volume " +echo " make forced-clean-sql # drop all PostgreSQL tables for clean environment " +echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB" +echo " make remove-docker-images # remove openmaptiles docker images" echo " cat .env # list PG database and MIN_ZOOM and MAX_ZOOM informations" -echo " cat quickstart.log # backup ./quickstart.log for Technical Support :) " +echo " cat ./quickstart.log # backup of the last ./quickstart.sh " +echo " .... # start lukasmartinelli/postgis-editor" +echo " " echo "-------------------------------------------------------------------------------------" echo " Acknowledgments " -echo " Thanks to all free, open source software developers and Open Data Contributors ! " +echo " Thanks to all free, open source software developers and Open Data Contributors ! " echo "-------------------------------------------------------------------------------------" From 9b8adcb25b76ae1581c46e6432d6530e665f364a Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Tue, 29 Nov 2016 00:27:44 +0100 Subject: [PATCH 17/39] pgclimb and other examples --- Makefile | 18 +++++++++++++++++- quickstart.sh | 11 ++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8f552c3..dd8a67c 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,12 @@ forced-clean-sql: docker-compose run --rm import-osm /usr/src/app/psql.sh -c "GRANT ALL ON SCHEMA public TO public;" docker-compose run --rm import-osm /usr/src/app/psql.sh -c "COMMENT ON SCHEMA public IS 'standard public schema';" +pgclimb-list-views: + docker-compose run --rm import-osm /usr/src/app/pgclimb.sh -c "select schemaname,viewname from pg_views where schemaname='public' order by viewname;" csv + +pgclimb-list-tables: + docker-compose run --rm import-osm /usr/src/app/pgclimb.sh -c "select schemaname,tablename from pg_tables where schemaname='public' order by tablename;" csv + import-sql-dev: docker-compose run --rm import-sql /bin/bash @@ -113,9 +119,19 @@ import-osm-dev: docker-compose run --rm import-osm /bin/bash download-geofabrik: + @echo =============== download-geofabrik ======================= @echo Download area : $(area) + @echo [[ example: make download-greofabrik area=albania ]] + @echo [[ list areas: make download-geofabrik-list ]] docker-compose run --rm import-osm ./download-geofabrik.sh $(area) - ls ./data/* + ls -la ./data/$(area).* + @echo "Generated config file: ./data/docker-compose-config.yml" + @echo " " + cat ./data/docker-compose-config.yml + @echo " " + +list: + docker-compose run --rm import-osm ./download-geofabrik-list.sh download-geofabrik-list: docker-compose run --rm import-osm ./download-geofabrik-list.sh diff --git a/quickstart.sh b/quickstart.sh index 69ed956..73cf4e0 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -93,9 +93,15 @@ if [ $DOCKER_VER "<" $MIN_DOCKER_VER ]; then exit 1 fi +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Docker Space info: " +docker info | grep Space + echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Pulling or Refreshing OpenMapTiles docker images " +#for developers is not perfect ... #make refresh-docker-images echo " " @@ -291,6 +297,7 @@ echo "Start experimenting ! And check the QUICKSTART.MD file !" echo " " echo "Hints for testing other areas" echo " make download-geofabrik-list # list actual geofabrik OSM extracts for download -> <> " +echo " make list # list actual geofabrik OSM extracts for download -> <> " echo " ./quickstart.sh <> # example: ./quickstart.sh madagascar " echo " " echo "Hints for designers:" @@ -299,7 +306,7 @@ echo " .... # start Tileserver-gl-light" echo " make start-mapbox-studio # start Mapbox Studio" echo " " echo "Hints for developers:" -echo " make download-geofabrik area=albania # download OSM data from geofabrik area=albania" +echo " make download-geofabrik area=albania # download OSM data from geofabrik, and create config file" echo " make psql # start PostgreSQL console " echo " make psql-list-tables # list all PostgreSQL tables " echo " make import-sql-dev # start import-sql /bin/bash terminal " @@ -308,6 +315,8 @@ echo " make clean-docker # remove docker containers, PG data echo " make forced-clean-sql # drop all PostgreSQL tables for clean environment " echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB" echo " make remove-docker-images # remove openmaptiles docker images" +echo " make pgclimb-list-views # list PostgreSQL public schema views" +echo " make pgclimb-list-tables # list PostgreSQL public schema tabless" echo " cat .env # list PG database and MIN_ZOOM and MAX_ZOOM informations" echo " cat ./quickstart.log # backup of the last ./quickstart.sh " echo " .... # start lukasmartinelli/postgis-editor" From 5c38d0283e9bd0507c0f320b40da6e444e510b63 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Tue, 29 Nov 2016 00:53:52 +0100 Subject: [PATCH 18/39] small fixes --- Makefile | 2 +- quickstart.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dd8a67c..976db2a 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ import-osm-dev: download-geofabrik: @echo =============== download-geofabrik ======================= @echo Download area : $(area) - @echo [[ example: make download-greofabrik area=albania ]] + @echo [[ example: make download-geofabrik area=albania ]] @echo [[ list areas: make download-geofabrik-list ]] docker-compose run --rm import-osm ./download-geofabrik.sh $(area) ls -la ./data/$(area).* diff --git a/quickstart.sh b/quickstart.sh index 73cf4e0..794bd53 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -139,7 +139,7 @@ if [ ! -f ./data/${testdata} ]; then echo "====> : Downloading testdata $testdata " rm -f ./data/* #wget $testdataurl -P ./data - docker-compose run --rm import-osm ./download-geofabrik.sh ${osm_area} + make download-geofabrik area=${osm_area} echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Osm metadata : $testdata " From a1ed4dd2fa6910df3a2eeb21366a7aae111d2622 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Tue, 29 Nov 2016 15:31:55 +0100 Subject: [PATCH 19/39] fix docker-compose volume --- docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8c20fc9..2b7cc9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,8 @@ version: "2" + volumes: -- pgdata: + pgdata: + services: postgres: image: "openmaptiles/postgis:2.2" From 0d826d6cc4e2f2168e88affa7a683e12baa35f3c Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 30 Nov 2016 03:34:19 +0100 Subject: [PATCH 20/39] Operating system checks and QUICKSTART.md improvements --- QUICKSTART.md | 483 +++++++++++++++++++++++++++----------------------- quickstart.sh | 43 +++-- 2 files changed, 286 insertions(+), 240 deletions(-) diff --git a/QUICKSTART.md b/QUICKSTART.md index 84557fc..9a63cc0 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -1,251 +1,288 @@ -## Quickstart - for small extracts +## Quickstart - for small extracts ### Req: -* linux x86_64 -* docker >=1.10 -* docker-compose >=1.7.1 +* CPU: AMD64 ( = Intel 64 bit) + * The base docker debian images are x86_64 based, so the ARM,MIPS currently not supported! +* Operating system + * Linux is suggested + * The development and the testing platform is Linux. + * If you are using FreeBSD, Solaris, Windows, ... + * Please give a feedback, share your experience, write a tutorial +* bash * git -* free disk space >= ~15Gb ( more is better! ) +* make +* docker >=1.10 + * https://www.docker.com/products/overview +* docker-compose >=1.7.1 + * https://docs.docker.com/compose/install/ +* disk space ( >= ~15Gb ) + * for small extracts >= ~15Gb + * for big extracts ( continents, planet) > 20 ... 1000 Gb + * And depends on + * OpenStreetMap data size + * Zoom level +* memory ( >= 3Gb ) + * for small extracts 3Gb-8Gb RAM + * for big extracts ( Europe, Planet) > 8-32 Gb +* internet connections + * for downloading docker images + * for downloading OpenStreetMap data from Geofabrik +Important: The ./quickstart.sh is for small extracts - not optimal for a Planet rendering !! -### First experiment - with albania ( small extracts! ) +### First experiment - with albania ( small extracts! ) ```bash git clone https://github.com/openmaptiles/openmaptiles.git cd openmaptiles -./quickstart.sh +./quickstart.sh ``` -if you have problems check the ./quickstart.log! - - - +If you have problems with the quickstart +* check the ./quickstart.log! +* check again the system requirements +* check the current issues : https://github.com/openmaptiles/openmaptiles/issues +* create new issues: + * create a new gist https://gist.github.com/ from your ./quickstart.log + * double check: if this is not contain any sensitive informations about your system + * create a new issues: https://github.com/openmaptiles/openmaptiles/issues + * describe the problems + * add any important informations your environment + * and link your (quickstart.log) gist ! + ### Check other extracts -IF the previous step is working, -THAN you can test other available quickstart extracts ( based on geofabrik extracts) ! +IF the previous step is working, +THAN you can test other available quickstart extracts ( based on geofabrik extracts) ! * We are using https://github.com/julien-noblet/download-geofabrik tool + * The current extract list, and more informations -> `make list` This is generating mbtiles for your area : [ MIN_ZOOM: "0" - MAX_ZOOM: "7" ] ```bash -./quickstart.sh africa -./quickstart.sh alabama -./quickstart.sh alaska -./quickstart.sh albania -./quickstart.sh alberta -./quickstart.sh alps -./quickstart.sh alsace -./quickstart.sh andorra -./quickstart.sh antarctica -./quickstart.sh aquitaine -./quickstart.sh argentina -./quickstart.sh arizona -./quickstart.sh arkansas -./quickstart.sh asia -./quickstart.sh australia -./quickstart.sh austria -./quickstart.sh auvergne -./quickstart.sh azerbaijan -./quickstart.sh azores -./quickstart.sh bangladesh -./quickstart.sh bayern -./quickstart.sh belarus -./quickstart.sh belgium -./quickstart.sh belize -./quickstart.sh berlin -./quickstart.sh bolivia -./quickstart.sh botswana -./quickstart.sh bourgogne -./quickstart.sh brandenburg -./quickstart.sh brazil -./quickstart.sh bremen -./quickstart.sh bretagne -./quickstart.sh buckinghamshire -./quickstart.sh bulgaria -./quickstart.sh california -./quickstart.sh cambridgeshire -./quickstart.sh cameroon -./quickstart.sh canada -./quickstart.sh centre -./quickstart.sh cheshire -./quickstart.sh chile -./quickstart.sh china -./quickstart.sh colombia -./quickstart.sh colorado -./quickstart.sh connecticut -./quickstart.sh cornwall -./quickstart.sh corse -./quickstart.sh croatia -./quickstart.sh cuba -./quickstart.sh cumbria -./quickstart.sh cyprus -./quickstart.sh dach -./quickstart.sh delaware -./quickstart.sh denmark -./quickstart.sh derbyshire -./quickstart.sh devon -./quickstart.sh dorset -./quickstart.sh ecuador -./quickstart.sh egypt -./quickstart.sh england -./quickstart.sh essex -./quickstart.sh estonia -./quickstart.sh ethiopia -./quickstart.sh europe -./quickstart.sh fiji -./quickstart.sh finland -./quickstart.sh florida -./quickstart.sh france -./quickstart.sh germany -./quickstart.sh gloucestershire -./quickstart.sh greece -./quickstart.sh greenland -./quickstart.sh guadeloupe -./quickstart.sh guatemala -./quickstart.sh guinea -./quickstart.sh guyane -./quickstart.sh hamburg -./quickstart.sh hampshire -./quickstart.sh hawaii -./quickstart.sh herefordshire -./quickstart.sh hertfordshire -./quickstart.sh hessen -./quickstart.sh hungary -./quickstart.sh iceland -./quickstart.sh idaho -./quickstart.sh illinois -./quickstart.sh india -./quickstart.sh indiana -./quickstart.sh indonesia -./quickstart.sh iowa -./quickstart.sh irak -./quickstart.sh iran -./quickstart.sh italy -./quickstart.sh japan -./quickstart.sh jordan -./quickstart.sh kansas -./quickstart.sh kazakhstan -./quickstart.sh kent -./quickstart.sh kentucky -./quickstart.sh kenya -./quickstart.sh kosovo -./quickstart.sh kyrgyzstan -./quickstart.sh lancashire -./quickstart.sh latvia -./quickstart.sh lebanon -./quickstart.sh leicestershire -./quickstart.sh lesotho -./quickstart.sh liberia -./quickstart.sh libya -./quickstart.sh liechtenstein -./quickstart.sh limousin -./quickstart.sh lithuania -./quickstart.sh lorraine -./quickstart.sh louisiana -./quickstart.sh luxembourg -./quickstart.sh macedonia -./quickstart.sh madagascar -./quickstart.sh maine -./quickstart.sh malta -./quickstart.sh manitoba -./quickstart.sh martinique -./quickstart.sh maryland -./quickstart.sh massachusetts -./quickstart.sh mayotte -./quickstart.sh mexico -./quickstart.sh michigan -./quickstart.sh minnesota -./quickstart.sh mississippi -./quickstart.sh missouri -./quickstart.sh mittelfranken -./quickstart.sh moldova -./quickstart.sh monaco -./quickstart.sh mongolia -./quickstart.sh montana -./quickstart.sh montenegro -./quickstart.sh morocco -./quickstart.sh nebraska -./quickstart.sh nepal -./quickstart.sh netherlands -./quickstart.sh nevada -./quickstart.sh niederbayern -./quickstart.sh niedersachsen -./quickstart.sh nigeria -./quickstart.sh norfolk -./quickstart.sh norway -./quickstart.sh nottinghamshire -./quickstart.sh nunavut -./quickstart.sh oberbayern -./quickstart.sh oberfranken -./quickstart.sh oberpfalz -./quickstart.sh ohio -./quickstart.sh oklahoma -./quickstart.sh ontario -./quickstart.sh oregon -./quickstart.sh oxfordshire -./quickstart.sh pakistan -./quickstart.sh paraguay -./quickstart.sh pennsylvania -./quickstart.sh peru -./quickstart.sh philippines -./quickstart.sh picardie -./quickstart.sh poland -./quickstart.sh portugal -./quickstart.sh quebec -./quickstart.sh reunion -./quickstart.sh romania -./quickstart.sh saarland -./quickstart.sh sachsen -./quickstart.sh saskatchewan -./quickstart.sh schwaben -./quickstart.sh scotland -./quickstart.sh serbia -./quickstart.sh shropshire -./quickstart.sh slovakia -./quickstart.sh slovenia -./quickstart.sh somalia -./quickstart.sh somerset -./quickstart.sh spain -./quickstart.sh staffordshire -./quickstart.sh suffolk -./quickstart.sh surrey -./quickstart.sh sweden -./quickstart.sh switzerland -./quickstart.sh syria -./quickstart.sh taiwan -./quickstart.sh tajikistan -./quickstart.sh tanzania -./quickstart.sh tennessee -./quickstart.sh texas -./quickstart.sh thailand -./quickstart.sh thueringen -./quickstart.sh turkey -./quickstart.sh turkmenistan -./quickstart.sh ukraine -./quickstart.sh unterfranken -./quickstart.sh uruguay -./quickstart.sh us -./quickstart.sh utah -./quickstart.sh uzbekistan -./quickstart.sh vermont -./quickstart.sh vietnam -./quickstart.sh virginia -./quickstart.sh wales -./quickstart.sh washington -./quickstart.sh wiltshire -./quickstart.sh wisconsin -./quickstart.sh wyoming -./quickstart.sh yukon +./quickstart.sh africa +./quickstart.sh alabama +./quickstart.sh alaska +./quickstart.sh albania +./quickstart.sh alberta +./quickstart.sh alps +./quickstart.sh alsace +./quickstart.sh andorra +./quickstart.sh antarctica +./quickstart.sh aquitaine +./quickstart.sh argentina +./quickstart.sh arizona +./quickstart.sh arkansas +./quickstart.sh asia +./quickstart.sh australia +./quickstart.sh austria +./quickstart.sh auvergne +./quickstart.sh azerbaijan +./quickstart.sh azores +./quickstart.sh bangladesh +./quickstart.sh bayern +./quickstart.sh belarus +./quickstart.sh belgium +./quickstart.sh belize +./quickstart.sh berlin +./quickstart.sh bolivia +./quickstart.sh botswana +./quickstart.sh bourgogne +./quickstart.sh brandenburg +./quickstart.sh brazil +./quickstart.sh bremen +./quickstart.sh bretagne +./quickstart.sh buckinghamshire +./quickstart.sh bulgaria +./quickstart.sh california +./quickstart.sh cambridgeshire +./quickstart.sh cameroon +./quickstart.sh canada +./quickstart.sh centre +./quickstart.sh cheshire +./quickstart.sh chile +./quickstart.sh china +./quickstart.sh colombia +./quickstart.sh colorado +./quickstart.sh connecticut +./quickstart.sh cornwall +./quickstart.sh corse +./quickstart.sh croatia +./quickstart.sh cuba +./quickstart.sh cumbria +./quickstart.sh cyprus +./quickstart.sh dach +./quickstart.sh delaware +./quickstart.sh denmark +./quickstart.sh derbyshire +./quickstart.sh devon +./quickstart.sh dorset +./quickstart.sh ecuador +./quickstart.sh egypt +./quickstart.sh england +./quickstart.sh essex +./quickstart.sh estonia +./quickstart.sh ethiopia +./quickstart.sh europe +./quickstart.sh fiji +./quickstart.sh finland +./quickstart.sh florida +./quickstart.sh france +./quickstart.sh germany +./quickstart.sh gloucestershire +./quickstart.sh greece +./quickstart.sh greenland +./quickstart.sh guadeloupe +./quickstart.sh guatemala +./quickstart.sh guinea +./quickstart.sh guyane +./quickstart.sh hamburg +./quickstart.sh hampshire +./quickstart.sh hawaii +./quickstart.sh herefordshire +./quickstart.sh hertfordshire +./quickstart.sh hessen +./quickstart.sh hungary +./quickstart.sh iceland +./quickstart.sh idaho +./quickstart.sh illinois +./quickstart.sh india +./quickstart.sh indiana +./quickstart.sh indonesia +./quickstart.sh iowa +./quickstart.sh irak +./quickstart.sh iran +./quickstart.sh italy +./quickstart.sh japan +./quickstart.sh jordan +./quickstart.sh kansas +./quickstart.sh kazakhstan +./quickstart.sh kent +./quickstart.sh kentucky +./quickstart.sh kenya +./quickstart.sh kosovo +./quickstart.sh kyrgyzstan +./quickstart.sh lancashire +./quickstart.sh latvia +./quickstart.sh lebanon +./quickstart.sh leicestershire +./quickstart.sh lesotho +./quickstart.sh liberia +./quickstart.sh libya +./quickstart.sh liechtenstein +./quickstart.sh limousin +./quickstart.sh lithuania +./quickstart.sh lorraine +./quickstart.sh louisiana +./quickstart.sh luxembourg +./quickstart.sh macedonia +./quickstart.sh madagascar +./quickstart.sh maine +./quickstart.sh malta +./quickstart.sh manitoba +./quickstart.sh martinique +./quickstart.sh maryland +./quickstart.sh massachusetts +./quickstart.sh mayotte +./quickstart.sh mexico +./quickstart.sh michigan +./quickstart.sh minnesota +./quickstart.sh mississippi +./quickstart.sh missouri +./quickstart.sh mittelfranken +./quickstart.sh moldova +./quickstart.sh monaco +./quickstart.sh mongolia +./quickstart.sh montana +./quickstart.sh montenegro +./quickstart.sh morocco +./quickstart.sh nebraska +./quickstart.sh nepal +./quickstart.sh netherlands +./quickstart.sh nevada +./quickstart.sh niederbayern +./quickstart.sh niedersachsen +./quickstart.sh nigeria +./quickstart.sh norfolk +./quickstart.sh norway +./quickstart.sh nottinghamshire +./quickstart.sh nunavut +./quickstart.sh oberbayern +./quickstart.sh oberfranken +./quickstart.sh oberpfalz +./quickstart.sh ohio +./quickstart.sh oklahoma +./quickstart.sh ontario +./quickstart.sh oregon +./quickstart.sh oxfordshire +./quickstart.sh pakistan +./quickstart.sh paraguay +./quickstart.sh pennsylvania +./quickstart.sh peru +./quickstart.sh philippines +./quickstart.sh picardie +./quickstart.sh poland +./quickstart.sh portugal +./quickstart.sh quebec +./quickstart.sh reunion +./quickstart.sh romania +./quickstart.sh saarland +./quickstart.sh sachsen +./quickstart.sh saskatchewan +./quickstart.sh schwaben +./quickstart.sh scotland +./quickstart.sh serbia +./quickstart.sh shropshire +./quickstart.sh slovakia +./quickstart.sh slovenia +./quickstart.sh somalia +./quickstart.sh somerset +./quickstart.sh spain +./quickstart.sh staffordshire +./quickstart.sh suffolk +./quickstart.sh surrey +./quickstart.sh sweden +./quickstart.sh switzerland +./quickstart.sh syria +./quickstart.sh taiwan +./quickstart.sh tajikistan +./quickstart.sh tanzania +./quickstart.sh tennessee +./quickstart.sh texas +./quickstart.sh thailand +./quickstart.sh thueringen +./quickstart.sh turkey +./quickstart.sh turkmenistan +./quickstart.sh ukraine +./quickstart.sh unterfranken +./quickstart.sh uruguay +./quickstart.sh us +./quickstart.sh utah +./quickstart.sh uzbekistan +./quickstart.sh vermont +./quickstart.sh vietnam +./quickstart.sh virginia +./quickstart.sh wales +./quickstart.sh washington +./quickstart.sh wiltshire +./quickstart.sh wisconsin +./quickstart.sh wyoming +./quickstart.sh yukon ``` ### Change MIN_ZOOM and MAX_ZOOM -modify the settings in the `.env` file +modify the settings in the `.env` file * QUICKSTART_MIN_ZOOM=0 -* QUICKSTART_MAX_ZOOM=7 +* QUICKSTART_MAX_ZOOM=7 and re-start `./quickstart.sh ` * the new config file re-generating to here ./data/docker-compose-config.yml + +Hints: +* Small increments! Never starts with the MAX_ZOOM = 14 +* The suggested MAX_ZOOM = 14 - use only with small extracts + + diff --git a/quickstart.sh b/quickstart.sh index edb05d5..1050e59 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -47,28 +47,37 @@ echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmap echo " : This will be logged to the $log_file file ( for debugging ) and to the screen" echo " : Git version: $githash / Started: $STARTDATE " echo " : Your bash version: $BASH_VERSION" -echo " : Your system is:" -lsb_release -a +echo " : Your OS: $OSTYPE" -echo " " -echo "-------------------------------------------------------------------------------------" -echo " : This is working on x86_64 ; Your kernel is:" -uname -r -uname -m -KERNEL_CPU_VER=$(uname -m) -if [ "$KERNEL_CPU_VER" != "x86_64" ]; then - echo "ERR: Sorry this is working only on x86_64!" - exit 1 +if [[ "$OSTYPE" == "linux-gnu" ]]; then + + echo " : Your system is:" + lsb_release -a + echo " " + echo "-------------------------------------------------------------------------------------" + echo " : This is working on x86_64 ; Your kernel is:" + uname -r + uname -m + + KERNEL_CPU_VER=$(uname -m) + if [ "$KERNEL_CPU_VER" != "x86_64" ]; then + echo "ERR: Sorry this is working only on x86_64!" + exit 1 + fi + echo " : --- Memory, CPU info ---- " + mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) + echo "system memory (GB): ${mem} " + grep SwapTotal /proc/meminfo + echo cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1) + cat /proc/meminfo | grep Free +else + echo " " + echo "Warning : This is not a Linux ... ( Less tested ... ) " + echo " " fi -echo " : --- Memory, CPU info ---- " -mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) -echo "system memory (GB): ${mem} " -grep SwapTotal /proc/meminfo -echo cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1) -cat /proc/meminfo | grep Free echo " " echo "-------------------------------------------------------------------------------------" From 631f7b7856b26b4f9e4e8f2dc43e820bcf0c23ef Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 30 Nov 2016 05:39:06 +0100 Subject: [PATCH 21/39] fixes , improvements --- Makefile | 65 ++++++++++++++++++++++++++----------------- QUICKSTART.md | 8 +++++- docker-compose.yml | 2 +- quickstart.sh | 69 ++++++++++++++-------------------------------- 4 files changed, 68 insertions(+), 76 deletions(-) diff --git a/Makefile b/Makefile index 547694b..969dc64 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,37 @@ all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql +help: + @echo "==============================================================================" + @echo " OpenMapTiles https://github.com/openmaptiles/openmaptiles " + @echo "Hints for testing areas " + @echo " make download-geofabrik-list # list actual geofabrik OSM extracts for download -> <> " + @echo " make list # list actual geofabrik OSM extracts for download -> <> " + @echo " ./quickstart.sh <> # example: ./quickstart.sh madagascar " + @echo " " + @echo "Hints for designers:" + @echo " ....TODO.... # start Maputnik " + @echo " ....TODO.... # start Tileserver-gl-light" + @echo " make start-mapbox-studio # start Mapbox Studio" + @echo " " + @echo "Hints for developers:" + @echo " make # build source code " + @echo " make download-geofabrik area=albania # download OSM data from geofabrik, and create config file" + @echo " make psql # start PostgreSQL console " + @echo " make psql-list-tables # list all PostgreSQL tables " + @echo " make import-sql-dev # start import-sql /bin/bash terminal " + @echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" + @echo " make clean-docker # remove docker containers, PG data volume " + @echo " make forced-clean-sql # drop all PostgreSQL tables for clean environment " + @echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB" + @echo " make remove-docker-images # remove openmaptiles docker images" + @echo " make pgclimb-list-views # list PostgreSQL public schema views" + @echo " make pgclimb-list-tables # list PostgreSQL public schema tabless" + @echo " cat .env # list PG database and MIN_ZOOM and MAX_ZOOM informations" + @echo " cat ./quickstart.log # backup of the last ./quickstart.sh " + @echo " ....TODO.... # start lukasmartinelli/postgis-editor" + @echo " make help # help about avaialable commands" + @echo "==============================================================================" + build/openmaptiles.tm2source/data.yml: mkdir -p build/openmaptiles.tm2source && generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > build/openmaptiles.tm2source/data.yml @@ -21,25 +53,10 @@ list-docker-images: docker images | grep openmaptiles refresh-docker-images: - docker pull openmaptiles/generate-vectortiles - docker pull openmaptiles/import-lakelines - docker pull openmaptiles/import-natural-earth - docker pull openmaptiles/import-osm - docker pull openmaptiles/import-sql - docker pull openmaptiles/import-water - docker pull openmaptiles/openmaptiles-tools - docker pull openmaptiles/postgis - docker pull osm2vectortiles/mapbox-studio + docker-compose pull remove-docker-images: - docker rmi openmaptiles/generate-vectortiles - docker rmi openmaptiles/import-lakelines - docker rmi openmaptiles/import-natural-earth - docker rmi openmaptiles/import-osm - docker rmi openmaptiles/import-sql - docker rmi openmaptiles/import-water - docker rmi openmaptiles/openmaptiles-tools - docker rmi openmaptiles/postgis + docker rmi -f $(docker images | grep "openmaptiles" | awk "{print \$3}") docker rmi osm2vectortiles/mapbox-studio psql: @@ -52,14 +69,9 @@ psql-pg-stat-reset: docker-compose run --rm import-osm /usr/src/app/psql.sh -P pager=off -c 'SELECT pg_stat_statements_reset();' forced-clean-sql: - docker-compose run --rm import-osm /usr/src/app/psql.sh -c "DROP SCHEMA IF EXISTS public CASCADE" - docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE SCHEMA IF NOT EXISTS public" - docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION hstore" - docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION postgis" - docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION pg_stat_statements" - docker-compose run --rm import-osm /usr/src/app/psql.sh -c "GRANT ALL ON SCHEMA public TO postgres;" - docker-compose run --rm import-osm /usr/src/app/psql.sh -c "GRANT ALL ON SCHEMA public TO public;" - docker-compose run --rm import-osm /usr/src/app/psql.sh -c "COMMENT ON SCHEMA public IS 'standard public schema';" + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "DROP SCHEMA IF EXISTS public CASCADE ; CREATE SCHEMA IF NOT EXISTS public; " + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION pg_stat_statements;" + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "GRANT ALL ON SCHEMA public TO public;COMMENT ON SCHEMA public IS 'standard public schema';" pgclimb-list-views: docker-compose run --rm import-osm /usr/src/app/pgclimb.sh -c "select schemaname,viewname from pg_views where schemaname='public' order by viewname;" csv @@ -85,15 +97,18 @@ download-geofabrik: cat ./data/docker-compose-config.yml @echo " " +# the `download-geofabrik` error message mention `list`, if the area parameter is wrong. so I created a similar make command list: docker-compose run --rm import-osm ./download-geofabrik-list.sh +# same as a `make list` download-geofabrik-list: docker-compose run --rm import-osm ./download-geofabrik-list.sh start-mapbox-studio: docker-compose up mapbox-studio +# work in progress - please don't remove test_etlgraph: generate-etlgraph layers/boundary/boundary.yaml generate-etlgraph layers/highway/highway.yaml diff --git a/QUICKSTART.md b/QUICKSTART.md index 9a63cc0..39827cf 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -272,6 +272,11 @@ This is generating mbtiles for your area : [ MIN_ZOOM: "0" - MAX_ZOOM: "7" ] ./quickstart.sh yukon ``` +### Check other commands + +`make help` + + ### Change MIN_ZOOM and MAX_ZOOM modify the settings in the `.env` file @@ -280,9 +285,10 @@ modify the settings in the `.env` file and re-start `./quickstart.sh ` * the new config file re-generating to here ./data/docker-compose-config.yml +* Known problems: + * If you use same area - then the ./data/docker-compose-config.yml not re-generating, so you have to modify by hand! Hints: * Small increments! Never starts with the MAX_ZOOM = 14 * The suggested MAX_ZOOM = 14 - use only with small extracts - diff --git a/docker-compose.yml b/docker-compose.yml index fd79506..194fbdd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: links: - postgres import-osm: - image: "openmaptiles/import-osm" + image: "openmaptiles/import-osm:add_tools" env_file: .env environment: DIFF_MODE: "false" diff --git a/quickstart.sh b/quickstart.sh index 1050e59..b3f9d2d 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -110,19 +110,18 @@ docker info | grep Space echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Pulling or Refreshing OpenMapTiles docker images " -#for developers is not perfect ... -#make refresh-docker-images - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Checking OpenMapTiles docker images " -docker images | grep openmaptiles +make refresh-docker-images echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Stopping running services & removing old containers " make clean-docker +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Checking OpenMapTiles docker images " +docker images | grep openmaptiles + echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Making directories - if they don't exist ( ./build ./data ./pgdata ) " @@ -135,13 +134,6 @@ echo "-------------------------------------------------------------------------- echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) " rm -f ./data/*.mbtiles -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Removing pgdata " -# rm -rf ./pgdata/* - - - if [ ! -f ./data/${testdata} ]; then echo " " echo "-------------------------------------------------------------------------------------" @@ -170,7 +162,6 @@ if [ ! -f ./data/${testdata} ]; then exit 404 fi - echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Remove old generated source files ( ./build/* ) ( if they exist ) " @@ -190,7 +181,11 @@ echo " : Source code: https://github.com/openmaptiles/postgis " echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" docker-compose up -d postgres -# Drop all PostgreSQL tables ... +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Drop and Recreate PostgreSQL public schema " +# Drop all PostgreSQL tables +# This is add an extra safe belt , if the user modify the docker volume seetings make forced-clean-sql echo " " @@ -281,12 +276,12 @@ echo "====> : (disk space) We have created a lot of docker images: " echo " : Hint: you can remove with: docker rmi IMAGE " docker images | grep openmaptiles -# -# echo " " -# echo "-------------------------------------------------------------------------------------" -# echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:" -# echo " : Hint: you can remove with : docker volume rm openmaptiles_pgdata " -# docker volume ls -q | grep openmaptiles + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:" +echo " : Hint: you can remove with : docker volume rm openmaptiles_pgdata " +docker volume ls -q | grep openmaptiles echo " " echo "-------------------------------------------------------------------------------------" @@ -303,33 +298,9 @@ echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete" echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !" echo " " echo "Start experimenting ! And check the QUICKSTART.MD file !" -echo " " -echo "Hints for testing other areas" -echo " make download-geofabrik-list # list actual geofabrik OSM extracts for download -> <> " -echo " make list # list actual geofabrik OSM extracts for download -> <> " -echo " ./quickstart.sh <> # example: ./quickstart.sh madagascar " -echo " " -echo "Hints for designers:" -echo " .... # start Maputnik " -echo " .... # start Tileserver-gl-light" -echo " make start-mapbox-studio # start Mapbox Studio" -echo " " -echo "Hints for developers:" -echo " make download-geofabrik area=albania # download OSM data from geofabrik, and create config file" -echo " make psql # start PostgreSQL console " -echo " make psql-list-tables # list all PostgreSQL tables " -echo " make import-sql-dev # start import-sql /bin/bash terminal " -echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" -echo " make clean-docker # remove docker containers, PG data volume " -echo " make forced-clean-sql # drop all PostgreSQL tables for clean environment " -echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB" -echo " make remove-docker-images # remove openmaptiles docker images" -echo " make pgclimb-list-views # list PostgreSQL public schema views" -echo " make pgclimb-list-tables # list PostgreSQL public schema tabless" -echo " cat .env # list PG database and MIN_ZOOM and MAX_ZOOM informations" -echo " cat ./quickstart.log # backup of the last ./quickstart.sh " -echo " .... # start lukasmartinelli/postgis-editor" -echo " " +echo "Avaialable help commands ( make help ) " +make help + echo "-------------------------------------------------------------------------------------" echo " Acknowledgments " echo " Thanks to all free, open source software developers and Open Data Contributors ! " From 5a6b6764e6c70a9d103d472bb1c803c6b505ef7b Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 30 Nov 2016 05:40:08 +0100 Subject: [PATCH 22/39] remove some refactored files --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index ee38ce1..d0244cc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,9 @@ data.xml *.swp *.pbf -# quickstart log chk +# quickstart quickstart.log -quickstart_checklist.chk + # imput / output data data/* From a106fa35e3165676abaab2c595c01a6c7c9bfb88 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Fri, 2 Dec 2016 17:44:14 +0100 Subject: [PATCH 23/39] add QA reports - markdown FREQ, TOPLENGTH, ... --- qa/layer_freq.sh | 32 +++++++++++++++ qa/layer_numvar_analyze.sh | 37 +++++++++++++++++ qa/layer_toplength.sh | 34 +++++++++++++++ qa/run.sh | 84 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 187 insertions(+) create mode 100755 qa/layer_freq.sh create mode 100755 qa/layer_numvar_analyze.sh create mode 100755 qa/layer_toplength.sh create mode 100755 qa/run.sh diff --git a/qa/layer_freq.sh b/qa/layer_freq.sh new file mode 100755 index 0000000..8265554 --- /dev/null +++ b/qa/layer_freq.sh @@ -0,0 +1,32 @@ + +#!/bin/bash + +layerid=$1 +classvars=$2 + +echo "# FREQ - $layerid group by : $classvars " + +for z in {0..15} +do +echo " " +echo "## $layerid z$z" + +SQL=$(generate-qadoc layers/${layerid}/${layerid}.yaml $z ) + +read -r -d '' SQLCODE <<- EOMSQL + select $classvars , count(*) as _count_ from + $SQL + GROUP BY $classvars + ORDER BY $classvars + ; +EOMSQL + +echo "\`\`\`SQL" +echo "$SQLCODE" +echo "\`\`\`" + +docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ + | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' + +done + diff --git a/qa/layer_numvar_analyze.sh b/qa/layer_numvar_analyze.sh new file mode 100755 index 0000000..a37c7e9 --- /dev/null +++ b/qa/layer_numvar_analyze.sh @@ -0,0 +1,37 @@ + +#!/bin/bash + +layerid=$1 +var=$2 + +echo "# NUMVAR-ANALYZE - $layerid - $var " + +for z in {0..15} +do +echo " " +echo "## $layerid z$z - $var " + +SQL=$(generate-qadoc layers/${layerid}/${layerid}.yaml $z ) + +read -r -d '' SQLCODE <<- EOMSQL + SELECT + count($var) as count_$var + ,min($var) as min_$var + ,max($var) as max_$var + ,avg($var) as avg_$var + ,stddev($var) as stddev_$var + ,variance($var) as variance_$var + FROM + $SQL + ; +EOMSQL + +echo "\`\`\`SQL" +echo "$SQLCODE" +echo "\`\`\`" + +docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ + | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' + +done + diff --git a/qa/layer_toplength.sh b/qa/layer_toplength.sh new file mode 100755 index 0000000..3624d30 --- /dev/null +++ b/qa/layer_toplength.sh @@ -0,0 +1,34 @@ + +#!/bin/bash + + +layerid=$1 +classvar=$2 + +echo "# TOP30LENGHT - $layerid - $classvar" + +for z in {0..15} +do +echo " " +echo "## $layerid z$z" + +SQL=$(generate-qadoc layers/${layerid}/${layerid}.yaml $z ) + +read -r -d '' SQLCODE <<- EOMSQL + select $classvar , length( $classvar ) as _length_ from + $SQL + WHERE length( $classvar ) > 0 + ORDER BY length( $classvar ) DESC NULLS LAST + LIMIT 30 + ; +EOMSQL + +echo "\`\`\`SQL" +echo "$SQLCODE" +echo "\`\`\`" + +docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ + | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' + +done + diff --git a/qa/run.sh b/qa/run.sh new file mode 100755 index 0000000..518986d --- /dev/null +++ b/qa/run.sh @@ -0,0 +1,84 @@ + +#!/bin/bash + +# example call from the parent folder : ./qa/run.sh + +export_path=./build/qadoc + +mkdir -p ${export_path} +rm -f ${export_path}/*.md + +# ---- freq + + +./qa/layer_freq.sh aeroway "class" > ${export_path}/freq_aeroway.md + +./qa/layer_freq.sh boundary "admin_level,disputed" > ${export_path}/freq_boundary_admin_level_disputed.md +./qa/layer_freq.sh boundary "admin_level" > ${export_path}/freq_boundary_admin_level.md +./qa/layer_freq.sh boundary "disputed" > ${export_path}/freq_boundary_disputed.md + +./qa/layer_freq.sh building "render_min_height" > ${export_path}/freq_building.md + +#./qa/layer_freq.sh housenumber "housenumber " > ${export_path}freq_housenumber.md +./qa/layer_freq.sh landcover "class, subclass" > ${export_path}/freq_landcover.md + +./qa/layer_freq.sh landuse "class " > ${export_path}/freq_landuse.md + +./qa/layer_freq.sh park "class " > ${export_path}/freq_park.md + +./qa/layer_freq.sh place "class " > ${export_path}/freq_place_class.md +./qa/layer_freq.sh place "class,capital " > ${export_path}/freq_place_class_capital.md +./qa/layer_freq.sh place "capital " > ${export_path}/freq_place_capital.md +./qa/layer_freq.sh place "class,capital,rank " > ${export_path}/freq_place_class_capital_rank.md +./qa/layer_freq.sh place "rank " > ${export_path}/freq_place_rank.md + +./qa/layer_freq.sh poi "class " > ${export_path}/freq_poi_class.md +./qa/layer_freq.sh poi "subclass " > ${export_path}/freq_poi_subclass.md +./qa/layer_freq.sh poi "rank " > ${export_path}/freq_poi_rank.md +./qa/layer_freq.sh poi "class,subclass " > ${export_path}/freq_poi_class_subclass.md +./qa/layer_freq.sh poi "class,subclass,rank" > ${export_path}/freq_poi_class_subclass_rank.md +./qa/layer_freq.sh poi "class,rank " > ${export_path}/freq_poi_class_rank.md + +./qa/layer_freq.sh transportation "class,subclass " > ${export_path}/freq_transportation_class_subclass.md +./qa/layer_freq.sh transportation "class,subclass,service,oneway,brunnel,ramp" > ${export_path}/freq_transportation_class_subclass_service_oneway_brunnel_ramp.md +./qa/layer_freq.sh transportation "subclass " > ${export_path}/freq_transportation_subclass.md +./qa/layer_freq.sh transportation "class " > ${export_path}/freq_transportation_class.md + +./qa/layer_freq.sh transportation_name "class " > ${export_path}/freq_transportation_name_class.md +./qa/layer_freq.sh transportation_name "reflength " > ${export_path}/freq_transportation_name_reflength.md + +./qa/layer_freq.sh water "class " > ${export_path}/freq_water.md + +./qa/layer_freq.sh water_name "class " > ${export_path}/freq_water_name.md + +./qa/layer_freq.sh waterway "class " > ${export_path}/freq_waterway.md + +# ---- toplength + +./layer_toplength.sh housenumber "housenumber" > ${export_path}/toplength_housenumber_housenumber.md + +./layer_toplength.sh place "name" > ${export_path}/toplength_place_name.md +./layer_toplength.sh place "name_en" > ${export_path}/toplength_place_name_en.md + +./layer_toplength.sh poi "name" > ${export_path}/toplength_poi_name.md +./layer_toplength.sh poi "name_en" > ${export_path}/toplength_poi_name_en.md + +./layer_toplength.sh transportation_name "name" > ${export_path}/toplength_transportation_name.md +./layer_toplength.sh transportation_name "ref" > ${export_path}/toplength_transportation_ref.md +./layer_toplength.sh transportation_name "network" > ${export_path}/toplength_transportation_network.md + +./layer_toplength.sh water_name "name" > ${export_path}/toplength_water_name_name.md +./layer_toplength.sh water_name "name_en" > ${export_path}/toplength_water_name_name_en.md + +./layer_toplength.sh waterway "name" > ${export_path}/toplength_waterway_name.md + + + + +# ---- numvar analyze + +./qa/layer_numvar_analyze.sh building "render_min_height" > ${export_path}/numvara_building_render_min_height.md +./qa/layer_numvar_analyze.sh building "render_max_height" > ${export_path}/numvara_building_render_max_height.md + +./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ${export_path}/numvara_transportation_name_ref_length.md + From 7ea5e72212e460fa02f503995724da902dbccddf Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 3 Dec 2016 02:11:50 +0100 Subject: [PATCH 24/39] qa fixes --- qa/layer_freq.sh | 8 ++-- qa/layer_numvar_analyze.sh | 18 +++---- qa/layer_toplength.sh | 8 ++-- qa/run.sh | 96 +++++++++++++++++++------------------- 4 files changed, 66 insertions(+), 64 deletions(-) diff --git a/qa/layer_freq.sh b/qa/layer_freq.sh index 8265554..df23cc1 100755 --- a/qa/layer_freq.sh +++ b/qa/layer_freq.sh @@ -9,7 +9,7 @@ echo "# FREQ - $layerid group by : $classvars " for z in {0..15} do echo " " -echo "## $layerid z$z" +echo "## $layerid z$z - freq" SQL=$(generate-qadoc layers/${layerid}/${layerid}.yaml $z ) @@ -21,9 +21,9 @@ read -r -d '' SQLCODE <<- EOMSQL ; EOMSQL -echo "\`\`\`SQL" -echo "$SQLCODE" -echo "\`\`\`" +#echo "\`\`\`SQL" +#echo "$SQLCODE" +#echo "\`\`\`" docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' diff --git a/qa/layer_numvar_analyze.sh b/qa/layer_numvar_analyze.sh index a37c7e9..c18669d 100755 --- a/qa/layer_numvar_analyze.sh +++ b/qa/layer_numvar_analyze.sh @@ -15,20 +15,20 @@ SQL=$(generate-qadoc layers/${layerid}/${layerid}.yaml $z ) read -r -d '' SQLCODE <<- EOMSQL SELECT - count($var) as count_$var - ,min($var) as min_$var - ,max($var) as max_$var - ,avg($var) as avg_$var - ,stddev($var) as stddev_$var - ,variance($var) as variance_$var + count($var) as count + ,min($var) as min + ,max($var) as max + ,avg($var) as avg + ,stddev($var) as stddev + ,variance($var) as variance FROM $SQL ; EOMSQL -echo "\`\`\`SQL" -echo "$SQLCODE" -echo "\`\`\`" +#echo "\`\`\`SQL" +#echo "$SQLCODE" +#echo "\`\`\`" docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' diff --git a/qa/layer_toplength.sh b/qa/layer_toplength.sh index 3624d30..10a612c 100755 --- a/qa/layer_toplength.sh +++ b/qa/layer_toplength.sh @@ -10,7 +10,7 @@ echo "# TOP30LENGHT - $layerid - $classvar" for z in {0..15} do echo " " -echo "## $layerid z$z" +echo "## $layerid z$z max length ($classvar)" SQL=$(generate-qadoc layers/${layerid}/${layerid}.yaml $z ) @@ -23,9 +23,9 @@ read -r -d '' SQLCODE <<- EOMSQL ; EOMSQL -echo "\`\`\`SQL" -echo "$SQLCODE" -echo "\`\`\`" +#echo "\`\`\`SQL" +#echo "$SQLCODE" +#echo "\`\`\`" docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' diff --git a/qa/run.sh b/qa/run.sh index 518986d..38f2be4 100755 --- a/qa/run.sh +++ b/qa/run.sh @@ -5,80 +5,82 @@ export_path=./build/qadoc -mkdir -p ${export_path} -rm -f ${export_path}/*.md + # ---- freq -./qa/layer_freq.sh aeroway "class" > ${export_path}/freq_aeroway.md + +mkdir -p ./build/qadoc +rm -f ./build/qadoc/*.md -./qa/layer_freq.sh boundary "admin_level,disputed" > ${export_path}/freq_boundary_admin_level_disputed.md -./qa/layer_freq.sh boundary "admin_level" > ${export_path}/freq_boundary_admin_level.md -./qa/layer_freq.sh boundary "disputed" > ${export_path}/freq_boundary_disputed.md +./qa/layer_freq.sh aeroway "class" > ./build/qadoc/freq_aeroway.md -./qa/layer_freq.sh building "render_min_height" > ${export_path}/freq_building.md +./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qadoc/freq_boundary_admin_level_disputed.md +./qa/layer_freq.sh boundary "admin_level" > ./build/qadoc/freq_boundary_admin_level.md +./qa/layer_freq.sh boundary "disputed" > ./build/qadoc/freq_boundary_disputed.md -#./qa/layer_freq.sh housenumber "housenumber " > ${export_path}freq_housenumber.md -./qa/layer_freq.sh landcover "class, subclass" > ${export_path}/freq_landcover.md +./qa/layer_freq.sh building "render_min_height" > ./build/qadoc/freq_building.md -./qa/layer_freq.sh landuse "class " > ${export_path}/freq_landuse.md +#./qa/layer_freq.sh housenumber "housenumber " > ./build/qadocfreq_housenumber.md +./qa/layer_freq.sh landcover "class, subclass" > ./build/qadoc/freq_landcover.md -./qa/layer_freq.sh park "class " > ${export_path}/freq_park.md +./qa/layer_freq.sh landuse "class " > ./build/qadoc/freq_landuse.md -./qa/layer_freq.sh place "class " > ${export_path}/freq_place_class.md -./qa/layer_freq.sh place "class,capital " > ${export_path}/freq_place_class_capital.md -./qa/layer_freq.sh place "capital " > ${export_path}/freq_place_capital.md -./qa/layer_freq.sh place "class,capital,rank " > ${export_path}/freq_place_class_capital_rank.md -./qa/layer_freq.sh place "rank " > ${export_path}/freq_place_rank.md +./qa/layer_freq.sh park "class " > ./build/qadoc/freq_park.md -./qa/layer_freq.sh poi "class " > ${export_path}/freq_poi_class.md -./qa/layer_freq.sh poi "subclass " > ${export_path}/freq_poi_subclass.md -./qa/layer_freq.sh poi "rank " > ${export_path}/freq_poi_rank.md -./qa/layer_freq.sh poi "class,subclass " > ${export_path}/freq_poi_class_subclass.md -./qa/layer_freq.sh poi "class,subclass,rank" > ${export_path}/freq_poi_class_subclass_rank.md -./qa/layer_freq.sh poi "class,rank " > ${export_path}/freq_poi_class_rank.md +./qa/layer_freq.sh place "class " > ./build/qadoc/freq_place_class.md +./qa/layer_freq.sh place "class,capital " > ./build/qadoc/freq_place_class_capital.md +./qa/layer_freq.sh place "capital " > ./build/qadoc/freq_place_capital.md +./qa/layer_freq.sh place "class,capital,rank " > ./build/qadoc/freq_place_class_capital_rank.md +./qa/layer_freq.sh place "rank " > ./build/qadoc/freq_place_rank.md -./qa/layer_freq.sh transportation "class,subclass " > ${export_path}/freq_transportation_class_subclass.md -./qa/layer_freq.sh transportation "class,subclass,service,oneway,brunnel,ramp" > ${export_path}/freq_transportation_class_subclass_service_oneway_brunnel_ramp.md -./qa/layer_freq.sh transportation "subclass " > ${export_path}/freq_transportation_subclass.md -./qa/layer_freq.sh transportation "class " > ${export_path}/freq_transportation_class.md +./qa/layer_freq.sh poi "class " > ./build/qadoc/freq_poi_class.md +./qa/layer_freq.sh poi "subclass " > ./build/qadoc/freq_poi_subclass.md +./qa/layer_freq.sh poi "rank " > ./build/qadoc/freq_poi_rank.md +./qa/layer_freq.sh poi "class,subclass " > ./build/qadoc/freq_poi_class_subclass.md +./qa/layer_freq.sh poi "class,subclass,rank" > ./build/qadoc/freq_poi_class_subclass_rank.md +./qa/layer_freq.sh poi "class,rank " > ./build/qadoc/freq_poi_class_rank.md -./qa/layer_freq.sh transportation_name "class " > ${export_path}/freq_transportation_name_class.md -./qa/layer_freq.sh transportation_name "reflength " > ${export_path}/freq_transportation_name_reflength.md +./qa/layer_freq.sh transportation "class, oneway, ramp, brunnel, service" > ./build/qadoc/freq_transportation_class_oneway_ramp_brunnel_service.md +./qa/layer_freq.sh transportation "oneway, ramp, brunnel, service " > ./build/qadoc/freq_transportation_oneway_ramp_brunnel_service.md +./qa/layer_freq.sh transportation "class " > ./build/qadoc/freq_transportation_class.md -./qa/layer_freq.sh water "class " > ${export_path}/freq_water.md +./qa/layer_freq.sh transportation_name "class " > ./build/qadoc/freq_transportation_name_class.md +./qa/layer_freq.sh transportation_name "reflength " > ./build/qadoc/freq_transportation_name_reflength.md + +./qa/layer_freq.sh water "class " > ./build/qadoc/freq_water.md + +./qa/layer_freq.sh water_name "class " > ./build/qadoc/freq_water_name.md + +./qa/layer_freq.sh waterway "class " > ./build/qadoc/freq_waterway.md -./qa/layer_freq.sh water_name "class " > ${export_path}/freq_water_name.md -./qa/layer_freq.sh waterway "class " > ${export_path}/freq_waterway.md # ---- toplength -./layer_toplength.sh housenumber "housenumber" > ${export_path}/toplength_housenumber_housenumber.md +./qa/layer_toplength.sh housenumber "housenumber" > ./build/qadoc/toplength_housenumber_housenumber.md -./layer_toplength.sh place "name" > ${export_path}/toplength_place_name.md -./layer_toplength.sh place "name_en" > ${export_path}/toplength_place_name_en.md +./qa/layer_toplength.sh place "name" > ./build/qadoc/toplength_place_name.md +./qa/layer_toplength.sh place "name_en" > ./build/qadoc/toplength_place_name_en.md -./layer_toplength.sh poi "name" > ${export_path}/toplength_poi_name.md -./layer_toplength.sh poi "name_en" > ${export_path}/toplength_poi_name_en.md +./qa/layer_toplength.sh poi "name" > ./build/qadoc/toplength_poi_name.md +./qa/layer_toplength.sh poi "name_en" > ./build/qadoc/toplength_poi_name_en.md -./layer_toplength.sh transportation_name "name" > ${export_path}/toplength_transportation_name.md -./layer_toplength.sh transportation_name "ref" > ${export_path}/toplength_transportation_ref.md -./layer_toplength.sh transportation_name "network" > ${export_path}/toplength_transportation_network.md - -./layer_toplength.sh water_name "name" > ${export_path}/toplength_water_name_name.md -./layer_toplength.sh water_name "name_en" > ${export_path}/toplength_water_name_name_en.md - -./layer_toplength.sh waterway "name" > ${export_path}/toplength_waterway_name.md +./qa/layer_toplength.sh transportation_name "name" > ./build/qadoc/toplength_transportation_name.md +./qa/layer_toplength.sh transportation_name "ref" > ./build/qadoc/toplength_transportation_ref.md +./qa/layer_toplength.sh transportation_name "network" > ./build/qadoc/toplength_transportation_network.md +./qa/layer_toplength.sh water_name "name" > ./build/qadoc/toplength_water_name_name.md +./qa/layer_toplength.sh water_name "name_en" > ./build/qadoc/toplength_water_name_name_en.md +./qa/layer_toplength.sh waterway "name" > ./build/qadoc/toplength_waterway_name.md # ---- numvar analyze -./qa/layer_numvar_analyze.sh building "render_min_height" > ${export_path}/numvara_building_render_min_height.md -./qa/layer_numvar_analyze.sh building "render_max_height" > ${export_path}/numvara_building_render_max_height.md +./qa/layer_numvar_analyze.sh building "render_min_height" > ./build/qadoc/numvara_building_render_min_height.md +./qa/layer_numvar_analyze.sh building "render_max_height" > ./build/qadoc/numvara_building_render_max_height.md -./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ${export_path}/numvara_transportation_name_ref_length.md +./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ./build/qadoc/numvara_transportation_name_ref_length.md From bf1b4e0a35ddae1a39a480466cbba5dcc367a239 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sat, 3 Dec 2016 02:45:01 +0100 Subject: [PATCH 25/39] some fixes --- qa/run.sh | 107 ++++++++++++++++++++++++++---------------------------- 1 file changed, 51 insertions(+), 56 deletions(-) diff --git a/qa/run.sh b/qa/run.sh index 38f2be4..5d820e4 100755 --- a/qa/run.sh +++ b/qa/run.sh @@ -3,84 +3,79 @@ # example call from the parent folder : ./qa/run.sh -export_path=./build/qadoc -# ---- freq - - +# ---- freq --------------------------------------------------------------- mkdir -p ./build/qadoc rm -f ./build/qadoc/*.md -./qa/layer_freq.sh aeroway "class" > ./build/qadoc/freq_aeroway.md +./qa/layer_freq.sh aeroway "class" > ./build/qadoc/freq_aeroway__class.md -./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qadoc/freq_boundary_admin_level_disputed.md -./qa/layer_freq.sh boundary "admin_level" > ./build/qadoc/freq_boundary_admin_level.md -./qa/layer_freq.sh boundary "disputed" > ./build/qadoc/freq_boundary_disputed.md +./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qadoc/freq_boundary__admin_level_disputed.md +./qa/layer_freq.sh boundary "admin_level" > ./build/qadoc/freq_boundary__admin_level.md +./qa/layer_freq.sh boundary "disputed" > ./build/qadoc/freq_boundary__disputed.md -./qa/layer_freq.sh building "render_min_height" > ./build/qadoc/freq_building.md +./qa/layer_freq.sh building "render_min_height" > ./build/qadoc/freq_building__render_min_height.md #./qa/layer_freq.sh housenumber "housenumber " > ./build/qadocfreq_housenumber.md -./qa/layer_freq.sh landcover "class, subclass" > ./build/qadoc/freq_landcover.md +./qa/layer_freq.sh landcover "class, subclass" > ./build/qadoc/freq_landcover__class_subclass.md -./qa/layer_freq.sh landuse "class " > ./build/qadoc/freq_landuse.md +./qa/layer_freq.sh landuse "class " > ./build/qadoc/freq_landuse__class.md -./qa/layer_freq.sh park "class " > ./build/qadoc/freq_park.md +./qa/layer_freq.sh park "class " > ./build/qadoc/freq_park__class.md -./qa/layer_freq.sh place "class " > ./build/qadoc/freq_place_class.md -./qa/layer_freq.sh place "class,capital " > ./build/qadoc/freq_place_class_capital.md -./qa/layer_freq.sh place "capital " > ./build/qadoc/freq_place_capital.md -./qa/layer_freq.sh place "class,capital,rank " > ./build/qadoc/freq_place_class_capital_rank.md -./qa/layer_freq.sh place "rank " > ./build/qadoc/freq_place_rank.md +./qa/layer_freq.sh place "class " > ./build/qadoc/freq_place__class.md +./qa/layer_freq.sh place "class,capital " > ./build/qadoc/freq_place__class_capital.md +./qa/layer_freq.sh place "capital " > ./build/qadoc/freq_place__capital.md +./qa/layer_freq.sh place "class,capital,rank " > ./build/qadoc/freq_place__class_capital_rank.md +./qa/layer_freq.sh place "rank " > ./build/qadoc/freq_place__rank.md -./qa/layer_freq.sh poi "class " > ./build/qadoc/freq_poi_class.md -./qa/layer_freq.sh poi "subclass " > ./build/qadoc/freq_poi_subclass.md -./qa/layer_freq.sh poi "rank " > ./build/qadoc/freq_poi_rank.md -./qa/layer_freq.sh poi "class,subclass " > ./build/qadoc/freq_poi_class_subclass.md -./qa/layer_freq.sh poi "class,subclass,rank" > ./build/qadoc/freq_poi_class_subclass_rank.md -./qa/layer_freq.sh poi "class,rank " > ./build/qadoc/freq_poi_class_rank.md +./qa/layer_freq.sh poi "class " > ./build/qadoc/freq_poi__class.md +./qa/layer_freq.sh poi "subclass " > ./build/qadoc/freq_poi__subclass.md +./qa/layer_freq.sh poi "rank " > ./build/qadoc/freq_poi__rank.md +./qa/layer_freq.sh poi "class,subclass " > ./build/qadoc/freq_poi__class_subclass.md +./qa/layer_freq.sh poi "class,subclass,rank" > ./build/qadoc/freq_poi__class_subclass_rank.md +./qa/layer_freq.sh poi "class,rank " > ./build/qadoc/freq_poi__class_rank.md -./qa/layer_freq.sh transportation "class, oneway, ramp, brunnel, service" > ./build/qadoc/freq_transportation_class_oneway_ramp_brunnel_service.md -./qa/layer_freq.sh transportation "oneway, ramp, brunnel, service " > ./build/qadoc/freq_transportation_oneway_ramp_brunnel_service.md -./qa/layer_freq.sh transportation "class " > ./build/qadoc/freq_transportation_class.md +./qa/layer_freq.sh transportation "class, oneway, ramp, brunnel, service" > ./build/qadoc/freq_transportation__class_oneway_ramp_brunnel_service.md +./qa/layer_freq.sh transportation "oneway, ramp, brunnel, service " > ./build/qadoc/freq_transportation__oneway_ramp_brunnel_service.md +./qa/layer_freq.sh transportation "class " > ./build/qadoc/freq_transportation__class.md -./qa/layer_freq.sh transportation_name "class " > ./build/qadoc/freq_transportation_name_class.md -./qa/layer_freq.sh transportation_name "reflength " > ./build/qadoc/freq_transportation_name_reflength.md +./qa/layer_freq.sh transportation_name "class " > ./build/qadoc/freq_transportation_name__class.md +./qa/layer_freq.sh transportation_name "ref_length" > ./build/qadoc/freq_transportation_name__ref_length.md -./qa/layer_freq.sh water "class " > ./build/qadoc/freq_water.md +./qa/layer_freq.sh water "class " > ./build/qadoc/freq_water__class.md -./qa/layer_freq.sh water_name "class " > ./build/qadoc/freq_water_name.md +./qa/layer_freq.sh water_name "class " > ./build/qadoc/freq_water_name__class.md -./qa/layer_freq.sh waterway "class " > ./build/qadoc/freq_waterway.md +./qa/layer_freq.sh waterway "class " > ./build/qadoc/freq_waterway__class.md + +# ---- toplength ------------------------------------------- + +./qa/layer_toplength.sh housenumber "housenumber" > ./build/qadoc/toplength_housenumber__housenumber.md + +./qa/layer_toplength.sh place "name" > ./build/qadoc/toplength_place__name.md +./qa/layer_toplength.sh place "name_en" > ./build/qadoc/toplength_place__name_en.md + +./qa/layer_toplength.sh poi "name" > ./build/qadoc/toplength_poi__name.md +./qa/layer_toplength.sh poi "name_en" > ./build/qadoc/toplength_poi__name_en.md + +./qa/layer_toplength.sh transportation_name "name" > ./build/qadoc/toplength_transportation_name__name.md +./qa/layer_toplength.sh transportation_name "ref" > ./build/qadoc/toplength_transportation_name__ref.md +./qa/layer_toplength.sh transportation_name "network" > ./build/qadoc/toplength_transportation_name__network.md + +./qa/layer_toplength.sh water_name "name" > ./build/qadoc/toplength_water_name__name.md +./qa/layer_toplength.sh water_name "name_en" > ./build/qadoc/toplength_water_name__name_en.md + +./qa/layer_toplength.sh waterway "name" > ./build/qadoc/toplength_waterway__name.md +# ---- numvar analyze ------------------------------------- -# ---- toplength +./qa/layer_numvar_analyze.sh building "render_min_height" > ./build/qadoc/numvara_building__render_min_height.md +./qa/layer_numvar_analyze.sh building "render_max_height" > ./build/qadoc/numvara_building__render_max_height.md -./qa/layer_toplength.sh housenumber "housenumber" > ./build/qadoc/toplength_housenumber_housenumber.md - -./qa/layer_toplength.sh place "name" > ./build/qadoc/toplength_place_name.md -./qa/layer_toplength.sh place "name_en" > ./build/qadoc/toplength_place_name_en.md - -./qa/layer_toplength.sh poi "name" > ./build/qadoc/toplength_poi_name.md -./qa/layer_toplength.sh poi "name_en" > ./build/qadoc/toplength_poi_name_en.md - -./qa/layer_toplength.sh transportation_name "name" > ./build/qadoc/toplength_transportation_name.md -./qa/layer_toplength.sh transportation_name "ref" > ./build/qadoc/toplength_transportation_ref.md -./qa/layer_toplength.sh transportation_name "network" > ./build/qadoc/toplength_transportation_network.md - -./qa/layer_toplength.sh water_name "name" > ./build/qadoc/toplength_water_name_name.md -./qa/layer_toplength.sh water_name "name_en" > ./build/qadoc/toplength_water_name_name_en.md - -./qa/layer_toplength.sh waterway "name" > ./build/qadoc/toplength_waterway_name.md - - -# ---- numvar analyze - -./qa/layer_numvar_analyze.sh building "render_min_height" > ./build/qadoc/numvara_building_render_min_height.md -./qa/layer_numvar_analyze.sh building "render_max_height" > ./build/qadoc/numvara_building_render_max_height.md - -./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ./build/qadoc/numvara_transportation_name_ref_length.md +./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ./build/qadoc/numvara_transportation_name__ref_length.md From 2e72698ed979985391818d34ef240209c66accca Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 4 Dec 2016 01:37:28 +0100 Subject: [PATCH 26/39] refactor to `make generate-qareports` and `make generate-devdoc` --- Makefile | 48 +++++++++++--------- docker-compose.yml | 6 +-- qa/layer_freq.sh | 5 ++- qa/layer_numvar_analyze.sh | 6 ++- qa/layer_toplength.sh | 5 ++- qa/run.sh | 91 +++++++++++++++++++------------------- 6 files changed, 87 insertions(+), 74 deletions(-) diff --git a/Makefile b/Makefile index 969dc64..a6d7cbf 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,12 @@ help: @echo " make start-mapbox-studio # start Mapbox Studio" @echo " " @echo "Hints for developers:" - @echo " make # build source code " + @echo " make # build source code " @echo " make download-geofabrik area=albania # download OSM data from geofabrik, and create config file" @echo " make psql # start PostgreSQL console " @echo " make psql-list-tables # list all PostgreSQL tables " + @echo " make generate-qareports # generate reports [./build/qareports]" + @echo " make generate-devdoc # generate devdoc [./build/devdoc]" @echo " make import-sql-dev # start import-sql /bin/bash terminal " @echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" @echo " make clean-docker # remove docker containers, PG data volume " @@ -29,7 +31,7 @@ help: @echo " cat .env # list PG database and MIN_ZOOM and MAX_ZOOM informations" @echo " cat ./quickstart.log # backup of the last ./quickstart.sh " @echo " ....TODO.... # start lukasmartinelli/postgis-editor" - @echo " make help # help about avaialable commands" + @echo " make help # help about avaialable commands" @echo "==============================================================================" build/openmaptiles.tm2source/data.yml: @@ -53,7 +55,7 @@ list-docker-images: docker images | grep openmaptiles refresh-docker-images: - docker-compose pull + echo docker-compose pull remove-docker-images: docker rmi -f $(docker images | grep "openmaptiles" | awk "{print \$3}") @@ -93,9 +95,9 @@ download-geofabrik: docker-compose run --rm import-osm ./download-geofabrik.sh $(area) ls -la ./data/$(area).* @echo "Generated config file: ./data/docker-compose-config.yml" - @echo " " - cat ./data/docker-compose-config.yml - @echo " " + @echo " " + cat ./data/docker-compose-config.yml + @echo " " # the `download-geofabrik` error message mention `list`, if the area parameter is wrong. so I created a similar make command list: @@ -108,18 +110,24 @@ download-geofabrik-list: start-mapbox-studio: docker-compose up mapbox-studio +generate-qareports: + ./qa/run.sh + # work in progress - please don't remove -test_etlgraph: - generate-etlgraph layers/boundary/boundary.yaml - generate-etlgraph layers/highway/highway.yaml - generate-etlgraph layers/housenumber/housenumber.yaml - generate-etlgraph layers/landuse/landuse.yaml - generate-etlgraph layers/poi/poi.yaml - generate-etlgraph layers/water/water.yaml - generate-etlgraph layers/waterway/waterway.yaml - generate-etlgraph layers/building/building.yaml - generate-etlgraph layers/highway_name/highway_name.yaml - generate-etlgraph layers/landcover/landcover.yaml - generate-etlgraph layers/place/place.yaml - generate-etlgraph layers/railway/railway.yaml - generate-etlgraph layers/water_name/water_name.yaml +generate-devdoc: + mkdir -p ./build/devdoc + generate-etlgraph layers/aeroway/aeroway.yaml ./build/devdoc + generate-etlgraph layers/boundary/boundary.yaml ./build/devdoc + generate-etlgraph layers/building/building.yaml ./build/devdoc + generate-etlgraph layers/housenumber/housenumber.yaml ./build/devdoc + generate-etlgraph layers/landcover/landcover.yaml ./build/devdoc + generate-etlgraph layers/landuse/landuse.yaml ./build/devdoc + generate-etlgraph layers/park/park.yaml ./build/devdoc + generate-etlgraph layers/place/place.yaml ./build/devdoc + generate-etlgraph layers/poi/poi.yaml ./build/devdoc + generate-etlgraph layers/transportation/transportation.yaml ./build/devdoc + generate-etlgraph layers/transportation_name/transportation_name.yaml ./build/devdoc + generate-etlgraph layers/water/water.yaml ./build/devdoc + generate-etlgraph layers/water_name/water_name.yaml ./build/devdoc + generate-etlgraph layers/waterway/waterway.yaml ./build/devdoc + diff --git a/docker-compose.yml b/docker-compose.yml index 1f741ca..3574e72 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,11 +25,7 @@ services: links: - postgres import-osm: -<<<<<<< HEAD - image: "openmaptiles/import-osm:add_tools" -======= - image: "openmaptiles/import-osm:0.1" ->>>>>>> master + image: "openmaptiles/import-osm:0.2" env_file: .env environment: DIFF_MODE: "false" diff --git a/qa/layer_freq.sh b/qa/layer_freq.sh index df23cc1..b54d4bf 100755 --- a/qa/layer_freq.sh +++ b/qa/layer_freq.sh @@ -1,5 +1,8 @@ #!/bin/bash +set -o errexit +set -o pipefail +set -o nounset layerid=$1 classvars=$2 @@ -11,7 +14,7 @@ do echo " " echo "## $layerid z$z - freq" -SQL=$(generate-qadoc layers/${layerid}/${layerid}.yaml $z ) +SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) read -r -d '' SQLCODE <<- EOMSQL select $classvars , count(*) as _count_ from diff --git a/qa/layer_numvar_analyze.sh b/qa/layer_numvar_analyze.sh index c18669d..cb5eb6d 100755 --- a/qa/layer_numvar_analyze.sh +++ b/qa/layer_numvar_analyze.sh @@ -1,5 +1,8 @@ #!/bin/bash +set -o errexit +set -o pipefail +set -o nounset layerid=$1 var=$2 @@ -11,7 +14,7 @@ do echo " " echo "## $layerid z$z - $var " -SQL=$(generate-qadoc layers/${layerid}/${layerid}.yaml $z ) +SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) read -r -d '' SQLCODE <<- EOMSQL SELECT @@ -34,4 +37,3 @@ docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P borde | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' done - diff --git a/qa/layer_toplength.sh b/qa/layer_toplength.sh index 10a612c..a4c7173 100755 --- a/qa/layer_toplength.sh +++ b/qa/layer_toplength.sh @@ -1,5 +1,8 @@ #!/bin/bash +set -o errexit +set -o pipefail +set -o nounset layerid=$1 @@ -12,7 +15,7 @@ do echo " " echo "## $layerid z$z max length ($classvar)" -SQL=$(generate-qadoc layers/${layerid}/${layerid}.yaml $z ) +SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) read -r -d '' SQLCODE <<- EOMSQL select $classvar , length( $classvar ) as _length_ from diff --git a/qa/run.sh b/qa/run.sh index 5d820e4..8909698 100755 --- a/qa/run.sh +++ b/qa/run.sh @@ -1,81 +1,82 @@ #!/bin/bash +set -o errexit +set -o pipefail +set -o nounset # example call from the parent folder : ./qa/run.sh - - - +# or make generate-qareports # ---- freq --------------------------------------------------------------- mkdir -p ./build/qadoc -rm -f ./build/qadoc/*.md +rm -f ./build/qareports/*.md -./qa/layer_freq.sh aeroway "class" > ./build/qadoc/freq_aeroway__class.md +./qa/layer_freq.sh aeroway "class" > ./build/qareports/freq_aeroway__class.md -./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qadoc/freq_boundary__admin_level_disputed.md -./qa/layer_freq.sh boundary "admin_level" > ./build/qadoc/freq_boundary__admin_level.md -./qa/layer_freq.sh boundary "disputed" > ./build/qadoc/freq_boundary__disputed.md +./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qareports/freq_boundary__admin_level_disputed.md +./qa/layer_freq.sh boundary "admin_level" > ./build/qareports/freq_boundary__admin_level.md +./qa/layer_freq.sh boundary "disputed" > ./build/qareports/freq_boundary__disputed.md -./qa/layer_freq.sh building "render_min_height" > ./build/qadoc/freq_building__render_min_height.md +./qa/layer_freq.sh building "render_min_height" > ./build/qareports/freq_building__render_min_height.md #./qa/layer_freq.sh housenumber "housenumber " > ./build/qadocfreq_housenumber.md -./qa/layer_freq.sh landcover "class, subclass" > ./build/qadoc/freq_landcover__class_subclass.md +./qa/layer_freq.sh landcover "class, subclass" > ./build/qareports/freq_landcover__class_subclass.md -./qa/layer_freq.sh landuse "class " > ./build/qadoc/freq_landuse__class.md +./qa/layer_freq.sh landuse "class " > ./build/qareports/freq_landuse__class.md -./qa/layer_freq.sh park "class " > ./build/qadoc/freq_park__class.md +./qa/layer_freq.sh park "class " > ./build/qareports/freq_park__class.md -./qa/layer_freq.sh place "class " > ./build/qadoc/freq_place__class.md -./qa/layer_freq.sh place "class,capital " > ./build/qadoc/freq_place__class_capital.md -./qa/layer_freq.sh place "capital " > ./build/qadoc/freq_place__capital.md -./qa/layer_freq.sh place "class,capital,rank " > ./build/qadoc/freq_place__class_capital_rank.md -./qa/layer_freq.sh place "rank " > ./build/qadoc/freq_place__rank.md +./qa/layer_freq.sh place "class " > ./build/qareports/freq_place__class.md +./qa/layer_freq.sh place "class,capital " > ./build/qareports/freq_place__class_capital.md +./qa/layer_freq.sh place "capital " > ./build/qareports/freq_place__capital.md +./qa/layer_freq.sh place "class,capital,rank " > ./build/qareports/freq_place__class_capital_rank.md +./qa/layer_freq.sh place "rank " > ./build/qareports/freq_place__rank.md -./qa/layer_freq.sh poi "class " > ./build/qadoc/freq_poi__class.md -./qa/layer_freq.sh poi "subclass " > ./build/qadoc/freq_poi__subclass.md -./qa/layer_freq.sh poi "rank " > ./build/qadoc/freq_poi__rank.md -./qa/layer_freq.sh poi "class,subclass " > ./build/qadoc/freq_poi__class_subclass.md -./qa/layer_freq.sh poi "class,subclass,rank" > ./build/qadoc/freq_poi__class_subclass_rank.md -./qa/layer_freq.sh poi "class,rank " > ./build/qadoc/freq_poi__class_rank.md +./qa/layer_freq.sh poi "class " > ./build/qareports/freq_poi__class.md +./qa/layer_freq.sh poi "subclass " > ./build/qareports/freq_poi__subclass.md +./qa/layer_freq.sh poi "rank " > ./build/qareports/freq_poi__rank.md +./qa/layer_freq.sh poi "class,subclass " > ./build/qareports/freq_poi__class_subclass.md +./qa/layer_freq.sh poi "class,subclass,rank" > ./build/qareports/freq_poi__class_subclass_rank.md +./qa/layer_freq.sh poi "class,rank " > ./build/qareports/freq_poi__class_rank.md -./qa/layer_freq.sh transportation "class, oneway, ramp, brunnel, service" > ./build/qadoc/freq_transportation__class_oneway_ramp_brunnel_service.md -./qa/layer_freq.sh transportation "oneway, ramp, brunnel, service " > ./build/qadoc/freq_transportation__oneway_ramp_brunnel_service.md -./qa/layer_freq.sh transportation "class " > ./build/qadoc/freq_transportation__class.md +./qa/layer_freq.sh transportation "class, oneway, ramp, brunnel, service" > ./build/qareports/freq_transportation__class_oneway_ramp_brunnel_service.md +./qa/layer_freq.sh transportation "oneway, ramp, brunnel, service " > ./build/qareports/freq_transportation__oneway_ramp_brunnel_service.md +./qa/layer_freq.sh transportation "class " > ./build/qareports/freq_transportation__class.md -./qa/layer_freq.sh transportation_name "class " > ./build/qadoc/freq_transportation_name__class.md -./qa/layer_freq.sh transportation_name "ref_length" > ./build/qadoc/freq_transportation_name__ref_length.md +./qa/layer_freq.sh transportation_name "class " > ./build/qareports/freq_transportation_name__class.md +./qa/layer_freq.sh transportation_name "ref_length" > ./build/qareports/freq_transportation_name__ref_length.md -./qa/layer_freq.sh water "class " > ./build/qadoc/freq_water__class.md +./qa/layer_freq.sh water "class " > ./build/qareports/freq_water__class.md -./qa/layer_freq.sh water_name "class " > ./build/qadoc/freq_water_name__class.md +./qa/layer_freq.sh water_name "class " > ./build/qareports/freq_water_name__class.md -./qa/layer_freq.sh waterway "class " > ./build/qadoc/freq_waterway__class.md +./qa/layer_freq.sh waterway "class " > ./build/qareports/freq_waterway__class.md # ---- toplength ------------------------------------------- -./qa/layer_toplength.sh housenumber "housenumber" > ./build/qadoc/toplength_housenumber__housenumber.md +./qa/layer_toplength.sh housenumber "housenumber" > ./build/qareports/toplength_housenumber__housenumber.md -./qa/layer_toplength.sh place "name" > ./build/qadoc/toplength_place__name.md -./qa/layer_toplength.sh place "name_en" > ./build/qadoc/toplength_place__name_en.md +./qa/layer_toplength.sh place "name" > ./build/qareports/toplength_place__name.md +./qa/layer_toplength.sh place "name_en" > ./build/qareports/toplength_place__name_en.md -./qa/layer_toplength.sh poi "name" > ./build/qadoc/toplength_poi__name.md -./qa/layer_toplength.sh poi "name_en" > ./build/qadoc/toplength_poi__name_en.md +./qa/layer_toplength.sh poi "name" > ./build/qareports/toplength_poi__name.md +./qa/layer_toplength.sh poi "name_en" > ./build/qareports/toplength_poi__name_en.md -./qa/layer_toplength.sh transportation_name "name" > ./build/qadoc/toplength_transportation_name__name.md -./qa/layer_toplength.sh transportation_name "ref" > ./build/qadoc/toplength_transportation_name__ref.md -./qa/layer_toplength.sh transportation_name "network" > ./build/qadoc/toplength_transportation_name__network.md +./qa/layer_toplength.sh transportation_name "name" > ./build/qareports/toplength_transportation_name__name.md +./qa/layer_toplength.sh transportation_name "ref" > ./build/qareports/toplength_transportation_name__ref.md +./qa/layer_toplength.sh transportation_name "network" > ./build/qareports/toplength_transportation_name__network.md -./qa/layer_toplength.sh water_name "name" > ./build/qadoc/toplength_water_name__name.md -./qa/layer_toplength.sh water_name "name_en" > ./build/qadoc/toplength_water_name__name_en.md +./qa/layer_toplength.sh water_name "name" > ./build/qareports/toplength_water_name__name.md +./qa/layer_toplength.sh water_name "name_en" > ./build/qareports/toplength_water_name__name_en.md -./qa/layer_toplength.sh waterway "name" > ./build/qadoc/toplength_waterway__name.md +./qa/layer_toplength.sh waterway "name" > ./build/qareports/toplength_waterway__name.md # ---- numvar analyze ------------------------------------- -./qa/layer_numvar_analyze.sh building "render_min_height" > ./build/qadoc/numvara_building__render_min_height.md -./qa/layer_numvar_analyze.sh building "render_max_height" > ./build/qadoc/numvara_building__render_max_height.md +./qa/layer_numvar_analyze.sh building "render_min_height" > ./build/qareports/numvara_building__render_min_height.md +./qa/layer_numvar_analyze.sh building "render_height" > ./build/qareports/numvara_building__render_height.md -./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ./build/qadoc/numvara_transportation_name__ref_length.md +./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ./build/qareports/numvara_transportation_name__ref_length.md From 3623b893fa1b6782d5df12ee02b4fa39d2ef3a52 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 4 Dec 2016 16:50:57 +0100 Subject: [PATCH 27/39] set new version of openmaptiles-tools, import-osm --- docker-compose.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 3574e72..d39505e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,14 @@ services: - postgres volumes: - ./build:/sql + openmaptiles-tools: + image: "openmaptiles/openmaptiles-tools:0.2" + env_file: .env + links: + - postgres + volumes: + - $(pwd):/tileset + - ./build:/sql mapbox-studio: image: "osm2vectortiles/mapbox-studio" volumes: From 5c724e50cef080b8ff41e5c2582c6204f5606ddf Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 4 Dec 2016 16:55:25 +0100 Subject: [PATCH 28/39] add `docker-compose pull` --- QUICKSTART.md | 1 + 1 file changed, 1 insertion(+) diff --git a/QUICKSTART.md b/QUICKSTART.md index 39827cf..725e324 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -37,6 +37,7 @@ Important: The ./quickstart.sh is for small extracts - not optimal for a Planet ```bash git clone https://github.com/openmaptiles/openmaptiles.git cd openmaptiles +docker-compose pull ./quickstart.sh ``` From 38329390de8f3570e86bb7da13e8cfe5dd2ae5ba Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 4 Dec 2016 17:11:16 +0100 Subject: [PATCH 29/39] small fixes --- Makefile | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a6d7cbf..ea2bd9a 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ list-docker-images: docker images | grep openmaptiles refresh-docker-images: - echo docker-compose pull + docker-compose pull --ignore-pull-failures remove-docker-images: docker rmi -f $(docker images | grep "openmaptiles" | awk "{print \$3}") diff --git a/docker-compose.yml b/docker-compose.yml index d39505e..09d9206 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,7 +47,7 @@ services: links: - postgres volumes: - - $(pwd):/tileset + - .:/tileset - ./build:/sql mapbox-studio: image: "osm2vectortiles/mapbox-studio" From 4cea3eec65a6ca488b0d7c6071eef676b2e097bb Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 4 Dec 2016 19:07:03 +0100 Subject: [PATCH 30/39] fix bash errors --- qa/layer_freq.sh | 18 +++++++++--------- qa/layer_numvar_analyze.sh | 29 +++++++++++++++-------------- qa/layer_toplength.sh | 29 +++++++++++++---------------- qa/run.sh | 12 ++++++++++-- 4 files changed, 47 insertions(+), 41 deletions(-) diff --git a/qa/layer_freq.sh b/qa/layer_freq.sh index b54d4bf..b62ba33 100755 --- a/qa/layer_freq.sh +++ b/qa/layer_freq.sh @@ -1,4 +1,3 @@ - #!/bin/bash set -o errexit set -o pipefail @@ -16,15 +15,16 @@ echo "## $layerid z$z - freq" SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) -read -r -d '' SQLCODE <<- EOMSQL - select $classvars , count(*) as _count_ from - $SQL - GROUP BY $classvars - ORDER BY $classvars - ; -EOMSQL +SQLCODE=$(cat <<-END +select $classvars , count(*) as _count_ from +$SQL +GROUP BY $classvars +ORDER BY $classvars +; +END +) -#echo "\`\`\`SQL" +#echo "\`\`\`sql" #echo "$SQLCODE" #echo "\`\`\`" diff --git a/qa/layer_numvar_analyze.sh b/qa/layer_numvar_analyze.sh index cb5eb6d..ad15510 100755 --- a/qa/layer_numvar_analyze.sh +++ b/qa/layer_numvar_analyze.sh @@ -1,4 +1,3 @@ - #!/bin/bash set -o errexit set -o pipefail @@ -16,20 +15,22 @@ echo "## $layerid z$z - $var " SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) -read -r -d '' SQLCODE <<- EOMSQL - SELECT - count($var) as count - ,min($var) as min - ,max($var) as max - ,avg($var) as avg - ,stddev($var) as stddev - ,variance($var) as variance - FROM - $SQL - ; -EOMSQL +SQLCODE=$(cat <<-END +SELECT + count($var) as count + ,min($var) as min + ,max($var) as max + ,avg($var) as avg + ,stddev($var) as stddev + ,variance($var) as variance +FROM + $SQL +; +END +) -#echo "\`\`\`SQL" + +#echo "\`\`\`sql" #echo "$SQLCODE" #echo "\`\`\`" diff --git a/qa/layer_toplength.sh b/qa/layer_toplength.sh index a4c7173..1fa85b2 100755 --- a/qa/layer_toplength.sh +++ b/qa/layer_toplength.sh @@ -1,37 +1,34 @@ - #!/bin/bash set -o errexit set -o pipefail set -o nounset - layerid=$1 classvar=$2 -echo "# TOP30LENGHT - $layerid - $classvar" - for z in {0..15} do echo " " echo "## $layerid z$z max length ($classvar)" -SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) +SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) -read -r -d '' SQLCODE <<- EOMSQL - select $classvar , length( $classvar ) as _length_ from - $SQL - WHERE length( $classvar ) > 0 - ORDER BY length( $classvar ) DESC NULLS LAST - LIMIT 30 - ; -EOMSQL +SQLCODE=$(cat <<-END +select $classvar , length( $classvar ) as _length_ from +$SQL +WHERE length( $classvar ) > 0 +ORDER BY length( $classvar ) DESC NULLS LAST +LIMIT 30 +; +END +) -#echo "\`\`\`SQL" +#echo "\`\`\`sql" #echo "$SQLCODE" #echo "\`\`\`" docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ - | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' - + | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' + done diff --git a/qa/run.sh b/qa/run.sh index 8909698..320e6d0 100755 --- a/qa/run.sh +++ b/qa/run.sh @@ -1,4 +1,3 @@ - #!/bin/bash set -o errexit set -o pipefail @@ -9,9 +8,18 @@ set -o nounset # ---- freq --------------------------------------------------------------- -mkdir -p ./build/qadoc +mkdir -p ./build/qareports rm -f ./build/qareports/*.md +./qa/layer_numvar_analyze.sh transportation_name "ref_length" +./qa/layer_toplength.sh place "name" +./qa/layer_freq.sh poi "class " + +echo "OKÉ" +exit + + +# ----- ./qa/layer_freq.sh aeroway "class" > ./build/qareports/freq_aeroway__class.md ./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qareports/freq_boundary__admin_level_disputed.md From d4c06a4eba26233510bb5097bbb638bb4c425982 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Mon, 5 Dec 2016 01:11:39 +0100 Subject: [PATCH 31/39] fix parameters qareports paramaters --- qa/layer_toplength.sh | 4 ++-- qa/run.sh | 15 ++++----------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/qa/layer_toplength.sh b/qa/layer_toplength.sh index 1fa85b2..e92e25c 100755 --- a/qa/layer_toplength.sh +++ b/qa/layer_toplength.sh @@ -14,10 +14,10 @@ echo "## $layerid z$z max length ($classvar)" SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) SQLCODE=$(cat <<-END -select $classvar , length( $classvar ) as _length_ from +SELECT DISTINCT $classvar , length( $classvar ) AS _length_ from $SQL WHERE length( $classvar ) > 0 -ORDER BY length( $classvar ) DESC NULLS LAST +ORDER BY length( $classvar ) DESC NULLS LAST LIMIT 30 ; END diff --git a/qa/run.sh b/qa/run.sh index 320e6d0..ea9f573 100755 --- a/qa/run.sh +++ b/qa/run.sh @@ -11,20 +11,13 @@ set -o nounset mkdir -p ./build/qareports rm -f ./build/qareports/*.md -./qa/layer_numvar_analyze.sh transportation_name "ref_length" -./qa/layer_toplength.sh place "name" -./qa/layer_freq.sh poi "class " - -echo "OKÉ" -exit - - # ----- + ./qa/layer_freq.sh aeroway "class" > ./build/qareports/freq_aeroway__class.md -./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qareports/freq_boundary__admin_level_disputed.md +###Todo:./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qareports/freq_boundary__admin_level_disputed.md ./qa/layer_freq.sh boundary "admin_level" > ./build/qareports/freq_boundary__admin_level.md -./qa/layer_freq.sh boundary "disputed" > ./build/qareports/freq_boundary__disputed.md +###Todo: ./qa/layer_freq.sh boundary "disputed" > ./build/qareports/freq_boundary__disputed.md ./qa/layer_freq.sh building "render_min_height" > ./build/qareports/freq_building__render_min_height.md @@ -73,7 +66,7 @@ exit ./qa/layer_toplength.sh transportation_name "name" > ./build/qareports/toplength_transportation_name__name.md ./qa/layer_toplength.sh transportation_name "ref" > ./build/qareports/toplength_transportation_name__ref.md -./qa/layer_toplength.sh transportation_name "network" > ./build/qareports/toplength_transportation_name__network.md +###Todo: ./qa/layer_toplength.sh transportation_name "network" > ./build/qareports/toplength_transportation_name__network.md ./qa/layer_toplength.sh water_name "name" > ./build/qareports/toplength_water_name__name.md ./qa/layer_toplength.sh water_name "name_en" > ./build/qareports/toplength_water_name__name_en.md From eef775fda94e71040dc3aa7496d101d28a9926ca Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Tue, 6 Dec 2016 18:43:32 +0100 Subject: [PATCH 32/39] fix some `make` commands --- Makefile | 14 ++++++++++++-- docker-compose.yml | 4 ++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index ea2bd9a..10c3212 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ help: @echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" @echo " make clean-docker # remove docker containers, PG data volume " @echo " make forced-clean-sql # drop all PostgreSQL tables for clean environment " + @echo " make docker-unnecessary-clean # clean unnecessary docker image(s) and container(s)" @echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB" @echo " make remove-docker-images # remove openmaptiles docker images" @echo " make pgclimb-list-views # list PostgreSQL public schema views" @@ -58,8 +59,17 @@ refresh-docker-images: docker-compose pull --ignore-pull-failures remove-docker-images: - docker rmi -f $(docker images | grep "openmaptiles" | awk "{print \$3}") - docker rmi osm2vectortiles/mapbox-studio + @echo "Deleting all openmaptiles related docker image(s)..." + @docker-compose down + @docker images | grep "" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi + @docker images | grep "openmaptiles" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi + @docker images | grep "osm2vectortiles/mapbox-studio" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi + +docker-unnecessary-clean: + @echo "Deleting unnecessary container(s)..." + @docker ps -a | grep Exited | awk -F" " '{print $$1}' | xargs --no-run-if-empty docker rm + @echo "Deleting unnecessary image(s)..." + @docker images | grep \ | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi psql: docker-compose run --rm import-osm /usr/src/app/psql.sh diff --git a/docker-compose.yml b/docker-compose.yml index 87c1e87..19c2a0a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: links: - postgres import-osm: - image: "openmaptiles/import-osm:0.2" + image: "openmaptiles/import-osm:latest" env_file: .env environment: DIFF_MODE: "false" @@ -42,7 +42,7 @@ services: volumes: - ./build:/sql openmaptiles-tools: - image: "openmaptiles/openmaptiles-tools:0.2" + image: "openmaptiles/openmaptiles-tools:latest" env_file: .env links: - postgres From f2f85be189cef7e7790cea6fb8c185a076a8f9a7 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Fri, 9 Dec 2016 16:59:47 +0100 Subject: [PATCH 33/39] :rocket: rstefactor quickstart.sh logging --- quickstart.sh | 98 +++++++++++++++++++++++++++------------------------ 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/quickstart.sh b/quickstart.sh index b3f9d2d..a53c309 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -32,54 +32,15 @@ testdata=${osm_area}.osm.pbf ## Min versions ... MIN_COMPOSE_VER=1.7.1 MIN_DOCKER_VER=1.10.0 - STARTTIME=$(date +%s) STARTDATE=$(date -Iminutes) githash=$( git rev-parse HEAD ) log_file=./quickstart.log rm -f $log_file -exec &> >(tee -a "$log_file") - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] " -echo " : This will be logged to the $log_file file ( for debugging ) and to the screen" -echo " : Git version: $githash / Started: $STARTDATE " -echo " : Your bash version: $BASH_VERSION" -echo " : Your OS: $OSTYPE" - - -if [[ "$OSTYPE" == "linux-gnu" ]]; then - - echo " : Your system is:" - lsb_release -a - echo " " - echo "-------------------------------------------------------------------------------------" - echo " : This is working on x86_64 ; Your kernel is:" - uname -r - uname -m - - KERNEL_CPU_VER=$(uname -m) - if [ "$KERNEL_CPU_VER" != "x86_64" ]; then - echo "ERR: Sorry this is working only on x86_64!" - exit 1 - fi - echo " : --- Memory, CPU info ---- " - mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) - echo "system memory (GB): ${mem} " - grep SwapTotal /proc/meminfo - echo cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1) - cat /proc/meminfo | grep Free -else - echo " " - echo "Warning : This is not a Linux ... ( Less tested ... ) " - echo " " -fi - - - echo " " +echo "=====================================================================================" +echo " Docker check & Download images " echo "-------------------------------------------------------------------------------------" echo "====> : Please check the docker and docker-compose version!" echo " : We are using docker-compose V2 file format ! see more: https://docs.docker.com/" @@ -104,13 +65,58 @@ fi echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : Docker Space info: " -docker info | grep Space +echo "====> : Pulling or Refreshing OpenMapTiles docker images " +make refresh-docker-images + + +##### backup log from here ... +exec &> >(tee -a "$log_file") + +echo " " +echo "=====================================================================================" +echo " Start processing " +echo "-------------------------------------------------------------------------------------" +echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] " +echo " : This will be logged to the $log_file file ( for debugging ) and to the screen" +echo " : Area : $osm_area " +echo " : Git version : $githash " +echo " : Started : $STARTDATE " +echo " : Your bash version: $BASH_VERSION" +echo " : Your OS : $OSTYPE" +docker --version +docker-compose --version + +if [[ "$OSTYPE" == "linux-gnu" ]]; then + + echo " : Your system is:" + lsb_release -a + echo " " + echo "-------------------------------------------------------------------------------------" + echo " : This is working on x86_64 ; Your kernel is:" + uname -r + uname -m + + KERNEL_CPU_VER=$(uname -m) + if [ "$KERNEL_CPU_VER" != "x86_64" ]; then + echo "ERR: Sorry this is working only on x86_64!" + exit 1 + fi + echo " : --- Memory, CPU info ---- " + mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) + echo "system memory (GB): ${mem} " + grep SwapTotal /proc/meminfo + echo cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1) + cat /proc/meminfo | grep Free +else + echo " " + echo "Warning : This is not a Linux ... ( Less tested ... ) " + echo " " +fi echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : Pulling or Refreshing OpenMapTiles docker images " -make refresh-docker-images +echo "====> : Docker Space info: " +docker info | grep Space echo " " echo "-------------------------------------------------------------------------------------" @@ -185,7 +191,7 @@ echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Drop and Recreate PostgreSQL public schema " # Drop all PostgreSQL tables -# This is add an extra safe belt , if the user modify the docker volume seetings +# This is add an extra safe belt , if the user modify the docker volume seetings make forced-clean-sql echo " " From 6c7b44f00fe3ca9a8b736b19d22ed0bedbdaf92e Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Fri, 9 Dec 2016 17:07:02 +0100 Subject: [PATCH 34/39] :rocket: small fixes --- QUICKSTART.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/QUICKSTART.md b/QUICKSTART.md index 725e324..36abe18 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -32,18 +32,17 @@ Important: The ./quickstart.sh is for small extracts - not optimal for a Planet rendering !! -### First experiment - with albania ( small extracts! ) +### First experiment - with `albania` ( small extracts! ) ```bash git clone https://github.com/openmaptiles/openmaptiles.git cd openmaptiles -docker-compose pull ./quickstart.sh ``` If you have problems with the quickstart * check the ./quickstart.log! -* check again the system requirements +* check again the system requirements! * check the current issues : https://github.com/openmaptiles/openmaptiles/issues * create new issues: * create a new gist https://gist.github.com/ from your ./quickstart.log @@ -60,7 +59,7 @@ THAN you can test other available quickstart extracts ( based on geofabrik extra * We are using https://github.com/julien-noblet/download-geofabrik tool * The current extract list, and more informations -> `make list` -This is generating mbtiles for your area : [ MIN_ZOOM: "0" - MAX_ZOOM: "7" ] +This is generating `.mbtiles` for your area : [ MIN_ZOOM: "0" - MAX_ZOOM: "7" ] ```bash ./quickstart.sh africa From 2ce6b22d3fa5f3e15585e6a2cf170c257f0ad809 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 14 Dec 2016 19:15:35 +0100 Subject: [PATCH 35/39] remove `docker info` .. --- quickstart.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/quickstart.sh b/quickstart.sh index a53c309..404478a 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -113,11 +113,6 @@ else echo " " fi -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Docker Space info: " -docker info | grep Space - echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Stopping running services & removing old containers " From 5744baa2d68a7c063837f0d412e7acc67518ac36 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 15 Dec 2016 12:59:59 +0100 Subject: [PATCH 36/39] refactor to the new generate-sqlquery --- qa/layer_freq.sh | 4 ++-- qa/layer_numvar_analyze.sh | 4 ++-- qa/layer_toplength.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/qa/layer_freq.sh b/qa/layer_freq.sh index b62ba33..1ba1a90 100755 --- a/qa/layer_freq.sh +++ b/qa/layer_freq.sh @@ -13,11 +13,11 @@ do echo " " echo "## $layerid z$z - freq" -SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) +SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) SQLCODE=$(cat <<-END select $classvars , count(*) as _count_ from -$SQL +( $SQL ) as t GROUP BY $classvars ORDER BY $classvars ; diff --git a/qa/layer_numvar_analyze.sh b/qa/layer_numvar_analyze.sh index ad15510..e786724 100755 --- a/qa/layer_numvar_analyze.sh +++ b/qa/layer_numvar_analyze.sh @@ -13,7 +13,7 @@ do echo " " echo "## $layerid z$z - $var " -SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) +SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) SQLCODE=$(cat <<-END SELECT @@ -24,7 +24,7 @@ SELECT ,stddev($var) as stddev ,variance($var) as variance FROM - $SQL +( $SQL ) as t ; END ) diff --git a/qa/layer_toplength.sh b/qa/layer_toplength.sh index e92e25c..a72e902 100755 --- a/qa/layer_toplength.sh +++ b/qa/layer_toplength.sh @@ -11,11 +11,11 @@ do echo " " echo "## $layerid z$z max length ($classvar)" -SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) +SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) SQLCODE=$(cat <<-END SELECT DISTINCT $classvar , length( $classvar ) AS _length_ from -$SQL +( $SQL ) as t WHERE length( $classvar ) > 0 ORDER BY length( $classvar ) DESC NULLS LAST LIMIT 30 From 973c937d29bd160943a8e6c18289cf3212b630e9 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 15 Dec 2016 13:05:07 +0100 Subject: [PATCH 37/39] add DIFF_MODE=false --- .env | 1 + 1 file changed, 1 insertion(+) diff --git a/.env b/.env index ed3cf2a..11b9f68 100644 --- a/.env +++ b/.env @@ -5,3 +5,4 @@ POSTGRES_HOST=postgres POSTGRES_PORT=5432 QUICKSTART_MIN_ZOOM=0 QUICKSTART_MAX_ZOOM=7 +DIFF_MODE=false \ No newline at end of file From 9e24e909d0ffc744774e24f039dc7dfcfa742596 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 15 Dec 2016 14:57:24 +0100 Subject: [PATCH 38/39] dockerize the generate-etlgraph procedure --- Makefile | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 10c3212..3a1da92 100644 --- a/Makefile +++ b/Makefile @@ -126,18 +126,17 @@ generate-qareports: # work in progress - please don't remove generate-devdoc: mkdir -p ./build/devdoc - generate-etlgraph layers/aeroway/aeroway.yaml ./build/devdoc - generate-etlgraph layers/boundary/boundary.yaml ./build/devdoc - generate-etlgraph layers/building/building.yaml ./build/devdoc - generate-etlgraph layers/housenumber/housenumber.yaml ./build/devdoc - generate-etlgraph layers/landcover/landcover.yaml ./build/devdoc - generate-etlgraph layers/landuse/landuse.yaml ./build/devdoc - generate-etlgraph layers/park/park.yaml ./build/devdoc - generate-etlgraph layers/place/place.yaml ./build/devdoc - generate-etlgraph layers/poi/poi.yaml ./build/devdoc - generate-etlgraph layers/transportation/transportation.yaml ./build/devdoc - generate-etlgraph layers/transportation_name/transportation_name.yaml ./build/devdoc - generate-etlgraph layers/water/water.yaml ./build/devdoc - generate-etlgraph layers/water_name/water_name.yaml ./build/devdoc - generate-etlgraph layers/waterway/waterway.yaml ./build/devdoc - + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/aeroway/aeroway.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/boundary/boundary.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/building/building.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/housenumber/housenumber.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/landcover/landcover.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/landuse/landuse.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/park/park.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/place/place.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/poi/poi.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/transportation/transportation.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/transportation_name/transportation_name.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/water/water.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/water_name/water_name.yaml ./build/devdoc + docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/waterway/waterway.yaml ./build/devdoc From 2b76c17a1185c19087ab51b7aaaabae178d3b94f Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 15 Dec 2016 15:39:01 +0100 Subject: [PATCH 39/39] fix make masking --- Makefile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 3a1da92..937060b 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ refresh-docker-images: remove-docker-images: @echo "Deleting all openmaptiles related docker image(s)..." @docker-compose down - @docker images | grep "" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi + @docker images | grep "" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi @docker images | grep "openmaptiles" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi @docker images | grep "osm2vectortiles/mapbox-studio" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi @@ -126,17 +126,17 @@ generate-qareports: # work in progress - please don't remove generate-devdoc: mkdir -p ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/aeroway/aeroway.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/boundary/boundary.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/building/building.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/housenumber/housenumber.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/landcover/landcover.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/landuse/landuse.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/park/park.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/place/place.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/poi/poi.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/transportation/transportation.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/transportation_name/transportation_name.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/water/water.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/water_name/water_name.yaml ./build/devdoc - docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/waterway/waterway.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/aeroway/aeroway.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/boundary/boundary.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/building/building.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/housenumber/housenumber.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/landcover/landcover.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/landuse/landuse.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/park/park.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/place/place.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/poi/poi.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/transportation/transportation.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/transportation_name/transportation_name.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/water/water.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/water_name/water_name.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/waterway/waterway.yaml ./build/devdoc