151 lines
6.3 KiB
Bash
Executable File
151 lines
6.3 KiB
Bash
Executable File
set -e
|
|
|
|
#area=europe
|
|
area=planet
|
|
|
|
make refresh-docker-images
|
|
|
|
# Show mem info
|
|
mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc )
|
|
echo "System memory (GB): ${mem}"
|
|
grep SwapTotal /proc/meminfo
|
|
echo "CPU number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)"
|
|
grep Free /proc/meminfo
|
|
|
|
|
|
make destroy-db
|
|
make init-dirs
|
|
|
|
# rm -f "./data/${area}.mbtiles"
|
|
#echo "====> : Downloading ${area} from ${osm_server:-any source}..."
|
|
#area=$area make "download${osm_server:+-${osm_server}}"
|
|
|
|
make clean
|
|
make all
|
|
|
|
make start-db
|
|
make import-data
|
|
|
|
# about 1.25hrs, Presumably Disk IO limited.
|
|
# If there is an option to double the amount of processes this might
|
|
# also help. Seeing about 50% usage per core
|
|
# [May 21 08:52:53] [INFO] Reading OSM data took: 22m9.040318577s
|
|
# [May 21 09:42:37] [INFO] Writing OSM data took: 49m44.555518107s
|
|
# [May 21 09:45:15] [INFO] Importing OSM data took: 52m21.813870474s
|
|
# [May 21 09:45:15] [INFO] [PostGIS] Rotating tables took: 38.69062ms
|
|
# [May 21 09:45:15] [INFO] Imposm took: 1h14m30.893125729s
|
|
make import-osm area=$area
|
|
|
|
make import-borders area=$area
|
|
|
|
|
|
make import-wikidata
|
|
|
|
# It not doing the materialized view in parallel despite postgres 12?!
|
|
# argh
|
|
make import-sql
|
|
|
|
echo " "
|
|
echo "-------------------------------------------------------------------------------------"
|
|
echo "====> : Analyze PostgreSQL tables"
|
|
make analyze-db
|
|
|
|
echo " "
|
|
echo "-------------------------------------------------------------------------------------"
|
|
echo "====> : Testing PostgreSQL tables to match layer definitions metadata"
|
|
# TODO: this right here shows us whats wrong. settings aren't optimal (see performance tips tabs)
|
|
# shared_buffersm work_mem, max_worker_processes,
|
|
# TODO, Ideal solution: Autoconfigure according to performance tips (and max_worker_processes == HT Cores ( or 1.5X HT Cores)
|
|
# (1.5x HT Cores might make sense as most work is probably memory-limited (latency), not CPU limited)
|
|
make test-perf-null
|
|
|
|
#echo " "
|
|
#echo "-------------------------------------------------------------------------------------"
|
|
#
|
|
#if [[ "$(source .env ; echo "$BBOX")" = "-180.0,-85.0511,180.0,85.0511" ]]; then
|
|
# if [[ "$area" != "planet" ]]; then
|
|
# echo "====> : Compute bounding box for tile generation"
|
|
# make generate-bbox-file ${MIN_ZOOM:+MIN_ZOOM="${MIN_ZOOM}"} ${MAX_ZOOM:+MAX_ZOOM="${MAX_ZOOM}"}
|
|
# else
|
|
# echo "====> : Skipping bbox calculation when generating the entire planet"
|
|
# fi
|
|
#
|
|
#else
|
|
# echo "====> : Bounding box is set in .env file"
|
|
#fi
|
|
#
|
|
##echo " "
|
|
##echo "-------------------------------------------------------------------------------------"
|
|
##echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. "
|
|
##echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml "
|
|
##echo " : Output MBTiles: ./data/${area}.mbtiles "
|
|
##echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/generate-vectortiles "
|
|
##echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox "
|
|
##echo " : Thank you https://www.mapbox.com !"
|
|
##echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-tiles "
|
|
##echo " : "
|
|
##echo " : You will see a lot of deprecated warning in the log! This is normal! "
|
|
##echo " : like : Mapnik LOG> ... is deprecated and will be removed in Mapnik 4.x ... "
|
|
make generate-tiles area=europe
|
|
#OVERLAY
|
|
##
|
|
##echo " "
|
|
##echo "-------------------------------------------------------------------------------------"
|
|
##echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )"
|
|
##make stop-db
|
|
##
|
|
##echo " "
|
|
##echo "-------------------------------------------------------------------------------------"
|
|
##echo "====> : Inputs - Outputs md5sum for debugging "
|
|
##rm -f ./data/quickstart_checklist.chk
|
|
##{
|
|
## find build -type f | sort | xargs md5sum
|
|
## find data -type f | sort | xargs md5sum
|
|
##} >> ./data/quickstart_checklist.chk
|
|
##cat ./data/quickstart_checklist.chk
|
|
##
|
|
##ENDTIME=$(date +%s)
|
|
##
|
|
##echo " "
|
|
##echo " "
|
|
##echo "-------------------------------------------------------------------------------------"
|
|
##echo "-- S u m m a r y --"
|
|
##echo "-------------------------------------------------------------------------------------"
|
|
##echo " "
|
|
##echo "-------------------------------------------------------------------------------------"
|
|
##echo "====> : (disk space) We have created a lot of docker images: "
|
|
##echo " : Hint: you can remove with: docker rmi IMAGE "
|
|
##docker images | grep openmaptiles
|
|
##
|
|
##echo " "
|
|
##echo "-------------------------------------------------------------------------------------"
|
|
##echo "====> : (disk space) We have created the new vectortiles ( ./data/${area}.mbtiles ) "
|
|
##echo " : Please respect the licenses (OdBL for OSM data) of the sources when distributing the MBTiles file."
|
|
##echo " : Data directory content:"
|
|
##ls -la ./data
|
|
##
|
|
##echo " "
|
|
##echo "-------------------------------------------------------------------------------------"
|
|
##echo "The ./quickstart.sh $area is finished! "
|
|
##echo "It took $((ENDTIME - STARTTIME)) seconds to complete"
|
|
##echo "We saved the log file to $log_file (for debugging) You can compare with the travis log !"
|
|
##echo " "
|
|
##echo "Start experimenting and check the QUICKSTART.MD file!"
|
|
##echo " "
|
|
##echo "* Use make start-maputnik to explore tile generation on request"
|
|
##echo "* Use make start-tileserver to view pre-generated tiles"
|
|
##echo " "
|
|
##echo "Available help commands (make help) "
|
|
##make help
|
|
##
|
|
##echo "-------------------------------------------------------------------------------------"
|
|
##echo " Acknowledgments "
|
|
##echo " Generated vector tiles are produced work of OpenStreetMap data. "
|
|
##echo " Such tiles are reusable under CC-BY license granted by OpenMapTiles team: "
|
|
##echo " https://github.com/openmaptiles/openmaptiles/#license "
|
|
##echo " Maps made with these vector tiles must display a visible credit: "
|
|
##echo " © OpenMapTiles © OpenStreetMap contributors "
|
|
##echo " "
|
|
##echo " Thanks to all free, open source software developers and Open Data Contributors! "
|
|
##echo "-------------------------------------------------------------------------------------"
|