Move "qa" tools from OMT to OMT-T (#1031)
With the new release of OMT-T (5.3) are available tools [`layer-stat`](https://github.com/openmaptiles/openmaptiles-tools/pull/293). With the new release of OMT-T can be replaced `make` target `generate-qareports` by `generate-qa` Used as: ``` make generate-qa STAT_FUNCTION=frequency LAYER=transportation ATTRIBUTE=class ```
This commit is contained in:
Tomas Pohanka
GitHub
parent
0aa6648bd9
commit
d180988f5f
13
Makefile
13
Makefile
@ -213,7 +213,7 @@ help:
|
|||||||
@echo " make psql-list-tables # list all PostgreSQL tables"
|
@echo " make psql-list-tables # list all PostgreSQL tables"
|
||||||
@echo " make vacuum-db # PostgreSQL: VACUUM ANALYZE"
|
@echo " make vacuum-db # PostgreSQL: VACUUM ANALYZE"
|
||||||
@echo " make analyze-db # PostgreSQL: ANALYZE"
|
@echo " make analyze-db # PostgreSQL: ANALYZE"
|
||||||
@echo " make generate-qareports # generate reports [./build/qareports]"
|
@echo " make generate-qa # statistics for a given layer's field"
|
||||||
@echo " make generate-devdoc # generate devdoc including graphs for all layers [./layers/...]"
|
@echo " make generate-devdoc # generate devdoc including graphs for all layers [./layers/...]"
|
||||||
@echo " make bash # start openmaptiles-tools /bin/bash terminal"
|
@echo " make bash # start openmaptiles-tools /bin/bash terminal"
|
||||||
@echo " make destroy-db # remove docker containers and PostgreSQL data volume"
|
@echo " make destroy-db # remove docker containers and PostgreSQL data volume"
|
||||||
@ -464,9 +464,14 @@ start-maputnik: stop-maputnik start-postserve
|
|||||||
stop-maputnik:
|
stop-maputnik:
|
||||||
-docker rm -f maputnik_editor
|
-docker rm -f maputnik_editor
|
||||||
|
|
||||||
.PHONY: generate-qareports
|
# STAT_FUNCTION=frequency|toplength|variance
|
||||||
generate-qareports: start-db
|
.PHONY: generate-qa
|
||||||
./qa/run.sh
|
generate-qa: all start-db-nowait
|
||||||
|
@echo " "
|
||||||
|
@echo "e.g. make generate-qa STAT_FUNCTION=frequency LAYER=transportation ATTRIBUTE=class"
|
||||||
|
@echo " "
|
||||||
|
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \
|
||||||
|
layer-stats $(STAT_FUNCTION) $(TILESET_FILE) $(LAYER) $(ATTRIBUTE) -m 0 -n 14 -v
|
||||||
|
|
||||||
# generate all etl and mapping graphs
|
# generate all etl and mapping graphs
|
||||||
.PHONY: generate-devdoc
|
.PHONY: generate-devdoc
|
||||||
|
|||||||
@ -433,7 +433,7 @@ Hints for developers:
|
|||||||
make psql-list-tables # list all PostgreSQL tables
|
make psql-list-tables # list all PostgreSQL tables
|
||||||
make psql-vacuum-analyze # PostgreSQL: VACUUM ANALYZE
|
make psql-vacuum-analyze # PostgreSQL: VACUUM ANALYZE
|
||||||
make psql-analyze # PostgreSQL: ANALYZE
|
make psql-analyze # PostgreSQL: ANALYZE
|
||||||
make generate-qareports # generate reports [./build/qareports]
|
make generate-qa # statistics for a given layer's field
|
||||||
make generate-devdoc # generate devdoc [./build/devdoc]
|
make generate-devdoc # generate devdoc [./build/devdoc]
|
||||||
make tools-dev # start import-sql /bin/bash terminal
|
make tools-dev # start import-sql /bin/bash terminal
|
||||||
make db-destroy # remove docker containers, PG data volume
|
make db-destroy # remove docker containers, PG data volume
|
||||||
|
|||||||
@ -1,35 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
set -o errexit
|
|
||||||
set -o pipefail
|
|
||||||
set -o nounset
|
|
||||||
|
|
||||||
layerid=$1
|
|
||||||
classvars=$2
|
|
||||||
|
|
||||||
echo "# FREQ - $layerid group by : $classvars "
|
|
||||||
|
|
||||||
for z in {0..15}
|
|
||||||
do
|
|
||||||
echo " "
|
|
||||||
echo "## $layerid z$z - freq"
|
|
||||||
|
|
||||||
SQL=$(docker-compose run --rm openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
|
|
||||||
|
|
||||||
SQLCODE=$(cat <<-END
|
|
||||||
select $classvars , count(*) as _count_ from
|
|
||||||
( $SQL ) as t
|
|
||||||
GROUP BY $classvars
|
|
||||||
ORDER BY $classvars
|
|
||||||
;
|
|
||||||
END
|
|
||||||
)
|
|
||||||
|
|
||||||
#echo "\`\`\`sql"
|
|
||||||
#echo "$SQLCODE"
|
|
||||||
#echo "\`\`\`"
|
|
||||||
|
|
||||||
docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \
|
|
||||||
| sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g'
|
|
||||||
|
|
||||||
done
|
|
||||||
|
|
||||||
@ -1,40 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
set -o errexit
|
|
||||||
set -o pipefail
|
|
||||||
set -o nounset
|
|
||||||
|
|
||||||
layerid=$1
|
|
||||||
var=$2
|
|
||||||
|
|
||||||
echo "# NUMVAR-ANALYZE - $layerid - $var "
|
|
||||||
|
|
||||||
for z in {0..15}
|
|
||||||
do
|
|
||||||
echo " "
|
|
||||||
echo "## $layerid z$z - $var "
|
|
||||||
|
|
||||||
SQL=$(docker-compose run --rm openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
|
|
||||||
|
|
||||||
SQLCODE=$(cat <<-END
|
|
||||||
SELECT
|
|
||||||
count($var) as count
|
|
||||||
,min($var) as min
|
|
||||||
,max($var) as max
|
|
||||||
,avg($var) as avg
|
|
||||||
,stddev($var) as stddev
|
|
||||||
,variance($var) as variance
|
|
||||||
FROM
|
|
||||||
( $SQL ) as t
|
|
||||||
;
|
|
||||||
END
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
#echo "\`\`\`sql"
|
|
||||||
#echo "$SQLCODE"
|
|
||||||
#echo "\`\`\`"
|
|
||||||
|
|
||||||
docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \
|
|
||||||
| sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g'
|
|
||||||
|
|
||||||
done
|
|
||||||
@ -1,34 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
set -o errexit
|
|
||||||
set -o pipefail
|
|
||||||
set -o nounset
|
|
||||||
|
|
||||||
layerid=$1
|
|
||||||
classvar=$2
|
|
||||||
|
|
||||||
for z in {0..15}
|
|
||||||
do
|
|
||||||
echo " "
|
|
||||||
echo "## $layerid z$z max length ($classvar)"
|
|
||||||
|
|
||||||
SQL=$(docker-compose run --rm openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
|
|
||||||
|
|
||||||
SQLCODE=$(cat <<-END
|
|
||||||
SELECT DISTINCT $classvar , length( $classvar ) AS _length_ from
|
|
||||||
( $SQL ) as t
|
|
||||||
WHERE length( $classvar ) > 0
|
|
||||||
ORDER BY length( $classvar ) DESC NULLS LAST
|
|
||||||
LIMIT 30
|
|
||||||
;
|
|
||||||
END
|
|
||||||
)
|
|
||||||
|
|
||||||
#echo "\`\`\`sql"
|
|
||||||
#echo "$SQLCODE"
|
|
||||||
#echo "\`\`\`"
|
|
||||||
|
|
||||||
docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \
|
|
||||||
| sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g'
|
|
||||||
|
|
||||||
done
|
|
||||||
|
|
||||||
83
qa/run.sh
83
qa/run.sh
@ -1,83 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
set -o errexit
|
|
||||||
set -o pipefail
|
|
||||||
set -o nounset
|
|
||||||
|
|
||||||
# example call from the parent folder : ./qa/run.sh
|
|
||||||
# or make generate-qareports
|
|
||||||
|
|
||||||
# ---- freq ---------------------------------------------------------------
|
|
||||||
|
|
||||||
mkdir -p ./build/qareports
|
|
||||||
rm -f ./build/qareports/*.md
|
|
||||||
|
|
||||||
# -----
|
|
||||||
|
|
||||||
./qa/layer_freq.sh aeroway "class" > ./build/qareports/freq_aeroway__class.md
|
|
||||||
|
|
||||||
###Todo:./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qareports/freq_boundary__admin_level_disputed.md
|
|
||||||
./qa/layer_freq.sh boundary "admin_level" > ./build/qareports/freq_boundary__admin_level.md
|
|
||||||
###Todo: ./qa/layer_freq.sh boundary "disputed" > ./build/qareports/freq_boundary__disputed.md
|
|
||||||
|
|
||||||
./qa/layer_freq.sh building "render_min_height" > ./build/qareports/freq_building__render_min_height.md
|
|
||||||
|
|
||||||
#./qa/layer_freq.sh housenumber "housenumber " > ./build/qadocfreq_housenumber.md
|
|
||||||
./qa/layer_freq.sh landcover "class, subclass" > ./build/qareports/freq_landcover__class_subclass.md
|
|
||||||
|
|
||||||
./qa/layer_freq.sh landuse "class " > ./build/qareports/freq_landuse__class.md
|
|
||||||
|
|
||||||
./qa/layer_freq.sh park "class " > ./build/qareports/freq_park__class.md
|
|
||||||
|
|
||||||
./qa/layer_freq.sh place "class " > ./build/qareports/freq_place__class.md
|
|
||||||
./qa/layer_freq.sh place "class,capital " > ./build/qareports/freq_place__class_capital.md
|
|
||||||
./qa/layer_freq.sh place "capital " > ./build/qareports/freq_place__capital.md
|
|
||||||
./qa/layer_freq.sh place "class,capital,rank " > ./build/qareports/freq_place__class_capital_rank.md
|
|
||||||
./qa/layer_freq.sh place "rank " > ./build/qareports/freq_place__rank.md
|
|
||||||
|
|
||||||
./qa/layer_freq.sh poi "class " > ./build/qareports/freq_poi__class.md
|
|
||||||
./qa/layer_freq.sh poi "subclass " > ./build/qareports/freq_poi__subclass.md
|
|
||||||
./qa/layer_freq.sh poi "rank " > ./build/qareports/freq_poi__rank.md
|
|
||||||
./qa/layer_freq.sh poi "class,subclass " > ./build/qareports/freq_poi__class_subclass.md
|
|
||||||
./qa/layer_freq.sh poi "class,subclass,rank" > ./build/qareports/freq_poi__class_subclass_rank.md
|
|
||||||
./qa/layer_freq.sh poi "class,rank " > ./build/qareports/freq_poi__class_rank.md
|
|
||||||
|
|
||||||
./qa/layer_freq.sh transportation "class, oneway, ramp, brunnel, service" > ./build/qareports/freq_transportation__class_oneway_ramp_brunnel_service.md
|
|
||||||
./qa/layer_freq.sh transportation "oneway, ramp, brunnel, service " > ./build/qareports/freq_transportation__oneway_ramp_brunnel_service.md
|
|
||||||
./qa/layer_freq.sh transportation "class " > ./build/qareports/freq_transportation__class.md
|
|
||||||
|
|
||||||
./qa/layer_freq.sh transportation_name "class " > ./build/qareports/freq_transportation_name__class.md
|
|
||||||
./qa/layer_freq.sh transportation_name "ref_length" > ./build/qareports/freq_transportation_name__ref_length.md
|
|
||||||
|
|
||||||
./qa/layer_freq.sh water "class " > ./build/qareports/freq_water__class.md
|
|
||||||
|
|
||||||
./qa/layer_freq.sh water_name "class " > ./build/qareports/freq_water_name__class.md
|
|
||||||
|
|
||||||
./qa/layer_freq.sh waterway "class " > ./build/qareports/freq_waterway__class.md
|
|
||||||
|
|
||||||
# ---- toplength -------------------------------------------
|
|
||||||
|
|
||||||
./qa/layer_toplength.sh housenumber "housenumber" > ./build/qareports/toplength_housenumber__housenumber.md
|
|
||||||
|
|
||||||
./qa/layer_toplength.sh place "name" > ./build/qareports/toplength_place__name.md
|
|
||||||
./qa/layer_toplength.sh place "name_en" > ./build/qareports/toplength_place__name_en.md
|
|
||||||
|
|
||||||
./qa/layer_toplength.sh poi "name" > ./build/qareports/toplength_poi__name.md
|
|
||||||
./qa/layer_toplength.sh poi "name_en" > ./build/qareports/toplength_poi__name_en.md
|
|
||||||
|
|
||||||
./qa/layer_toplength.sh transportation_name "name" > ./build/qareports/toplength_transportation_name__name.md
|
|
||||||
./qa/layer_toplength.sh transportation_name "ref" > ./build/qareports/toplength_transportation_name__ref.md
|
|
||||||
###Todo: ./qa/layer_toplength.sh transportation_name "network" > ./build/qareports/toplength_transportation_name__network.md
|
|
||||||
|
|
||||||
./qa/layer_toplength.sh water_name "name" > ./build/qareports/toplength_water_name__name.md
|
|
||||||
./qa/layer_toplength.sh water_name "name_en" > ./build/qareports/toplength_water_name__name_en.md
|
|
||||||
|
|
||||||
./qa/layer_toplength.sh waterway "name" > ./build/qareports/toplength_waterway__name.md
|
|
||||||
|
|
||||||
|
|
||||||
# ---- numvar analyze -------------------------------------
|
|
||||||
|
|
||||||
./qa/layer_numvar_analyze.sh building "render_min_height" > ./build/qareports/numvara_building__render_min_height.md
|
|
||||||
./qa/layer_numvar_analyze.sh building "render_height" > ./build/qareports/numvara_building__render_height.md
|
|
||||||
|
|
||||||
./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ./build/qareports/numvara_transportation_name__ref_length.md
|
|
||||||
|
|
||||||
@ -1,140 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
import sys
|
|
||||||
import argparse
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument('--noan', action='store_true', help='Not to run make psql-analyze')
|
|
||||||
|
|
||||||
TOTAL_SIZE_SQL = """SELECT
|
|
||||||
pg_size_pretty(sum(size)) AS size
|
|
||||||
FROM (
|
|
||||||
SELECT
|
|
||||||
relname as "Table",
|
|
||||||
pg_total_relation_size(relid) as "size"
|
|
||||||
FROM pg_catalog.pg_statio_user_tables
|
|
||||||
WHERE schemaname='public'
|
|
||||||
) a
|
|
||||||
;""".replace('\"', '\\\"')
|
|
||||||
|
|
||||||
TABLE_SIZES_SQL = """SELECT
|
|
||||||
a.relname as "table",
|
|
||||||
pg_table_size(a.relid) as "size",
|
|
||||||
b.n_live_tup as "rows"
|
|
||||||
FROM pg_catalog.pg_statio_user_tables a
|
|
||||||
LEFT JOIN pg_stat_user_tables b ON (a.relid = b.relid)
|
|
||||||
WHERE
|
|
||||||
a.schemaname='public'
|
|
||||||
ORDER BY a.relname;
|
|
||||||
""".replace('\"', '\\\"')
|
|
||||||
|
|
||||||
TABLES_SQL = """SELECT
|
|
||||||
a.relname
|
|
||||||
FROM pg_catalog.pg_statio_user_tables a
|
|
||||||
WHERE
|
|
||||||
a.schemaname='public'
|
|
||||||
ORDER BY a.relname;
|
|
||||||
"""
|
|
||||||
|
|
||||||
COLUMN_NAMES_SQL = """SELECT a.attname
|
|
||||||
FROM pg_class As c
|
|
||||||
INNER JOIN pg_attribute As a ON c.oid = a.attrelid
|
|
||||||
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
||||||
LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
|
|
||||||
WHERE
|
|
||||||
c.relkind IN('r', 'v', 'm') AND
|
|
||||||
a.attnum > 0 AND
|
|
||||||
n.nspname = 'public' AND
|
|
||||||
c.relname = '{0}' AND
|
|
||||||
a.attisdropped = FALSE
|
|
||||||
ORDER BY a.attname;
|
|
||||||
"""
|
|
||||||
|
|
||||||
COLUMNS_SQL = """select
|
|
||||||
sum(pg_column_size(t.*)) as "all",
|
|
||||||
{0}
|
|
||||||
from {1} t;
|
|
||||||
""".replace('\"', '\\\"')
|
|
||||||
|
|
||||||
|
|
||||||
def print_column_sizes(tables):
|
|
||||||
for table in tables:
|
|
||||||
print("Column sizes of table " + table)
|
|
||||||
cmds = [
|
|
||||||
'docker-compose run --rm import-osm',
|
|
||||||
'/usr/src/app/psql.sh -t -A -F\",\" -P pager=off',
|
|
||||||
'-c \"' + COLUMN_NAMES_SQL.format(table).replace('\n', ' ').replace('\r',
|
|
||||||
'') + '\"'
|
|
||||||
]
|
|
||||||
# print " ".join(cmds)
|
|
||||||
output = subprocess.check_output(" ".join(cmds), shell=True)
|
|
||||||
columns = filter(lambda c: len(c) > 0,
|
|
||||||
map(lambda l: l.strip(), output.split('\n')))
|
|
||||||
|
|
||||||
# print columns
|
|
||||||
|
|
||||||
col_sql = ",\n".join(
|
|
||||||
map(lambda c: "sum(pg_column_size(\\\"" + c + "\\\")) as \\\"" + c + "\\\"",
|
|
||||||
columns))
|
|
||||||
|
|
||||||
# print COLUMNS_SQL.format(col_sql, table);
|
|
||||||
|
|
||||||
cmds = [
|
|
||||||
'docker-compose run --rm import-osm',
|
|
||||||
'/usr/src/app/psql.sh -F\",\" --no-align -P pager=off',
|
|
||||||
'-c \"' + COLUMNS_SQL.format(col_sql, table).replace('\n', ' ').replace(
|
|
||||||
'\r', '') + '\"'
|
|
||||||
]
|
|
||||||
# print " ".join(cmds)
|
|
||||||
col_csv = subprocess.check_output(" ".join(cmds), shell=True)
|
|
||||||
print(col_csv)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
try:
|
|
||||||
|
|
||||||
if not args.noan:
|
|
||||||
print("Running make psql-analyze")
|
|
||||||
subprocess.check_output("make psql-analyze", shell=True)
|
|
||||||
|
|
||||||
print("Total size of tables")
|
|
||||||
cmds = [
|
|
||||||
'docker-compose run --rm import-osm',
|
|
||||||
'/usr/src/app/psql.sh -F\",\" --no-align -P pager=off',
|
|
||||||
'-c \"' + TOTAL_SIZE_SQL.replace('\n', ' ').replace('\r', '') + '\"'
|
|
||||||
]
|
|
||||||
# print " ".join(cmds)
|
|
||||||
TOTAL_SIZE_CSV = subprocess.check_output(" ".join(cmds), shell=True)
|
|
||||||
print(TOTAL_SIZE_CSV)
|
|
||||||
print("\n")
|
|
||||||
|
|
||||||
print("Table sizes")
|
|
||||||
cmds = [
|
|
||||||
'docker-compose run --rm import-osm',
|
|
||||||
'/usr/src/app/psql.sh -F\",\" --no-align -P pager=off',
|
|
||||||
'-c \"' + TABLE_SIZES_SQL.replace('\n', ' ').replace('\r', '') + '\"'
|
|
||||||
]
|
|
||||||
# print " ".join(cmds)
|
|
||||||
TABLE_SIZES_CSV = subprocess.check_output(" ".join(cmds), shell=True)
|
|
||||||
print(TABLE_SIZES_CSV)
|
|
||||||
print("\n")
|
|
||||||
|
|
||||||
print("Column sizes")
|
|
||||||
cmds = [
|
|
||||||
'docker-compose run --rm import-osm',
|
|
||||||
'/usr/src/app/psql.sh -t -A -F\",\" -P pager=off',
|
|
||||||
'-c \"' + TABLES_SQL.replace('\n', ' ').replace('\r', '') + '\"'
|
|
||||||
]
|
|
||||||
# print " ".join(cmds)
|
|
||||||
output = subprocess.check_output(" ".join(cmds), shell=True)
|
|
||||||
tables = filter(lambda t: len(t) > 0,
|
|
||||||
map(lambda l: l.strip(), output.split('\n')))
|
|
||||||
|
|
||||||
print_column_sizes(tables);
|
|
||||||
|
|
||||||
# print tables
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
print("Error:\n", e.output)
|
|
||||||
sys.exit(0)
|
|
||||||
Loading…
x
Reference in New Issue
Block a user