From 29d33dff3cd8d098c6069b722152ef8a993d0ea2 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Mon, 7 Nov 2016 15:37:17 +0100 Subject: [PATCH 01/19] example graphviz based ETL graph --- gen_etl_graph.sh | 21 +++++++++++++++++++ layers/highway/highway.sql | 33 +++++++++++++++++++++++++++++- layers/highway/mapping.yaml | 24 ++++++++++++++++++++++ layers/highway/ne_global_roads.sql | 18 ++++++++++++++++ layers/highway/types.sql | 16 +++++++++++++++ 5 files changed, 111 insertions(+), 1 deletion(-) create mode 100755 gen_etl_graph.sh diff --git a/gen_etl_graph.sh b/gen_etl_graph.sh new file mode 100755 index 0000000..ddd3520 --- /dev/null +++ b/gen_etl_graph.sh @@ -0,0 +1,21 @@ + + +DOT=highway.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/highway/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT +cat ./layers/highway/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/highway/ne_global_roads.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/highway/highway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_highway_etl.png diff --git a/layers/highway/highway.sql b/layers/highway/highway.sql index a6c14a2..d233f8c 100644 --- a/layers/highway/highway.sql +++ b/layers/highway/highway.sql @@ -1,7 +1,19 @@ + + +-- etldoc: highway_sql[label="highway.sql", shape=note ] +-- etldoc: function_highway_is_link[label="FUNCTION highway_is_link"] +-- etldoc: highway_sql-> function_highway_is_link +-- etldoc: function_highway_is_link -> postgreSQL + CREATE OR REPLACE FUNCTION highway_is_link(highway TEXT) RETURNS BOOLEAN AS $$ SELECT highway LIKE '%_link'; $$ LANGUAGE SQL IMMUTABLE STRICT; + +-- etldoc: postgreSQL -> layer_highway +-- etldoc: highway_sql-> layer_highway +-- etldoc: layer_highway[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label=" layer_highway (highway.sql) | z4-z7 | z8 | z9 | z10 | z11 | z12| z13| z14" ] ; CREATE OR REPLACE FUNCTION layer_highway(bbox geometry, zoom_level int) RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass text, properties highway_properties) AS $$ SELECT @@ -9,29 +21,41 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass te to_highway_class(highway) AS class, highway AS subclass, to_highway_properties(is_bridge, is_tunnel, is_ford, is_ramp, is_oneway) AS properties FROM ( - SELECT + + -- etldoc: ne_10m_global_roads -> layer_highway:z4z7 + SELECT NULL::bigint AS osm_id, geometry, highway, FALSE AS is_bridge, FALSE AS is_tunnel, FALSE AS is_ford, FALSE AS is_ramp, FALSE AS is_oneway, 0 AS z_order FROM ne_10m_global_roads WHERE zoom_level BETWEEN 4 AND 7 AND scalerank <= 1 + zoom_level UNION ALL + + -- etldoc: osm_highway_linestring_gen4 -> layer_highway:z8 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring_gen4 WHERE zoom_level = 8 UNION ALL + + -- etldoc: osm_highway_linestring_gen3 -> layer_highway:z9 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring_gen3 WHERE zoom_level = 9 UNION ALL + + -- etldoc: osm_highway_linestring_gen2 -> layer_highway:z10 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring_gen2 WHERE zoom_level = 10 UNION ALL + + -- etldoc: osm_highway_linestring_gen1 -> layer_highway:z11 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring_gen1 WHERE zoom_level = 11 UNION ALL + + -- etldoc: osm_highway_linestring -> layer_highway:z12 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring WHERE zoom_level = 12 @@ -39,18 +63,25 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass te AND NOT highway_is_link(highway) AND NOT is_area UNION ALL + + -- etldoc: osm_highway_linestring -> layer_highway:z13 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring WHERE zoom_level = 13 AND to_highway_class(highway) < 'path'::highway_class AND NOT is_area UNION ALL + + -- etldoc: osm_highway_linestring -> layer_highway:z14 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring WHERE zoom_level >= 14 AND NOT is_area UNION ALL + -- NOTE: We limit the selection of polys because we need to be careful to net get false positives here because -- it is possible that closed linestrings appear both as highway linestrings and as polygon + -- etldoc: osm_highway_polygon -> layer_highway:z13 + -- etldoc: osm_highway_polygon -> layer_highway:z14 SELECT osm_id, geometry, highway, FALSE AS is_bridge, FALSE AS is_tunnel, FALSE AS is_ford, FALSE AS is_ramp, FALSE AS is_oneway, z_order FROM osm_highway_polygon -- We do not want underground pedestrian areas for now diff --git a/layers/highway/mapping.yaml b/layers/highway/mapping.yaml index b11bb57..1f76e47 100644 --- a/layers/highway/mapping.yaml +++ b/layers/highway/mapping.yaml @@ -1,21 +1,43 @@ + + + +# etldoc: imposm3 [fillcolor=yellow, style="rounded,filled", shape=diamond ]; +# etldoc: osm [fillcolor=lightblue, +# etldoc: style="rounded,filled", +# etldoc: shape=box , label="OpenStreetMap", +# etldoc: url="http://www.openstreetmap.org"]; +# etldoc: mapping_yml[label="mapping.yml", shape=note ] +# etldoc: mapping_yml -> imposm3 +# etldoc: osm -> imposm3 + generalized_tables: + +# etldoc: imposm3 -> osm_highway_linestring_gen4 highway_linestring_gen4: source: highway_linestring_gen3 sql_filter: highway IN ('motorway','trunk') AND NOT is_area tolerance: 200.0 + +# etldoc: imposm3 -> osm_highway_linestring_gen3 highway_linestring_gen3: source: highway_linestring_gen2 sql_filter: highway IN ('motorway','trunk', 'primary') AND NOT is_area tolerance: 120.0 + +# etldoc: imposm3 -> osm_highway_linestring_gen2 highway_linestring_gen2: source: highway_linestring_gen1 sql_filter: highway IN ('motorway','trunk', 'primary', 'secondary') AND NOT is_area tolerance: 50.0 + +# etldoc: imposm3 -> osm_highway_linestring_gen1 highway_linestring_gen1: source: highway_linestring sql_filter: highway IN ('motorway','trunk', 'primary', 'secondary', 'tertiary') AND NOT is_area tolerance: 20.0 + tables: +# etldoc: imposm3 -> osm_highway_linestring highway_linestring: type: linestring fields: @@ -85,6 +107,8 @@ tables: - corridor - crossing - pedestrian + +# etldoc: imposm3 -> osm_highway_polygon highway_polygon: type: polygon fields: diff --git a/layers/highway/ne_global_roads.sql b/layers/highway/ne_global_roads.sql index f52bb0f..42ed1bd 100644 --- a/layers/highway/ne_global_roads.sql +++ b/layers/highway/ne_global_roads.sql @@ -1,3 +1,10 @@ + + +-- etldoc: ne_global_roads_sql [label="ne_global_roads.sql", shape=note ] +-- etldoc: function_ne_highway[label="FUNCTION ne_highway"] +-- etldoc: ne_global_roads_sql -> function_ne_highway +-- etldoc: function_ne_highway -> postgreSQL + CREATE OR REPLACE FUNCTION ne_highway(type VARCHAR) RETURNS VARCHAR AS $$ SELECT CASE type WHEN 'Major Highway' THEN 'motorway' @@ -7,13 +14,24 @@ CREATE OR REPLACE FUNCTION ne_highway(type VARCHAR) RETURNS VARCHAR AS $$ END; $$ LANGUAGE SQL IMMUTABLE; + +-- etldoc: natural_earth [fillcolor=lightblue, style="rounded,filled", shape=box , label="Natural Earth" ]; +-- etldoc: natural_earth -> ne_10m_roads +-- etldoc: natural_earth -> ne_10m_roads_north_america + +-- etldoc: ne_global_roads_sql -> ne_10m_global_roads ; CREATE TABLE IF NOT EXISTS ne_10m_global_roads AS ( + + -- etldoc: ne_10m_roads -> ne_10m_global_roads SELECT geom AS geometry, scalerank, ne_highway(type) AS highway FROM ne_10m_roads WHERE continent <> 'North America' AND featurecla = 'Road' AND type IN ('Major Highway', 'Secondary Highway', 'Road') UNION ALL + + -- etldoc: function_ne_highway -> ne_10m_global_roads + -- etldoc: ne_10m_roads_north_america -> ne_10m_global_roads SELECT geom AS geometry, scalerank, ne_highway(type) AS highway FROM ne_10m_roads_north_america WHERE type IN ('Major Highway', 'Secondary Highway', 'Road') diff --git a/layers/highway/types.sql b/layers/highway/types.sql index f7346fa..a1e8443 100644 --- a/layers/highway/types.sql +++ b/layers/highway/types.sql @@ -1,4 +1,11 @@ + + +-- etldoc: types_sql[label="types.sql", shape=note ] + DO $$ +-- etldoc: type_highway_class[label="TYPE highway_class"] +-- etldoc: types_sql-> type_highway_class +-- etldoc: type_highway_class -> postgreSQL BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'highway_class') THEN CREATE TYPE highway_class AS ENUM ('motorway', 'major_road', 'minor_road', 'path'); @@ -7,6 +14,9 @@ END $$; DO $$ +-- etldoc: type_highway_properties[label="TYPE highway_properties"] +-- etldoc: types_sql-> type_highway_properties +-- etldoc: type_highway_properties -> postgreSQL BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'highway_properties') THEN CREATE TYPE highway_properties AS ENUM ('bridge:oneway', 'tunnel:oneway', 'ramp', 'ford', 'bridge', 'tunnel', 'oneway'); @@ -15,6 +25,9 @@ END $$; +-- etldoc: function_to_highway_class[label="FUNCTION to_highway_class"] +-- etldoc: types_sql-> function_to_highway_class +-- etldoc: function_to_highway_class -> postgreSQL CREATE OR REPLACE FUNCTION to_highway_class(highway TEXT) RETURNS highway_class AS $$ SELECT CASE WHEN highway IN ('motorway', 'motorway_link') THEN 'motorway'::highway_class @@ -29,6 +42,9 @@ CREATE OR REPLACE FUNCTION to_highway_class(highway TEXT) RETURNS highway_class END; $$ LANGUAGE SQL IMMUTABLE STRICT; +-- etldoc: function_to_highway_properties[label="FUNCTION to_highway_properties"] +-- etldoc: types_sql-> function_to_highway_properties +-- etldoc: function_to_highway_properties -> postgreSQL CREATE OR REPLACE FUNCTION to_highway_properties(is_bridge boolean, is_tunnel boolean, is_ford boolean, is_ramp boolean, is_oneway boolean) RETURNS highway_properties AS $$ SELECT CASE WHEN is_bridge AND is_oneway THEN 'bridge:oneway'::highway_properties From 598c931e1758e18d7e8d6d25af1921d2a0b7eaf7 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 9 Nov 2016 22:06:24 +0100 Subject: [PATCH 02/19] boundary ETL graph comments - only sql tables and views --- gen_etl_graph_boundary.sh | 21 +++++++++++++++++++ layers/boundary/boundary.sql | 40 ++++++++++++++++++++++++++++++++++++ layers/boundary/mapping.yaml | 12 +++++++++++ 3 files changed, 73 insertions(+) create mode 100755 gen_etl_graph_boundary.sh diff --git a/gen_etl_graph_boundary.sh b/gen_etl_graph_boundary.sh new file mode 100755 index 0000000..1430eee --- /dev/null +++ b/gen_etl_graph_boundary.sh @@ -0,0 +1,21 @@ + + + + +DOT=boundary.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/boundary/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT +cat ./layers/boundary/boundary.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_boundary_etl.png diff --git a/layers/boundary/boundary.sql b/layers/boundary/boundary.sql index 1e77514..f7cd745 100644 --- a/layers/boundary/boundary.sql +++ b/layers/boundary/boundary.sql @@ -1,8 +1,15 @@ + + +-- etldoc: ne_110m_admin_0_boundary_lines_land -> boundary_z0 + CREATE OR REPLACE VIEW boundary_z0 AS ( SELECT geom, 2 AS admin_level FROM ne_110m_admin_0_boundary_lines_land ); +-- etldoc: ne_50m_admin_0_boundary_lines_land -> boundary_z1 +-- etldoc: ne_50m_admin_1_states_provinces_lines -> boundary_z1 + CREATE OR REPLACE VIEW boundary_z1 AS ( SELECT geom, 2 AS admin_level FROM ne_50m_admin_0_boundary_lines_land @@ -12,6 +19,10 @@ CREATE OR REPLACE VIEW boundary_z1 AS ( WHERE scalerank <= 2 ); + +-- etldoc: ne_50m_admin_0_boundary_lines_land -> boundary_z3 +-- etldoc: ne_50m_admin_1_states_provinces_lines -> boundary_z3 + CREATE OR REPLACE VIEW boundary_z3 AS ( SELECT geom, 2 AS admin_level FROM ne_50m_admin_0_boundary_lines_land @@ -20,6 +31,10 @@ CREATE OR REPLACE VIEW boundary_z3 AS ( FROM ne_50m_admin_1_states_provinces_lines ); + +-- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z4 +-- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z4 + CREATE OR REPLACE VIEW boundary_z4 AS ( SELECT geom, 2 AS admin_level FROM ne_10m_admin_0_boundary_lines_land @@ -29,6 +44,9 @@ CREATE OR REPLACE VIEW boundary_z4 AS ( WHERE scalerank <= 3 AND featurecla = 'Adm-1 boundary' ); +-- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z5 +-- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z5 + CREATE OR REPLACE VIEW boundary_z5 AS ( SELECT geom, 2 AS admin_level FROM ne_10m_admin_0_boundary_lines_land @@ -38,6 +56,8 @@ CREATE OR REPLACE VIEW boundary_z5 AS ( WHERE scalerank <= 7 AND featurecla = 'Adm-1 boundary' ); +-- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z7 +-- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z7 CREATE OR REPLACE VIEW boundary_z7 AS ( SELECT geom, 2 AS admin_level FROM ne_10m_admin_0_boundary_lines_land @@ -48,60 +68,80 @@ CREATE OR REPLACE VIEW boundary_z7 AS ( ); +-- etldoc: osm_boundary_linestring_gen5 -> boundary_z8 CREATE OR REPLACE VIEW boundary_z8 AS ( SELECT geometry AS geom, admin_level FROM osm_boundary_linestring_gen5 WHERE admin_level <= 4 AND ST_Length(geometry) > 1000 ); +-- etldoc: osm_boundary_linestring_gen4 -> boundary_z9 CREATE OR REPLACE VIEW boundary_z9 AS ( SELECT geometry AS geom, admin_level FROM osm_boundary_linestring_gen4 WHERE admin_level <= 6 ); +-- etldoc: osm_boundary_linestring_gen3 -> boundary_z10 CREATE OR REPLACE VIEW boundary_z10 AS ( SELECT geometry AS geom, admin_level FROM osm_boundary_linestring_gen3 WHERE admin_level <= 6 ); +-- etldoc: osm_boundary_linestring_gen2 -> boundary_z11 CREATE OR REPLACE VIEW boundary_z11 AS ( SELECT geometry AS geom, admin_level FROM osm_boundary_linestring_gen2 WHERE admin_level <= 8 ); +-- etldoc: osm_boundary_linestring_gen1 -> boundary_z12 CREATE OR REPLACE VIEW boundary_z12 AS ( SELECT geometry AS geom, admin_level FROM osm_boundary_linestring_gen1 ); +-- etldoc: layer_boundary[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label=" layer_boundary | z0 | z1_2 | z3 | z4 | z5_6 | z7 | z8 | z9 | z10 | z11 | z12| z13_ " ] + CREATE OR REPLACE FUNCTION layer_boundary (bbox geometry, zoom_level int) RETURNS TABLE(geometry geometry, admin_level int) AS $$ SELECT geom, admin_level FROM ( + -- etldoc: boundary_z0 -> layer_boundary:z0 SELECT * FROM boundary_z0 WHERE geom && bbox AND zoom_level = 0 UNION ALL + -- etldoc: boundary_z1 -> layer_boundary:z1_2 SELECT * FROM boundary_z1 WHERE geom && bbox AND zoom_level BETWEEN 1 AND 2 UNION ALL + -- etldoc: boundary_z3 -> layer_boundary:z3 SELECT * FROM boundary_z3 WHERE geom && bbox AND zoom_level = 3 UNION ALL + -- etldoc: boundary_z4 -> layer_boundary:z4 SELECT * FROM boundary_z4 WHERE geom && bbox AND zoom_level = 4 UNION ALL + -- etldoc: boundary_z5 -> layer_boundary:z5_6 SELECT * FROM boundary_z5 WHERE geom && bbox AND zoom_level BETWEEN 5 AND 6 UNION ALL + -- etldoc: boundary_z7 -> layer_boundary:z7 SELECT * FROM boundary_z7 WHERE geom && bbox AND zoom_level = 7 UNION ALL + -- etldoc: boundary_z8 -> layer_boundary:z8 SELECT * FROM boundary_z8 WHERE geom && bbox AND zoom_level = 8 UNION ALL + -- etldoc: boundary_z9 -> layer_boundary:z9 SELECT * FROM boundary_z9 WHERE geom && bbox AND zoom_level = 9 UNION ALL + -- etldoc: boundary_z10 -> layer_boundary:z10 SELECT * FROM boundary_z10 WHERE geom && bbox AND zoom_level = 10 UNION ALL + -- etldoc: boundary_z11 -> layer_boundary:z11 SELECT * FROM boundary_z11 WHERE geom && bbox AND zoom_level = 11 UNION ALL + -- etldoc: boundary_z12 -> layer_boundary:z12 SELECT * FROM boundary_z12 WHERE geom && bbox AND zoom_level = 12 UNION ALL + -- etldoc: boundary_z12 -> layer_boundary:z13_ SELECT * FROM boundary_z12 WHERE geom && bbox AND zoom_level >= 13 ) AS zoom_levels; $$ LANGUAGE SQL IMMUTABLE; diff --git a/layers/boundary/mapping.yaml b/layers/boundary/mapping.yaml index a6330c7..6223c54 100644 --- a/layers/boundary/mapping.yaml +++ b/layers/boundary/mapping.yaml @@ -1,20 +1,32 @@ generalized_tables: + + # etldoc: imposm3 -> osm_boundary_linestring_gen5 boundary_linestring_gen5: source: boundary_linestring tolerance: 420.0 + + # etldoc: imposm3 -> osm_boundary_linestring_gen4 boundary_linestring_gen4: source: boundary_linestring tolerance: 320.0 + + # etldoc: imposm3 -> osm_boundary_linestring_gen3 boundary_linestring_gen3: source: boundary_linestring tolerance: 150.0 + + # etldoc: imposm3 -> osm_boundary_linestring_gen2 boundary_linestring_gen2: source: boundary_linestring tolerance: 100.0 + + # etldoc: imposm3 -> osm_boundary_linestring_gen1 boundary_linestring_gen1: source: boundary_linestring tolerance: 50.0 + tables: + # etldoc: imposm3 -> osm_boundary_linestring boundary_linestring: fields: - name: osm_id From 7a0ec0e40d2a7ca4a9ea6de0c57ae802611d0f12 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 9 Nov 2016 23:39:44 +0100 Subject: [PATCH 03/19] add layer_building etldoc comments --- gen_etl_graph_building.sh | 21 +++++++++++++++++++++ layers/building/building.sql | 9 +++++++++ layers/building/mapping.yaml | 5 +++++ 3 files changed, 35 insertions(+) create mode 100755 gen_etl_graph_building.sh diff --git a/gen_etl_graph_building.sh b/gen_etl_graph_building.sh new file mode 100755 index 0000000..a74b0e6 --- /dev/null +++ b/gen_etl_graph_building.sh @@ -0,0 +1,21 @@ + + + + +DOT=building.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/building/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT +cat ./layers/building/building.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_building_etl.png diff --git a/layers/building/building.sql b/layers/building/building.sql index 8f0c7fe..9de11ac 100644 --- a/layers/building/building.sql +++ b/layers/building/building.sql @@ -1,11 +1,20 @@ + + + +-- etldoc: layer_building[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_building | z13 | z14_ " ] ; + CREATE OR REPLACE FUNCTION layer_building(bbox geometry, zoom_level int) RETURNS TABLE(geom geometry, osm_id bigint, render_height numeric) AS $$ SELECT geometry, osm_id, least(greatest(3, COALESCE(height, levels*3.66,5)),400)^.7::int AS render_height FROM ( + + -- etldoc: osm_building_polygon_gen1 -> layer_building:z13 SELECT osm_id, geometry, height, levels FROM osm_building_polygon_gen1 WHERE zoom_level = 13 AND geometry && bbox AND area > 1400 UNION ALL + -- etldoc: osm_building_polygon -> layer_building:z14_ SELECT osm_id, geometry, height, levels FROM osm_building_polygon WHERE zoom_level >= 14 AND geometry && bbox ) AS zoom_levels diff --git a/layers/building/mapping.yaml b/layers/building/mapping.yaml index 3743f80..8974234 100644 --- a/layers/building/mapping.yaml +++ b/layers/building/mapping.yaml @@ -1,9 +1,14 @@ generalized_tables: + + # etldoc: imposm3 -> osm_building_polygon_gen1 building_polygon_gen1: source: building_polygon sql_filter: area>1400.0 tolerance: 10.0 + tables: + + # etldoc: imposm3 -> osm_building_polygon building_polygon: fields: - name: osm_id From cafd338b1b0d73345edc838e971144a53547e9e7 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 9 Nov 2016 23:55:37 +0100 Subject: [PATCH 04/19] add etldoc to layer_highway ( v2 ) --- gen_etl_graph.sh => gen_etl_graph_highway.sh | 0 layers/highway/highway.sql | 15 ++++----------- layers/highway/mapping.yaml | 11 ----------- layers/highway/ne_global_roads.sql | 10 ---------- layers/highway/types.sql | 15 --------------- 5 files changed, 4 insertions(+), 47 deletions(-) rename gen_etl_graph.sh => gen_etl_graph_highway.sh (100%) diff --git a/gen_etl_graph.sh b/gen_etl_graph_highway.sh similarity index 100% rename from gen_etl_graph.sh rename to gen_etl_graph_highway.sh diff --git a/layers/highway/highway.sql b/layers/highway/highway.sql index d233f8c..921dce2 100644 --- a/layers/highway/highway.sql +++ b/layers/highway/highway.sql @@ -1,19 +1,12 @@ - - --- etldoc: highway_sql[label="highway.sql", shape=note ] --- etldoc: function_highway_is_link[label="FUNCTION highway_is_link"] --- etldoc: highway_sql-> function_highway_is_link --- etldoc: function_highway_is_link -> postgreSQL + CREATE OR REPLACE FUNCTION highway_is_link(highway TEXT) RETURNS BOOLEAN AS $$ SELECT highway LIKE '%_link'; $$ LANGUAGE SQL IMMUTABLE STRICT; --- etldoc: postgreSQL -> layer_highway --- etldoc: highway_sql-> layer_highway -- etldoc: layer_highway[shape=record fillcolor=lightpink, style="rounded,filled", --- etldoc: label=" layer_highway (highway.sql) | z4-z7 | z8 | z9 | z10 | z11 | z12| z13| z14" ] ; +-- etldoc: label=" layer_highway | z4-z7 | z8 | z9 | z10 | z11 | z12| z13| z14_" ] ; CREATE OR REPLACE FUNCTION layer_highway(bbox geometry, zoom_level int) RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass text, properties highway_properties) AS $$ SELECT @@ -72,7 +65,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass te AND NOT is_area UNION ALL - -- etldoc: osm_highway_linestring -> layer_highway:z14 + -- etldoc: osm_highway_linestring -> layer_highway:z14_ SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring WHERE zoom_level >= 14 AND NOT is_area @@ -81,7 +74,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass te -- NOTE: We limit the selection of polys because we need to be careful to net get false positives here because -- it is possible that closed linestrings appear both as highway linestrings and as polygon -- etldoc: osm_highway_polygon -> layer_highway:z13 - -- etldoc: osm_highway_polygon -> layer_highway:z14 + -- etldoc: osm_highway_polygon -> layer_highway:z14_ SELECT osm_id, geometry, highway, FALSE AS is_bridge, FALSE AS is_tunnel, FALSE AS is_ford, FALSE AS is_ramp, FALSE AS is_oneway, z_order FROM osm_highway_polygon -- We do not want underground pedestrian areas for now diff --git a/layers/highway/mapping.yaml b/layers/highway/mapping.yaml index 1f76e47..cbd8fed 100644 --- a/layers/highway/mapping.yaml +++ b/layers/highway/mapping.yaml @@ -1,15 +1,4 @@ - - -# etldoc: imposm3 [fillcolor=yellow, style="rounded,filled", shape=diamond ]; -# etldoc: osm [fillcolor=lightblue, -# etldoc: style="rounded,filled", -# etldoc: shape=box , label="OpenStreetMap", -# etldoc: url="http://www.openstreetmap.org"]; -# etldoc: mapping_yml[label="mapping.yml", shape=note ] -# etldoc: mapping_yml -> imposm3 -# etldoc: osm -> imposm3 - generalized_tables: # etldoc: imposm3 -> osm_highway_linestring_gen4 diff --git a/layers/highway/ne_global_roads.sql b/layers/highway/ne_global_roads.sql index 42ed1bd..71fdfc8 100644 --- a/layers/highway/ne_global_roads.sql +++ b/layers/highway/ne_global_roads.sql @@ -1,10 +1,5 @@ --- etldoc: ne_global_roads_sql [label="ne_global_roads.sql", shape=note ] --- etldoc: function_ne_highway[label="FUNCTION ne_highway"] --- etldoc: ne_global_roads_sql -> function_ne_highway --- etldoc: function_ne_highway -> postgreSQL - CREATE OR REPLACE FUNCTION ne_highway(type VARCHAR) RETURNS VARCHAR AS $$ SELECT CASE type WHEN 'Major Highway' THEN 'motorway' @@ -15,10 +10,6 @@ CREATE OR REPLACE FUNCTION ne_highway(type VARCHAR) RETURNS VARCHAR AS $$ $$ LANGUAGE SQL IMMUTABLE; --- etldoc: natural_earth [fillcolor=lightblue, style="rounded,filled", shape=box , label="Natural Earth" ]; --- etldoc: natural_earth -> ne_10m_roads --- etldoc: natural_earth -> ne_10m_roads_north_america - -- etldoc: ne_global_roads_sql -> ne_10m_global_roads ; CREATE TABLE IF NOT EXISTS ne_10m_global_roads AS ( @@ -30,7 +21,6 @@ CREATE TABLE IF NOT EXISTS ne_10m_global_roads AS ( AND type IN ('Major Highway', 'Secondary Highway', 'Road') UNION ALL - -- etldoc: function_ne_highway -> ne_10m_global_roads -- etldoc: ne_10m_roads_north_america -> ne_10m_global_roads SELECT geom AS geometry, scalerank, ne_highway(type) AS highway FROM ne_10m_roads_north_america diff --git a/layers/highway/types.sql b/layers/highway/types.sql index a1e8443..10e6c6b 100644 --- a/layers/highway/types.sql +++ b/layers/highway/types.sql @@ -1,11 +1,6 @@ --- etldoc: types_sql[label="types.sql", shape=note ] - DO $$ --- etldoc: type_highway_class[label="TYPE highway_class"] --- etldoc: types_sql-> type_highway_class --- etldoc: type_highway_class -> postgreSQL BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'highway_class') THEN CREATE TYPE highway_class AS ENUM ('motorway', 'major_road', 'minor_road', 'path'); @@ -14,9 +9,6 @@ END $$; DO $$ --- etldoc: type_highway_properties[label="TYPE highway_properties"] --- etldoc: types_sql-> type_highway_properties --- etldoc: type_highway_properties -> postgreSQL BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'highway_properties') THEN CREATE TYPE highway_properties AS ENUM ('bridge:oneway', 'tunnel:oneway', 'ramp', 'ford', 'bridge', 'tunnel', 'oneway'); @@ -24,10 +16,6 @@ BEGIN END $$; - --- etldoc: function_to_highway_class[label="FUNCTION to_highway_class"] --- etldoc: types_sql-> function_to_highway_class --- etldoc: function_to_highway_class -> postgreSQL CREATE OR REPLACE FUNCTION to_highway_class(highway TEXT) RETURNS highway_class AS $$ SELECT CASE WHEN highway IN ('motorway', 'motorway_link') THEN 'motorway'::highway_class @@ -42,9 +30,6 @@ CREATE OR REPLACE FUNCTION to_highway_class(highway TEXT) RETURNS highway_class END; $$ LANGUAGE SQL IMMUTABLE STRICT; --- etldoc: function_to_highway_properties[label="FUNCTION to_highway_properties"] --- etldoc: types_sql-> function_to_highway_properties --- etldoc: function_to_highway_properties -> postgreSQL CREATE OR REPLACE FUNCTION to_highway_properties(is_bridge boolean, is_tunnel boolean, is_ford boolean, is_ramp boolean, is_oneway boolean) RETURNS highway_properties AS $$ SELECT CASE WHEN is_bridge AND is_oneway THEN 'bridge:oneway'::highway_properties From 3a5c47e3ddf8602be27b41598f0c1841b1711d3a Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 10 Nov 2016 00:31:08 +0100 Subject: [PATCH 05/19] add etldoc to layer_highway_name --- gen_etl_graph_highway_name.sh | 19 +++++++++++++++++++ layers/highway_name/layer.sql | 18 ++++++++++++++++++ layers/highway_name/merge_highways.sql | 5 +++++ 3 files changed, 42 insertions(+) create mode 100755 gen_etl_graph_highway_name.sh diff --git a/gen_etl_graph_highway_name.sh b/gen_etl_graph_highway_name.sh new file mode 100755 index 0000000..39afd79 --- /dev/null +++ b/gen_etl_graph_highway_name.sh @@ -0,0 +1,19 @@ + + +DOT=highway_name.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/highway_name/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/highway_name/merge_highways.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_highway_names_etl.png diff --git a/layers/highway_name/layer.sql b/layers/highway_name/layer.sql index c9c27a8..100b1fc 100644 --- a/layers/highway_name/layer.sql +++ b/layers/highway_name/layer.sql @@ -1,31 +1,49 @@ + +-- etldoc: layer_highway_name[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_highway_name | z8 | z9 | z10 | z11 | z12| z13| z14_" ] ; + CREATE OR REPLACE FUNCTION layer_highway_name(bbox geometry, zoom_level integer) RETURNS TABLE(osm_id bigint, geometry geometry, name text, ref text, ref_length int, class highway_class, subclass text) AS $$ SELECT osm_id, geometry, name, NULLIF(ref, ''), NULLIF(LENGTH(ref), 0) AS ref_length, to_highway_class(highway) AS class, highway AS subclass FROM ( + + -- etldoc: osm_highway_name_linestring_gen3 -> layer_highway_name:z8 SELECT * FROM osm_highway_name_linestring_gen3 WHERE zoom_level = 8 UNION ALL + + -- etldoc: osm_highway_name_linestring_gen2 -> layer_highway_name:z9 SELECT * FROM osm_highway_name_linestring_gen2 WHERE zoom_level = 9 UNION ALL + + -- etldoc: osm_highway_name_linestring_gen1 -> layer_highway_name:z10 + -- etldoc: osm_highway_name_linestring_gen1 -> layer_highway_name:z11 SELECT * FROM osm_highway_name_linestring_gen1 WHERE zoom_level BETWEEN 10 AND 11 UNION ALL + + -- etldoc: osm_highway_name_linestring -> layer_highway_name:z12 SELECT * FROM osm_highway_name_linestring WHERE zoom_level = 12 AND LineLabel(zoom_level, COALESCE(NULLIF(name, ''), ref), geometry) AND to_highway_class(highway) < 'minor_road'::highway_class AND NOT highway_is_link(highway) UNION ALL + + -- etldoc: osm_highway_name_linestring -> layer_highway_name:z13 SELECT * FROM osm_highway_name_linestring WHERE zoom_level = 13 AND LineLabel(zoom_level, COALESCE(NULLIF(name, ''), ref), geometry) AND to_highway_class(highway) < 'path'::highway_class UNION ALL + + -- etldoc: osm_highway_name_linestring -> layer_highway_name:z14_ SELECT * FROM osm_highway_name_linestring WHERE zoom_level >= 14 + ) AS zoom_levels WHERE geometry && bbox ORDER BY z_order ASC; diff --git a/layers/highway_name/merge_highways.sql b/layers/highway_name/merge_highways.sql index 0f03c29..a96f6d1 100644 --- a/layers/highway_name/merge_highways.sql +++ b/layers/highway_name/merge_highways.sql @@ -2,6 +2,8 @@ -- stitch together the touching ways with the same name -- to allow for nice label rendering -- Because this works well for roads that do not have relations as well + +-- etldoc: osm_highway_linestring -> osm_highway_name_linestring CREATE TABLE IF NOT EXISTS osm_highway_name_linestring AS ( SELECT (ST_Dump(geometry)).geom AS geometry, @@ -30,6 +32,7 @@ CREATE TABLE IF NOT EXISTS osm_highway_name_linestring AS ( CREATE INDEX IF NOT EXISTS osm_highway_name_linestring_geometry_idx ON osm_highway_name_linestring USING gist(geometry); +-- etldoc: osm_highway_name_linestring -> osm_highway_name_linestring_gen1 CREATE TABLE IF NOT EXISTS osm_highway_name_linestring_gen1 AS ( SELECT ST_Simplify(geometry, 50) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order FROM osm_highway_name_linestring @@ -37,6 +40,7 @@ CREATE TABLE IF NOT EXISTS osm_highway_name_linestring_gen1 AS ( ); CREATE INDEX IF NOT EXISTS osm_highway_name_linestring_gen1_geometry_idx ON osm_highway_name_linestring_gen1 USING gist(geometry); +-- etldoc: osm_highway_name_linestring_gen1 -> osm_highway_name_linestring_gen2 CREATE TABLE IF NOT EXISTS osm_highway_name_linestring_gen2 AS ( SELECT ST_Simplify(geometry, 120) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order FROM osm_highway_name_linestring_gen1 @@ -44,6 +48,7 @@ CREATE TABLE IF NOT EXISTS osm_highway_name_linestring_gen2 AS ( ); CREATE INDEX IF NOT EXISTS osm_highway_name_linestring_gen2_geometry_idx ON osm_highway_name_linestring_gen2 USING gist(geometry); +-- etldoc: osm_highway_name_linestring_gen2 -> osm_highway_name_linestring_gen3 CREATE TABLE IF NOT EXISTS osm_highway_name_linestring_gen3 AS ( SELECT ST_Simplify(geometry, 120) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order FROM osm_highway_name_linestring_gen2 From 63cdc3362d08c49dcba23c15dafde29d8d010b4e Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 10 Nov 2016 01:11:11 +0100 Subject: [PATCH 06/19] add etldoc to layer_housenumber --- gen_etl_graph_housenumber.sh | 20 ++++++++++++++++++++ layers/housenumber/housenumber_centroid.sql | 2 ++ layers/housenumber/layer.sql | 5 +++++ layers/housenumber/mapping.yaml | 3 +++ 4 files changed, 30 insertions(+) create mode 100755 gen_etl_graph_housenumber.sh diff --git a/gen_etl_graph_housenumber.sh b/gen_etl_graph_housenumber.sh new file mode 100755 index 0000000..37c12a9 --- /dev/null +++ b/gen_etl_graph_housenumber.sh @@ -0,0 +1,20 @@ + + +DOT=housenumber.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/housenumber/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT +cat ./layers/housenumber/housenumber_centroid.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/housenumber/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_housenumber_etl.png diff --git a/layers/housenumber/housenumber_centroid.sql b/layers/housenumber/housenumber_centroid.sql index c490a24..8a81d6a 100644 --- a/layers/housenumber/housenumber_centroid.sql +++ b/layers/housenumber/housenumber_centroid.sql @@ -1,2 +1,4 @@ + +-- etldoc: osm_housenumber_point -> osm_housenumber_point UPDATE osm_housenumber_point SET geometry=topoint(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; diff --git a/layers/housenumber/layer.sql b/layers/housenumber/layer.sql index 518f891..21f98f3 100644 --- a/layers/housenumber/layer.sql +++ b/layers/housenumber/layer.sql @@ -1,5 +1,10 @@ + +-- etldoc: layer_housenumber[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_housenumber | z14_" ] ; + CREATE OR REPLACE FUNCTION layer_housenumber(bbox geometry, zoom_level integer) RETURNS TABLE(osm_id bigint, geometry geometry, housenumber text) AS $$ + -- etldoc: osm_housenumber_point -> layer_housenumber:z14_ SELECT osm_id, geometry, housenumber FROM osm_housenumber_point WHERE zoom_level >= 14 AND geometry && bbox; $$ LANGUAGE SQL IMMUTABLE; diff --git a/layers/housenumber/mapping.yaml b/layers/housenumber/mapping.yaml index 939c953..cf5f255 100644 --- a/layers/housenumber/mapping.yaml +++ b/layers/housenumber/mapping.yaml @@ -1,4 +1,7 @@ + tables: + + # etldoc: imposm3 -> osm_housenumber_point housenumber_point: type: geometry fields: From 9e27faf9e3c36f1b824f4f000e47feeba9379c7a Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 10 Nov 2016 01:45:17 +0100 Subject: [PATCH 07/19] add etldoc to layer_landcover --- gen_etl_graph_landcover.sh | 19 +++++++++++++++++++ layers/landcover/landcover.sql | 25 +++++++++++++++++++++++++ layers/landcover/mapping.yaml | 13 +++++++++++++ 3 files changed, 57 insertions(+) create mode 100755 gen_etl_graph_landcover.sh diff --git a/gen_etl_graph_landcover.sh b/gen_etl_graph_landcover.sh new file mode 100755 index 0000000..008b0ce --- /dev/null +++ b/gen_etl_graph_landcover.sh @@ -0,0 +1,19 @@ + + +DOT=landcover.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/landcover/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT +cat ./layers/landcover/landcover.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_landcover_etl.png diff --git a/layers/landcover/landcover.sql b/layers/landcover/landcover.sql index ca84f93..a354924 100644 --- a/layers/landcover/landcover.sql +++ b/layers/landcover/landcover.sql @@ -20,83 +20,108 @@ CREATE OR REPLACE FUNCTION landcover_class(landuse VARCHAR, "natural" VARCHAR, w END; $$ LANGUAGE SQL IMMUTABLE; +-- etldoc: ne_110m_glaciated_areas -> landcover_z0 CREATE OR REPLACE VIEW landcover_z0 AS ( SELECT NULL::int AS osm_id, geom AS geometry, NULL AS landuse, 'glacier' AS "natural", NULL::text AS wetland FROM ne_110m_glaciated_areas ); CREATE OR REPLACE VIEW landcover_z2 AS ( + -- etldoc: ne_50m_glaciated_areas -> landcover_z2 SELECT NULL::bigint AS osm_id, geom AS geometry, NULL::text AS landuse, 'glacier' AS "natural", NULL::text AS wetland FROM ne_50m_glaciated_areas UNION ALL + -- etldoc: ne_50m_antarctic_ice_shelves_polys -> landcover_z2 SELECT NULL::bigint AS osm_id, geom AS geometry, NULL::text AS landuse, 'ice_shelf' AS "natural", NULL::text AS wetland FROM ne_50m_antarctic_ice_shelves_polys ); CREATE OR REPLACE VIEW landcover_z5 AS ( + -- etldoc: ne_10m_glaciated_areas -> landcover_z5 SELECT NULL::bigint AS osm_id, geom AS geometry, NULL::text AS landuse, 'glacier' AS "natural", NULL::text AS wetland FROM ne_10m_glaciated_areas UNION ALL + -- etldoc: ne_10m_antarctic_ice_shelves_polys -> landcover_z5 SELECT NULL::bigint AS osm_id, geom AS geometry, NULL::text AS landuse, 'ice_shelf' AS "natural", NULL::text AS wetland FROM ne_10m_antarctic_ice_shelves_polys ); CREATE OR REPLACE VIEW landcover_z8 AS ( + -- etldoc: osm_landcover_polygon_gen5 -> landcover_z8 SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon_gen5 ); CREATE OR REPLACE VIEW landcover_z9 AS ( + -- etldoc: osm_landcover_polygon_gen4 -> landcover_z9 SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon_gen4 ); CREATE OR REPLACE VIEW landcover_z10 AS ( + -- etldoc: osm_landcover_polygon_gen3 -> landcover_z10 SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon_gen3 ); CREATE OR REPLACE VIEW landcover_z11 AS ( + -- etldoc: osm_landcover_polygon_gen2 -> landcover_z11 SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon_gen2 ); CREATE OR REPLACE VIEW landcover_z12 AS ( + -- etldoc: osm_landcover_polygon_gen1 -> landcover_z12 SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon_gen1 ); CREATE OR REPLACE VIEW landcover_z13 AS ( + -- etldoc: osm_landcover_polygon -> landcover_z13 SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon WHERE ST_Area(geometry) > 60000 ); CREATE OR REPLACE VIEW landcover_z14 AS ( + -- etldoc: osm_landcover_polygon -> landcover_z14 SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon ); +-- etldoc: layer_landcover[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_landcover | z0 | z2_4 | z5_7 | z8 | z9 | z10 | z11 | z12| z13| z14_" ] ; + CREATE OR REPLACE FUNCTION layer_landcover(bbox geometry, zoom_level int) RETURNS TABLE(osm_id bigint, geometry geometry, class text, subclass text) AS $$ SELECT osm_id, geometry, landcover_class(landuse, "natural", wetland) AS class, COALESCE(NULLIF("natural", ''), NULLIF(landuse, ''), NULLIF('wetland', '')) AS subclass FROM ( + -- etldoc: landcover_z0 -> layer_landcover:z0 SELECT * FROM landcover_z0 WHERE zoom_level BETWEEN 0 AND 1 AND geometry && bbox UNION ALL + -- etldoc: landcover_z2 -> layer_landcover:z2_4 SELECT * FROM landcover_z2 WHERE zoom_level BETWEEN 2 AND 4 AND geometry && bbox UNION ALL + -- etldoc: landcover_z5 -> layer_landcover:z5_7 SELECT * FROM landcover_z5 WHERE zoom_level BETWEEN 5 AND 7 AND geometry && bbox UNION ALL + -- etldoc: landcover_z8 -> layer_landcover:z8 SELECT osm_id, geometry, landuse, "natural", wetland FROM landcover_z8 WHERE zoom_level = 8 AND geometry && bbox UNION ALL + -- etldoc: landcover_z9 -> layer_landcover:z9 SELECT osm_id, geometry, landuse, "natural", wetland FROM landcover_z9 WHERE zoom_level = 9 AND geometry && bbox UNION ALL + -- etldoc: landcover_z10 -> layer_landcover:z10 SELECT osm_id, geometry, landuse, "natural", wetland FROM landcover_z10 WHERE zoom_level = 10 AND geometry && bbox UNION ALL + -- etldoc: landcover_z11 -> layer_landcover:z11 SELECT osm_id, geometry, landuse, "natural", wetland FROM landcover_z11 WHERE zoom_level = 11 AND geometry && bbox UNION ALL + -- etldoc: landcover_z12 -> layer_landcover:z12 SELECT osm_id, geometry, landuse, "natural", wetland FROM landcover_z12 WHERE zoom_level = 12 AND geometry && bbox UNION ALL + -- etldoc: landcover_z13 -> layer_landcover:z13 SELECT osm_id, ST_Simplify(geometry, 10) AS geometry, landuse, "natural", wetland FROM landcover_z13 WHERE zoom_level = 13 AND geometry && bbox UNION ALL + -- etldoc: landcover_z14 -> layer_landcover:z14_ SELECT osm_id, geometry, landuse, "natural", wetland FROM landcover_z14 WHERE zoom_level >= 14 AND geometry && bbox ) AS zoom_levels; diff --git a/layers/landcover/mapping.yaml b/layers/landcover/mapping.yaml index 112cb75..2ac8741 100644 --- a/layers/landcover/mapping.yaml +++ b/layers/landcover/mapping.yaml @@ -1,25 +1,38 @@ generalized_tables: + + # etldoc: imposm3 -> osm_landcover_polygon_gen5 landcover_polygon_gen5: source: landcover_polygon_gen4 sql_filter: area>15000000 tolerance: 300.0 + + # etldoc: imposm3 -> osm_landcover_polygon_gen4 landcover_polygon_gen4: source: landcover_polygon_gen3 sql_filter: area>4200000 tolerance: 200.0 + + # etldoc: imposm3 -> osm_landcover_polygon_gen3 landcover_polygon_gen3: source: landcover_polygon_gen2 sql_filter: area>1200000 tolerance: 120.0 + + # etldoc: imposm3 -> osm_landcover_polygon_gen2 landcover_polygon_gen2: source: landcover_polygon_gen1 sql_filter: area>480000 tolerance: 80.0 + + # etldoc: imposm3 -> osm_landcover_polygon_gen1 landcover_polygon_gen1: source: landcover_polygon sql_filter: area>240000 tolerance: 50.0 + tables: + + # etldoc: imposm3 -> osm_landcover_polygon landcover_polygon: fields: - name: osm_id From 5580f42458af104278d77aa0046ce36e694be06c Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 10 Nov 2016 02:12:39 +0100 Subject: [PATCH 08/19] add etldoc to layer_landuse --- gen_etl_graph_landuse.sh | 19 +++++++++++++++++++ layers/landuse/landuse.sql | 27 +++++++++++++++++++++++++++ layers/landuse/mapping.yaml | 13 +++++++++++++ 3 files changed, 59 insertions(+) create mode 100755 gen_etl_graph_landuse.sh diff --git a/gen_etl_graph_landuse.sh b/gen_etl_graph_landuse.sh new file mode 100755 index 0000000..25ff4f9 --- /dev/null +++ b/gen_etl_graph_landuse.sh @@ -0,0 +1,19 @@ + + +DOT=landuse.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/landuse/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT +cat ./layers/landuse/landuse.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_landuse_etl.png diff --git a/layers/landuse/landuse.sql b/layers/landuse/landuse.sql index 661360d..c2cafba 100644 --- a/layers/landuse/landuse.sql +++ b/layers/landuse/landuse.sql @@ -8,78 +8,105 @@ CREATE OR REPLACE FUNCTION landuse_class(landuse TEXT, amenity TEXT, leisure TEX END; $$ LANGUAGE SQL IMMUTABLE; +-- etldoc: ne_50m_urban_areas -> landuse_z4 CREATE OR REPLACE VIEW landuse_z4 AS ( SELECT NULL::bigint AS osm_id, geom AS geometry, 'residential' AS landuse, NULL::text AS amenity, NULL::text AS leisure, NULL::text AS boundary, scalerank FROM ne_50m_urban_areas WHERE scalerank <= 2 ); +-- etldoc: ne_50m_urban_areas -> landuse_z5 CREATE OR REPLACE VIEW landuse_z5 AS ( SELECT NULL::bigint AS osm_id, geom AS geometry, 'residential' AS landuse, NULL::text AS amenity, NULL::text AS leisure, NULL::text AS boundary, scalerank FROM ne_50m_urban_areas ); +-- etldoc: ne_10m_urban_areas -> landuse_z6 CREATE OR REPLACE VIEW landuse_z6 AS ( SELECT NULL::bigint AS osm_id, geom AS geometry, 'residential' AS landuse, NULL::text AS amenity, NULL::text AS leisure, NULL::text AS boundary, scalerank FROM ne_10m_urban_areas ); +-- etldoc: osm_landuse_polygon_gen5 -> landuse_z8 CREATE OR REPLACE VIEW landuse_z8 AS ( SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon_gen5 ); +-- etldoc: osm_landuse_polygon_gen4 -> landuse_z9 CREATE OR REPLACE VIEW landuse_z9 AS ( SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon_gen4 ); +-- etldoc: osm_landuse_polygon_gen3 -> landuse_z10 CREATE OR REPLACE VIEW landuse_z10 AS ( SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon_gen3 ); +-- etldoc: osm_landuse_polygon_gen2 -> landuse_z11 CREATE OR REPLACE VIEW landuse_z11 AS ( SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon_gen2 ); +-- etldoc: osm_landuse_polygon_gen1 -> landuse_z12 CREATE OR REPLACE VIEW landuse_z12 AS ( SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon_gen1 ); +-- etldoc: osm_landuse_polygon -> landuse_z13 CREATE OR REPLACE VIEW landuse_z13 AS ( SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon WHERE ST_Area(geometry) > 60000 ); +-- etldoc: osm_landuse_polygon -> landuse_z14 CREATE OR REPLACE VIEW landuse_z14 AS ( SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon ); +-- etldoc: layer_landuse[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_landuse | z4|z5|z6|z7| z8 | z9 | z10 | z11 | z12| z13| z14_" ] ; + CREATE OR REPLACE FUNCTION layer_landuse(bbox geometry, zoom_level int) RETURNS TABLE(osm_id bigint, geometry geometry, class text, subclass text) AS $$ SELECT osm_id, geometry, landuse_class(landuse, amenity, leisure, boundary) AS class, COALESCE(NULLIF(landuse, ''), NULLIF(amenity, ''), NULLIF(leisure, ''), NULLIF(boundary, '')) AS subclass FROM ( + -- etldoc: landuse_z4 -> layer_landuse:z4 SELECT * FROM landuse_z4 WHERE zoom_level = 4 UNION ALL + -- etldoc: landuse_z5 -> layer_landuse:z5 SELECT * FROM landuse_z5 WHERE zoom_level = 5 UNION ALL + -- etldoc: landuse_z6 -> layer_landuse:z6 + -- etldoc: landuse_z6 -> layer_landuse:z7 + -- etldoc: landuse_z6 -> layer_landuse:z8 + -- etldoc: landuse_z6 -> layer_landuse:z9 + -- etldoc: landuse_z6 -> layer_landuse:z10 SELECT * FROM landuse_z6 WHERE zoom_level BETWEEN 6 AND 10 AND scalerank-1 <= zoom_level UNION ALL + -- etldoc: landuse_z8 -> layer_landuse:z8 SELECT * FROM landuse_z8 WHERE zoom_level = 8 UNION ALL + -- etldoc: landuse_z9 -> layer_landuse:z9 SELECT * FROM landuse_z9 WHERE zoom_level = 9 UNION ALL + -- etldoc: landuse_z10 -> layer_landuse:z10 SELECT * FROM landuse_z10 WHERE zoom_level = 10 UNION ALL + -- etldoc: landuse_z11 -> layer_landuse:z11 SELECT * FROM landuse_z11 WHERE zoom_level = 11 UNION ALL + -- etldoc: landuse_z12 -> layer_landuse:z12 SELECT * FROM landuse_z12 WHERE zoom_level = 12 UNION ALL + -- etldoc: landuse_z13 -> layer_landuse:z13 SELECT * FROM landuse_z13 WHERE zoom_level = 13 UNION ALL + -- etldoc: landuse_z14 -> layer_landuse:z14_ SELECT * FROM landuse_z14 WHERE zoom_level >= 14 ) AS zoom_levels WHERE geometry && bbox; diff --git a/layers/landuse/mapping.yaml b/layers/landuse/mapping.yaml index 262c897..76bffe8 100644 --- a/layers/landuse/mapping.yaml +++ b/layers/landuse/mapping.yaml @@ -1,25 +1,38 @@ generalized_tables: + + # etldoc: imposm3 -> osm_landuse_polygon_gen5 landuse_polygon_gen5: source: landuse_polygon_gen4 sql_filter: area>15000000 tolerance: 300.0 + + # etldoc: imposm3 -> osm_landuse_polygon_gen4 landuse_polygon_gen4: source: landuse_polygon_gen3 sql_filter: area>4200000 tolerance: 200.0 + + # etldoc: imposm3 -> osm_landuse_polygon_gen3 landuse_polygon_gen3: source: landuse_polygon_gen2 sql_filter: area>1200000 tolerance: 120.0 + + # etldoc: imposm3 -> osm_landuse_polygon_gen2 landuse_polygon_gen2: source: landuse_polygon_gen1 sql_filter: area>480000 tolerance: 80.0 + + # etldoc: imposm3 -> osm_landuse_polygon_gen1 landuse_polygon_gen1: source: landuse_polygon sql_filter: area>240000 tolerance: 50.0 + tables: + + # etldoc: imposm3 -> osm_landuse_polygon landuse_polygon: type: polygon fields: From 7d7ca7f47a17261d05d9848fbdf6d29451294366 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 10 Nov 2016 03:09:43 +0100 Subject: [PATCH 09/19] add etldoc to layer_place --- gen_etl_graph_place.sh | 27 +++++++++++++++++++++++++++ layers/place/city.sql | 17 +++++++++++++++++ layers/place/country.sql | 2 ++ layers/place/mapping.yaml | 6 ++++++ layers/place/merge_city_rank.sql | 4 ++++ layers/place/merge_country_rank.sql | 3 +++ layers/place/merge_state_rank.sql | 3 +++ layers/place/place.sql | 5 +++++ layers/place/state.sql | 3 +++ 9 files changed, 70 insertions(+) create mode 100755 gen_etl_graph_place.sh diff --git a/gen_etl_graph_place.sh b/gen_etl_graph_place.sh new file mode 100755 index 0000000..5056cfa --- /dev/null +++ b/gen_etl_graph_place.sh @@ -0,0 +1,27 @@ + + +DOT=place.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/place/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT + +cat ./layers/place/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/place/city.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/place/country.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/place/state.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/place/merge_country_rank.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/place/merge_city_rank.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/place/merge_state_rank.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/place/place.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_place_etl.png diff --git a/layers/place/city.sql b/layers/place/city.sql index d64b0cc..6b3faec 100644 --- a/layers/place/city.sql +++ b/layers/place/city.sql @@ -1,5 +1,15 @@ + +-- etldoc: layer_city[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_city | z2 | z3 | z4 | z5| z6 | z7 | z8 | z9 | z10 | z11 | z12| z13| z14_" ] ; + CREATE OR REPLACE FUNCTION layer_city(bbox geometry, zoom_level int, pixel_width numeric) RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class city_class, "rank" int) AS $$ + -- etldoc: osm_city_point -> layer_city:z2 + -- etldoc: osm_city_point -> layer_city:z3 + -- etldoc: osm_city_point -> layer_city:z4 + -- etldoc: osm_city_point -> layer_city:z5 + -- etldoc: osm_city_point -> layer_city:z6 + -- etldoc: osm_city_point -> layer_city:z7 SELECT osm_id, geometry, name, COALESCE(NULLIF(name_en, ''), name) AS name_en, place AS class, "rank" FROM osm_city_point WHERE geometry && bbox @@ -20,6 +30,13 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class c population DESC NULLS LAST, length(name) ASC )::int AS gridrank + -- etldoc: osm_city_point -> layer_city:z8 + -- etldoc: osm_city_point -> layer_city:z9 + -- etldoc: osm_city_point -> layer_city:z10 + -- etldoc: osm_city_point -> layer_city:z11 + -- etldoc: osm_city_point -> layer_city:z12 + -- etldoc: osm_city_point -> layer_city:z13 + -- etldoc: osm_city_point -> layer_city:z14_ FROM osm_city_point WHERE geometry && bbox AND ((zoom_level = 8 AND place <= 'town'::city_class) diff --git a/layers/place/country.sql b/layers/place/country.sql index 896e38e..092aed2 100644 --- a/layers/place/country.sql +++ b/layers/place/country.sql @@ -1,3 +1,5 @@ + +-- etldoc: osm_country_point -> layer_country CREATE OR REPLACE FUNCTION layer_country(bbox geometry, zoom_level int) RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, "rank" int) AS $$ SELECT osm_id, geometry, name, COALESCE(NULLIF(name_en, ''), name) AS name_en, "rank" FROM osm_country_point diff --git a/layers/place/mapping.yaml b/layers/place/mapping.yaml index 6f6264a..2e34232 100644 --- a/layers/place/mapping.yaml +++ b/layers/place/mapping.yaml @@ -11,6 +11,8 @@ rank_field: &rank key: rank type: integer tables: + + # etldoc: imposm3 -> osm_country_point country_point: type: point fields: @@ -27,6 +29,8 @@ tables: mapping: place: - country + + # etldoc: imposm3 -> osm_state_point state_point: type: point fields: @@ -52,6 +56,8 @@ tables: mapping: place: - state + + # etldoc: imposm3 -> osm_city_point city_point: type: point fields: diff --git a/layers/place/merge_city_rank.sql b/layers/place/merge_city_rank.sql index 8a08b5d..025f058 100644 --- a/layers/place/merge_city_rank.sql +++ b/layers/place/merge_city_rank.sql @@ -1,3 +1,7 @@ + +-- etldoc: ne_10m_populated_places -> osm_city_point +-- etldoc: osm_city_point -> osm_city_point + WITH important_city_point AS ( SELECT osm.geometry, osm.osm_id, osm.name, osm.name_en, ne.scalerank, ne.labelrank FROM ne_10m_populated_places AS ne, osm_city_point AS osm diff --git a/layers/place/merge_country_rank.sql b/layers/place/merge_country_rank.sql index 40e905e..adf0027 100644 --- a/layers/place/merge_country_rank.sql +++ b/layers/place/merge_country_rank.sql @@ -1,5 +1,8 @@ ALTER TABLE osm_country_point DROP CONSTRAINT IF EXISTS osm_country_point_rank_constraint; +-- etldoc: ne_10m_admin_0_countries -> osm_country_point +-- etldoc: osm_country_point -> osm_country_point + WITH important_country_point AS ( SELECT osm.geometry, osm.osm_id, osm.name, COALESCE(NULLIF(osm.name_en, ''), ne.name) AS name_en, ne.scalerank, ne.labelrank FROM ne_10m_admin_0_countries AS ne, osm_country_point AS osm diff --git a/layers/place/merge_state_rank.sql b/layers/place/merge_state_rank.sql index 0e2a8b4..e297ae9 100644 --- a/layers/place/merge_state_rank.sql +++ b/layers/place/merge_state_rank.sql @@ -1,5 +1,8 @@ ALTER TABLE osm_state_point DROP CONSTRAINT IF EXISTS osm_state_point_rank_constraint; +-- etldoc: ne_10m_admin_1_states_provinces_shp -> osm_state_point +-- etldoc: osm_state_point -> osm_state_point + WITH important_state_point AS ( SELECT osm.geometry, osm.osm_id, osm.name, COALESCE(NULLIF(osm.name_en, ''), ne.name) AS name_en, ne.scalerank, ne.labelrank, ne.datarank FROM ne_10m_admin_1_states_provinces_shp AS ne, osm_state_point AS osm diff --git a/layers/place/place.sql b/layers/place/place.sql index af3f404..320dbfa 100644 --- a/layers/place/place.sql +++ b/layers/place/place.sql @@ -1,3 +1,8 @@ + +-- etldoc: layer_country -> layer_place +-- etldoc: layer_state -> layer_place +-- etldoc: layer_city -> layer_place + CREATE OR REPLACE FUNCTION layer_place(bbox geometry, zoom_level int, pixel_width numeric) RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class text, "rank" int) AS $$ SELECT osm_id, geometry, name, name_en, 'country' AS class, "rank" FROM layer_country(bbox, zoom_level) diff --git a/layers/place/state.sql b/layers/place/state.sql index d4ba508..64037f9 100644 --- a/layers/place/state.sql +++ b/layers/place/state.sql @@ -1,3 +1,6 @@ + +-- etldoc: osm_state_point -> layer_state + CREATE OR REPLACE FUNCTION layer_state(bbox geometry, zoom_level int) RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, "rank" int) AS $$ SELECT osm_id, geometry, name, COALESCE(NULLIF(name_en, ''), name) AS name_en, "rank" From b0c762cd2bf9a8083233588455d4792836da1a00 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 10 Nov 2016 03:23:52 +0100 Subject: [PATCH 10/19] simplified etldoc for layer_place --- layers/place/city.sql | 17 +++-------------- layers/place/country.sql | 3 +++ layers/place/place.sql | 3 +++ layers/place/state.sql | 3 +++ 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/layers/place/city.sql b/layers/place/city.sql index 6b3faec..af23866 100644 --- a/layers/place/city.sql +++ b/layers/place/city.sql @@ -1,15 +1,10 @@ -- etldoc: layer_city[shape=record fillcolor=lightpink, style="rounded,filled", --- etldoc: label="layer_city | z2 | z3 | z4 | z5| z6 | z7 | z8 | z9 | z10 | z11 | z12| z13| z14_" ] ; +-- etldoc: label="layer_city | z2-z7 | z8_z14_ " ] ; CREATE OR REPLACE FUNCTION layer_city(bbox geometry, zoom_level int, pixel_width numeric) RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class city_class, "rank" int) AS $$ - -- etldoc: osm_city_point -> layer_city:z2 - -- etldoc: osm_city_point -> layer_city:z3 - -- etldoc: osm_city_point -> layer_city:z4 - -- etldoc: osm_city_point -> layer_city:z5 - -- etldoc: osm_city_point -> layer_city:z6 - -- etldoc: osm_city_point -> layer_city:z7 + -- etldoc: osm_city_point -> layer_city:z2_7 SELECT osm_id, geometry, name, COALESCE(NULLIF(name_en, ''), name) AS name_en, place AS class, "rank" FROM osm_city_point WHERE geometry && bbox @@ -30,13 +25,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class c population DESC NULLS LAST, length(name) ASC )::int AS gridrank - -- etldoc: osm_city_point -> layer_city:z8 - -- etldoc: osm_city_point -> layer_city:z9 - -- etldoc: osm_city_point -> layer_city:z10 - -- etldoc: osm_city_point -> layer_city:z11 - -- etldoc: osm_city_point -> layer_city:z12 - -- etldoc: osm_city_point -> layer_city:z13 - -- etldoc: osm_city_point -> layer_city:z14_ + -- etldoc: osm_city_point -> layer_city:z8_14_ FROM osm_city_point WHERE geometry && bbox AND ((zoom_level = 8 AND place <= 'town'::city_class) diff --git a/layers/place/country.sql b/layers/place/country.sql index 092aed2..3f276fa 100644 --- a/layers/place/country.sql +++ b/layers/place/country.sql @@ -1,4 +1,7 @@ +-- etldoc: layer_country[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_country | z0-z14_ " ] ; + -- etldoc: osm_country_point -> layer_country CREATE OR REPLACE FUNCTION layer_country(bbox geometry, zoom_level int) RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, "rank" int) AS $$ diff --git a/layers/place/place.sql b/layers/place/place.sql index 320dbfa..8e11d21 100644 --- a/layers/place/place.sql +++ b/layers/place/place.sql @@ -1,4 +1,7 @@ +-- etldoc: layer_place[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_place | z0-z14_ " ] ; + -- etldoc: layer_country -> layer_place -- etldoc: layer_state -> layer_place -- etldoc: layer_city -> layer_place diff --git a/layers/place/state.sql b/layers/place/state.sql index 64037f9..de11478 100644 --- a/layers/place/state.sql +++ b/layers/place/state.sql @@ -1,4 +1,7 @@ +-- etldoc: layer_state[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_state | z0-z14_ " ] ; + -- etldoc: osm_state_point -> layer_state CREATE OR REPLACE FUNCTION layer_state(bbox geometry, zoom_level int) From dd2fc0e2dd731f91cf07213c912e58a9b03f0e25 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 10 Nov 2016 13:11:20 +0100 Subject: [PATCH 11/19] add etldoc to layer_poi --- gen_etl_graph_poi.sh | 22 ++++++++++++++++++++++ layers/poi/layer.sql | 5 +++++ layers/poi/mapping.yaml | 1 + 3 files changed, 28 insertions(+) create mode 100755 gen_etl_graph_poi.sh diff --git a/gen_etl_graph_poi.sh b/gen_etl_graph_poi.sh new file mode 100755 index 0000000..f565ba8 --- /dev/null +++ b/gen_etl_graph_poi.sh @@ -0,0 +1,22 @@ + + +ID=poi +DOT=${ID}.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT + +cat ./layers/${ID}/class.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/${ID}/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/layers/poi/layer.sql b/layers/poi/layer.sql index 0a0a603..b82dc41 100644 --- a/layers/poi/layer.sql +++ b/layers/poi/layer.sql @@ -1,3 +1,7 @@ + +-- etldoc: layer_poi[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_poi | z14_" ] ; + CREATE OR REPLACE FUNCTION layer_poi(bbox geometry, zoom_level integer, pixel_width numeric) RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class text, subclass text, "rank" int) AS $$ SELECT osm_id, geometry, name, NULLIF(name_en, ''), poi_class(subclass) AS class, subclass, @@ -5,6 +9,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class t PARTITION BY LabelGrid(geometry, 100 * pixel_width) ORDER BY poi_class_rank(poi_class(subclass)) ASC, length(name) DESC )::int AS "rank" + -- etldoc: osm_poi_point -> layer_poi:z14_ FROM osm_poi_point WHERE geometry && bbox AND zoom_level >= 14 diff --git a/layers/poi/mapping.yaml b/layers/poi/mapping.yaml index 71cb792..37c7193 100644 --- a/layers/poi/mapping.yaml +++ b/layers/poi/mapping.yaml @@ -1,4 +1,5 @@ tables: + # etldoc: imposm3 -> osm_poi_point poi_point: type: point fields: From 36d7fa62277bef4a8c7b2356284b647897a16f33 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 10 Nov 2016 13:24:18 +0100 Subject: [PATCH 12/19] add etldoc to layer_railway --- gen_etl_graph_railway.sh | 22 ++++++++++++++++++++++ layers/railway/mapping.yaml | 1 + layers/railway/railway.sql | 5 +++++ 3 files changed, 28 insertions(+) create mode 100755 gen_etl_graph_railway.sh diff --git a/gen_etl_graph_railway.sh b/gen_etl_graph_railway.sh new file mode 100755 index 0000000..9d82d8b --- /dev/null +++ b/gen_etl_graph_railway.sh @@ -0,0 +1,22 @@ + + +ID=railway +DOT=${ID}.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT + +cat ./layers/${ID}/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/${ID}/railway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/layers/railway/mapping.yaml b/layers/railway/mapping.yaml index 6abb3d2..2d21cf2 100644 --- a/layers/railway/mapping.yaml +++ b/layers/railway/mapping.yaml @@ -1,4 +1,5 @@ tables: + # etldoc: imposm3 -> osm_railway_linestring railway_linestring: type: linestring fields: diff --git a/layers/railway/railway.sql b/layers/railway/railway.sql index 3d1e77c..7902211 100644 --- a/layers/railway/railway.sql +++ b/layers/railway/railway.sql @@ -13,6 +13,9 @@ CREATE OR REPLACE FUNCTION railway_brunnel(is_bridge boolean, is_tunnel boolean) END; $$ LANGUAGE SQL IMMUTABLE; +-- etldoc: layer_railway[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_railway | z13 | z14_" ] ; + CREATE OR REPLACE FUNCTION layer_railway(bbox geometry, zoom_level int) RETURNS TABLE(osm_id bigint, geometry geometry, class text, subclass text, properties railway_properties) AS $$ SELECT osm_id, geometry, @@ -20,9 +23,11 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class text, subclass text, prope railway AS subclass, to_railway_properties(is_bridge, is_tunnel) AS properties FROM ( + -- etldoc: osm_railway_linestring -> layer_railway :z13 SELECT * FROM osm_railway_linestring WHERE zoom_level = 13 AND railway = 'rail' AND service='' UNION ALL + -- etldoc: osm_railway_linestring -> layer_railway :z14_ SELECT * FROM osm_railway_linestring WHERE zoom_level >= 14 ) AS zoom_levels WHERE geometry && bbox From 05992e066cebf043b278836399b5779043e59f87 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 10 Nov 2016 14:08:39 +0100 Subject: [PATCH 13/19] add etldoc to layer_water --- gen_etl_graph_water.sh | 21 ++++++++++++++++++++ layers/water/mapping.yaml | 9 +++++++++ layers/water/water.sql | 42 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100755 gen_etl_graph_water.sh diff --git a/gen_etl_graph_water.sh b/gen_etl_graph_water.sh new file mode 100755 index 0000000..86547c7 --- /dev/null +++ b/gen_etl_graph_water.sh @@ -0,0 +1,21 @@ + + +ID=water +DOT=${ID}.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT + +cat ./layers/${ID}/water.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/layers/water/mapping.yaml b/layers/water/mapping.yaml index e77772a..feee49b 100644 --- a/layers/water/mapping.yaml +++ b/layers/water/mapping.yaml @@ -1,17 +1,26 @@ generalized_tables: + + # etldoc: imposm3 -> osm_water_polygon_gen3 water_polygon_gen3: source: water_polygon sql_filter: area>15000000.0 AND waterway = '' tolerance: 350.0 + + # etldoc: imposm3 -> osm_water_polygon_gen2 water_polygon_gen2: source: water_polygon sql_filter: area>2000000 AND waterway = '' tolerance: 200.0 + + # etldoc: imposm3 -> osm_water_polygon_gen1 water_polygon_gen1: source: water_polygon sql_filter: area>500000 AND waterway = '' tolerance: 100.0 + tables: + + # etldoc: imposm3 -> osm_water_polygon water_polygon: fields: - name: osm_id diff --git a/layers/water/water.sql b/layers/water/water.sql index dab2aa9..e1955f9 100644 --- a/layers/water/water.sql +++ b/layers/water/water.sql @@ -3,101 +3,141 @@ CREATE OR REPLACE FUNCTION water_class(waterway TEXT) RETURNS TEXT AS $$ $$ LANGUAGE SQL IMMUTABLE; + CREATE OR REPLACE VIEW water_z0 AS ( + -- etldoc: ne_110m_ocean -> water_z0 SELECT geom, 'ocean' AS class FROM ne_110m_ocean UNION ALL + -- etldoc: ne_110m_lakes -> water_z0 SELECT geom, 'lake' AS class FROM ne_110m_lakes ); CREATE OR REPLACE VIEW water_z1 AS ( + -- etldoc: ne_110m_ocean -> water_z1 SELECT geom, 'ocean' AS class FROM ne_110m_ocean - UNION ALL + UNION ALL + -- etldoc: ne_110m_lakes -> water_z1 SELECT geom, 'lake' AS class FROM ne_110m_lakes ); CREATE OR REPLACE VIEW water_z2 AS ( + -- etldoc: ne_50m_ocean -> water_z2 SELECT geom, 'ocean' AS class FROM ne_50m_ocean UNION ALL + -- etldoc: ne_110m_lakes -> water_z2 SELECT geom, 'lake' AS class FROM ne_110m_lakes ); CREATE OR REPLACE VIEW water_z4 AS ( + -- etldoc: ne_50m_ocean -> water_z4 SELECT geom, 'ocean' AS class FROM ne_50m_ocean UNION ALL + -- etldoc: ne_50m_lakes -> water_z4 SELECT geom, 'lake' AS class FROM ne_50m_lakes ); CREATE OR REPLACE VIEW water_z5 AS ( + -- etldoc: ne_10m_ocean -> water_z5 SELECT geom, 'ocean' AS class FROM ne_10m_ocean UNION ALL + -- etldoc: ne_10m_lakes -> water_z5 SELECT geom, 'lake' AS class FROM ne_10m_lakes ); CREATE OR REPLACE VIEW water_z6 AS ( + -- etldoc: ne_10m_ocean -> water_z6 SELECT geom, 'ocean' AS class FROM ne_10m_ocean UNION ALL + -- etldoc: ne_10m_lakes -> water_z6 SELECT geom, 'lake' AS class FROM ne_10m_lakes ); CREATE OR REPLACE VIEW water_z7 AS ( + -- etldoc: ne_10m_ocean -> water_z7 SELECT geom, 'ocean' AS class FROM ne_10m_ocean UNION ALL + -- etldoc: osm_water_polygon_gen3 -> water_z7 SELECT geometry AS geom, 'lake' AS class FROM osm_water_polygon_gen3 ); CREATE OR REPLACE VIEW water_z8 AS ( + -- etldoc: ne_10m_ocean -> water_z8 SELECT geom, 'ocean' AS class FROM ne_10m_ocean UNION ALL + -- etldoc: osm_water_polygon_gen2 -> water_z8 SELECT geometry AS geom, 'lake' AS class FROM osm_water_polygon_gen2 ); CREATE OR REPLACE VIEW water_z9 AS ( + -- etldoc: osm_water_polygon_gen1 -> water_z9 SELECT geometry AS geom, 'lake'::text AS class FROM osm_water_polygon_gen1 ); CREATE OR REPLACE VIEW water_z11 AS ( + -- etldoc: osm_water_polygon -> water_z11 SELECT geometry AS geom, water_class(waterway) AS class FROM osm_water_polygon WHERE area > 40000 ); CREATE OR REPLACE VIEW water_z12 AS ( + -- etldoc: osm_water_polygon -> water_z12 SELECT geometry AS geom, water_class(waterway) AS class FROM osm_water_polygon WHERE area > 10000 ); CREATE OR REPLACE VIEW water_z13 AS ( + -- etldoc: osm_water_polygon -> water_z13 SELECT geometry AS geom, water_class(waterway) AS class FROM osm_water_polygon WHERE area > 5000 ); CREATE OR REPLACE VIEW water_z14 AS ( + -- etldoc: osm_water_polygon -> water_z14 SELECT geometry AS geom, water_class(waterway) AS class FROM osm_water_polygon ); +-- etldoc: layer_water [shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_water | z0|z1|z2|z3 | z4|z5|z6|z7| z8 | z9 | z10 | z11 | z12| z13| z14_" ] ; + CREATE OR REPLACE FUNCTION layer_water (bbox geometry, zoom_level int) RETURNS TABLE(geom geometry, class text) AS $$ SELECT geom, class::text FROM ( + -- etldoc: water_z0 -> layer_water:z0 SELECT * FROM water_z0 WHERE zoom_level = 0 UNION ALL + -- etldoc: water_z1 -> layer_water:z1 SELECT * FROM water_z1 WHERE zoom_level = 1 UNION ALL + -- etldoc: water_z2 -> layer_water:z2 + -- etldoc: water_z2 -> layer_water:z3 SELECT * FROM water_z2 WHERE zoom_level BETWEEN 2 AND 3 UNION ALL + -- etldoc: water_z4 -> layer_water:z4 SELECT * FROM water_z4 WHERE zoom_level = 4 UNION ALL + -- etldoc: water_z5 -> layer_water:z5 SELECT * FROM water_z5 WHERE zoom_level = 5 UNION ALL + -- etldoc: water_z6 -> layer_water:z6 SELECT * FROM water_z6 WHERE zoom_level = 6 UNION ALL + -- etldoc: water_z7 -> layer_water:z7 SELECT * FROM water_z7 WHERE zoom_level = 7 UNION ALL + -- etldoc: water_z8 -> layer_water:z8 SELECT * FROM water_z8 WHERE zoom_level = 8 UNION ALL + -- etldoc: water_z9 -> layer_water:z9 + -- etldoc: water_z9 -> layer_water:z10 SELECT * FROM water_z9 WHERE zoom_level BETWEEN 9 AND 10 UNION ALL + -- etldoc: water_z11 -> layer_water:z11 SELECT * FROM water_z11 WHERE zoom_level = 11 UNION ALL + -- etldoc: water_z12 -> layer_water:z2 SELECT * FROM water_z12 WHERE zoom_level = 12 UNION ALL + -- etldoc: water_z13 -> layer_water:z13 SELECT * FROM water_z13 WHERE zoom_level = 13 UNION ALL + -- etldoc: water_z14 -> layer_water:z14_ SELECT * FROM water_z14 WHERE zoom_level >= 14 ) AS zoom_levels WHERE geom && bbox; From 906c61418710b23f8357bdc27039ec9e85fcc209 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 10 Nov 2016 14:28:49 +0100 Subject: [PATCH 14/19] add etldoc to layer_water_name --- gen_etl_graph_water_name.sh | 20 ++++++++++++++++++++ layers/water_name/layer.sql | 6 ++++++ layers/water_name/merge_lakelines.sql | 2 ++ 3 files changed, 28 insertions(+) create mode 100755 gen_etl_graph_water_name.sh diff --git a/gen_etl_graph_water_name.sh b/gen_etl_graph_water_name.sh new file mode 100755 index 0000000..4245c1a --- /dev/null +++ b/gen_etl_graph_water_name.sh @@ -0,0 +1,20 @@ + + +ID=water_name +DOT=${ID}.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/${ID}/merge_lakelines.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/${ID}/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/layers/water_name/layer.sql b/layers/water_name/layer.sql index 7236961..8c9e866 100644 --- a/layers/water_name/layer.sql +++ b/layers/water_name/layer.sql @@ -1,5 +1,11 @@ + +-- etldoc: layer_water_name[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_water_name | z9_13 | z14_" ] ; + CREATE OR REPLACE FUNCTION layer_water_name(bbox geometry, zoom_level integer) RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class text) AS $$ + -- etldoc: osm_water_lakeline -> layer_water_name:z9_13 + -- etldoc: osm_water_lakeline -> layer_water_name:z14_ SELECT osm_id, geometry, name, name_en, 'lake'::text AS class FROM osm_water_lakeline WHERE geometry && bbox diff --git a/layers/water_name/merge_lakelines.sql b/layers/water_name/merge_lakelines.sql index 125f802..b74b709 100644 --- a/layers/water_name/merge_lakelines.sql +++ b/layers/water_name/merge_lakelines.sql @@ -1,3 +1,5 @@ + +-- etldoc: osm_water_polygon -> osm_water_lakeline CREATE TABLE IF NOT EXISTS osm_water_lakeline AS ( SELECT wp.osm_id, ll.wkb_geometry AS geometry, From 2a0ef3a95388d9877a0566650f97a376b1271f18 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 10 Nov 2016 15:00:13 +0100 Subject: [PATCH 15/19] add etldoc to layer_waterway --- gen_etl_graph_waterway.sh | 21 +++++++++++++++++++++ layers/waterway/mapping.yaml | 3 +++ layers/waterway/waterway.sql | 23 +++++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100755 gen_etl_graph_waterway.sh diff --git a/gen_etl_graph_waterway.sh b/gen_etl_graph_waterway.sh new file mode 100755 index 0000000..4f7b480 --- /dev/null +++ b/gen_etl_graph_waterway.sh @@ -0,0 +1,21 @@ + + +ID=waterway +DOT=${ID}.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT + +cat ./layers/${ID}//waterway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/layers/waterway/mapping.yaml b/layers/waterway/mapping.yaml index c4e4ba0..5238b98 100644 --- a/layers/waterway/mapping.yaml +++ b/layers/waterway/mapping.yaml @@ -1,13 +1,16 @@ generalized_tables: + # etldoc: imposm3 -> osm_waterway_linestring_gen0 waterway_linestring_gen0: source: waterway_linestring_gen1 sql_filter: waterway IN ('river') AND ST_Length(geometry) > 10000 tolerance: 200.0 + # etldoc: imposm3 -> osm_waterway_linestring_gen1 waterway_linestring_gen1: source: waterway_linestring sql_filter: waterway IN ('river') AND ST_Length(geometry) > 5000 tolerance: 100.0 tables: + # etldoc: imposm3 -> osm_waterway_linestring waterway_linestring: type: linestring fields: diff --git a/layers/waterway/waterway.sql b/layers/waterway/waterway.sql index 7201073..29cb62c 100644 --- a/layers/waterway/waterway.sql +++ b/layers/waterway/waterway.sql @@ -1,66 +1,89 @@ + +-- etldoc: ne_110m_rivers_lake_centerlines -> waterway_z3 CREATE OR REPLACE VIEW waterway_z3 AS ( SELECT geom AS geometry, 'river' AS class FROM ne_110m_rivers_lake_centerlines WHERE featurecla = 'River' ); +-- etldoc: ne_50m_rivers_lake_centerlines -> waterway_z4 CREATE OR REPLACE VIEW waterway_z4 AS ( SELECT geom AS geometry, 'river' AS class FROM ne_50m_rivers_lake_centerlines WHERE featurecla = 'River' ); +-- etldoc: ne_10m_rivers_lake_centerlines -> waterway_z6 CREATE OR REPLACE VIEW waterway_z6 AS ( SELECT geom AS geometry, 'river' AS class FROM ne_10m_rivers_lake_centerlines WHERE featurecla = 'River' ); +-- etldoc: osm_waterway_linestring -> waterway_z8 CREATE OR REPLACE VIEW waterway_z8 AS ( SELECT geometry, waterway AS class FROM osm_waterway_linestring WHERE waterway IN ('river') AND ST_Length(geometry) > 10000 ); +-- etldoc: osm_waterway_linestring -> waterway_z9 CREATE OR REPLACE VIEW waterway_z9 AS ( SELECT geometry, waterway AS class FROM osm_waterway_linestring WHERE waterway IN ('river') AND ST_Length(geometry) > 5000 ); +-- etldoc: osm_waterway_linestring -> waterway_z11 CREATE OR REPLACE VIEW waterway_z11 AS ( SELECT geometry, waterway AS class FROM osm_waterway_linestring WHERE waterway IN ('river') AND ST_Length(geometry) > 5000 ); +-- etldoc: osm_waterway_linestring -> waterway_z12 CREATE OR REPLACE VIEW waterway_z12 AS ( SELECT geometry, waterway AS class FROM osm_waterway_linestring WHERE waterway IN ('river', 'canal') OR (waterway = 'stream' AND ST_Length(geometry) > 1000) ); +-- etldoc: osm_waterway_linestring -> waterway_z13 CREATE OR REPLACE VIEW waterway_z13 AS ( SELECT geometry, waterway AS class FROM osm_waterway_linestring WHERE waterway IN ('river', 'canal', 'stream', 'drain', 'ditch') AND ST_Length(geometry) > 300 ); +-- etldoc: osm_waterway_linestring -> waterway_z14 CREATE OR REPLACE VIEW waterway_z14 AS ( SELECT geometry, waterway AS class FROM osm_waterway_linestring ); + +-- etldoc: layer_waterway[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label="layer_waterway | z3 | z4_5 | z6_7 | z8 | z9_10 | z11 | z12| z13| z14_" ] ; + CREATE OR REPLACE FUNCTION layer_waterway(bbox geometry, zoom_level int) RETURNS TABLE(geometry geometry, class text) AS $$ SELECT geometry, class FROM ( + -- etldoc: waterway_z3 -> layer_waterway:z3 SELECT * FROM waterway_z3 WHERE zoom_level = 3 UNION ALL + -- etldoc: waterway_z4 -> layer_waterway:z4_5 SELECT * FROM waterway_z4 WHERE zoom_level BETWEEN 4 AND 5 UNION ALL + -- etldoc: waterway_z6 -> layer_waterway:z6_7 SELECT * FROM waterway_z6 WHERE zoom_level BETWEEN 6 AND 7 UNION ALL + -- etldoc: waterway_z8 -> layer_waterway:z8 SELECT * FROM waterway_z8 WHERE zoom_level = 8 UNION ALL + -- etldoc: waterway_z9 -> layer_waterway:z9_10 SELECT * FROM waterway_z9 WHERE zoom_level BETWEEN 9 AND 10 UNION ALL + -- etldoc: waterway_z11 -> layer_waterway:z11 SELECT * FROM waterway_z11 WHERE zoom_level = 11 UNION ALL + -- etldoc: waterway_z12 -> layer_waterway:z12 SELECT * FROM waterway_z12 WHERE zoom_level = 12 UNION ALL + -- etldoc: waterway_z13 -> layer_waterway:z13 SELECT * FROM waterway_z13 WHERE zoom_level = 13 UNION ALL + -- etldoc: waterway_z14 -> layer_waterway:z14_ SELECT * FROM waterway_z14 WHERE zoom_level >= 14 ) AS zoom_levels WHERE geometry && bbox; From 34afba471f55a745551c2876c0f89e1cf9bd5910 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Thu, 10 Nov 2016 21:08:43 +0100 Subject: [PATCH 16/19] rename gen_etl_graph* codes to sh/* dir --- gen_etl_graph_boundary.sh => sh/gen_etl_graph_boundary.sh | 0 gen_etl_graph_building.sh => sh/gen_etl_graph_building.sh | 0 gen_etl_graph_highway.sh => sh/gen_etl_graph_highway.sh | 0 gen_etl_graph_highway_name.sh => sh/gen_etl_graph_highway_name.sh | 0 gen_etl_graph_housenumber.sh => sh/gen_etl_graph_housenumber.sh | 0 gen_etl_graph_landcover.sh => sh/gen_etl_graph_landcover.sh | 0 gen_etl_graph_landuse.sh => sh/gen_etl_graph_landuse.sh | 0 gen_etl_graph_place.sh => sh/gen_etl_graph_place.sh | 0 gen_etl_graph_poi.sh => sh/gen_etl_graph_poi.sh | 0 gen_etl_graph_railway.sh => sh/gen_etl_graph_railway.sh | 0 gen_etl_graph_water.sh => sh/gen_etl_graph_water.sh | 0 gen_etl_graph_water_name.sh => sh/gen_etl_graph_water_name.sh | 0 gen_etl_graph_waterway.sh => sh/gen_etl_graph_waterway.sh | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename gen_etl_graph_boundary.sh => sh/gen_etl_graph_boundary.sh (100%) rename gen_etl_graph_building.sh => sh/gen_etl_graph_building.sh (100%) rename gen_etl_graph_highway.sh => sh/gen_etl_graph_highway.sh (100%) rename gen_etl_graph_highway_name.sh => sh/gen_etl_graph_highway_name.sh (100%) rename gen_etl_graph_housenumber.sh => sh/gen_etl_graph_housenumber.sh (100%) rename gen_etl_graph_landcover.sh => sh/gen_etl_graph_landcover.sh (100%) rename gen_etl_graph_landuse.sh => sh/gen_etl_graph_landuse.sh (100%) rename gen_etl_graph_place.sh => sh/gen_etl_graph_place.sh (100%) rename gen_etl_graph_poi.sh => sh/gen_etl_graph_poi.sh (100%) rename gen_etl_graph_railway.sh => sh/gen_etl_graph_railway.sh (100%) rename gen_etl_graph_water.sh => sh/gen_etl_graph_water.sh (100%) rename gen_etl_graph_water_name.sh => sh/gen_etl_graph_water_name.sh (100%) rename gen_etl_graph_waterway.sh => sh/gen_etl_graph_waterway.sh (100%) diff --git a/gen_etl_graph_boundary.sh b/sh/gen_etl_graph_boundary.sh similarity index 100% rename from gen_etl_graph_boundary.sh rename to sh/gen_etl_graph_boundary.sh diff --git a/gen_etl_graph_building.sh b/sh/gen_etl_graph_building.sh similarity index 100% rename from gen_etl_graph_building.sh rename to sh/gen_etl_graph_building.sh diff --git a/gen_etl_graph_highway.sh b/sh/gen_etl_graph_highway.sh similarity index 100% rename from gen_etl_graph_highway.sh rename to sh/gen_etl_graph_highway.sh diff --git a/gen_etl_graph_highway_name.sh b/sh/gen_etl_graph_highway_name.sh similarity index 100% rename from gen_etl_graph_highway_name.sh rename to sh/gen_etl_graph_highway_name.sh diff --git a/gen_etl_graph_housenumber.sh b/sh/gen_etl_graph_housenumber.sh similarity index 100% rename from gen_etl_graph_housenumber.sh rename to sh/gen_etl_graph_housenumber.sh diff --git a/gen_etl_graph_landcover.sh b/sh/gen_etl_graph_landcover.sh similarity index 100% rename from gen_etl_graph_landcover.sh rename to sh/gen_etl_graph_landcover.sh diff --git a/gen_etl_graph_landuse.sh b/sh/gen_etl_graph_landuse.sh similarity index 100% rename from gen_etl_graph_landuse.sh rename to sh/gen_etl_graph_landuse.sh diff --git a/gen_etl_graph_place.sh b/sh/gen_etl_graph_place.sh similarity index 100% rename from gen_etl_graph_place.sh rename to sh/gen_etl_graph_place.sh diff --git a/gen_etl_graph_poi.sh b/sh/gen_etl_graph_poi.sh similarity index 100% rename from gen_etl_graph_poi.sh rename to sh/gen_etl_graph_poi.sh diff --git a/gen_etl_graph_railway.sh b/sh/gen_etl_graph_railway.sh similarity index 100% rename from gen_etl_graph_railway.sh rename to sh/gen_etl_graph_railway.sh diff --git a/gen_etl_graph_water.sh b/sh/gen_etl_graph_water.sh similarity index 100% rename from gen_etl_graph_water.sh rename to sh/gen_etl_graph_water.sh diff --git a/gen_etl_graph_water_name.sh b/sh/gen_etl_graph_water_name.sh similarity index 100% rename from gen_etl_graph_water_name.sh rename to sh/gen_etl_graph_water_name.sh diff --git a/gen_etl_graph_waterway.sh b/sh/gen_etl_graph_waterway.sh similarity index 100% rename from gen_etl_graph_waterway.sh rename to sh/gen_etl_graph_waterway.sh From d8cab7ad04cfb41c09dc6a58132dfe59c90d2af4 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Fri, 11 Nov 2016 17:26:53 +0100 Subject: [PATCH 17/19] refactor ETL graph --- sh/gen_doc.sh | 23 ++++++++++++++++ sh/gen_etl_graph.sh | 45 ++++++++++++++++++++++++++++++++ sh/gen_etl_graph_boundary.sh | 21 --------------- sh/gen_etl_graph_building.sh | 21 --------------- sh/gen_etl_graph_highway.sh | 21 --------------- sh/gen_etl_graph_highway_name.sh | 19 -------------- sh/gen_etl_graph_housenumber.sh | 20 -------------- sh/gen_etl_graph_landcover.sh | 19 -------------- sh/gen_etl_graph_landuse.sh | 19 -------------- sh/gen_etl_graph_place.sh | 27 ------------------- sh/gen_etl_graph_poi.sh | 22 ---------------- sh/gen_etl_graph_railway.sh | 22 ---------------- sh/gen_etl_graph_water.sh | 21 --------------- sh/gen_etl_graph_water_name.sh | 20 -------------- sh/gen_etl_graph_waterway.sh | 21 --------------- 15 files changed, 68 insertions(+), 273 deletions(-) create mode 100755 sh/gen_doc.sh create mode 100755 sh/gen_etl_graph.sh delete mode 100755 sh/gen_etl_graph_boundary.sh delete mode 100755 sh/gen_etl_graph_building.sh delete mode 100755 sh/gen_etl_graph_highway.sh delete mode 100755 sh/gen_etl_graph_highway_name.sh delete mode 100755 sh/gen_etl_graph_housenumber.sh delete mode 100755 sh/gen_etl_graph_landcover.sh delete mode 100755 sh/gen_etl_graph_landuse.sh delete mode 100755 sh/gen_etl_graph_place.sh delete mode 100755 sh/gen_etl_graph_poi.sh delete mode 100755 sh/gen_etl_graph_railway.sh delete mode 100755 sh/gen_etl_graph_water.sh delete mode 100755 sh/gen_etl_graph_water_name.sh delete mode 100755 sh/gen_etl_graph_waterway.sh diff --git a/sh/gen_doc.sh b/sh/gen_doc.sh new file mode 100755 index 0000000..a8dae5d --- /dev/null +++ b/sh/gen_doc.sh @@ -0,0 +1,23 @@ + + +# Work in progress ... +# +# generate ETL graph from all layers +# start from the root : +# ./sh/gen_doc.sh +# +# outut -> ./doc/ + + +mkdir -p ./doc/ +rm -f ./doc/* +for f in ./layers/* +do + echo "Processing : $f" + layer_id=$(echo "$f" | rev | cut -d"/" -f1 | rev ) + echo "layer_id = $layer_id" + ./sh/gen_etl_graph.sh $layer_id + +done + + diff --git a/sh/gen_etl_graph.sh b/sh/gen_etl_graph.sh new file mode 100755 index 0000000..572fe83 --- /dev/null +++ b/sh/gen_etl_graph.sh @@ -0,0 +1,45 @@ + + + +# Work in progress ... +# +# generate ETL graph from a single layer +# start from the root +# parameter1: "layer name" +# ./sh/gen_etl_graph.sh waterway +# +# outut -> ./doc/ + +ID=$1 +## ID=waterway + +mkdir -p ./doc +layer_dir=./layers/${ID}/ +DOT=./doc/dot_${ID}.dot + + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +if [ -f ${layer_dir}/mapping.yaml ]; then + echo "processing imposm3 mapping file ${layer_dir}/mapping.yaml " + cat ${layer_dir}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT +fi + + +for f in ${layer_dir}*.sql +do + echo "Processing : $f" + cat $f | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +done + +echo " +} +" >> $DOT + +##cat $DOT + +dot -Tpng $DOT > ./doc/etl_layer_${ID}.png +dot -Tsvg $DOT > ./doc/etl_layer_${ID}.svg diff --git a/sh/gen_etl_graph_boundary.sh b/sh/gen_etl_graph_boundary.sh deleted file mode 100755 index 1430eee..0000000 --- a/sh/gen_etl_graph_boundary.sh +++ /dev/null @@ -1,21 +0,0 @@ - - - - -DOT=boundary.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/boundary/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -cat ./layers/boundary/boundary.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_boundary_etl.png diff --git a/sh/gen_etl_graph_building.sh b/sh/gen_etl_graph_building.sh deleted file mode 100755 index a74b0e6..0000000 --- a/sh/gen_etl_graph_building.sh +++ /dev/null @@ -1,21 +0,0 @@ - - - - -DOT=building.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/building/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -cat ./layers/building/building.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_building_etl.png diff --git a/sh/gen_etl_graph_highway.sh b/sh/gen_etl_graph_highway.sh deleted file mode 100755 index ddd3520..0000000 --- a/sh/gen_etl_graph_highway.sh +++ /dev/null @@ -1,21 +0,0 @@ - - -DOT=highway.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/highway/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -cat ./layers/highway/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/highway/ne_global_roads.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/highway/highway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_highway_etl.png diff --git a/sh/gen_etl_graph_highway_name.sh b/sh/gen_etl_graph_highway_name.sh deleted file mode 100755 index 39afd79..0000000 --- a/sh/gen_etl_graph_highway_name.sh +++ /dev/null @@ -1,19 +0,0 @@ - - -DOT=highway_name.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/highway_name/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/highway_name/merge_highways.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_highway_names_etl.png diff --git a/sh/gen_etl_graph_housenumber.sh b/sh/gen_etl_graph_housenumber.sh deleted file mode 100755 index 37c12a9..0000000 --- a/sh/gen_etl_graph_housenumber.sh +++ /dev/null @@ -1,20 +0,0 @@ - - -DOT=housenumber.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/housenumber/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -cat ./layers/housenumber/housenumber_centroid.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/housenumber/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_housenumber_etl.png diff --git a/sh/gen_etl_graph_landcover.sh b/sh/gen_etl_graph_landcover.sh deleted file mode 100755 index 008b0ce..0000000 --- a/sh/gen_etl_graph_landcover.sh +++ /dev/null @@ -1,19 +0,0 @@ - - -DOT=landcover.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/landcover/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -cat ./layers/landcover/landcover.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_landcover_etl.png diff --git a/sh/gen_etl_graph_landuse.sh b/sh/gen_etl_graph_landuse.sh deleted file mode 100755 index 25ff4f9..0000000 --- a/sh/gen_etl_graph_landuse.sh +++ /dev/null @@ -1,19 +0,0 @@ - - -DOT=landuse.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/landuse/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -cat ./layers/landuse/landuse.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_landuse_etl.png diff --git a/sh/gen_etl_graph_place.sh b/sh/gen_etl_graph_place.sh deleted file mode 100755 index 5056cfa..0000000 --- a/sh/gen_etl_graph_place.sh +++ /dev/null @@ -1,27 +0,0 @@ - - -DOT=place.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/place/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT - -cat ./layers/place/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/city.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/country.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/state.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/merge_country_rank.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/merge_city_rank.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/merge_state_rank.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/place.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_place_etl.png diff --git a/sh/gen_etl_graph_poi.sh b/sh/gen_etl_graph_poi.sh deleted file mode 100755 index f565ba8..0000000 --- a/sh/gen_etl_graph_poi.sh +++ /dev/null @@ -1,22 +0,0 @@ - - -ID=poi -DOT=${ID}.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT - -cat ./layers/${ID}/class.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/${ID}/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/sh/gen_etl_graph_railway.sh b/sh/gen_etl_graph_railway.sh deleted file mode 100755 index 9d82d8b..0000000 --- a/sh/gen_etl_graph_railway.sh +++ /dev/null @@ -1,22 +0,0 @@ - - -ID=railway -DOT=${ID}.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT - -cat ./layers/${ID}/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/${ID}/railway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/sh/gen_etl_graph_water.sh b/sh/gen_etl_graph_water.sh deleted file mode 100755 index 86547c7..0000000 --- a/sh/gen_etl_graph_water.sh +++ /dev/null @@ -1,21 +0,0 @@ - - -ID=water -DOT=${ID}.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT - -cat ./layers/${ID}/water.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/sh/gen_etl_graph_water_name.sh b/sh/gen_etl_graph_water_name.sh deleted file mode 100755 index 4245c1a..0000000 --- a/sh/gen_etl_graph_water_name.sh +++ /dev/null @@ -1,20 +0,0 @@ - - -ID=water_name -DOT=${ID}.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/${ID}/merge_lakelines.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/${ID}/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/sh/gen_etl_graph_waterway.sh b/sh/gen_etl_graph_waterway.sh deleted file mode 100755 index 4f7b480..0000000 --- a/sh/gen_etl_graph_waterway.sh +++ /dev/null @@ -1,21 +0,0 @@ - - -ID=waterway -DOT=${ID}.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT - -cat ./layers/${ID}//waterway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_${ID}_etl.png From c1c811a11540e750d2b74ef2fc693552c6762417 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Fri, 11 Nov 2016 17:28:37 +0100 Subject: [PATCH 18/19] add travis and quickstart.sh file for basic testing --- .travis.yml | 29 ++++++++++++++++++++ docker-compose-test-override.yml | 8 ++++++ quickstart.sh | 46 ++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 .travis.yml create mode 100644 docker-compose-test-override.yml create mode 100755 quickstart.sh diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..8bb8d4d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,29 @@ +sudo: required + +dist: precise + +language: bash + +services: + - docker + +before_install: + - sudo sh -c 'echo "deb https://apt.dockerproject.org/repo ubuntu-precise main" > /etc/apt/sources.list.d/docker.list' + - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D + - sudo apt-get clean + - sudo apt-get update + - sudo apt-key update + - sudo apt-get -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine=1.12.0-0~precise + - sudo rm -f /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/1.8.0/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin + - sudo service docker restart + - docker-compose -v + - docker -v + +script: + - sudo service docker restart + - docker-compose config + - sudo ./quickstart.sh + diff --git a/docker-compose-test-override.yml b/docker-compose-test-override.yml new file mode 100644 index 0000000..0ae9e4a --- /dev/null +++ b/docker-compose-test-override.yml @@ -0,0 +1,8 @@ +version: "2" + +services: + generate-vectortiles: + environment: + BBOX: "8.25,46.97,9.58,47.52" + MIN_ZOOM: "0" + MAX_ZOOM: "7" diff --git a/quickstart.sh b/quickstart.sh new file mode 100755 index 0000000..ffcfa8a --- /dev/null +++ b/quickstart.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + +#Check installed versions +docker --version +docker-compose --version + +#Remove +docker-compose down +docker-compose rm -fv +echo "Remove old volume" +docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true + +echo "Make directories " +mkdir -p build +mkdir -p data + +testdata=zurich_switzerland.osm.pbf +if [ ! -f ./data/${testdata} ]; then + echo "Download $testdata " + rm -f ./data/* + wget https://s3.amazonaws.com/metro-extracts.mapzen.com/zurich_switzerland.osm.pbf -P ./data +fi + + +docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make +docker-compose up -d postgres +sleep 30 + +docker-compose run --rm import-water +docker-compose run --rm import-natural-earth +docker-compose run --rm import-lakelines +docker-compose run --rm import-osm +docker-compose run --rm import-sql + +docker-compose -f docker-compose.yml -f docker-compose-test-override.yml run --rm generate-vectortiles + +docker-compose stop postgres +echo "The vectortiles created from $testdata " +ls ./data/*.mbtiles -la +echo "Hello ... start experimenting - see docs ! " + + + From 5c1e9a2f58f6fdc0c8780d1818d741003c47ce99 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Fri, 11 Nov 2016 20:21:43 +0100 Subject: [PATCH 19/19] remove sh/gen_*.sh - later I will move this to a separate tool --- sh/gen_doc.sh | 23 ----------------------- sh/gen_etl_graph.sh | 45 --------------------------------------------- 2 files changed, 68 deletions(-) delete mode 100755 sh/gen_doc.sh delete mode 100755 sh/gen_etl_graph.sh diff --git a/sh/gen_doc.sh b/sh/gen_doc.sh deleted file mode 100755 index a8dae5d..0000000 --- a/sh/gen_doc.sh +++ /dev/null @@ -1,23 +0,0 @@ - - -# Work in progress ... -# -# generate ETL graph from all layers -# start from the root : -# ./sh/gen_doc.sh -# -# outut -> ./doc/ - - -mkdir -p ./doc/ -rm -f ./doc/* -for f in ./layers/* -do - echo "Processing : $f" - layer_id=$(echo "$f" | rev | cut -d"/" -f1 | rev ) - echo "layer_id = $layer_id" - ./sh/gen_etl_graph.sh $layer_id - -done - - diff --git a/sh/gen_etl_graph.sh b/sh/gen_etl_graph.sh deleted file mode 100755 index 572fe83..0000000 --- a/sh/gen_etl_graph.sh +++ /dev/null @@ -1,45 +0,0 @@ - - - -# Work in progress ... -# -# generate ETL graph from a single layer -# start from the root -# parameter1: "layer name" -# ./sh/gen_etl_graph.sh waterway -# -# outut -> ./doc/ - -ID=$1 -## ID=waterway - -mkdir -p ./doc -layer_dir=./layers/${ID}/ -DOT=./doc/dot_${ID}.dot - - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -if [ -f ${layer_dir}/mapping.yaml ]; then - echo "processing imposm3 mapping file ${layer_dir}/mapping.yaml " - cat ${layer_dir}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -fi - - -for f in ${layer_dir}*.sql -do - echo "Processing : $f" - cat $f | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -done - -echo " -} -" >> $DOT - -##cat $DOT - -dot -Tpng $DOT > ./doc/etl_layer_${ID}.png -dot -Tsvg $DOT > ./doc/etl_layer_${ID}.svg