Merge pull request #17 from openmaptiles/datadoc

Documenting :  graphviz based  ETL graph

.travis.yml

@@ -0,0 +1,29 @@
+sudo: required
+
+dist: precise
+
+language: bash
+
+services:
+  - docker
+  
+before_install:
+  - sudo sh -c 'echo "deb https://apt.dockerproject.org/repo ubuntu-precise main" > /etc/apt/sources.list.d/docker.list'
+  - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+  - sudo apt-get clean
+  - sudo apt-get update
+  - sudo apt-key update
+  - sudo apt-get -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine=1.12.0-0~precise
+  - sudo rm -f /usr/local/bin/docker-compose
+  - curl -L https://github.com/docker/compose/releases/download/1.8.0/docker-compose-`uname -s`-`uname -m` > docker-compose
+  - chmod +x docker-compose
+  - sudo mv docker-compose /usr/local/bin
+  - sudo service docker restart
+  - docker-compose -v
+  - docker -v
+ 
+script:
+  - sudo service docker restart
+  - docker-compose config
+  - sudo ./quickstart.sh
+  

docker-compose-test-override.yml

@@ -0,0 +1,8 @@
+version: "2"
+
+services:
+  generate-vectortiles:
+    environment:
+      BBOX: "8.25,46.97,9.58,47.52"
+      MIN_ZOOM: "0"
+      MAX_ZOOM: "7"

layers/boundary/boundary.sql

@@ -1,8 +1,15 @@
+
+
+-- etldoc: ne_110m_admin_0_boundary_lines_land  -> boundary_z0
+
 CREATE OR REPLACE VIEW boundary_z0 AS (
     SELECT geom, 2 AS admin_level
     FROM ne_110m_admin_0_boundary_lines_land
 );
 
+-- etldoc: ne_50m_admin_0_boundary_lines_land  -> boundary_z1
+-- etldoc: ne_50m_admin_1_states_provinces_lines -> boundary_z1
+
 CREATE OR REPLACE VIEW boundary_z1 AS (
     SELECT geom, 2 AS admin_level
     FROM ne_50m_admin_0_boundary_lines_land
@@ -12,6 +19,10 @@ CREATE OR REPLACE VIEW boundary_z1 AS (
     WHERE scalerank <= 2
 );
 
+
+-- etldoc: ne_50m_admin_0_boundary_lines_land -> boundary_z3
+-- etldoc: ne_50m_admin_1_states_provinces_lines -> boundary_z3
+
 CREATE OR REPLACE VIEW boundary_z3 AS (
     SELECT geom, 2 AS admin_level
     FROM ne_50m_admin_0_boundary_lines_land
@@ -20,6 +31,10 @@ CREATE OR REPLACE VIEW boundary_z3 AS (
     FROM ne_50m_admin_1_states_provinces_lines
 );
 
+
+-- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z4
+-- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z4
+
 CREATE OR REPLACE VIEW boundary_z4 AS (
     SELECT geom, 2 AS admin_level
     FROM ne_10m_admin_0_boundary_lines_land
@@ -29,6 +44,9 @@ CREATE OR REPLACE VIEW boundary_z4 AS (
     WHERE scalerank <= 3 AND featurecla = 'Adm-1 boundary'
 );
 
+-- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z5
+-- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z5
+
 CREATE OR REPLACE VIEW boundary_z5 AS (
     SELECT geom, 2 AS admin_level
     FROM ne_10m_admin_0_boundary_lines_land
@@ -38,6 +56,8 @@ CREATE OR REPLACE VIEW boundary_z5 AS (
     WHERE scalerank <= 7 AND featurecla = 'Adm-1 boundary'
 );
 
+-- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z7
+-- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z7
 CREATE OR REPLACE VIEW boundary_z7 AS (
     SELECT geom, 2 AS admin_level
     FROM ne_10m_admin_0_boundary_lines_land
@@ -48,60 +68,80 @@ CREATE OR REPLACE VIEW boundary_z7 AS (
 
 );
 
+-- etldoc: osm_boundary_linestring_gen5 -> boundary_z8
 CREATE OR REPLACE VIEW boundary_z8 AS (
     SELECT geometry AS geom, admin_level
     FROM osm_boundary_linestring_gen5
     WHERE admin_level <= 4 AND ST_Length(geometry) > 1000
 );
 
+-- etldoc: osm_boundary_linestring_gen4 -> boundary_z9
 CREATE OR REPLACE VIEW boundary_z9 AS (
     SELECT geometry AS geom, admin_level
     FROM osm_boundary_linestring_gen4
     WHERE admin_level <= 6
 );
 
+-- etldoc: osm_boundary_linestring_gen3 -> boundary_z10
 CREATE OR REPLACE VIEW boundary_z10 AS (
     SELECT geometry AS geom, admin_level
     FROM osm_boundary_linestring_gen3
     WHERE admin_level <= 6
 );
 
+-- etldoc: osm_boundary_linestring_gen2 -> boundary_z11
 CREATE OR REPLACE VIEW boundary_z11 AS (
     SELECT geometry AS geom, admin_level
     FROM osm_boundary_linestring_gen2
     WHERE admin_level <= 8
 );
 
+-- etldoc: osm_boundary_linestring_gen1 -> boundary_z12
 CREATE OR REPLACE VIEW boundary_z12 AS (
     SELECT geometry AS geom, admin_level
     FROM osm_boundary_linestring_gen1
 );
 
+-- etldoc: layer_boundary[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="<sql> layer_boundary |<z0> z0 |<z1_2> z1_2 | <z3> z3 | <z4> z4 |<z5_6> z5_6 | <z7> z7 | <z8> z8 | <z9> z9    |<z10> z10 |<z11> z11 |<z12> z12|<z13_> z13_ " ] 
+
 CREATE OR REPLACE FUNCTION layer_boundary (bbox geometry, zoom_level int)
 RETURNS TABLE(geometry geometry, admin_level int) AS $$
     SELECT geom, admin_level FROM (
+        -- etldoc: boundary_z0 ->  layer_boundary:z0
         SELECT * FROM boundary_z0 WHERE geom && bbox AND zoom_level = 0
         UNION ALL
+        -- etldoc: boundary_z1 ->  layer_boundary:z1_2
         SELECT * FROM boundary_z1 WHERE geom && bbox AND zoom_level BETWEEN 1 AND 2
         UNION ALL
+        -- etldoc: boundary_z3 ->  layer_boundary:z3
         SELECT * FROM boundary_z3 WHERE geom && bbox AND zoom_level = 3
         UNION ALL
+        -- etldoc: boundary_z4 ->  layer_boundary:z4
         SELECT * FROM boundary_z4 WHERE geom && bbox AND zoom_level = 4
         UNION ALL
+        -- etldoc: boundary_z5 ->  layer_boundary:z5_6
         SELECT * FROM boundary_z5 WHERE geom && bbox AND zoom_level BETWEEN 5 AND 6
         UNION ALL
+        -- etldoc: boundary_z7 ->  layer_boundary:z7
         SELECT * FROM boundary_z7 WHERE geom && bbox AND zoom_level = 7
         UNION ALL
+        -- etldoc: boundary_z8 ->  layer_boundary:z8
         SELECT * FROM boundary_z8 WHERE geom && bbox AND zoom_level = 8
         UNION ALL
+        -- etldoc: boundary_z9 ->  layer_boundary:z9
         SELECT * FROM boundary_z9 WHERE geom && bbox AND zoom_level = 9
         UNION ALL
+        -- etldoc: boundary_z10 ->  layer_boundary:z10
         SELECT * FROM boundary_z10 WHERE geom && bbox AND zoom_level = 10
         UNION ALL
+        -- etldoc: boundary_z11 ->  layer_boundary:z11
         SELECT * FROM boundary_z11 WHERE geom && bbox AND zoom_level = 11
         UNION ALL
+        -- etldoc: boundary_z12 ->  layer_boundary:z12
         SELECT * FROM boundary_z12 WHERE geom && bbox AND zoom_level = 12
         UNION ALL
+        -- etldoc: boundary_z12 -> layer_boundary:z13_
         SELECT * FROM boundary_z12 WHERE geom && bbox AND zoom_level >= 13
     ) AS zoom_levels;
 $$ LANGUAGE SQL IMMUTABLE;

layers/boundary/mapping.yaml

@@ -1,20 +1,32 @@
 generalized_tables:
+
+  # etldoc: imposm3 -> osm_boundary_linestring_gen5 
   boundary_linestring_gen5:
     source: boundary_linestring
     tolerance: 420.0
+
+  # etldoc: imposm3 -> osm_boundary_linestring_gen4
   boundary_linestring_gen4:
     source: boundary_linestring
     tolerance: 320.0
+
+  # etldoc: imposm3 -> osm_boundary_linestring_gen3    
   boundary_linestring_gen3:
     source: boundary_linestring
     tolerance: 150.0
+
+  # etldoc: imposm3 -> osm_boundary_linestring_gen2    
   boundary_linestring_gen2:
     source: boundary_linestring
     tolerance: 100.0
+
+  # etldoc: imposm3 -> osm_boundary_linestring_gen1    
   boundary_linestring_gen1:
     source: boundary_linestring
     tolerance: 50.0
+
 tables:
+  # etldoc: imposm3 -> osm_boundary_linestring
   boundary_linestring:
     fields:
     - name: osm_id

layers/building/building.sql

@@ -1,11 +1,20 @@
+
+
+
+-- etldoc: layer_building[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_building | <z13> z13 | <z14_> z14_ " ] ;
+
 CREATE OR REPLACE FUNCTION layer_building(bbox geometry, zoom_level int)
 RETURNS TABLE(geom geometry, osm_id bigint, render_height numeric) AS $$
     SELECT geometry, osm_id,
     least(greatest(3, COALESCE(height, levels*3.66,5)),400)^.7::int AS render_height
     FROM (
+
+        -- etldoc: osm_building_polygon_gen1 -> layer_building:z13
         SELECT osm_id, geometry, height, levels FROM osm_building_polygon_gen1
         WHERE zoom_level = 13 AND geometry && bbox AND area > 1400
         UNION ALL
+        -- etldoc: osm_building_polygon -> layer_building:z14_
         SELECT osm_id, geometry, height, levels FROM osm_building_polygon
         WHERE zoom_level >= 14 AND geometry && bbox
     ) AS zoom_levels

layers/building/mapping.yaml

@@ -1,9 +1,14 @@
 generalized_tables:
+
+  # etldoc: imposm3 -> osm_building_polygon_gen1
   building_polygon_gen1:
     source: building_polygon
     sql_filter: area>1400.0
     tolerance: 10.0
+
 tables:
+
+  # etldoc: imposm3 -> osm_building_polygon
   building_polygon:
     fields:
     - name: osm_id

layers/highway/highway.sql

@@ -1,7 +1,12 @@
+ 
+
 CREATE OR REPLACE FUNCTION highway_is_link(highway TEXT) RETURNS BOOLEAN AS $$
     SELECT highway LIKE '%_link';
 $$ LANGUAGE SQL IMMUTABLE STRICT;
 
+
+-- etldoc: layer_highway[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="<sql> layer_highway |<z4z7> z4-z7 |<z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14_> z14_" ] ;
 CREATE OR REPLACE FUNCTION layer_highway(bbox geometry, zoom_level int)
 RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass text, properties highway_properties) AS $$
     SELECT
@@ -9,29 +14,41 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass te
 		to_highway_class(highway) AS class, highway AS subclass,
 		to_highway_properties(is_bridge, is_tunnel, is_ford, is_ramp, is_oneway) AS properties
 	FROM (
-		SELECT
+
+		-- etldoc: ne_10m_global_roads ->  layer_highway:z4z7
+		SELECT 
 			NULL::bigint AS osm_id, geometry, highway,
 			FALSE AS is_bridge, FALSE AS is_tunnel, FALSE AS is_ford, FALSE AS is_ramp, FALSE AS is_oneway,
 			0 AS z_order
 		FROM ne_10m_global_roads
 		WHERE zoom_level BETWEEN 4 AND 7 AND scalerank <= 1 + zoom_level
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring_gen4  ->  layer_highway:z8
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring_gen4
 		WHERE zoom_level = 8
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring_gen3  ->  layer_highway:z9
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring_gen3
 		WHERE zoom_level = 9
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring_gen2  ->  layer_highway:z10
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring_gen2
 		WHERE zoom_level = 10
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring_gen1  ->  layer_highway:z11
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring_gen1
 		WHERE zoom_level = 11
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring       ->  layer_highway:z12
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring
 		WHERE zoom_level = 12
@@ -39,18 +56,25 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass te
             AND NOT highway_is_link(highway)
             AND NOT is_area
         UNION ALL
+
+		-- etldoc: osm_highway_linestring       ->  layer_highway:z13
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring
 		WHERE zoom_level = 13
             AND to_highway_class(highway) < 'path'::highway_class
             AND NOT is_area
         UNION ALL
+
+		-- etldoc: osm_highway_linestring       ->  layer_highway:z14_
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring
 		WHERE zoom_level >= 14 AND NOT is_area
         UNION ALL
+
         -- NOTE: We limit the selection of polys because we need to be careful to net get false positives here because
         -- it is possible that closed linestrings appear both as highway linestrings and as polygon
+        -- etldoc: osm_highway_polygon          ->  layer_highway:z13
+        -- etldoc: osm_highway_polygon          ->  layer_highway:z14_
 		SELECT osm_id, geometry, highway, FALSE AS is_bridge, FALSE AS is_tunnel, FALSE AS is_ford, FALSE AS is_ramp, FALSE AS is_oneway, z_order
         FROM osm_highway_polygon
         -- We do not want underground pedestrian areas for now

layers/highway/mapping.yaml

@@ -1,21 +1,32 @@
+
 generalized_tables:
+
+# etldoc: imposm3 -> osm_highway_linestring_gen4
   highway_linestring_gen4:
     source: highway_linestring_gen3
     sql_filter: highway IN ('motorway','trunk') AND NOT is_area
     tolerance: 200.0
+
+# etldoc: imposm3 -> osm_highway_linestring_gen3    
   highway_linestring_gen3:
     source: highway_linestring_gen2
     sql_filter: highway IN ('motorway','trunk', 'primary') AND NOT is_area
     tolerance: 120.0
+
+# etldoc: imposm3 -> osm_highway_linestring_gen2    
   highway_linestring_gen2:
     source: highway_linestring_gen1
     sql_filter: highway IN ('motorway','trunk', 'primary', 'secondary') AND NOT is_area
     tolerance: 50.0
+
+# etldoc: imposm3 -> osm_highway_linestring_gen1    
   highway_linestring_gen1:
     source: highway_linestring
     sql_filter: highway IN ('motorway','trunk', 'primary', 'secondary', 'tertiary') AND NOT is_area
     tolerance: 20.0
+
 tables:
+# etldoc: imposm3 -> osm_highway_linestring
   highway_linestring:
     type: linestring
     fields:
@@ -85,6 +96,8 @@ tables:
       - corridor
       - crossing
       - pedestrian
+
+# etldoc: imposm3 -> osm_highway_polygon     
   highway_polygon:
     type: polygon
     fields:

layers/highway/ne_global_roads.sql

@@ -1,3 +1,5 @@
+
+
 CREATE OR REPLACE FUNCTION ne_highway(type VARCHAR) RETURNS VARCHAR AS $$
   SELECT CASE type
     WHEN 'Major Highway' THEN 'motorway'
@@ -7,13 +9,19 @@ CREATE OR REPLACE FUNCTION ne_highway(type VARCHAR) RETURNS VARCHAR AS $$
   END;
 $$ LANGUAGE SQL IMMUTABLE;
 
+
+-- etldoc: ne_global_roads_sql -> ne_10m_global_roads  ;
 CREATE TABLE IF NOT EXISTS ne_10m_global_roads AS (
+
+    -- etldoc:  ne_10m_roads -> ne_10m_global_roads
     SELECT geom AS geometry, scalerank, ne_highway(type) AS highway
     FROM ne_10m_roads
     WHERE continent <> 'North America'
       AND featurecla = 'Road'
       AND type IN ('Major Highway', 'Secondary Highway', 'Road')
     UNION ALL
+
+    -- etldoc: ne_10m_roads_north_america ->  ne_10m_global_roads  
     SELECT geom AS geometry, scalerank, ne_highway(type) AS highway
     FROM ne_10m_roads_north_america
     WHERE type IN ('Major Highway', 'Secondary Highway', 'Road')

layers/highway/types.sql

@@ -1,3 +1,5 @@
+
+
 DO $$
 BEGIN
     IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'highway_class') THEN
@@ -14,7 +16,6 @@ BEGIN
 END
 $$;
 
-
 CREATE OR REPLACE FUNCTION to_highway_class(highway TEXT) RETURNS highway_class AS $$
     SELECT CASE
         WHEN highway IN ('motorway', 'motorway_link') THEN 'motorway'::highway_class

layers/highway_name/layer.sql

@@ -1,31 +1,49 @@
+
+-- etldoc: layer_highway_name[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_highway_name | <z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14_> z14_" ] ;
+
 CREATE OR REPLACE FUNCTION layer_highway_name(bbox geometry, zoom_level integer)
 RETURNS TABLE(osm_id bigint, geometry geometry, name text, ref text, ref_length int, class highway_class, subclass text) AS $$
     SELECT osm_id, geometry, name,
       NULLIF(ref, ''), NULLIF(LENGTH(ref), 0) AS ref_length,
       to_highway_class(highway) AS class, highway AS subclass
     FROM (
+
+        -- etldoc: osm_highway_name_linestring_gen3 ->  layer_highway_name:z8
         SELECT * FROM osm_highway_name_linestring_gen3
         WHERE zoom_level = 8
         UNION ALL
+
+        -- etldoc: osm_highway_name_linestring_gen2 ->  layer_highway_name:z9
         SELECT * FROM osm_highway_name_linestring_gen2
         WHERE zoom_level = 9
         UNION ALL
+
+        -- etldoc: osm_highway_name_linestring_gen1 ->  layer_highway_name:z10 
+        -- etldoc: osm_highway_name_linestring_gen1 ->  layer_highway_name:z11          
         SELECT * FROM osm_highway_name_linestring_gen1
         WHERE zoom_level BETWEEN 10 AND 11
         UNION ALL
+
+        -- etldoc: osm_highway_name_linestring ->  layer_highway_name:z12        
         SELECT * FROM osm_highway_name_linestring
         WHERE zoom_level = 12
             AND LineLabel(zoom_level, COALESCE(NULLIF(name, ''), ref), geometry)
             AND to_highway_class(highway) < 'minor_road'::highway_class
             AND NOT highway_is_link(highway)
         UNION ALL
+
+        -- etldoc: osm_highway_name_linestring ->  layer_highway_name:z13
         SELECT * FROM osm_highway_name_linestring
         WHERE zoom_level = 13
             AND LineLabel(zoom_level, COALESCE(NULLIF(name, ''), ref), geometry)
             AND to_highway_class(highway) < 'path'::highway_class
         UNION ALL
+
+        -- etldoc: osm_highway_name_linestring ->  layer_highway_name:z14_
         SELECT * FROM osm_highway_name_linestring
         WHERE zoom_level >= 14
+
     ) AS zoom_levels
     WHERE geometry && bbox
     ORDER BY z_order ASC;

layers/highway_name/merge_highways.sql

@@ -2,6 +2,8 @@
 -- stitch together the touching ways with the same name
 -- to allow for nice label rendering
 -- Because this works well for roads that do not have relations as well
+
+-- etldoc: osm_highway_linestring ->  osm_highway_name_linestring
 CREATE TABLE IF NOT EXISTS osm_highway_name_linestring AS (
 	SELECT
 		(ST_Dump(geometry)).geom AS geometry,
@@ -30,6 +32,7 @@ CREATE TABLE IF NOT EXISTS osm_highway_name_linestring AS (
 
 CREATE INDEX IF NOT EXISTS osm_highway_name_linestring_geometry_idx ON osm_highway_name_linestring USING gist(geometry);
 
+-- etldoc: osm_highway_name_linestring -> osm_highway_name_linestring_gen1 
 CREATE TABLE IF NOT EXISTS osm_highway_name_linestring_gen1 AS (
     SELECT ST_Simplify(geometry, 50) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order
     FROM osm_highway_name_linestring
@@ -37,6 +40,7 @@ CREATE TABLE IF NOT EXISTS osm_highway_name_linestring_gen1 AS (
 );
 CREATE INDEX IF NOT EXISTS osm_highway_name_linestring_gen1_geometry_idx ON osm_highway_name_linestring_gen1 USING gist(geometry);
 
+-- etldoc: osm_highway_name_linestring_gen1 -> osm_highway_name_linestring_gen2
 CREATE TABLE IF NOT EXISTS osm_highway_name_linestring_gen2 AS (
     SELECT ST_Simplify(geometry, 120) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order
     FROM osm_highway_name_linestring_gen1
@@ -44,6 +48,7 @@ CREATE TABLE IF NOT EXISTS osm_highway_name_linestring_gen2 AS (
 );
 CREATE INDEX IF NOT EXISTS osm_highway_name_linestring_gen2_geometry_idx ON osm_highway_name_linestring_gen2 USING gist(geometry);
 
+-- etldoc: osm_highway_name_linestring_gen2 -> osm_highway_name_linestring_gen3
 CREATE TABLE IF NOT EXISTS osm_highway_name_linestring_gen3 AS (
     SELECT ST_Simplify(geometry, 120) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order
     FROM osm_highway_name_linestring_gen2

layers/housenumber/housenumber_centroid.sql

@@ -1,2 +1,4 @@
+
+-- etldoc: osm_housenumber_point -> osm_housenumber_point
 UPDATE osm_housenumber_point SET geometry=topoint(geometry)
 WHERE ST_GeometryType(geometry) <> 'ST_Point';

layers/housenumber/layer.sql

@@ -1,5 +1,10 @@
+
+-- etldoc: layer_housenumber[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_housenumber | <z14_> z14_" ] ;
+
 CREATE OR REPLACE FUNCTION layer_housenumber(bbox geometry, zoom_level integer)
 RETURNS TABLE(osm_id bigint, geometry geometry, housenumber text) AS $$
+   -- etldoc: osm_housenumber_point -> layer_housenumber:z14_
     SELECT osm_id, geometry, housenumber FROM osm_housenumber_point
     WHERE zoom_level >= 14 AND geometry && bbox;
 $$ LANGUAGE SQL IMMUTABLE;

layers/housenumber/mapping.yaml

@@ -1,4 +1,7 @@
+
 tables:
+
+  # etldoc: imposm3 -> osm_housenumber_point
   housenumber_point:
     type: geometry
     fields:

layers/landcover/landcover.sql

@@ -20,83 +20,108 @@ CREATE OR REPLACE FUNCTION landcover_class(landuse VARCHAR, "natural" VARCHAR, w
     END;
 $$ LANGUAGE SQL IMMUTABLE;
 
+-- etldoc: ne_110m_glaciated_areas ->  landcover_z0 
 CREATE OR REPLACE VIEW landcover_z0 AS (
     SELECT NULL::int AS osm_id, geom AS geometry, NULL AS landuse, 'glacier' AS "natural", NULL::text AS wetland FROM ne_110m_glaciated_areas
 );
 
 CREATE OR REPLACE VIEW landcover_z2 AS (
+    -- etldoc: ne_50m_glaciated_areas ->  landcover_z2 
     SELECT NULL::bigint AS osm_id, geom AS geometry, NULL::text AS landuse, 'glacier' AS "natural", NULL::text AS wetland FROM ne_50m_glaciated_areas
     UNION ALL
+    -- etldoc: ne_50m_antarctic_ice_shelves_polys ->  landcover_z2     
     SELECT NULL::bigint AS osm_id, geom AS geometry, NULL::text AS landuse, 'ice_shelf' AS "natural", NULL::text AS wetland FROM ne_50m_antarctic_ice_shelves_polys
 );
 
 CREATE OR REPLACE VIEW landcover_z5 AS (
+    -- etldoc: ne_10m_glaciated_areas ->  landcover_z5
     SELECT NULL::bigint AS osm_id, geom AS geometry, NULL::text AS landuse, 'glacier' AS "natural", NULL::text AS wetland FROM ne_10m_glaciated_areas
     UNION ALL
+    -- etldoc: ne_10m_antarctic_ice_shelves_polys ->  landcover_z5 
     SELECT NULL::bigint AS osm_id, geom AS geometry, NULL::text AS landuse, 'ice_shelf' AS "natural", NULL::text AS wetland FROM ne_10m_antarctic_ice_shelves_polys
 );
 
 CREATE OR REPLACE VIEW landcover_z8 AS (
+    -- etldoc: osm_landcover_polygon_gen5 ->  landcover_z8  
     SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon_gen5
 );
 
 CREATE OR REPLACE VIEW landcover_z9 AS (
+    -- etldoc: osm_landcover_polygon_gen4 ->  landcover_z9    
     SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon_gen4
 );
 
 CREATE OR REPLACE VIEW landcover_z10 AS (
+    -- etldoc: osm_landcover_polygon_gen3 ->  landcover_z10    
     SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon_gen3
 );
 
 CREATE OR REPLACE VIEW landcover_z11 AS (
+    -- etldoc: osm_landcover_polygon_gen2 ->  landcover_z11    
     SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon_gen2
 );
 
 CREATE OR REPLACE VIEW landcover_z12 AS (
+    -- etldoc: osm_landcover_polygon_gen1 ->  landcover_z12    
     SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon_gen1
 );
 
 CREATE OR REPLACE VIEW landcover_z13 AS (
+    -- etldoc: osm_landcover_polygon ->  landcover_z13
     SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon WHERE ST_Area(geometry) > 60000
 );
 
 CREATE OR REPLACE VIEW landcover_z14 AS (
+    -- etldoc: osm_landcover_polygon ->  landcover_z14    
     SELECT osm_id, geometry, landuse, "natural", wetland FROM osm_landcover_polygon
 );
 
+-- etldoc: layer_landcover[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_landcover | <z0> z0  | <z2_4> z2_4  | <z5_7> z5_7 | <z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14_> z14_" ] ;
+
 CREATE OR REPLACE FUNCTION layer_landcover(bbox geometry, zoom_level int)
 RETURNS TABLE(osm_id bigint, geometry geometry, class text, subclass text) AS $$
     SELECT osm_id, geometry,
         landcover_class(landuse, "natural", wetland) AS class,
         COALESCE(NULLIF("natural", ''), NULLIF(landuse, ''), NULLIF('wetland', '')) AS subclass
         FROM (
+        -- etldoc:  landcover_z0 -> layer_landcover:z0
         SELECT * FROM landcover_z0
         WHERE zoom_level BETWEEN 0 AND 1 AND geometry && bbox
         UNION ALL
+        -- etldoc:  landcover_z2 -> layer_landcover:z2_4        
         SELECT * FROM landcover_z2
         WHERE zoom_level BETWEEN 2 AND 4 AND geometry && bbox
         UNION ALL
+        -- etldoc:  landcover_z5 -> layer_landcover:z5_7
         SELECT * FROM landcover_z5
         WHERE zoom_level BETWEEN 5 AND 7 AND geometry && bbox
         UNION ALL
+        -- etldoc:  landcover_z8 -> layer_landcover:z8
         SELECT osm_id, geometry, landuse, "natural", wetland
         FROM landcover_z8 WHERE zoom_level = 8 AND geometry && bbox
         UNION ALL
+        -- etldoc:  landcover_z9 -> layer_landcover:z9        
         SELECT osm_id, geometry, landuse, "natural", wetland
         FROM landcover_z9 WHERE zoom_level = 9 AND geometry && bbox
         UNION ALL
+        -- etldoc:  landcover_z10 -> layer_landcover:z10
         SELECT osm_id, geometry, landuse, "natural", wetland
         FROM landcover_z10 WHERE zoom_level = 10 AND geometry && bbox
         UNION ALL
+        -- etldoc:  landcover_z11 -> layer_landcover:z11        
         SELECT osm_id, geometry, landuse, "natural", wetland
         FROM landcover_z11 WHERE zoom_level = 11 AND geometry && bbox
         UNION ALL
+        -- etldoc:  landcover_z12 -> layer_landcover:z12        
         SELECT osm_id, geometry, landuse, "natural", wetland
         FROM landcover_z12 WHERE zoom_level = 12 AND geometry && bbox
         UNION ALL
+        -- etldoc:  landcover_z13 -> layer_landcover:z13        
         SELECT osm_id, ST_Simplify(geometry, 10) AS geometry, landuse, "natural", wetland
         FROM landcover_z13 WHERE zoom_level = 13 AND geometry && bbox
         UNION ALL
+        -- etldoc:  landcover_z14 -> layer_landcover:z14_        
         SELECT osm_id, geometry, landuse, "natural", wetland
         FROM landcover_z14 WHERE zoom_level >= 14 AND geometry && bbox
     ) AS zoom_levels;

layers/landcover/mapping.yaml

@@ -1,25 +1,38 @@
 generalized_tables:
+  
+  # etldoc: imposm3 -> osm_landcover_polygon_gen5
   landcover_polygon_gen5:
     source: landcover_polygon_gen4
     sql_filter: area>15000000
     tolerance: 300.0
+
+  # etldoc: imposm3 -> osm_landcover_polygon_gen4
   landcover_polygon_gen4:
     source: landcover_polygon_gen3
     sql_filter: area>4200000
     tolerance: 200.0
+
+  # etldoc: imposm3 -> osm_landcover_polygon_gen3  
   landcover_polygon_gen3:
     source: landcover_polygon_gen2
     sql_filter: area>1200000
     tolerance: 120.0
+
+  # etldoc: imposm3 -> osm_landcover_polygon_gen2  
   landcover_polygon_gen2:
     source: landcover_polygon_gen1
     sql_filter: area>480000
     tolerance: 80.0
+
+  # etldoc: imposm3 -> osm_landcover_polygon_gen1  
   landcover_polygon_gen1:
     source: landcover_polygon
     sql_filter: area>240000
     tolerance: 50.0
+
 tables:
+  
+  # etldoc: imposm3 -> osm_landcover_polygon 
   landcover_polygon:
     fields:
     - name: osm_id

layers/landuse/landuse.sql

@@ -8,78 +8,105 @@ CREATE OR REPLACE FUNCTION landuse_class(landuse TEXT, amenity TEXT, leisure TEX
 	 END;
 $$ LANGUAGE SQL IMMUTABLE;
 
+-- etldoc: ne_50m_urban_areas -> landuse_z4
 CREATE OR REPLACE VIEW landuse_z4 AS (
     SELECT NULL::bigint AS osm_id, geom AS geometry, 'residential' AS landuse, NULL::text AS amenity, NULL::text AS leisure, NULL::text AS boundary, scalerank
     FROM ne_50m_urban_areas
     WHERE scalerank <= 2
 );
 
+-- etldoc: ne_50m_urban_areas -> landuse_z5
 CREATE OR REPLACE VIEW landuse_z5 AS (
     SELECT NULL::bigint AS osm_id, geom AS geometry, 'residential' AS landuse, NULL::text AS amenity, NULL::text AS leisure, NULL::text AS boundary, scalerank
     FROM ne_50m_urban_areas
 );
 
+-- etldoc: ne_10m_urban_areas -> landuse_z6
 CREATE OR REPLACE VIEW landuse_z6 AS (
     SELECT NULL::bigint AS osm_id, geom AS geometry, 'residential' AS landuse, NULL::text AS amenity, NULL::text AS leisure, NULL::text AS boundary, scalerank
     FROM ne_10m_urban_areas
 );
 
+-- etldoc: osm_landuse_polygon_gen5 -> landuse_z8
 CREATE OR REPLACE VIEW landuse_z8 AS (
     SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon_gen5
 );
 
+-- etldoc: osm_landuse_polygon_gen4 -> landuse_z9
 CREATE OR REPLACE VIEW landuse_z9 AS (
     SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon_gen4
 );
 
+-- etldoc: osm_landuse_polygon_gen3 -> landuse_z10
 CREATE OR REPLACE VIEW landuse_z10 AS (
     SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon_gen3
 );
 
+-- etldoc: osm_landuse_polygon_gen2 -> landuse_z11
 CREATE OR REPLACE VIEW landuse_z11 AS (
     SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon_gen2
 );
 
+-- etldoc: osm_landuse_polygon_gen1 -> landuse_z12
 CREATE OR REPLACE VIEW landuse_z12 AS (
     SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon_gen1
 );
 
+-- etldoc: osm_landuse_polygon -> landuse_z13
 CREATE OR REPLACE VIEW landuse_z13 AS (
     SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon
     WHERE ST_Area(geometry) > 60000
 );
 
+-- etldoc: osm_landuse_polygon -> landuse_z14
 CREATE OR REPLACE VIEW landuse_z14 AS (
     SELECT osm_id, geometry, landuse, amenity, leisure, boundary, NULL::int as scalerank FROM osm_landuse_polygon
 );
 
+-- etldoc: layer_landuse[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_landuse |<z4> z4|<z5>z5|<z6>z6|<z7>z7| <z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14_> z14_" ] ;
+
 CREATE OR REPLACE FUNCTION layer_landuse(bbox geometry, zoom_level int)
 RETURNS TABLE(osm_id bigint, geometry geometry, class text, subclass text) AS $$
     SELECT osm_id, geometry,
         landuse_class(landuse, amenity, leisure, boundary) AS class,
         COALESCE(NULLIF(landuse, ''), NULLIF(amenity, ''), NULLIF(leisure, ''), NULLIF(boundary, '')) AS subclass
         FROM (
+        -- etldoc: landuse_z4 -> layer_landuse:z4
         SELECT * FROM landuse_z4
         WHERE zoom_level = 4
         UNION ALL
+        -- etldoc: landuse_z5 -> layer_landuse:z5        
         SELECT * FROM landuse_z5
         WHERE zoom_level = 5
         UNION ALL
+        -- etldoc: landuse_z6 -> layer_landuse:z6
+        -- etldoc: landuse_z6 -> layer_landuse:z7
+        -- etldoc: landuse_z6 -> layer_landuse:z8
+        -- etldoc: landuse_z6 -> layer_landuse:z9
+        -- etldoc: landuse_z6 -> layer_landuse:z10                       
         SELECT * FROM landuse_z6
         WHERE zoom_level BETWEEN 6 AND 10 AND scalerank-1 <= zoom_level
         UNION ALL
+        -- etldoc: landuse_z8 -> layer_landuse:z8
         SELECT * FROM landuse_z8 WHERE zoom_level = 8
         UNION ALL
+        -- etldoc: landuse_z9 -> layer_landuse:z9        
         SELECT * FROM landuse_z9 WHERE zoom_level = 9
         UNION ALL
+        -- etldoc: landuse_z10 -> layer_landuse:z10        
         SELECT * FROM landuse_z10 WHERE zoom_level = 10
         UNION ALL
+        -- etldoc: landuse_z11 -> layer_landuse:z11        
         SELECT * FROM landuse_z11 WHERE zoom_level = 11
         UNION ALL
+        -- etldoc: landuse_z12 -> layer_landuse:z12        
         SELECT * FROM landuse_z12 WHERE zoom_level = 12
         UNION ALL
+        -- etldoc: landuse_z13 -> layer_landuse:z13        
         SELECT * FROM landuse_z13 WHERE zoom_level = 13
         UNION ALL
+        -- etldoc: landuse_z14 -> layer_landuse:z14_        
         SELECT * FROM landuse_z14 WHERE zoom_level >= 14
     ) AS zoom_levels
     WHERE geometry && bbox;

layers/landuse/mapping.yaml

@@ -1,25 +1,38 @@
 generalized_tables:
+
+  # etldoc: imposm3 -> osm_landuse_polygon_gen5
   landuse_polygon_gen5:
     source: landuse_polygon_gen4
     sql_filter: area>15000000
     tolerance: 300.0
+
+  # etldoc: imposm3 -> osm_landuse_polygon_gen4    
   landuse_polygon_gen4:
     source: landuse_polygon_gen3
     sql_filter: area>4200000
     tolerance: 200.0
+
+  # etldoc: imposm3 -> osm_landuse_polygon_gen3    
   landuse_polygon_gen3:
     source: landuse_polygon_gen2
     sql_filter: area>1200000
     tolerance: 120.0
+
+  # etldoc: imposm3 -> osm_landuse_polygon_gen2    
   landuse_polygon_gen2:
     source: landuse_polygon_gen1
     sql_filter: area>480000
     tolerance: 80.0
+
+  # etldoc: imposm3 -> osm_landuse_polygon_gen1    
   landuse_polygon_gen1:
     source: landuse_polygon
     sql_filter: area>240000
     tolerance: 50.0
+
 tables:
+
+  # etldoc: imposm3 -> osm_landuse_polygon
   landuse_polygon:
     type: polygon
     fields:

layers/place/city.sql

@@ -1,5 +1,10 @@
+
+-- etldoc: layer_city[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_city | <z2_7> z2-z7 | <z8_14_> z8_z14_   " ] ;
+
 CREATE OR REPLACE FUNCTION layer_city(bbox geometry, zoom_level int, pixel_width numeric)
 RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class city_class, "rank" int) AS $$
+    -- etldoc: osm_city_point -> layer_city:z2_7              
     SELECT osm_id, geometry, name, COALESCE(NULLIF(name_en, ''), name) AS name_en, place AS class, "rank"
     FROM osm_city_point
     WHERE geometry && bbox
@@ -20,6 +25,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class c
         population DESC NULLS LAST,
         length(name) ASC
       )::int AS gridrank
+    -- etldoc: osm_city_point -> layer_city:z8_14_                          
         FROM osm_city_point
         WHERE geometry && bbox
           AND ((zoom_level = 8 AND place <= 'town'::city_class)

layers/place/country.sql

@@ -1,3 +1,8 @@
+
+-- etldoc: layer_country[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_country | <zall> z0-z14_ " ] ;
+
+-- etldoc: osm_country_point -> layer_country
 CREATE OR REPLACE FUNCTION layer_country(bbox geometry, zoom_level int)
 RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, "rank" int) AS $$
     SELECT osm_id, geometry, name, COALESCE(NULLIF(name_en, ''), name) AS name_en, "rank" FROM osm_country_point

layers/place/mapping.yaml

@@ -11,6 +11,8 @@ rank_field: &rank
   key: rank
   type: integer
 tables:
+
+  # etldoc: imposm3 -> osm_country_point
   country_point:
     type: point
     fields:
@@ -27,6 +29,8 @@ tables:
     mapping:
       place:
       - country
+
+  # etldoc: imposm3 -> osm_state_point 
   state_point:
     type: point
     fields:
@@ -52,6 +56,8 @@ tables:
     mapping:
       place:
       - state
+
+  # etldoc: imposm3 -> osm_city_point      
   city_point:
     type: point
     fields:

layers/place/merge_city_rank.sql

@@ -1,3 +1,7 @@
+
+-- etldoc: ne_10m_populated_places -> osm_city_point
+-- etldoc: osm_city_point          -> osm_city_point
+
 WITH important_city_point AS (
     SELECT osm.geometry, osm.osm_id, osm.name, osm.name_en, ne.scalerank, ne.labelrank
     FROM ne_10m_populated_places AS ne, osm_city_point AS osm

layers/place/merge_country_rank.sql

@@ -1,5 +1,8 @@
 ALTER TABLE osm_country_point DROP CONSTRAINT IF EXISTS osm_country_point_rank_constraint;
 
+-- etldoc: ne_10m_admin_0_countries   -> osm_country_point
+-- etldoc: osm_country_point          -> osm_country_point
+
 WITH important_country_point AS (
     SELECT osm.geometry, osm.osm_id, osm.name, COALESCE(NULLIF(osm.name_en, ''), ne.name) AS name_en, ne.scalerank, ne.labelrank
     FROM ne_10m_admin_0_countries AS ne, osm_country_point AS osm

layers/place/merge_state_rank.sql

@@ -1,5 +1,8 @@
 ALTER TABLE osm_state_point DROP CONSTRAINT IF EXISTS osm_state_point_rank_constraint;
 
+-- etldoc: ne_10m_admin_1_states_provinces_shp   -> osm_state_point
+-- etldoc: osm_state_point                       -> osm_state_point
+
 WITH important_state_point AS (
     SELECT osm.geometry, osm.osm_id, osm.name, COALESCE(NULLIF(osm.name_en, ''), ne.name) AS name_en, ne.scalerank, ne.labelrank, ne.datarank
     FROM ne_10m_admin_1_states_provinces_shp AS ne, osm_state_point AS osm

layers/place/place.sql

@@ -1,3 +1,11 @@
+
+-- etldoc: layer_place[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_place | <zall> z0-z14_ " ] ;
+
+-- etldoc: layer_country -> layer_place
+-- etldoc: layer_state   -> layer_place
+-- etldoc: layer_city    -> layer_place
+
 CREATE OR REPLACE FUNCTION layer_place(bbox geometry, zoom_level int, pixel_width numeric)
 RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class text, "rank" int) AS $$
     SELECT osm_id, geometry, name, name_en, 'country' AS class, "rank" FROM layer_country(bbox, zoom_level)

layers/place/state.sql

@@ -1,3 +1,9 @@
+
+-- etldoc: layer_state[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_state | <zall> z0-z14_ " ] ;
+
+-- etldoc:  osm_state_point -> layer_state
+
 CREATE OR REPLACE FUNCTION layer_state(bbox geometry, zoom_level int)
 RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, "rank" int) AS $$
     SELECT osm_id, geometry, name, COALESCE(NULLIF(name_en, ''), name) AS name_en, "rank"

layers/poi/layer.sql

@@ -1,3 +1,7 @@
+
+-- etldoc: layer_poi[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_poi | <z14_> z14_" ] ;
+
 CREATE OR REPLACE FUNCTION layer_poi(bbox geometry, zoom_level integer, pixel_width numeric)
 RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class text, subclass text, "rank" int) AS $$
     SELECT osm_id, geometry, name, NULLIF(name_en, ''), poi_class(subclass) AS class, subclass,
@@ -5,6 +9,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class t
             PARTITION BY LabelGrid(geometry, 100 * pixel_width)
             ORDER BY poi_class_rank(poi_class(subclass)) ASC, length(name) DESC
         )::int AS "rank"
+    -- etldoc: osm_poi_point ->  layer_poi:z14_        
     FROM osm_poi_point
     WHERE geometry && bbox
       AND zoom_level >= 14

layers/poi/mapping.yaml

@@ -1,4 +1,5 @@
 tables:
+  # etldoc: imposm3 -> osm_poi_point
   poi_point:
     type: point
     fields:

layers/railway/mapping.yaml

@@ -1,4 +1,5 @@
 tables:
+  # etldoc: imposm3 -> osm_railway_linestring
   railway_linestring:
     type: linestring
     fields:

layers/railway/railway.sql

@@ -13,6 +13,9 @@ CREATE OR REPLACE FUNCTION railway_brunnel(is_bridge boolean, is_tunnel boolean)
     END;
 $$ LANGUAGE SQL IMMUTABLE;
 
+-- etldoc: layer_railway[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_railway | <z13> z13 | <z14_> z14_" ] ;
+
 CREATE OR REPLACE FUNCTION layer_railway(bbox geometry, zoom_level int)
 RETURNS TABLE(osm_id bigint, geometry geometry, class text, subclass text, properties railway_properties) AS $$
     SELECT osm_id, geometry,
@@ -20,9 +23,11 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class text, subclass text, prope
         railway AS subclass,
         to_railway_properties(is_bridge, is_tunnel) AS properties
     FROM (
+        -- etldoc: osm_railway_linestring ->  layer_railway :z13 
         SELECT * FROM osm_railway_linestring
         WHERE zoom_level = 13 AND railway = 'rail' AND service=''
         UNION ALL
+        -- etldoc: osm_railway_linestring ->  layer_railway :z14_        
         SELECT * FROM osm_railway_linestring WHERE zoom_level >= 14
     ) AS zoom_levels
     WHERE geometry && bbox

layers/water/mapping.yaml

@@ -1,17 +1,26 @@
 generalized_tables:
+
+  # etldoc: imposm3 -> osm_water_polygon_gen3
   water_polygon_gen3:
     source: water_polygon
     sql_filter: area>15000000.0 AND waterway = ''
     tolerance: 350.0
+
+  # etldoc: imposm3 -> osm_water_polygon_gen2    
   water_polygon_gen2:
     source: water_polygon
     sql_filter: area>2000000 AND waterway = ''
     tolerance: 200.0
+
+  # etldoc: imposm3 -> osm_water_polygon_gen1    
   water_polygon_gen1:
     source: water_polygon
     sql_filter: area>500000 AND waterway = ''
     tolerance: 100.0
+
 tables:
+
+  # etldoc: imposm3 -> osm_water_polygon
   water_polygon:
     fields:
     - name: osm_id

layers/water/water.sql

@@ -3,101 +3,141 @@ CREATE OR REPLACE FUNCTION water_class(waterway TEXT) RETURNS TEXT AS $$
 $$ LANGUAGE SQL IMMUTABLE;
 
 
+
 CREATE OR REPLACE VIEW water_z0 AS (
+    -- etldoc:  ne_110m_ocean ->  water_z0 
     SELECT geom, 'ocean' AS class FROM ne_110m_ocean
     UNION ALL
+    -- etldoc:  ne_110m_lakes ->  water_z0     
     SELECT geom, 'lake' AS class FROM ne_110m_lakes
 );
 
 CREATE OR REPLACE VIEW water_z1 AS (
+    -- etldoc:  ne_110m_ocean ->  water_z1    
     SELECT geom, 'ocean' AS class FROM ne_110m_ocean
-    UNION ALL
+    UNION ALL   
+    -- etldoc:  ne_110m_lakes ->  water_z1    
     SELECT geom, 'lake' AS class FROM ne_110m_lakes
 );
 
 CREATE OR REPLACE VIEW water_z2 AS (
+    -- etldoc:  ne_50m_ocean ->  water_z2    
     SELECT geom, 'ocean' AS class FROM ne_50m_ocean
     UNION ALL
+    -- etldoc:  ne_110m_lakes ->  water_z2     
     SELECT geom, 'lake' AS class FROM ne_110m_lakes
 );
 
 CREATE OR REPLACE VIEW water_z4 AS (
+    -- etldoc:  ne_50m_ocean ->  water_z4       
     SELECT geom, 'ocean' AS class FROM ne_50m_ocean
     UNION ALL
+    -- etldoc:  ne_50m_lakes ->  water_z4     
     SELECT geom, 'lake' AS class FROM ne_50m_lakes
 );
 
 CREATE OR REPLACE VIEW water_z5 AS (
+    -- etldoc:  ne_10m_ocean ->  water_z5    
     SELECT geom, 'ocean' AS class FROM ne_10m_ocean
     UNION ALL
+    -- etldoc:  ne_10m_lakes ->  water_z5    
     SELECT geom, 'lake' AS class FROM ne_10m_lakes
 );
 
 CREATE OR REPLACE VIEW water_z6 AS (
+    -- etldoc:  ne_10m_ocean ->  water_z6     
     SELECT geom, 'ocean' AS class FROM ne_10m_ocean
     UNION ALL
+   -- etldoc:  ne_10m_lakes ->  water_z6    
     SELECT geom, 'lake' AS class FROM ne_10m_lakes
 );
 
 CREATE OR REPLACE VIEW water_z7 AS (
+    -- etldoc:  ne_10m_ocean ->  water_z7     
     SELECT geom, 'ocean' AS class FROM ne_10m_ocean
     UNION ALL
+    -- etldoc:  osm_water_polygon_gen3 ->  water_z7     
     SELECT geometry AS geom, 'lake' AS class FROM osm_water_polygon_gen3
 );
 
 CREATE OR REPLACE VIEW water_z8 AS (
+    -- etldoc:  ne_10m_ocean ->  water_z8     
     SELECT geom, 'ocean' AS class FROM ne_10m_ocean
     UNION ALL
+    -- etldoc:  osm_water_polygon_gen2 ->  water_z8         
     SELECT geometry AS geom, 'lake' AS class FROM osm_water_polygon_gen2
 );
 
 CREATE OR REPLACE VIEW water_z9 AS (
+    -- etldoc:  osm_water_polygon_gen1 ->  water_z9    
     SELECT geometry AS geom, 'lake'::text AS class FROM osm_water_polygon_gen1
 );
 
 CREATE OR REPLACE VIEW water_z11 AS (
+    -- etldoc:  osm_water_polygon ->  water_z11 
     SELECT geometry AS geom, water_class(waterway) AS class FROM osm_water_polygon WHERE area > 40000
 );
 
 CREATE OR REPLACE VIEW water_z12 AS (
+    -- etldoc:  osm_water_polygon ->  water_z12     
     SELECT geometry AS geom, water_class(waterway) AS class FROM osm_water_polygon WHERE area > 10000
 );
 
 CREATE OR REPLACE VIEW water_z13 AS (
+    -- etldoc:  osm_water_polygon ->  water_z13    
     SELECT geometry AS geom, water_class(waterway) AS class FROM osm_water_polygon WHERE area > 5000
 );
 
 CREATE OR REPLACE VIEW water_z14 AS (
+    -- etldoc:  osm_water_polygon ->  water_z14    
     SELECT geometry AS geom, water_class(waterway) AS class FROM osm_water_polygon
 );
 
+-- etldoc: layer_water [shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_water |<z0> z0|<z1>z1|<z2>z2|<z3>z3 |<z4> z4|<z5>z5|<z6>z6|<z7>z7| <z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14_> z14_" ] ;
+
 CREATE OR REPLACE FUNCTION layer_water (bbox geometry, zoom_level int)
 RETURNS TABLE(geom geometry, class text) AS $$
     SELECT geom, class::text FROM (
+        -- etldoc: water_z0 ->  layer_water:z0
         SELECT * FROM water_z0 WHERE zoom_level = 0
         UNION ALL
+        -- etldoc: water_z1 ->  layer_water:z1        
         SELECT * FROM water_z1 WHERE zoom_level = 1
         UNION ALL
+        -- etldoc: water_z2 ->  layer_water:z2
+        -- etldoc: water_z2 ->  layer_water:z3          
         SELECT * FROM water_z2 WHERE zoom_level BETWEEN 2 AND 3
         UNION ALL
+        -- etldoc: water_z4 ->  layer_water:z4        
         SELECT * FROM water_z4 WHERE zoom_level = 4
         UNION ALL
+        -- etldoc: water_z5 ->  layer_water:z5        
         SELECT * FROM water_z5 WHERE zoom_level = 5
         UNION ALL
+        -- etldoc: water_z6 ->  layer_water:z6       
         SELECT * FROM water_z6 WHERE zoom_level = 6
         UNION ALL
+        -- etldoc: water_z7 ->  layer_water:z7        
         SELECT * FROM water_z7 WHERE zoom_level = 7
         UNION ALL
+        -- etldoc: water_z8 ->  layer_water:z8        
         SELECT * FROM water_z8 WHERE zoom_level = 8
         UNION ALL
+        -- etldoc: water_z9 ->  layer_water:z9   
+        -- etldoc: water_z9 ->  layer_water:z10                   
         SELECT * FROM water_z9 WHERE zoom_level BETWEEN 9 AND 10
         UNION ALL
+        -- etldoc: water_z11 ->  layer_water:z11        
         SELECT * FROM water_z11 WHERE zoom_level = 11
         UNION ALL
+        -- etldoc: water_z12 ->  layer_water:z2        
         SELECT * FROM water_z12 WHERE zoom_level = 12
         UNION ALL
+        -- etldoc: water_z13 ->  layer_water:z13        
         SELECT * FROM water_z13 WHERE zoom_level = 13
         UNION ALL
+        -- etldoc: water_z14 ->  layer_water:z14_        
         SELECT * FROM water_z14 WHERE zoom_level >= 14
     ) AS zoom_levels
     WHERE geom && bbox;

layers/water_name/layer.sql

@@ -1,5 +1,11 @@
+
+-- etldoc: layer_water_name[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_water_name | <z9_13> z9_13 | <z14_> z14_" ] ;
+
 CREATE OR REPLACE FUNCTION layer_water_name(bbox geometry, zoom_level integer)
 RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class text) AS $$
+    -- etldoc: osm_water_lakeline ->  layer_water_name:z9_13
+    -- etldoc: osm_water_lakeline ->  layer_water_name:z14_    
     SELECT osm_id, geometry, name, name_en, 'lake'::text AS class
     FROM osm_water_lakeline
     WHERE geometry && bbox

layers/water_name/merge_lakelines.sql

@@ -1,3 +1,5 @@
+
+-- etldoc:  osm_water_polygon ->  osm_water_lakeline
 CREATE TABLE IF NOT EXISTS osm_water_lakeline AS (
 	SELECT wp.osm_id,
 		ll.wkb_geometry AS geometry,

layers/waterway/mapping.yaml

@@ -1,13 +1,16 @@
 generalized_tables:
+  # etldoc: imposm3 -> osm_waterway_linestring_gen0 
   waterway_linestring_gen0:
     source: waterway_linestring_gen1
     sql_filter: waterway IN ('river') AND ST_Length(geometry) > 10000
     tolerance: 200.0
+  # etldoc: imposm3 -> osm_waterway_linestring_gen1    
   waterway_linestring_gen1:
     source: waterway_linestring
     sql_filter: waterway IN ('river') AND ST_Length(geometry) > 5000
     tolerance: 100.0
 tables:
+  # etldoc: imposm3 -> osm_waterway_linestring
   waterway_linestring:
     type: linestring
     fields:

layers/waterway/waterway.sql

@@ -1,66 +1,89 @@
+
+-- etldoc: ne_110m_rivers_lake_centerlines ->  waterway_z3
 CREATE OR REPLACE VIEW waterway_z3 AS (
     SELECT geom AS geometry, 'river' AS class FROM ne_110m_rivers_lake_centerlines
     WHERE featurecla = 'River'
 );
 
+-- etldoc: ne_50m_rivers_lake_centerlines ->  waterway_z4
 CREATE OR REPLACE VIEW waterway_z4 AS (
     SELECT geom AS geometry, 'river' AS class FROM ne_50m_rivers_lake_centerlines
     WHERE featurecla = 'River'
 );
 
+-- etldoc: ne_10m_rivers_lake_centerlines ->  waterway_z6
 CREATE OR REPLACE VIEW waterway_z6 AS (
     SELECT geom AS geometry, 'river' AS class FROM ne_10m_rivers_lake_centerlines
     WHERE featurecla = 'River'
 );
 
+-- etldoc: osm_waterway_linestring ->  waterway_z8
 CREATE OR REPLACE VIEW waterway_z8 AS (
     SELECT geometry, waterway AS class FROM osm_waterway_linestring
     WHERE waterway IN ('river') AND ST_Length(geometry) > 10000
 );
 
+-- etldoc: osm_waterway_linestring ->  waterway_z9
 CREATE OR REPLACE VIEW waterway_z9 AS (
     SELECT geometry, waterway AS class FROM osm_waterway_linestring
     WHERE waterway IN ('river') AND ST_Length(geometry) > 5000
 );
 
+-- etldoc: osm_waterway_linestring ->  waterway_z11
 CREATE OR REPLACE VIEW waterway_z11 AS (
     SELECT geometry, waterway AS class FROM osm_waterway_linestring
     WHERE waterway IN ('river') AND ST_Length(geometry) > 5000
 );
 
+-- etldoc: osm_waterway_linestring ->  waterway_z12
 CREATE OR REPLACE VIEW waterway_z12 AS (
     SELECT geometry, waterway AS class FROM osm_waterway_linestring
     WHERE waterway IN ('river', 'canal') OR (waterway = 'stream' AND ST_Length(geometry) > 1000)
 );
 
+-- etldoc: osm_waterway_linestring ->  waterway_z13
 CREATE OR REPLACE VIEW waterway_z13 AS (
     SELECT geometry, waterway AS class FROM osm_waterway_linestring
     WHERE waterway IN ('river', 'canal', 'stream', 'drain', 'ditch') AND ST_Length(geometry) > 300
 );
 
+-- etldoc: osm_waterway_linestring ->  waterway_z14
 CREATE OR REPLACE VIEW waterway_z14 AS (
     SELECT geometry, waterway AS class FROM osm_waterway_linestring
 );
 
+
+-- etldoc: layer_waterway[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="layer_waterway | <z3> z3 |<z4_5> z4_5 |<z6_7> z6_7 | <z8> z8 |<z9_10> z9_10 |<z11> z11 |<z12> z12|<z13> z13|<z14_> z14_" ] ;
+
 CREATE OR REPLACE FUNCTION layer_waterway(bbox geometry, zoom_level int)
 RETURNS TABLE(geometry geometry, class text) AS $$
     SELECT geometry, class FROM (
+        -- etldoc: waterway_z3 ->  layer_waterway:z3        
         SELECT * FROM waterway_z3 WHERE zoom_level = 3
         UNION ALL
+        -- etldoc: waterway_z4 ->  layer_waterway:z4_5        
         SELECT * FROM waterway_z4 WHERE zoom_level BETWEEN 4 AND 5
         UNION ALL
+        -- etldoc: waterway_z6 ->  layer_waterway:z6_7           
         SELECT * FROM waterway_z6 WHERE zoom_level BETWEEN 6 AND 7
         UNION ALL
+        -- etldoc: waterway_z8 ->  layer_waterway:z8           
         SELECT * FROM waterway_z8 WHERE zoom_level = 8
         UNION ALL
+        -- etldoc: waterway_z9 ->  layer_waterway:z9_10        
         SELECT * FROM waterway_z9 WHERE zoom_level BETWEEN 9 AND 10
         UNION ALL
+        -- etldoc: waterway_z11 ->  layer_waterway:z11        
         SELECT * FROM waterway_z11 WHERE zoom_level = 11
         UNION ALL
+        -- etldoc: waterway_z12 ->  layer_waterway:z12        
         SELECT * FROM waterway_z12 WHERE zoom_level = 12
         UNION ALL
+        -- etldoc: waterway_z13 ->  layer_waterway:z13        
         SELECT * FROM waterway_z13 WHERE zoom_level = 13
         UNION ALL
+        -- etldoc: waterway_z14 ->  layer_waterway:z14_        
         SELECT * FROM waterway_z14 WHERE zoom_level >= 14
     ) AS zoom_levels
     WHERE geometry && bbox;

quickstart.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -o errexit
+set -o pipefail
+set -o nounset
+
+#Check installed versions
+docker --version
+docker-compose --version
+
+#Remove 
+docker-compose down
+docker-compose rm -fv
+echo "Remove old volume"
+docker volume ls -q | grep openmaptiles  | xargs -r docker volume rm || true
+
+echo "Make directories "
+mkdir -p build
+mkdir -p data
+
+testdata=zurich_switzerland.osm.pbf
+if [ !  -f ./data/${testdata} ]; then
+    echo "Download $testdata   "
+    rm -f ./data/*
+    wget https://s3.amazonaws.com/metro-extracts.mapzen.com/zurich_switzerland.osm.pbf -P ./data
+fi
+
+
+docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make
+docker-compose up   -d postgres
+sleep 30
+
+docker-compose run --rm import-water
+docker-compose run --rm import-natural-earth
+docker-compose run --rm import-lakelines
+docker-compose run --rm import-osm
+docker-compose run --rm import-sql
+
+docker-compose -f docker-compose.yml -f docker-compose-test-override.yml  run --rm generate-vectortiles
+
+docker-compose stop postgres
+echo "The vectortiles created from $testdata  "
+ls ./data/*.mbtiles -la
+echo "Hello ... start experimenting   - see docs !   "
+
+
+