Merge pull request #1 from galewis2/patch-1

Better updates

Dockerfile

@@ -139,7 +139,8 @@ RUN adduser --disabled-password --gecos "" renderer
 # Install python libraries
 RUN pip3 install \
  requests \
- pyyaml
+ pyyaml \
+ osmium
 
 # Configure Apache
 RUN mkdir /var/lib/mod_tile \
@@ -160,6 +161,8 @@ RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \
 && mkdir /var/log/tiles \
 && chmod a+rw /var/log/tiles \
 && ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \
+&& echo "SHELL=/bin/bash" > /etc/crontab \
+&& echo "BASH_ENV=/container.env" >> /etc/crontab \
 && echo "* * * * *   renderer    openstreetmap-tiles-update-expire\n" >> /etc/crontab
 
 RUN mkdir /nodes \

README.md

@@ -29,9 +29,12 @@ If your import is an extract of the planet and has polygonal bounds associated w
 ```
 docker run \
     -e UPDATES=enabled \
+    -e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \
+    -e MAX_INTERVAL_SECONDS=60 \
     -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \
     -v /absolute/path/to/luxembourg.poly:/data.poly \
     -v openstreetmap-data:/var/lib/postgresql/12/main \
+    -v openstreetmap-rendered-tiles:/var/lib/mod_tile \
     overv/openstreetmap-tile-server \
     import
 ```
@@ -92,6 +95,8 @@ Given that you've set up your import as described in the *Automatic updates* sec
 ```
 docker run \
     -p 8080:80 \
+    -e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \
+    -e MAX_INTERVAL_SECONDS=60 \
     -e UPDATES=enabled \
     -v openstreetmap-data:/var/lib/postgresql/12/main \
     -v openstreetmap-rendered-tiles:/var/lib/mod_tile \

openstreetmap-tiles-update-expire

@@ -30,7 +30,6 @@ OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/
 #------------------------------------------------------------------------------
 TRIM_POLY_FILE="/var/lib/mod_tile/data.poly"
 TRIM_OPTIONS="-d $DBNAME"
-#TRIM_REGION_OPTIONS="-b -14.17 48.85 2.12 61.27"
 TRIM_REGION_OPTIONS="-p $TRIM_POLY_FILE"
 
 BASE_DIR=/var/lib/mod_tile
@@ -60,6 +59,10 @@ EXPIRY_TOUCHFROM=13
 EXPIRY_DELETEFROM=19
 EXPIRY_MAXZOOM=20
 
+
+REPLICATION_URL=${REPLICATION_URL:="https://planet.openstreetmap.org/replication/hour/"}
+MAX_INTERVAL_SECONDS=${MAX_INTERVAL_SECONDS:="3600"}
+
 #*************************************************************************
 #*************************************************************************
 
@@ -107,11 +110,18 @@ freelock()
 
 if [ $# -eq 1 ] ; then
     m_info "Initialising Osmosis replication system to $1"
-    mkdir $WORKOSM_DIR
+    mkdir -p $WORKOSM_DIR
-    $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG"
+    $OSMOSIS_BIN -v 5 --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG"
-    wget "https://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt
+
-    mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt
+    init_seq=$(pyosmium-get-changes --server $REPLICATION_URL -D $1)
-    sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt
+    url_dynamicPart=$(printf %09d $init_seq | sed 's_\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)_\1/\2/\3_')
+    wget $REPLICATION_URL/$url_dynamicPart.state.txt -O $WORKOSM_DIR/state.txt
+
+        cat > /var/lib/mod_tile/.osmosis/configuration.txt <<- EOM
+baseUrl=$REPLICATION_URL
+maxInterval=$MAX_INTERVAL_SECONDS
+EOM
+
 else
 # make sure the lockfile is removed when we exit and then claim it
 
@@ -142,7 +152,7 @@ fi
     /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt
     m_ok "downloading diff"
 
-    if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then
+    if ! $OSMOSIS_BIN -v 5 --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then
         m_error "Osmosis error"
     fi
 
@@ -167,7 +177,7 @@ fi
 
 #------------------------------------------------------------------------------
 # The lockfile is normally removed before we expire tiles because that is
-# something thatcan be done in parallel with further processing.  In order to
+# something that can be done in parallel with further processing.  In order to
 # avoid rework, if actually rerendering is done rather than just deleting or
 # dirtying, it makes sense to move it lower down.
 #------------------------------------------------------------------------------
@@ -198,7 +208,4 @@ fi
 
     m_ok "Done with import"
 
-
-
-
 fi

run.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+declare -p | grep -Ev 'BASHOPTS|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID' > /container.env
+
 function createPostgresConfig() {
   cp /etc/postgresql/12/main/postgresql.custom.conf.tmpl /etc/postgresql/12/main/conf.d/postgresql.custom.conf
   sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/12/main/conf.d/postgresql.custom.conf
@@ -62,11 +64,12 @@ if [ "$1" = "import" ]; then
     if [ "${UPDATES:-}" = "enabled" ]; then
         # determine and set osmosis_replication_timestamp (for consecutive updates)
         osmium fileinfo /data.osm.pbf > /var/lib/mod_tile/data.osm.pbf.info
-        osmium fileinfo /data.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44 > /var/lib/mod_tile/replication_timestamp.txt
+        osmium fileinfo --get=header.option.timestamp /data.osm.pbf > /var/lib/mod_tile/replication_timestamp.txt
         REPLICATION_TIMESTAMP=$(cat /var/lib/mod_tile/replication_timestamp.txt)
 
         # initial setup of osmosis workspace (for consecutive updates)
-        sudo -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP
+        sudo -E -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP
+
     fi
 
     # copy polygon file if available