refactor tile_gen in Python

Author: hyperknot

init-server.py

@@ -134,12 +134,14 @@ def debug(hostname, user, port):
     # sudo_cmd(c, f'{VENV_BIN}/python -u /data/ofm/http_host/bin/host_manager.py nginx-sync')
 
     # put(c, SCRIPTS_DIR / 'tile_gen' / 'upload_manager.py', f'{TILE_GEN_BIN}')
-    put_dir(c, SCRIPTS_DIR / 'loadbalancer', '/data/ofm/loadbalancer')
-    put_dir(
-        c,
-        SCRIPTS_DIR / 'loadbalancer' / 'loadbalancer_lib',
-        '/data/ofm/loadbalancer/loadbalancer_lib',
-    )
+    # put_dir(c, SCRIPTS_DIR / 'loadbalancer', '/data/ofm/loadbalancer')
+    # put_dir(
+    #     c,
+    #     SCRIPTS_DIR / 'loadbalancer' / 'loadbalancer_lib',
+    #     '/data/ofm/loadbalancer/loadbalancer_lib',
+    # )
+
+    prepare_tile_gen(c)
 
 
 if __name__ == '__main__':

scripts/debug_proxy/src/index.js

@@ -31,12 +31,12 @@ export default {
       url.pathname = '/styles/bright'
     }
 
-    // no failure, just warning
-    if (request.method !== 'GET') {
-      const warningMessage = `Non-GET request ${request.method} ${url.pathname} ${userIP}`
-      console.error(warningMessage)
-      await sendTelegramMessage(warningMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
-    }
+    // // no failure, just warning
+    // if (request.method !== 'GET') {
+    //   const warningMessage = `Non-GET request ${request.method} ${url.pathname} ${userIP}`
+    //   console.error(warningMessage)
+    //   await sendTelegramMessage(warningMessage, env.TELEGRAM_TOKEN, env.TELEGRAM_CHAT_ID)
+    // }
 
     if (!url.pathname.startsWith('/styles')) {
       const errorMessage = 'Bad path'

scripts/tile_gen/extract_btrfs.sh

@@ -0,0 +1,2 @@
+These are self contained Python scripts, they can be run outside of this project's environment.
+

scripts/tile_gen/planetiler_monaco.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+import json
+import subprocess
+from pathlib import Path
+
+import click
+from tile_gen_lib.config import config
+from tile_gen_lib.extract import make_btrfs
+from tile_gen_lib.planetiler import run_planetiler
+from tile_gen_lib.upload import make_indexes, upload_rclone
+
+
+@click.group()
+def cli():
+    """
+    Generates tiles and uploads to CloudFlare
+    """
+
+
+@cli.command()
+@click.argument('area', required=True)
+def make_tiles(area):
+    """
+    Generate tiles for a given area
+    """
+
+    # run_planetiler(area)
+    make_btrfs(Path('/data/ofm/tile_gen/runs/monaco/20240826_230406_pt'))
+
+
+@cli.command()
+def upload_runs():
+    """
+    Upload all runs present in system
+    """
+
+    print('running upload_runs')
+
+    for area in config.areas:
+        if not (config.runs_dir / area).exists():
+            continue
+
+        p = subprocess.run(
+            [
+                'rclone',
+                'lsjson',
+                '--dirs-only',
+                '--fast-list',
+                f'remote:ofm-{area}',
+            ],
+            text=True,
+            capture_output=True,
+            check=True,
+            env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
+        )
+        rclone_json = json.loads(p.stdout)
+        runs_remote = {p['Path'] for p in rclone_json}
+        runs_local = {p.name for p in (config.runs_dir / area).iterdir()}
+
+        runs_to_upload = runs_local - runs_remote
+        for run in runs_to_upload:
+            print(f'uploading {area} {run}')
+            upload_rclone(area, run)
+
+    make_indexes()
+
+
+@cli.command()
+def index():
+    """
+    Run index on Cloudflare buckets
+    """
+
+    make_indexes()
+
+
+if __name__ == '__main__':
+    cli()

scripts/tile_gen/planetiler_planet.sh

@@ -0,0 +1,18 @@
+from pathlib import Path
+
+
+class Configuration:
+    tile_gen_dir = Path('/data/ofm/tile_gen')
+
+    tile_gen_bin = tile_gen_dir / 'bin'
+    tile_gen_scripts_dir = tile_gen_bin / 'scripts'
+
+    planetiler_bin = tile_gen_dir / 'planetiler'
+    planetiler_path = planetiler_bin / 'planetiler.jar'
+
+    runs_dir = tile_gen_dir / 'runs'
+
+    areas = ['planet', 'monaco']
+
+
+config = Configuration()