upload by python

Author: hyperknot

init-server.py

@@ -3,7 +3,7 @@
 import click
 from fabric import Config, Connection
 
-from ssh_lib import SCRIPTS_DIR, VENV_BIN, dotenv_val
+from ssh_lib import SCRIPTS_DIR, TILE_GEN_BIN, VENV_BIN, dotenv_val
 from ssh_lib.planetiler import planetiler
 from ssh_lib.tasks import (
     prepare_http_host,
@@ -120,7 +120,7 @@ def debug(hostname, user, port):
     # upload_http_host_files(c)
     # sudo_cmd(c, f'{VENV_BIN}/python -u /data/ofm/http_host/bin/host_manager.py nginx-sync')
 
-    planetiler(c)
+    put(c, SCRIPTS_DIR / 'tile_gen' / 'upload_manager.py', f'{TILE_GEN_BIN}')
 
 
 if __name__ == '__main__':

scripts/tile_gen/cloudflare_upload.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+import json
+import pathlib
+import subprocess
+
+import click
+
+
+AREAS = ['planet', 'monaco']
+
+RUNS_DIR = pathlib.Path('/data/ofm/tile_gen/runs')
+
+
+def upload_rclone(area, run):
+    subprocess.run(
+        [
+            'rclone',
+            'sync',
+            '--transfers=8',
+            '--multi-thread-streams=8',
+            '--fast-list',
+            '-v',
+            '--stats-file-name-length',
+            '0',
+            '--stats-one-line',
+            '--log-file',
+            RUNS_DIR / area / run / 'logs' / 'rclone.log',
+            '--exclude',
+            'logs/**',
+            RUNS_DIR / area / run,
+            f'remote:ofm-{area}/{run}',
+        ],
+        env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
+        check=True,
+    )
+
+
+@click.group()
+def cli():
+    """
+    Uploads runs to Cloudflare
+    """
+
+
+@cli.command()
+def upload_runs():
+    """
+    Upload all runs present in system
+    """
+
+    print('running upload_runs')
+
+    for area in AREAS:
+        if not (RUNS_DIR / area).exists():
+            continue
+
+        p = subprocess.run(
+            [
+                'rclone',
+                'lsjson',
+                '--dirs-only',
+                '--fast-list',
+                f'remote:ofm-{area}',
+            ],
+            text=True,
+            capture_output=True,
+            check=True,
+            env=dict(RCLONE_CONFIG='/data/ofm/config/rclone.conf'),
+        )
+        rclone_json = json.loads(p.stdout)
+        runs_remote = {p['Path'] for p in rclone_json}
+        runs_local = {p.name for p in (RUNS_DIR / area).iterdir()}
+
+        runs_to_upload = runs_local - runs_remote
+        for run in runs_to_upload:
+            print(f'uploading {area} {run}')
+            upload_rclone(area, run)
+
+
+if __name__ == '__main__':
+    cli()