Skip to content

Commit 0b06eda

Browse files
Update limit (#83)
* Update setting file structure * Fix critical rename operation bugs (Phase 1A) Bug #1: Fix revision increment gap in mysql.rs - Changed revision increment logic to prevent gaps - Both tombstone and active rows now use same revision (current+1) - Maintains sequential revision tracking without N+2 jumps Bug #2: Fix cross-watcher MOVE broadcast in rename.rs - Use server IDs instead of client IDs for cross-watcher moves - Ensures other devices can properly locate moved files - Fixes synchronization failure across devices All tests passing (17/17). Ready for production deployment. * Remove unwrap() in encryption migration path - Convert String::from_utf8_lossy unwrap to proper error handling - Return StorageError::Database if plain_path is missing - Prevents potential production panic during encryption migration Phase 1B: unwrap removal (1/37) * Remove unsafe unwrap() calls in file, quota, and service modules - mysql_file.rs: Use match pattern for row_opt instead of is_none() + unwrap() - mysql_quota.rs: Replace unwrap() with expect() for valid date calculations - server/service.rs: Replace unwrap() with expect() for midnight time Phase 1B: unwrap removal (2-8/37) * Remove unsafe unwrap() calls in version and usage services - version_service.rs: Use map_or() instead of is_some()+unwrap() pattern - usage_service.rs: Use if-let for safe downcast in warning update Phase 1B: unwrap removal (9-10/37) * Optimize decrypt_text signature to avoid clone - Change decrypt_text parameter from Vec<u8> to &[u8] - Remove 20 unnecessary clone() calls in mysql_file.rs - Reduces heap allocations during file decryption operations Phase 3: Clone optimization (1/n) * Update setting file structure * Update setting file structure * Optimize server logging - reduce redundancy and improve efficiency * Refine log levels: info for requests, debug for internal operations * Add encryption key invalidation feature - Add InvalidateKeyId RPC for key lifecycle management - Block downloads for files with invalidated keys (KEY_INVALIDATED error) - Add error_code field to DownloadFileResponse - Add mysql_key.rs storage layer with idempotent invalidation * Update cloud_test env * Add soft delete feature with multi-device LWW sync support --------- Co-authored-by: Claude Opus 4.5 <[email protected]>
1 parent 2ef2930 commit 0b06eda

File tree

14 files changed

+1525
-59
lines changed

14 files changed

+1525
-59
lines changed

.env.cloud_test

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Temporary cloud DB configuration for migration testing
2+
DB_USER="genesis76"
3+
DB_PASS="VC1OGzY4ukM5DaCrVZJb6832XWYc08U072k4yp0bZ"
4+
DB_NAME="cosmic_sync"
5+
DB_HOST="127.0.0.1"
6+
DB_PORT=63306
7+
DB_POOL=5
8+
9+
SERVER_HOST=0.0.0.0
10+
SERVER_PORT=50051
11+
WORKER_THREADS=4
12+
HEARTBEAT_INTERVAL_SECS=30
13+
AUTH_TOKEN_EXPIRY_HOURS=24
14+
15+
OAUTH_CLIENT_ID=cosmic-sync
16+
OAUTH_CLIENT_SECRET=cosmicsecretsocmicsecret
17+
OAUTH_REDIRECT_URI=http://10.241.62.167:8080/oauth/callback
18+
OAUTH_AUTH_URL=http://10.241.62.167:4000/oauth/authorize
19+
OAUTH_TOKEN_URL=http://10.241.62.167:4000/oauth/token
20+
OAUTH_USER_INFO_URL=http://10.241.62.167:4000/api/settings
21+
OAUTH_SCOPE=profile:read
22+
23+
MAX_CONCURRENT_REQUESTS=100
24+
MAX_FILE_SIZE=52428800
25+
26+
RUST_LOG=cosmic_sync_server=debug,info
27+
LOG_LEVEL=debug
28+
LOG_TO_FILE=false
29+
30+
STORAGE_TYPE="s3"
31+
AWS_REGION="us-east-2"
32+
S3_BUCKET="cosmic-sync-files"
33+
S3_KEY_PREFIX="files/"
34+
AWS_ACCESS_KEY_ID="minioadmin"
35+
AWS_SECRET_ACCESS_KEY="minioadmin"
36+
S3_ENDPOINT_URL="http://127.0.0.1:9000"
37+
S3_FORCE_PATH_STYLE="true"
38+
S3_TIMEOUT_SECONDS="30"
39+
S3_MAX_RETRIES="3"
40+
41+
COSMIC_SYNC_DEV_MODE="1"
42+
COSMIC_SYNC_TEST_MODE="1"
43+
COSMIC_SYNC_DEBUG_MODE="1"
44+
45+
SERVER_ENCODE_KEY=c3e15e2f727cf777380f23a9f9fa8156c5f4f7f3e697f6dc95a47372e76ac6bf
46+
47+
RABBITMQ_ENABLED=false

env_home.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
# Switch to home network (192.168.50.100)
3+
4+
sed -i 's/10.17.89.63/192.168.50.100/g' .env
5+
6+
mysql -h 127.0.0.1 -P 3306 -u root -precognizer --ssl-mode=DISABLED recognizer_dev -e \
7+
"UPDATE oauth_applications SET redirect_uri = 'http://192.168.50.100:8080/oauth/callback' WHERE redirect_uri LIKE '%/oauth/callback';"
8+
9+
echo "Switched to home network (192.168.50.100)"

env_office.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
# Switch to office network (10.17.89.63)
3+
4+
sed -i 's/192.168.50.100/10.17.89.63/g' .env
5+
6+
mysql -h 127.0.0.1 -P 3306 -u root -precognizer --ssl-mode=DISABLED recognizer_dev -e \
7+
"UPDATE oauth_applications SET redirect_uri = 'http://10.17.89.63:8080/oauth/callback' WHERE redirect_uri LIKE '%/oauth/callback';"
8+
9+
echo "Switched to office network (10.17.89.63)"
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
-- Migration: Add key_status table for encryption key invalidation feature
2+
-- Date: 2025-12-12
3+
4+
CREATE TABLE IF NOT EXISTS key_status (
5+
id BIGINT AUTO_INCREMENT PRIMARY KEY,
6+
account_hash VARCHAR(64) NOT NULL,
7+
key_id VARCHAR(32) NOT NULL,
8+
status VARCHAR(20) NOT NULL DEFAULT 'active',
9+
reason VARCHAR(50),
10+
invalidated_at TIMESTAMP NULL,
11+
invalidated_by_device VARCHAR(64),
12+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
13+
UNIQUE INDEX idx_key_status_account_key (account_hash, key_id),
14+
INDEX idx_key_status_account (account_hash)
15+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

migrations/add_soft_delete.sql

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
-- Soft Delete Migration for Watcher Sync
2+
-- This migration adds is_deleted and deleted_at columns to support soft delete functionality
3+
-- for watchers, watcher_groups, and files tables.
4+
5+
-- ============================================================================
6+
-- 1. watcher_groups 테이블 수정
7+
-- ============================================================================
8+
ALTER TABLE watcher_groups
9+
ADD COLUMN is_deleted BOOLEAN NOT NULL DEFAULT FALSE,
10+
ADD COLUMN deleted_at TIMESTAMP NULL;
11+
12+
-- ============================================================================
13+
-- 2. watchers 테이블 수정
14+
-- ============================================================================
15+
ALTER TABLE watchers
16+
ADD COLUMN is_deleted BOOLEAN NOT NULL DEFAULT FALSE,
17+
ADD COLUMN deleted_at TIMESTAMP NULL;
18+
19+
-- ============================================================================
20+
-- 3. files 테이블 수정
21+
-- ============================================================================
22+
ALTER TABLE files
23+
ADD COLUMN is_deleted BOOLEAN NOT NULL DEFAULT FALSE,
24+
ADD COLUMN deleted_at TIMESTAMP NULL;
25+
26+
-- ============================================================================
27+
-- 4. 성능 인덱스 추가 (조회 최적화)
28+
-- ============================================================================
29+
30+
-- watcher_groups: account별 활성 그룹 조회용
31+
CREATE INDEX idx_watcher_groups_account_active
32+
ON watcher_groups(account_hash, is_deleted);
33+
34+
-- watchers: account별 활성 watcher 조회용
35+
CREATE INDEX idx_watchers_account_active
36+
ON watchers(account_hash, is_deleted);
37+
38+
-- watchers: group별 활성 watcher 조회용
39+
CREATE INDEX idx_watchers_group_active
40+
ON watchers(group_id, is_deleted);
41+
42+
-- files: watcher별 활성 파일 조회용
43+
CREATE INDEX idx_files_watcher_active
44+
ON files(server_watcher_id, is_deleted);
45+
46+
-- files: account별 활성 파일 조회용
47+
CREATE INDEX idx_files_account_active
48+
ON files(account_hash, is_deleted);
49+
50+
-- ============================================================================
51+
-- 5. Cleanup job용 인덱스 (삭제 대상 조회 최적화)
52+
-- ============================================================================
53+
54+
-- watcher_groups: 삭제 대상 조회용
55+
CREATE INDEX idx_watcher_groups_deleted_at
56+
ON watcher_groups(deleted_at);
57+
58+
-- watchers: 삭제 대상 조회용
59+
CREATE INDEX idx_watchers_deleted_at
60+
ON watchers(deleted_at);
61+
62+
-- files: 삭제 대상 조회용
63+
CREATE INDEX idx_files_deleted_at
64+
ON files(deleted_at);

proto/sync.proto

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -864,6 +864,7 @@ message SyncConfigurationRequest {
864864
bool incremental = 6; // Incremental sync vs full sync
865865
bool force_update = 7; // Force update flag
866866
int64 client_timestamp = 8; // Client timestamp
867+
int64 last_sync_timestamp = 9; // Last successful sync timestamp (for Item-level LWW)
867868
}
868869

869870
message SyncConfigurationResponse {
@@ -877,16 +878,37 @@ message SyncConfigurationResponse {
877878
repeated string conflict_details = 8; // Conflict details
878879
ActionTaken action_taken = 9; // Action taken during synchronization
879880
bool is_new_account = 10; // Whether this is a new account setup
881+
int32 retention_days = 11; // Soft-deleted items retention period (days)
882+
repeated DeletedItemInfo deleted_items = 12; // Items soft-deleted in this sync
883+
repeated WatcherGroupData recently_added = 13; // Items added by other devices (client should merge)
884+
repeated DeletedItemInfo recently_deleted = 14; // Items deleted by other devices (client should remove)
885+
}
886+
887+
// Information about a soft-deleted item
888+
message DeletedItemInfo {
889+
enum ItemType {
890+
GROUP = 0;
891+
WATCHER = 1;
892+
}
893+
ItemType type = 1; // Type of deleted item
894+
int32 group_id = 2; // Deleted group_id (or parent group_id for watcher)
895+
int32 watcher_id = 3; // Deleted watcher_id (0 if type is GROUP)
896+
string title = 4; // Name of the deleted item
897+
int32 affected_files = 5; // Number of files affected by deletion
880898
}
881899

882900
message SyncStats {
883901
int32 groups_updated = 1; // Number of updated groups
884902
int32 groups_created = 2; // Number of created groups
885-
int32 groups_deleted = 3; // Number of deleted groups
903+
int32 groups_deleted = 3; // Number of deleted groups (soft delete)
886904
int32 presets_updated = 4; // Number of updated presets
887905
int64 sync_timestamp = 5; // Synchronization timestamp
888906
int32 total_operations = 6; // Total number of operations
889907
double sync_duration_ms = 7; // Synchronization duration (milliseconds)
908+
int32 watchers_created = 8; // Number of created watchers
909+
int32 watchers_updated = 9; // Number of updated watchers
910+
int32 watchers_deleted = 10; // Number of deleted watchers (soft delete)
911+
int32 files_soft_deleted = 11; // Number of files soft-deleted due to watcher deletion
890912
}
891913

892914
// File history lookup request

src/handlers/admin_handler.rs

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
//! Admin API handlers for maintenance operations
2+
3+
use actix_web::{web, HttpResponse, Result as ActixResult};
4+
use serde::Serialize;
5+
use std::sync::Arc;
6+
use tracing::{error, info};
7+
8+
use crate::server::app_state::AppState;
9+
use crate::storage::CleanupStats;
10+
11+
/// Default retention period for soft-deleted items (in days)
12+
const DEFAULT_RETENTION_DAYS: i32 = 30;
13+
14+
/// Response for cleanup operation
15+
#[derive(Debug, Serialize)]
16+
pub struct CleanupResponse {
17+
pub success: bool,
18+
pub message: String,
19+
pub retention_days: i32,
20+
pub stats: CleanupStats,
21+
}
22+
23+
/// Run cleanup job to permanently delete soft-deleted records older than retention period
24+
///
25+
/// This endpoint permanently deletes:
26+
/// - Files (including S3 objects)
27+
/// - Watchers
28+
/// - Watcher groups
29+
///
30+
/// that have been soft-deleted for longer than the retention period (default 30 days).
31+
///
32+
/// # Request
33+
/// POST /admin/cleanup
34+
///
35+
/// # Response
36+
/// ```json
37+
/// {
38+
/// "success": true,
39+
/// "message": "Cleanup completed successfully",
40+
/// "retention_days": 30,
41+
/// "stats": {
42+
/// "files_deleted": 10,
43+
/// "watchers_deleted": 5,
44+
/// "groups_deleted": 2,
45+
/// "s3_objects_deleted": 10,
46+
/// "errors": []
47+
/// }
48+
/// }
49+
/// ```
50+
pub async fn run_cleanup(app_state: web::Data<Arc<AppState>>) -> ActixResult<HttpResponse> {
51+
info!("Admin cleanup job started");
52+
53+
match app_state
54+
.storage
55+
.cleanup_soft_deleted(DEFAULT_RETENTION_DAYS)
56+
.await
57+
{
58+
Ok(stats) => {
59+
info!(
60+
"Cleanup completed: {} files, {} watchers, {} groups deleted",
61+
stats.files_deleted, stats.watchers_deleted, stats.groups_deleted
62+
);
63+
64+
Ok(HttpResponse::Ok().json(CleanupResponse {
65+
success: true,
66+
message: "Cleanup completed successfully".to_string(),
67+
retention_days: DEFAULT_RETENTION_DAYS,
68+
stats,
69+
}))
70+
}
71+
Err(e) => {
72+
error!("Cleanup failed: {}", e);
73+
74+
Ok(HttpResponse::InternalServerError().json(CleanupResponse {
75+
success: false,
76+
message: format!("Cleanup failed: {}", e),
77+
retention_days: DEFAULT_RETENTION_DAYS,
78+
stats: CleanupStats::default(),
79+
}))
80+
}
81+
}
82+
}
83+
84+
/// Get cleanup status and statistics (read-only, no actual cleanup)
85+
///
86+
/// # Request
87+
/// GET /admin/cleanup/status
88+
///
89+
/// # Response
90+
/// Returns information about pending cleanup items
91+
pub async fn cleanup_status(app_state: web::Data<Arc<AppState>>) -> ActixResult<HttpResponse> {
92+
// For now, just return the retention configuration
93+
// In the future, could query counts of items pending cleanup
94+
let _ = app_state; // Will be used when we add pending count queries
95+
96+
Ok(HttpResponse::Ok().json(serde_json::json!({
97+
"retention_days": DEFAULT_RETENTION_DAYS,
98+
"description": "Items soft-deleted more than 30 days ago will be permanently deleted on cleanup",
99+
"trigger_endpoint": "POST /admin/cleanup"
100+
})))
101+
}

src/handlers/mod.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ pub mod metrics;
2929
// Usage tracking handlers
3030
pub mod usage_handler;
3131

32+
// Admin handlers (cleanup, maintenance)
33+
pub mod admin_handler;
34+
3235
use crate::sync::HealthCheckRequest;
3336
use crate::sync::HealthCheckResponse;
3437
use tonic::{Request, Response, Status};

0 commit comments

Comments
 (0)