fix: add backfill migration for routing response denormalized tables

packages/prisma/migrations/20250618093846_routing_form_response_denormalize_backfill/migration.sql

@@ -0,0 +1,197 @@
+DO $$
+DECLARE
+    chunk_size INTEGER := 1000;
+    start_id INTEGER := 1;      -- Starting ID
+    end_id INTEGER;             -- Will be set dynamically
+    current_id INTEGER;
+
+    processed_count INTEGER := 0;
+    chunk_updated_count INTEGER := 0;
+    total_count INTEGER;
+BEGIN
+    -- Get the maximum ID and total count from the App_RoutingForms_FormResponse table
+    SELECT COALESCE(MAX(id), 0) INTO end_id FROM "App_RoutingForms_FormResponse";
+    SELECT COUNT(*) INTO total_count FROM "App_RoutingForms_FormResponse";
+
+    -- Handle case where there are no records to process
+    IF total_count = 0 THEN
+        RAISE NOTICE 'No records found in App_RoutingForms_FormResponse table. Migration completed.';
+        RETURN;
+    END IF;
+
+    RAISE NOTICE 'Starting migration: processing up to ID % (total responses: %)', end_id, total_count;
+
+    FOR current_id IN SELECT * FROM generate_series(start_id, end_id, chunk_size)
+    LOOP
+        -- Use UPSERT to only update records that are missing or have incorrect data
+        WITH expected_data AS (
+            SELECT 
+                r.id,
+                r."formId" as expected_form_id,
+                f.name as expected_form_name,
+                f."teamId" as expected_form_team_id,
+                f."userId" as expected_form_user_id,
+                b.uid as expected_booking_uid,
+                b.id as expected_booking_id,
+                b.status as expected_booking_status,
+                calculate_booking_status_order(b.status::text) as expected_booking_status_order,
+                b."createdAt" as expected_booking_created_at,
+                b."startTime" as expected_booking_start_time,
+                b."endTime" as expected_booking_end_time,
+                b."userId" as expected_booking_user_id,
+                u.name as expected_booking_user_name,
+                u.email as expected_booking_user_email,
+                u."avatarUrl" as expected_booking_user_avatar_url,
+                COALESCE(
+                    (
+                        SELECT ar."reasonString"
+                        FROM "AssignmentReason" ar
+                        WHERE ar."bookingId" = b.id
+                        LIMIT 1
+                    ),
+                    ''
+                ) as expected_booking_assignment_reason,
+                et.id as expected_event_type_id,
+                et."parentId" as expected_event_type_parent_id,
+                et."schedulingType"::text as expected_event_type_scheduling_type,
+                r."createdAt" as expected_created_at,
+                t.utm_source as expected_utm_source,
+                t.utm_medium as expected_utm_medium,
+                t.utm_campaign as expected_utm_campaign,
+                t.utm_term as expected_utm_term,
+                t.utm_content as expected_utm_content
+            FROM "App_RoutingForms_FormResponse" r
+            INNER JOIN "App_RoutingForms_Form" f ON r."formId" = f.id
+            LEFT JOIN "users" u ON f."userId" = u.id
+            LEFT JOIN "Booking" b ON b.uid = r."routedToBookingUid"
+            LEFT JOIN "EventType" et ON b."eventTypeId" = et.id
+            LEFT JOIN "Tracking" t ON t."bookingId" = b.id
+            WHERE r.id BETWEEN current_id AND current_id + chunk_size - 1
+        ),
+        records_to_update AS (
+            SELECT e.*
+            FROM expected_data e
+            LEFT JOIN "RoutingFormResponseDenormalized" d ON e.id = d.id
+            WHERE CASE 
+                WHEN d.id IS NULL THEN true  -- Include missing records
+                WHEN d."formId" IS DISTINCT FROM e.expected_form_id THEN true
+                WHEN d."formName" IS DISTINCT FROM e.expected_form_name THEN true
+                WHEN d."formTeamId" IS DISTINCT FROM e.expected_form_team_id THEN true
+                WHEN d."formUserId" IS DISTINCT FROM e.expected_form_user_id THEN true
+                WHEN d."bookingUid" IS DISTINCT FROM e.expected_booking_uid THEN true
+                WHEN d."bookingId" IS DISTINCT FROM e.expected_booking_id THEN true
+                WHEN d."bookingStatus" IS DISTINCT FROM e.expected_booking_status THEN true
+                WHEN d."bookingStatusOrder" IS DISTINCT FROM e.expected_booking_status_order THEN true
+                WHEN d."bookingCreatedAt" IS DISTINCT FROM e.expected_booking_created_at THEN true
+                WHEN d."bookingStartTime" IS DISTINCT FROM e.expected_booking_start_time THEN true
+                WHEN d."bookingEndTime" IS DISTINCT FROM e.expected_booking_end_time THEN true
+                WHEN d."bookingUserId" IS DISTINCT FROM e.expected_booking_user_id THEN true
+                WHEN d."bookingUserName" IS DISTINCT FROM e.expected_booking_user_name THEN true
+                WHEN d."bookingUserEmail" IS DISTINCT FROM e.expected_booking_user_email THEN true
+                WHEN d."bookingUserAvatarUrl" IS DISTINCT FROM e.expected_booking_user_avatar_url THEN true
+                WHEN d."bookingAssignmentReason" IS DISTINCT FROM e.expected_booking_assignment_reason THEN true
+                WHEN d."eventTypeId" IS DISTINCT FROM e.expected_event_type_id THEN true
+                WHEN d."eventTypeParentId" IS DISTINCT FROM e.expected_event_type_parent_id THEN true
+                WHEN d."eventTypeSchedulingType"::text IS DISTINCT FROM e.expected_event_type_scheduling_type THEN true
+                WHEN d."createdAt" IS DISTINCT FROM e.expected_created_at THEN true
+                WHEN d."utm_source" IS DISTINCT FROM e.expected_utm_source THEN true
+                WHEN d."utm_medium" IS DISTINCT FROM e.expected_utm_medium THEN true
+                WHEN d."utm_campaign" IS DISTINCT FROM e.expected_utm_campaign THEN true
+                WHEN d."utm_term" IS DISTINCT FROM e.expected_utm_term THEN true
+                WHEN d."utm_content" IS DISTINCT FROM e.expected_utm_content THEN true
+                ELSE false  -- Don't include valid records
+            END
+        )
+        INSERT INTO "RoutingFormResponseDenormalized" (
+            id,
+            "formId",
+            "formName",
+            "formTeamId",
+            "formUserId",
+            "bookingUid",
+            "bookingId",
+            "bookingStatus",
+            "bookingStatusOrder",
+            "bookingCreatedAt",
+            "bookingStartTime",
+            "bookingEndTime",
+            "bookingUserId",
+            "bookingUserName",
+            "bookingUserEmail",
+            "bookingUserAvatarUrl",
+            "bookingAssignmentReason",
+            "eventTypeId",
+            "eventTypeParentId",
+            "eventTypeSchedulingType",
+            "createdAt",
+            "utm_source",
+            "utm_medium",
+            "utm_campaign",
+            "utm_term",
+            "utm_content"
+        )
+        SELECT 
+            r.id,
+            r.expected_form_id,
+            r.expected_form_name,
+            r.expected_form_team_id,
+            r.expected_form_user_id,
+            r.expected_booking_uid,
+            r.expected_booking_id,
+            r.expected_booking_status,
+            r.expected_booking_status_order,
+            r.expected_booking_created_at,
+            r.expected_booking_start_time,
+            r.expected_booking_end_time,
+            r.expected_booking_user_id,
+            r.expected_booking_user_name,
+            r.expected_booking_user_email,
+            r.expected_booking_user_avatar_url,
+            r.expected_booking_assignment_reason,
+            r.expected_event_type_id,
+            r.expected_event_type_parent_id,
+            r.expected_event_type_scheduling_type::text,
+            r.expected_created_at,
+            r.expected_utm_source,
+            r.expected_utm_medium,
+            r.expected_utm_campaign,
+            r.expected_utm_term,
+            r.expected_utm_content
+        FROM records_to_update r
+        ON CONFLICT (id) DO UPDATE SET
+            "formId" = EXCLUDED."formId",
+            "formName" = EXCLUDED."formName",
+            "formTeamId" = EXCLUDED."formTeamId",
+            "formUserId" = EXCLUDED."formUserId",
+            "bookingUid" = EXCLUDED."bookingUid",
+            "bookingId" = EXCLUDED."bookingId",
+            "bookingStatus" = EXCLUDED."bookingStatus",
+            "bookingStatusOrder" = EXCLUDED."bookingStatusOrder",
+            "bookingCreatedAt" = EXCLUDED."bookingCreatedAt",
+            "bookingStartTime" = EXCLUDED."bookingStartTime",
+            "bookingEndTime" = EXCLUDED."bookingEndTime",
+            "bookingUserId" = EXCLUDED."bookingUserId",
+            "bookingUserName" = EXCLUDED."bookingUserName",
+            "bookingUserEmail" = EXCLUDED."bookingUserEmail",
+            "bookingUserAvatarUrl" = EXCLUDED."bookingUserAvatarUrl",
+            "bookingAssignmentReason" = EXCLUDED."bookingAssignmentReason",
+            "eventTypeId" = EXCLUDED."eventTypeId",
+            "eventTypeParentId" = EXCLUDED."eventTypeParentId",
+            "eventTypeSchedulingType" = EXCLUDED."eventTypeSchedulingType",
+            "createdAt" = EXCLUDED."createdAt",
+            "utm_source" = EXCLUDED."utm_source",
+            "utm_medium" = EXCLUDED."utm_medium",
+            "utm_campaign" = EXCLUDED."utm_campaign",
+            "utm_term" = EXCLUDED."utm_term",
+            "utm_content" = EXCLUDED."utm_content";
+
+        GET DIAGNOSTICS chunk_updated_count = ROW_COUNT;
+        processed_count := processed_count + chunk_updated_count;
+
+        RAISE NOTICE 'Chunk processed: IDs %-% (updated/inserted: % records, total updated: %)', 
+            current_id, current_id + chunk_size - 1, chunk_updated_count, processed_count;
+    END LOOP;
+
+    RAISE NOTICE 'Migration completed: processed up to ID % (total updated/inserted: % records out of % total records)', 
+        end_id, processed_count, total_count;
+END $$;

packages/prisma/migrations/20250618093923_routing_form_response_field_backfill/migration.sql

@@ -0,0 +1,125 @@
+DO $$
+DECLARE
+    chunk_size INTEGER := 1000;
+    sleep_interval FLOAT := 1;
+    start_id INTEGER := 1;
+    end_id INTEGER;
+    current_id INTEGER;
+    response_record RECORD;
+    processed_count INTEGER := 0;
+    chunk_updated_count INTEGER := 0;
+    total_count INTEGER;
+BEGIN
+    -- Get the maximum ID and total count from the App_RoutingForms_FormResponse table
+    SELECT COALESCE(MAX(id), 0) INTO end_id FROM "App_RoutingForms_FormResponse";
+    SELECT COUNT(*) INTO total_count FROM "App_RoutingForms_FormResponse";
+
+    -- Handle case where there are no records to process
+    IF total_count = 0 THEN
+        RAISE NOTICE 'No records found in App_RoutingForms_FormResponse table. Migration completed.';
+        RETURN;
+    END IF;
+
+    RAISE NOTICE 'Starting migration: processing up to ID % (total responses: %)', end_id, total_count;
+
+    FOR current_id IN SELECT * FROM generate_series(start_id, end_id, chunk_size)
+    LOOP
+        -- Process only responses that have missing or incorrect field data
+        FOR response_record IN 
+            WITH form_response_fields AS (
+                -- Extract expected fields from the response data
+                SELECT
+                    r.id as "responseId",
+                    r."formId",
+                    field->>'id' as "fieldId",
+                    field->>'type' as "fieldType",
+                    response_value->>'value' as raw_value,
+                    jsonb_typeof(response_value->'value') as value_type,
+                    -- Extract array values for multiselect
+                    CASE
+                        WHEN field->>'type' = 'multiselect' AND jsonb_typeof(response_value->'value') = 'array'
+                        THEN ARRAY(SELECT jsonb_array_elements_text(response_value->'value'))
+                    END as expected_array,
+                    -- Extract first element for select type when it's an array
+                    CASE
+                        WHEN field->>'type' = 'select' AND jsonb_typeof(response_value->'value') = 'array'
+                        THEN (response_value->'value'->0)::text
+                        WHEN field->>'type' = 'select' AND jsonb_typeof(response_value->'value') = 'string'
+                        THEN response_value->>'value'
+                    END as expected_select_value
+                FROM "App_RoutingForms_FormResponse" r
+                CROSS JOIN LATERAL jsonb_array_elements(
+                    (
+                        SELECT fields::jsonb
+                        FROM "App_RoutingForms_Form" f
+                        WHERE f.id = r."formId"
+                    )
+                ) as field
+                CROSS JOIN LATERAL (
+                    SELECT r.response::jsonb->(field->>'id') as response_value
+                ) as rv
+                WHERE r.id BETWEEN current_id AND current_id + chunk_size - 1
+                AND r.response::jsonb ? (field->>'id')  -- Only include fields that exist in response
+            ),
+            responses_needing_update AS (
+                SELECT DISTINCT f."responseId"
+                FROM form_response_fields f
+                LEFT JOIN "RoutingFormResponseField" rf ON
+                    rf."responseId" = f."responseId" AND
+                    rf."fieldId" = f."fieldId"
+                WHERE
+                    -- Case 1: Field doesn't exist in denormalized table
+                    rf.id IS NULL OR
+                    -- Case 2: Multiselect field validation - compare actual array contents
+                    (f."fieldType" = 'multiselect' AND f.value_type = 'array' AND (
+                        rf."valueStringArray" IS NULL OR
+                        rf."valueStringArray" != f.expected_array OR
+                        array_length(rf."valueStringArray", 1) != array_length(f.expected_array, 1)
+                    )) OR
+                    -- Case 3: Number field validation - compare as numbers
+                    (f."fieldType" = 'number' AND f.value_type = 'number' AND (
+                        rf."valueNumber" IS NULL OR
+                        rf."valueNumber" != (f.raw_value)::decimal
+                    )) OR
+                    -- Case 4: Select field validation - compare expected select value
+                    (f."fieldType" = 'select' AND (
+                        rf."valueString" IS NULL OR
+                        rf."valueString" != f.expected_select_value
+                    )) OR
+                    -- Case 5: Other string field validation
+                    (f."fieldType" NOT IN ('multiselect', 'number', 'select') AND (
+                        rf."valueString" IS NULL OR
+                        rf."valueString" != f.raw_value
+                    ))
+            )
+            SELECT r.id
+            FROM responses_needing_update rnu
+            INNER JOIN "App_RoutingForms_FormResponse" r ON r.id = rnu."responseId"
+            ORDER BY r.id
+        LOOP
+            BEGIN
+                -- Use the reprocess_routing_form_response_fields function
+                PERFORM reprocess_routing_form_response_fields(response_record.id);
+                chunk_updated_count := chunk_updated_count + 1;
+
+            EXCEPTION WHEN OTHERS THEN
+                RAISE WARNING 'Failed to process responseId %: %', response_record.id, SQLERRM;
+                CONTINUE;
+            END;
+        END LOOP;
+
+        processed_count := processed_count + chunk_updated_count;
+
+        RAISE NOTICE 'Chunk processed: IDs %-% (updated: % records, total updated: %)', 
+            current_id, current_id + chunk_size - 1, chunk_updated_count, processed_count;
+
+        -- Reset chunk counter for next iteration
+        chunk_updated_count := 0;
+
+        -- Sleep after each chunk (outside transaction to avoid holding locks)
+        PERFORM pg_sleep(sleep_interval);
+    END LOOP;
+
+    RAISE NOTICE 'Migration completed: processed up to ID % (total updated: % records out of % total records)', 
+        end_id, processed_count, total_count;
+END $$;