Merge branch 'main' into feature/user-polling-options

Author: Haarolean

.dev/dev_arm64.yaml renamed to .dev/dev.yaml

@@ -1,9 +1,3 @@
-# This is a compose file designed for arm64/Apple Silicon systems
-# To adapt this to x86 please find and replace ".arm64" with empty
-
-# ARM64 supported images for kafka can be found here
-# https://hub.docker.com/r/confluentinc/cp-kafka/tags?page=1&name=arm64
----
 version: '3.8'
 name: "kafbat-ui-dev"
 
@@ -32,8 +26,7 @@ services:
       KAFKA_CLUSTERS_0_AUDIT_CONSOLEAUDITENABLED: 'true'
 
   kafka0:
-    image: confluentinc/cp-kafka:7.8.0.arm64
-    user: "0:0"
+    image: confluentinc/cp-kafka:7.8.0
     hostname: kafka0
     container_name: kafka0
     ports:
@@ -60,7 +53,7 @@ services:
       CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk'
 
   schema-registry0:
-    image: confluentinc/cp-schema-registry:7.8.0.arm64
+    image: confluentinc/cp-schema-registry:7.8.0
     ports:
       - 8085:8085
     depends_on:
@@ -76,7 +69,7 @@ services:
       SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
 
   kafka-connect0:
-    image: confluentinc/cp-kafka-connect:7.8.0.arm64
+    image: confluentinc/cp-kafka-connect:7.8.0
     ports:
       - 8083:8083
     depends_on:
@@ -101,7 +94,7 @@ services:
       CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components,/usr/local/share/kafka/plugins,/usr/share/filestream-connectors"
 
   ksqldb0:
-    image: confluentinc/cp-ksqldb-server:7.8.0.arm64
+    image: confluentinc/cp-ksqldb-server:7.8.0
     depends_on:
       - kafka0
       - kafka-connect0
@@ -119,7 +112,7 @@ services:
       KSQL_CACHE_MAX_BYTES_BUFFERING: 0
 
   kafka-init-topics:
-    image: confluentinc/cp-kafka:7.8.0.arm64
+    image: confluentinc/cp-kafka:7.8.0
     volumes:
       - ../documentation/compose/data/message.json:/data/message.json
     depends_on:

.github/FUNDING.yml

@@ -1 +1,2 @@
 github: [kafbat]
+open_collective: kafka-ui

.github/workflows/cve_checks.yml

@@ -1,5 +1,9 @@
 name: "Infra: CVE checks"
 on:
+  pull_request:
+    types: [ "opened", "reopened", "synchronize" ]
+  push:
+    branches: [ "main" ]
   workflow_dispatch:
   schedule:
     # * is a special character in YAML so you have to quote this string
@@ -71,7 +75,7 @@ jobs:
 
   notify:
     needs: check-cves
-    if: ${{ always() && needs.build-and-test.result == 'failure' }}
+    if: ${{ always() && needs.build-and-test.result == 'failure' && github.event_name == 'schedule' }}
     uses: ./.github/workflows/infra_discord_hook.yml
     with:
       message: "Attention! CVE checks run failed! Please fix them CVEs :("

.github/workflows/docker_publish.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        registry: [ 'docker.io', 'ghcr.io', 'ecr' ]
+        registry: [ 'docker.io', 'ghcr.io', 'public.ecr.aws' ]
 
     runs-on: ubuntu-latest
     steps:
@@ -31,7 +31,8 @@ jobs:
           name: image
           path: /tmp
 
-      # setup containerd to preserve provenance attestations :https://docs.docker.com/build/attestations/#creating-attestations
+      # setup containerd to preserve provenance attestations:
+      # https://docs.docker.com/build/attestations/#creating-attestations
       - name: Setup docker with containerd
         uses: crazy-max/ghaction-setup-docker@v3
         with:
@@ -63,33 +64,33 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Configure AWS credentials
-        if: matrix.registry == 'ecr'
+        if: matrix.registry == 'public.ecr.aws'
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: us-east-1 # This region only for public ECR
           role-to-assume: ${{ secrets.AWS_ROLE }}
 
       - name: Login to public ECR
-        if: matrix.registry == 'ecr'
+        if: matrix.registry == 'public.ecr.aws'
         id: login-ecr-public
         uses: aws-actions/amazon-ecr-login@v2
         with:
           registry-type: public
 
-      - name: define env vars
+      - name: Define env vars for container registry URL
         run: |
-          if [ ${{matrix.registry }} == 'docker.io' ]; then
-            echo "REGISTRY=${{ matrix.registry }}" >> $GITHUB_ENV
-            echo "REPOSITORY=${{ github.repository }}" >> $GITHUB_ENV
-          elif [ ${{ matrix.registry }} == 'ghcr.io' ]; then
-            echo "REGISTRY=${{ matrix.registry }}" >> $GITHUB_ENV
-            echo "REPOSITORY=${{ github.repository }}" >> $GITHUB_ENV
-          elif [ ${{ matrix.registry }} == 'ecr' ]; then
+          if [ ${{ matrix.registry }} == 'public.ecr.aws' ]; then
+            # vars.ECR_REGISTRY value is expected to be of the `public.ecr.aws/<public_ecr_id>` form
+            # The `public_ecr_id` must be a *default* alias associated with public regsitry (rather
+            # than a custom alias)
             echo "REGISTRY=${{ vars.ECR_REGISTRY }}" >> $GITHUB_ENV
+            # Trim GH Org name so that resulting Public ECR URL has no duplicate org name
+            # Public ECR default alias: public.ecr.aws/<public_ecr_id>/kafka-ui
+            # Public ECR custom alias:  public.ecr.aws/kafbat/kafka-ui
+            echo "REPOSITORY=$(basename ${{ github.repository }})" >> $GITHUB_ENV
+          else # this covers the case of docker.io and ghcr.io
+            echo "REGISTRY=${{ matrix.registry }}" >> $GITHUB_ENV
             echo "REPOSITORY=${{ github.repository }}" >> $GITHUB_ENV
-          else
-            echo "REGISTRY=" >> $GITHUB_ENV
-            echo "REPOSITORY=notworking" >> $GITHUB_ENV
           fi
 
       - name: Push images to ${{ matrix.registry }}

.github/workflows/frontend_tests.yml

@@ -23,12 +23,12 @@ jobs:
 
       - uses: pnpm/[email protected]
         with:
-          version: 9.15.0
+          version: 9.15.4
 
       - name: Install node
         uses: actions/[email protected]
         with:
-          node-version: "18.17.1"
+          node-version: "22.12.0"
           cache: "pnpm"
           cache-dependency-path: "./frontend/pnpm-lock.yaml"
 

.mvn/jvm.config

@@ -0,0 +1 @@
+-Djava.net.useSystemProxies=true

api/pom.xml

@@ -212,6 +212,19 @@
             <version>${okhttp3.mockwebserver.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-clients</artifactId>
+            <version>${confluent.version}-ccs</version>
+            <classifier>test</classifier>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcpkix-jdk18on</artifactId>
+            <version>1.80</version>
+            <scope>test</scope>
+        </dependency>
 
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -329,7 +342,7 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
                 <configuration>
-                    <argLine>@{argLine} --illegal-access=permit</argLine>
+                    <argLine>@{argLine}</argLine>
                 </configuration>
             </plugin>
             <plugin>
@@ -492,6 +505,7 @@
                                 </goals>
                                 <configuration>
                                     <arguments>build</arguments>
+                                    <pnpmInheritsProxyConfigFromMaven>false</pnpmInheritsProxyConfigFromMaven>
                                 </configuration>
                             </execution>
                         </executions>

api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java

@@ -12,7 +12,7 @@
 import io.kafbat.ui.connect.model.ConnectorTopics;
 import io.kafbat.ui.connect.model.NewConnector;
 import io.kafbat.ui.connect.model.TaskStatus;
-import io.kafbat.ui.exception.KafkaConnectConflictReponseException;
+import io.kafbat.ui.exception.KafkaConnectConflictResponseException;
 import io.kafbat.ui.exception.ValidationException;
 import io.kafbat.ui.util.WebClientConfigurator;
 import jakarta.validation.constraints.NotNull;
@@ -48,7 +48,7 @@ private static Retry conflictCodeRetry() {
         .fixedDelay(MAX_RETRIES, RETRIES_DELAY)
         .filter(e -> e instanceof WebClientResponseException.Conflict)
         .onRetryExhaustedThrow((spec, signal) ->
-            new KafkaConnectConflictReponseException(
+            new KafkaConnectConflictResponseException(
                 (WebClientResponseException.Conflict) signal.failure()));
   }
 
@@ -238,6 +238,16 @@ public Mono<ResponseEntity<Void>> pauseConnectorWithHttpInfo(String connectorNam
     return withRetryOnConflictOrRebalance(super.pauseConnectorWithHttpInfo(connectorName));
   }
 
+  @Override
+  public Mono<Void> stopConnector(String connectorName) throws WebClientResponseException {
+    return withRetryOnConflictOrRebalance(super.stopConnector(connectorName));
+  }
+
+  @Override
+  public Mono<ResponseEntity<Void>> stopConnectorWithHttpInfo(String connectorName) throws WebClientResponseException {
+    return withRetryOnConflictOrRebalance(super.stopConnectorWithHttpInfo(connectorName));
+  }
+
   @Override
   public Mono<Void> restartConnector(String connectorName, Boolean includeTasks, Boolean onlyFailed)
       throws WebClientResponseException {
@@ -261,6 +271,18 @@ public Mono<ResponseEntity<Void>> restartConnectorTaskWithHttpInfo(String connec
     return withRetryOnConflictOrRebalance(super.restartConnectorTaskWithHttpInfo(connectorName, taskId));
   }
 
+  @Override
+  public Mono<Void> resetConnectorOffsets(String connectorName)
+      throws WebClientResponseException {
+    return withRetryOnConflictOrRebalance(super.resetConnectorOffsets(connectorName));
+  }
+
+  @Override
+  public Mono<ResponseEntity<Void>> resetConnectorOffsetsWithHttpInfo(String connectorName)
+      throws WebClientResponseException {
+    return withRetryOnConflictOrRebalance(super.resetConnectorOffsetsWithHttpInfo(connectorName));
+  }
+
   @Override
   public Mono<Void> resumeConnector(String connectorName) throws WebClientResponseException {
     return withRetryOnRebalance(super.resumeConnector(connectorName));

api/src/main/java/io/kafbat/ui/config/auth/BasicAuthSecurityConfig.java

@@ -1,8 +1,6 @@
 package io.kafbat.ui.config.auth;
 
-import io.kafbat.ui.util.EmptyRedirectStrategy;
 import io.kafbat.ui.util.StaticFileWebFilter;
-import java.net.URI;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
@@ -12,8 +10,6 @@
 import org.springframework.security.config.web.server.SecurityWebFiltersOrder;
 import org.springframework.security.config.web.server.ServerHttpSecurity;
 import org.springframework.security.web.server.SecurityWebFilterChain;
-import org.springframework.security.web.server.authentication.RedirectServerAuthenticationSuccessHandler;
-import org.springframework.security.web.server.authentication.logout.RedirectServerLogoutSuccessHandler;
 import org.springframework.security.web.server.util.matcher.ServerWebExchangeMatchers;
 
 @Configuration

api/src/main/java/io/kafbat/ui/config/auth/LdapSecurityConfig.java

@@ -3,10 +3,13 @@
 import io.kafbat.ui.service.rbac.AccessControlService;
 import io.kafbat.ui.service.rbac.extractor.RbacActiveDirectoryAuthoritiesExtractor;
 import io.kafbat.ui.service.rbac.extractor.RbacLdapAuthoritiesExtractor;
+import io.kafbat.ui.util.CustomSslSocketFactory;
 import io.kafbat.ui.util.StaticFileWebFilter;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Stream;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -47,6 +50,9 @@
 @RequiredArgsConstructor
 @Slf4j
 public class LdapSecurityConfig extends AbstractAuthSecurityConfig {
+  private static final Map<String, Object> BASE_ENV_PROPS = Map.of(
+      "java.naming.ldap.factory.socket", CustomSslSocketFactory.class.getName()
+  );
 
   private final LdapProperties props;
 
@@ -63,13 +69,10 @@ public AbstractLdapAuthenticationProvider authenticationProvider(LdapAuthorities
 
     AbstractLdapAuthenticationProvider authProvider;
 
-    if (!props.isActiveDirectory()) {
-      authProvider = new LdapAuthenticationProvider(ba, authoritiesExtractor);
+    if (props.isActiveDirectory()) {
+      authProvider = activeDirectoryProvider(authoritiesExtractor);
     } else {
-      authProvider = new ActiveDirectoryLdapAuthenticationProvider(props.getActiveDirectoryDomain(),
-          props.getUrls());
-      authProvider.setUseAuthenticationRequestCredentials(true);
-      ((ActiveDirectoryLdapAuthenticationProvider) authProvider).setAuthoritiesPopulator(authoritiesExtractor);
+      authProvider = new LdapAuthenticationProvider(ba, authoritiesExtractor);
     }
 
     if (rbacEnabled) {
@@ -80,7 +83,7 @@ public AbstractLdapAuthenticationProvider authenticationProvider(LdapAuthorities
   }
 
   @Bean
-  @ConditionalOnProperty(value = "oauth2.ldap.activeDirectory", havingValue = "false")
+  @ConditionalOnProperty(value = "oauth2.ldap.activeDirectory", havingValue = "false", matchIfMissing = true)
   public BindAuthenticator ldapBindAuthentication(LdapContextSource ldapContextSource) {
     BindAuthenticator ba = new BindAuthenticator(ldapContextSource);
 
@@ -159,6 +162,22 @@ public SecurityWebFilterChain configureLdap(ServerHttpSecurity http) {
     return builder.build();
   }
 
+  private ActiveDirectoryLdapAuthenticationProvider activeDirectoryProvider(LdapAuthoritiesPopulator populator) {
+    ActiveDirectoryLdapAuthenticationProvider provider = new ActiveDirectoryLdapAuthenticationProvider(
+        props.getActiveDirectoryDomain(),
+        props.getUrls()
+    );
+
+    provider.setUseAuthenticationRequestCredentials(true);
+    provider.setAuthoritiesPopulator(populator);
+
+    if (Stream.of(props.getUrls().split(",")).anyMatch(url -> url.startsWith("ldaps://"))) {
+      provider.setContextEnvironmentProperties(BASE_ENV_PROPS);
+    }
+
+    return provider;
+  }
+
   private static class RbacUserDetailsMapper extends LdapUserDetailsMapper {
     @Override
     public UserDetails mapUserFromContext(DirContextOperations ctx, String username,

api/src/main/java/io/kafbat/ui/config/auth/condition/ActiveDirectoryCondition.java

@@ -40,7 +40,7 @@ public Mono<Void> handle(WebFilterExchange exchange, Authentication authenticati
         requestUri.getPath(), requestUri.getQuery());
 
     final UriComponents baseUrl = UriComponentsBuilder
-        .fromHttpUrl(fullUrl)
+        .fromUriString(fullUrl)
         .replacePath("/")
         .replaceQuery(null)
         .fragment(null)