Added SpannerMetadata Submodule and Extension Loader

Added SpannerMetadata Submodule and ExtensionLoader

Added SpannerMetadata Submodule and ExtensionLoader

changes

Author: Komal Yadav

cdap-SpannerMetadata/src/main/java/io/cdap/cdap/metadata/spanner/SpannerMetadataStorage.java

@@ -106,43 +106,9 @@
  */
 public class SpannerMetadataStorage implements MetadataStorage {
 
-
-    private static final Logger LOG = LoggerFactory.getLogger(SpannerMetadataStorage.class);
-
-    private  String instanceId = "instance";
-    private  String databaseId = "database";
-    private  String projectId = "project";
-    private volatile SpannerOptions options;
-    private volatile Spanner spanner;
-    private volatile DatabaseClient dbClient;
-    private volatile DatabaseAdminClient adminClient;
-
-    private static final Gson gson = new GsonBuilder()
-            .registerTypeAdapter(ScopedName.class, new ScopedNameTypeAdapter())
-            .registerTypeAdapter(ScopedNameOfKind.class, new ScopedNameOfKindTypeAdapter())
-            .create();
-
-
     @Override
     public void initialize(MetadataStorageContext context) throws Exception {
-        LOG.info("Initializing SpannerMetadataStorage...");
-        Map<String, String> conf = context.getConfiguration();
-        LOG.info(conf.toString());
-        projectId=getProjectId(conf);
-        databaseId=getDatabaseId(conf);
-        instanceId=getInstanceId(conf);
-        LOG.info(projectId,databaseId,instanceId);
-        try {
-            options = SpannerOptions.newBuilder().setProjectId(projectId).build();
-            spanner = options.getService();
-            dbClient = spanner.getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId));
-            adminClient = spanner.getDatabaseAdminClient();
-            LOG.info("Successfully initialized Spanner client for instance: {}, database: {}", instanceId, databaseId);
-        } catch (SpannerException e) {
-            LOG.error("Error initializing Spanner client: {}", e.getMessage(), e);
-            throw new RuntimeException("Failed to initialize Spanner client", e);
-        }
-        LOG.info("SpannerMetadataStorage initialized.");
+
     }
 
     @Override
@@ -215,14 +181,4 @@ public Object getDatasetMetadata(String datasetName) {
         return Collections.emptyMap();
     }
 
-    public String getProjectId(Map<String, String> conf){
-         return conf.get(projectId);
-    }
-    public String getDatabaseId(Map<String, String> conf){
-        return conf.get(databaseId);
-    }
-    public String getInstanceId(Map<String, String> conf){
-        return conf.get(instanceId);
-    }
-
 }

cdap-app-fabric/src/main/java/io/cdap/cdap/metadata/MetadataHttpHandler.java

@@ -361,7 +361,6 @@ public void searchMetadata(HttpRequest request, HttpResponder responder,
       @Nullable @QueryParam("entityScope") String entityScope,
       @Nullable @QueryParam("responseFormat") @DefaultValue("v5") String responseFormat)
       throws Exception {
-
     SearchRequest searchRequest = getValidatedSearchRequest(scope, namespaces, searchQuery, targets,
         sort,
         offset, limit, numCursors, cursorRequested, cursor,

cdap-common/src/main/java/io/cdap/cdap/extension/AbstractExtensionLoader.java

@@ -274,7 +274,6 @@ private Map<EXTENSION_TYPE, EXTENSION> getAllExtensions(ServiceLoader<EXTENSION>
     Map<EXTENSION_TYPE, EXTENSION> extensions = new HashMap<>();
     Iterator<EXTENSION> iterator = serviceLoader.iterator();
     // Cannot use for each loop here, because we want to catch exceptions during iterator.next().
-    LOG.info(iterator.toString());
     while (iterator.hasNext()) {
       try {
         EXTENSION extension = iterator.next();
@@ -284,6 +283,7 @@ private Map<EXTENSION_TYPE, EXTENSION> getAllExtensions(ServiceLoader<EXTENSION>
 
         for (EXTENSION_TYPE type : getSupportedTypesForProvider(extension)) {
           if (extensions.containsKey(type)) {
+            LOG.info("Ignoring extension {} for type {}", extension, type);
           } else {
             extensions.put(type, extension);
           }
@@ -335,6 +335,7 @@ private ServiceLoader<EXTENSION> createServiceLoader(File dir) {
    */
   private ClassLoader getExtensionParentClassLoader() {
     FilterClassLoader.Filter filter = getExtensionParentClassLoaderFilter();
+
     // SLF4j resources are always coming from parent.
     return new FilterClassLoader(getClass().getClassLoader(), new FilterClassLoader.Filter() {
       @Override

cdap-data-fabric/src/main/java/io/cdap/cdap/data/runtime/MetadataStorageExtensionLoader.java

@@ -14,8 +14,6 @@
 import java.util.Collections;
 import java.util.Set;
 
-import static io.cdap.cdap.common.conf.Constants.Metadata.Metadata_Storage_EXT_DIR;
-
 /**
  * Extension loader for {@link MetadataStorage} implementations.
  */
@@ -26,7 +24,7 @@ public class MetadataStorageExtensionLoader extends AbstractExtensionLoader<Stri
     private static final Set<String> ALLOWED_RESOURCES = createAllowedResources();
     private static final Set<String> ALLOWED_PACKAGES = createPackageSets(ALLOWED_RESOURCES);
 
-    private final CConfiguration cConf;
+    private final boolean ismetadatastorageEnabled;
 
     /**
      * Constructs a {@link MetadataStorageExtensionLoader} to manage the loading of SpannerMetadata
@@ -37,9 +35,10 @@ public class MetadataStorageExtensionLoader extends AbstractExtensionLoader<Stri
      */
     @Inject
     public MetadataStorageExtensionLoader(CConfiguration cConf) {
-        super(Metadata_Storage_EXT_DIR);
-        this.cConf=cConf;
-        LOG.debug("Metadata Storage extensions directory: {}", Metadata_Storage_EXT_DIR);
+        super(cConf.get(Constants.Metadata.Metadata_Storage_EXT_DIR));
+        this.ismetadatastorageEnabled = cConf.getBoolean(Constants.Metadata.Metadata_Storage_Enabled);
+        LOG.debug("Metadata Storage extensions directory: {}",
+                cConf.get(Constants.Metadata.Metadata_Storage_EXT_DIR));
     }
 
     private static Set<String> createAllowedResources() {
@@ -53,7 +52,7 @@ private static Set<String> createAllowedResources() {
 
     @Override
     protected Set<String> getSupportedTypesForProvider(MetadataStorage metadataStorage) {
-        if (!cConf.getBoolean(Constants.Metadata.Metadata_Storage_Enabled)) {
+        if (ismetadatastorageEnabled) {
             LOG.info("metadataStorage is not Empty",Collections.singleton(metadataStorage.getName()));
             return Collections.singleton(metadataStorage.getName());
         }

cdap-data-fabric/src/main/java/io/cdap/cdap/data2/metadata/AuditMetadataStorage.java

@@ -133,7 +133,7 @@ public MetadataChange apply(MetadataMutation mutation, MutationOptions options)
 
   @Override
   public List<MetadataChange> batch(List<? extends MetadataMutation> mutations,
-                                    MutationOptions options) throws IOException {
+      MutationOptions options) throws IOException {
     List<MetadataChange> changes;
     try {
       changes = storage.batch(mutations, options);

cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/common/DefaultStorageProvider.java

@@ -54,7 +54,6 @@ public class DefaultStorageProvider implements StorageProvider {
     this.extensionLoader = extensionLoader;
     this.metricsCollector = metricsCollectionService.getContext(
         Constants.Metrics.STORAGE_METRICS_TAGS);
-    this.extensionLoader.getAll();
   }
 
   @Override

cdap-data-fabric/src/main/java/io/cdap/cdap/spi/data/common/StorageProviderExtensionLoader.java

@@ -30,7 +30,6 @@
 /**
  * A extension loader for {@link StorageProvider}.
  */
-
 final class StorageProviderExtensionLoader extends
     AbstractExtensionLoader<String, StorageProvider> {
 

cdap-metadata-spi/src/main/java/io/cdap/cdap/spi/metadata/MetadataStorage.java

@@ -17,7 +17,6 @@
 package io.cdap.cdap.spi.metadata;
 
 import io.cdap.cdap.api.annotation.Beta;
-
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.List;
@@ -28,63 +27,62 @@
 @Beta
 public interface MetadataStorage extends Closeable {
 
-    String getName();
-
-    default void initialize(MetadataStorageContext context) throws Exception {
-        // no-op
-    }
+  /**
+   * Create all tables or indexes required for operations.
+   */
+  void createIndex() throws IOException;
 
-    Object getDatasetMetadata(String datasetName);
+  /**
+   * Drop all tables or indexes required for operations.
+   */
+  void dropIndex() throws IOException;
 
-    /**
-     * Create all tables or indexes required for operations.
-     */
-    void createIndex() throws IOException;
+  /**
+   * Apply the given mutation to the metadata state.
+   *
+   * @param mutation the mutation to perform
+   * @return the change effected by this mutation
+   */
+  MetadataChange apply(MetadataMutation mutation, MutationOptions options) throws IOException;
 
-    /**
-     * Drop all tables or indexes required for operations.
-     */
-    void dropIndex() throws IOException;
+  /**
+   * Apply a batch of mutations to the metadata state.
+   *
+   * @param mutations the mutations to perform. They are applied in the order given by the
+   *     list.
+   * @return the changes effected by each of the mutations, in the same order as the batch of
+   *     mutations.
+   */
+  List<MetadataChange> batch(List<? extends MetadataMutation> mutations, MutationOptions options)
+      throws IOException;
 
-    /**
-     * Apply the given mutation to the metadata state.
-     *
-     * @param mutation the mutation to perform
-     * @return the change effected by this mutation
-     */
-    MetadataChange apply(MetadataMutation mutation, MutationOptions options) throws IOException;
+  /**
+   * Retrieve the metadata for an entity.
+   *
+   * @param read the read operation to perform
+   * @return the metadata for the entity, never null.
+   */
+  Metadata read(Read read) throws IOException;
 
-    /**
-     * Apply a batch of mutations to the metadata state.
-     *
-     * @param mutations the mutations to perform. They are applied in the order given by the
-     *                  list.
-     * @return the changes effected by each of the mutations, in the same order as the batch of
-     * mutations.
-     */
-    List<MetadataChange> batch(List<? extends MetadataMutation> mutations, MutationOptions options)
-            throws IOException;
+  /**
+   * Search the metadata and return matching entities.
+   *
+   * @param request the search request
+   * @return the result of the search, never null.
+   */
+  SearchResponse search(SearchRequest request) throws IOException;
 
-    /**
-     * Retrieve the metadata for an entity.
-     *
-     * @param read the read operation to perform
-     * @return the metadata for the entity, never null.
-     */
-    Metadata read(Read read) throws IOException;
+  /**
+   * Close the storage provider. Do not throw exceptions - this will be called when the Metadata
+   * service shuts down.
+   */
+  @Override
+  void close();
 
-    /**
-     * Search the metadata and return matching entities.
-     *
-     * @param request the search request
-     * @return the result of the search, never null.
-     */
-    SearchResponse search(SearchRequest request) throws IOException;
+  String getName();
+  Object getDatasetMetadata(String datasetName);
 
-    /**
-     * Close the storage provider. Do not throw exceptions - this will be called when the Metadata
-     * service shuts down.
-     */
-    @Override
-    void close();
+  default void initialize(MetadataStorageContext context) throws Exception {
+    // no-op
+  }
 }