Merge pull request #129 from AllenNeuralDynamics/release-v1.0.0

Release v1.0.0

Author: helen-m-lin

.github/workflows/add_issue_to_project_board.yml

@@ -0,0 +1,15 @@
+name: Auto add issues to project board
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+        with:
+          project-url: https://github.com/orgs/AllenNeuralDynamics/projects/9
+          github-token: ${{ secrets.SERVICE_TOKEN }}

.readthedocs.yaml

@@ -5,6 +5,9 @@ build:
   tools:
     python: "3.8"
 
+sphinx:
+  configuration: docs/source/conf.py
+
 python:
   install:
     - method: pip

README.md

@@ -10,4 +10,31 @@ API to interact with a few AIND databases. We have two primary databases:
    unstructured json documents. The DocDB contains AIND metadata.
 2. A relational database to store structured tables.
 
+## Installation
+
+Basic installation:
+```bash
+pip install aind-data-access-api
+```
+
+The package includes optional features that require additional dependencies:
+
+### Document Database (DocDB)
+To use the `MetadataDbClient` and other DocDB features:
+```bash
+pip install "aind-data-access-api[docdb]"
+```
+Note: The quotes are required when using zsh or other shells that interpret square brackets.
+
+### Relational Database
+For RDS functionality:
+```bash
+pip install "aind-data-access-api[rds]"
+```
+
+### Other Install Options
+- AWS Secrets management: `pip install "aind-data-access-api[secrets]"`
+- Helpers: `pip install "aind-data-access-api[helpers]"`
+- All features: `pip install "aind-data-access-api[full]"`
+
 More information can be found at [readthedocs](https://aind-data-access-api.readthedocs.io).

docs/source/ExamplesDocDBDirectConnection.rst

@@ -25,7 +25,7 @@ Count Example 1: Get # of records with a certain subject_id
 
   credentials = DocumentDbSSHCredentials()
   with DocumentDbSSHClient(credentials=credentials) as doc_db_client:
-      filter = {"subject.subject_id": "689418"}
+      filter = {"subject.subject_id": "731015"}
       count = doc_db_client.collection.count_documents(filter)
       print(count)
 
@@ -35,7 +35,7 @@ Filter Example 1: Get records with a certain subject_id
 .. code:: python
 
   with DocumentDbSSHClient(credentials=credentials) as doc_db_client:
-      filter = {"subject.subject_id": "689418"}
+      filter = {"subject.subject_id": "731015"}
       records = list(
           doc_db_client.collection.find(filter=filter)
       )
@@ -47,7 +47,7 @@ With projection (recommended):
 .. code:: python
 
   with DocumentDbSSHClient(credentials=credentials) as doc_db_client:
-      filter = {"subject.subject_id": "689418"}
+      filter = {"subject.subject_id": "731015"}
       projection = {
           "name": 1,
           "created": 1,
@@ -68,7 +68,7 @@ Filter Example 2: Get records with a certain breeding group
 
   with DocumentDbSSHClient(credentials=credentials) as doc_db_client:
       filter = {
-          "subject.breeding_info.breeding_group": "Chat-IRES-Cre_Jax006410"
+          "subject.breeding_info.breeding_group": "Slc17a6-IRES-Cre;Ai230-hyg(ND)"
       }
       records = list(
           doc_db_client.collection.find(filter=filter)
@@ -82,7 +82,7 @@ With projection (recommended):
 
   with DocumentDbSSHClient(credentials=credentials) as doc_db_client:
       filter = {
-          "subject.breeding_info.breeding_group": "Chat-IRES-Cre_Jax006410"
+          "subject.breeding_info.breeding_group": "Slc17a6-IRES-Cre;Ai230-hyg(ND)"
       }
       projection = {
           "name": 1,
@@ -125,37 +125,19 @@ https://www.mongodb.com/docs/manual/aggregation/
 Updating Metadata
 ~~~~~~~~~~~~~~~~~~~~~~
 
-We provide several utility functions for interacting with DocDB within the
-``aind_data_access_api.utils`` module. Below is an example of how to use these
-functions to update records in DocDB.
+Below is an example of how to update records in DocDB using ``DocumentDbSSHClient``.
 
 .. code:: python
 
-  import json
   import logging
-  from typing import List, Optional
 
   from aind_data_access_api.document_db_ssh import (
       DocumentDbSSHClient,
       DocumentDbSSHCredentials,
   )
-  from aind_data_schema.core.metadata import Metadata
-
-  from aind_data_access_api.utils import paginate_docdb, is_dict_corrupt
 
   logging.basicConfig(level="INFO")
 
-  def _process_docdb_records(records: List[dict], doc_db_client: DocumentDbSSHClient, dryrun: bool) -> None:
-      """
-      Process records.
-      Parameters
-      ----------
-      records : List[dict]
-
-      """
-      for record in records:
-          _process_docdb_record(record=record, doc_db_client=doc_db_client, dryrun=dryrun)
-
   def _process_docdb_record(record: dict, doc_db_client: DocumentDbSSHClient, dryrun: bool) -> None:
       """
       Process record. This example updates the data_description.name field
@@ -171,58 +153,17 @@ functions to update records in DocDB.
       location = record.get("location")
       if _id:
           if record.get("data_description") and record["data_description"].get("name") != name:
-              # Option 1: update specific fields(s) only
+              # update specific fields(s) only
               new_fields = {
                   "data_description.name": name
               }
               update_docdb_record_partial(record_id=_id, new_fields=new_fields, doc_db_client=doc_db_client, dryrun=dryrun)
-              # Option 2: build new record Metadata.py and replace entire document with new record
-              # new_record = build_new_docdb_record(record=record)
-              # if new_record is not None:
-              #     update_docdb_record_entire(record_id=_id, new_record=new_record, doc_db_client=doc_db_client, dryrun=dryrun)
           # else:
           #     logging.info(f"Record for {location} does not need to be updated.")
       else:
           logging.warning(f"Record for {location} does not have an _id field! Skipping.")
 
 
-  def build_new_docdb_record(record: Optional[dict]) -> Optional[dict]:
-      """Build new record from existing record. This example updates the
-      data_description.name field if it does not match the record.name field.
-
-      Parameters
-      ----------
-      record : Optional[dict]
-
-      Returns
-      -------
-      Optional[dict]
-          The new record, or None if the record cannot be constructed.
-      """
-      # Example: Update record.data_description.name if not matching record.name
-      new_record = None
-      if record.get("data_description") and record["data_description"].get("name") != name:
-          _id = record.get("_id")
-          name = record.get("name")
-          location = record.get("location")
-          created = record.get("created")
-          if _id is None or name is None or location is None or created is None:
-              logging.warning(f"Record does not have _id, name, location, or created! Skipping.")
-              return None
-          try:
-              new_record = record.copy()
-              new_record["data_description"]["name"] = name
-              new_record_str = Metadata.model_construct(
-                  **new_record
-              ).model_dump_json(warnings=False, by_alias=True)
-              new_record = json.loads(new_record_str)
-              if is_dict_corrupt(new_record):
-                  logging.warning(f"New record for {location} is corrupt! Skipping.")
-                  new_record = None
-          except Exception:
-              new_record = None
-      return new_record
-
   def update_docdb_record_partial(record_id: str, new_fields: dict, doc_db_client: DocumentDbSSHClient, dryrun: bool) -> None:
       """
       Update record in docdb by updating specific fields only.
@@ -244,54 +185,24 @@ functions to update records in DocDB.
               upsert=False,
           )
           logging.info(response.raw_result)
-
-
-  def update_docdb_record_entire(record_id: str, new_record: dict, doc_db_client: DocumentDbSSHClient, dryrun: bool) -> None:
-      """
-      Update record in docdb by replacing the entire document with new record.
-      Parameters
-      ----------
-      record_id : str
-          The _id of the record to update.
-      new_record : dict
-          The new record to replace the existing record with.
-
-      """
-      if is_dict_corrupt(new_record) or record_id != new_record.get("_id"):
-          logging.warning(f"Attempting to update corrupt record {record_id}! Skipping.")
-          return
-      if dryrun:
-          logging.info(f"(dryrun) doc_db_client.collection.update_one: {record_id}")
-      else:
-          logging.info(f"doc_db_client.collection.update_one: {record_id}")
-          response = doc_db_client.collection.update_one(
-              {"_id": record_id},
-              {"$set": new_record},
-              upsert=False,
-          )
-          logging.info(response.raw_result)
             
           
   if __name__ == "__main__":
       credentials = DocumentDbSSHCredentials()    # credentials in environment
       dryrun = True
-      filter = {"location": {"$regex": ".*s3://codeocean-s3datasetsbucket.*"}}
+      filter = {"location": {"$regex": ".*s3://aind-open-data.*"}}
       projection = None
       
       with DocumentDbSSHClient(credentials=credentials) as doc_db_client:
           db_name = doc_db_client.database_name
           col_name = doc_db_client.collection_name
           # count = doc_db_client.collection.count_documents(filter)
-          # logging.info(f"{db_name}.{col_name}: Found {count} records with {filter}: {count}")
+          # logging.info(f"{db_name}.{col_name}: Found {count} records with {filter}")
 
           logging.info(f"{db_name}.{col_name}: Starting to scan for {filter}.")
-          docdb_pages = paginate_docdb(
-              db_name=doc_db_client.database_name,
-              collection_name=doc_db_client.collection_name,
-              docdb_client=doc_db_client._client,
-              page_size=500,
-              filter_query=filter,
+          records = doc_db_client.collection.find(
+              filter=filter,
           )
-          for page in docdb_pages:
-              _process_docdb_records(records=page, doc_db_client=doc_db_client, dryrun=dryrun)
+          for record in records:
+              _process_docdb_record(record=record, doc_db_client=doc_db_client, dryrun=dryrun)
           logging.info(f"{db_name}.{col_name}:Finished scanning through DocDb.")