full switch to not using datasets

Author: hemidactylus

AstraDB_langchain_quickstart_1.ipynb

@@ -36,9 +36,7 @@
     "!pip install --quiet \\\n",
     "    \"langchain>=0.3,<0.4\" \\\n",
     "    \"langchain-astradb>=0.6,<0.7\" \\\n",
-    "    \"langchain-openai>=0.3,<0.4\" \\\n",
-    "    \"datasets>=3.5,<4.0\" \\\n",
-    "    \"numpy<2.0\"  # this fixes a python 3.12 issue"
+    "    \"langchain-openai>=0.3,<0.4\""
    ]
   },
   {
@@ -65,15 +63,14 @@
    "outputs": [],
    "source": [
     "import os\n",
+    "import requests\n",
     "from getpass import getpass\n",
     "\n",
     "from astrapy.info import VectorServiceOptions\n",
     "from langchain_astradb import AstraDBVectorStore\n",
     "\n",
     "from langchain_core.documents import Document\n",
-    "from langchain_openai import OpenAIEmbeddings\n",
-    "\n",
-    "from datasets import load_dataset"
+    "from langchain_openai import OpenAIEmbeddings"
    ]
   },
   {
@@ -96,7 +93,9 @@
     "os.environ[\"ASTRA_DB_API_ENDPOINT\"] = input(\"ASTRA_DB_API_ENDPOINT =\")\n",
     "os.environ[\"ASTRA_DB_APPLICATION_TOKEN\"] = getpass(\"ASTRA_DB_APPLICATION_TOKEN =\")\n",
     "\n",
-    "os.environ[\"ASTRA_DB_KEYSPACE\"] = input(\"ASTRA_DB_KEYSPACE (optional) =\")\n",
+    "if _keyspace := input(\"ASTRA_DB_KEYSPACE (optional) =\"):\n",
+    "    os.environ[\"ASTRA_DB_KEYSPACE\"] = _keyspace\n",
+    "\n",
     "os.environ[\"ASTRA_DB_API_KEY_NAME\"] = input(\"ASTRA_DB_API_KEY_NAME (required for 'vectorize') =\")"
    ]
   },
@@ -159,7 +158,7 @@
    "source": [
     "ASTRA_DB_APPLICATION_TOKEN = os.environ[\"ASTRA_DB_APPLICATION_TOKEN\"]\n",
     "ASTRA_DB_API_ENDPOINT = os.environ[\"ASTRA_DB_API_ENDPOINT\"]\n",
-    "ASTRA_DB_KEYSPACE = os.environ.get(\"ASTRA_DB_KEYSPACE\") or None\n",
+    "ASTRA_DB_KEYSPACE = os.environ.get(\"ASTRA_DB_KEYSPACE\")\n",
     "ASTRA_DB_API_KEY_NAME = os.environ.get(\"ASTRA_DB_API_KEY_NAME\") or None\n",
     "\n",
     "OPENAI_API_KEY = os.environ.get(\"OPENAI_API_KEY\") or None"
@@ -262,7 +261,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "philo_dataset = load_dataset(\"datastax/philosopher-quotes\")[\"train\"]\n",
+    "philo_dataset = requests.get(\n",
+    "    \"https://raw.githubusercontent.com/\"\n",
+    "    \"datastaxdevs/mini-demo-astradb-langchain/\"\n",
+    "    \"refs/heads/main/data/philosopher-quotes.json\"\n",
+    ").json()\n",
     "\n",
     "print(\"An example entry:\")\n",
     "print(philo_dataset[16])"
@@ -288,14 +291,13 @@
     "documents_to_insert = []\n",
     "\n",
     "for entry_idx, entry in enumerate(philo_dataset):\n",
-    "    metadata = {\"author\": entry[\"author\"]}\n",
-    "    if entry[\"tags\"]:\n",
-    "        # Add metadata tags to the metadata dictionary\n",
-    "        for tag in entry[\"tags\"].split(\";\"):\n",
-    "            metadata[tag] = \"y\"\n",
+    "    metadata = {\n",
+    "        \"author\": entry[\"author\"],\n",
+    "        **entry[\"metadata\"],\n",
+    "    }\n",
     "    # Construct the Document, with the quote and metadata tags\n",
     "    new_document = Document(\n",
-    "        id=f\"{entry['author'][:4]}_{entry_idx:03}\",\n",
+    "        id=entry[\"_id\"],\n",
     "        page_content=entry[\"quote\"],\n",
     "        metadata=metadata,\n",
     "    )\n",
@@ -632,7 +634,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.8"
+   "version": "3.12.0"
   }
  },
  "nbformat": 4,

integrate_explicit_embeddings.py

@@ -11,8 +11,8 @@
 
 
 # Import dependencies
-import json
 import os
+import requests
 from getpass import getpass
 
 from astrapy.info import VectorServiceOptions
@@ -21,7 +21,6 @@
 from langchain_core.documents import Document
 from langchain_openai import OpenAIEmbeddings
 
-from datasets import load_dataset
 from dotenv import load_dotenv
 
 
@@ -48,7 +47,11 @@
 
 
 # Load data
-philo_dataset = json.load(open("data/philosopher-quotes.json"))
+philo_dataset = requests.get(
+    "https://raw.githubusercontent.com/"
+    "datastaxdevs/mini-demo-astradb-langchain/"
+    "refs/heads/main/data/philosopher-quotes.json"
+).json()
 
 print("An example entry:")
 print(philo_dataset[16])

integrate_vectorize.py

@@ -11,8 +11,8 @@
 
 
 # Import dependencies
-import json
 import os
+import requests
 from getpass import getpass
 
 from astrapy.info import VectorServiceOptions
@@ -21,7 +21,6 @@
 from langchain_core.documents import Document
 from langchain_openai import OpenAIEmbeddings
 
-from datasets import load_dataset
 from dotenv import load_dotenv
 
 
@@ -52,7 +51,11 @@
 
 
 # Load data
-philo_dataset = json.load(open("data/philosopher-quotes.json"))
+philo_dataset = requests.get(
+    "https://raw.githubusercontent.com/"
+    "datastaxdevs/mini-demo-astradb-langchain/"
+    "refs/heads/main/data/philosopher-quotes.json"
+).json()
 
 print("An example entry:")
 print(philo_dataset[16])