Fix tests now that we do indexing in background (#120)

Author: mangalaman93

README.md

@@ -90,6 +90,14 @@ op = pydgraph.Operation(schema=schema)
 client.alter(op)
 ```
 
+Starting Dgraph version `20.3.0`, indexes are computed in the background.
+This requires that you wait for indexing to complete before running queries.
+You could do that using following code.
+
+```python
+pydgraph.util.wait_for_indexing(client, "name", ["exact"], False, False)
+```
+
 `Operation` contains other fields as well, including drop predicate and drop all.
 Drop all is useful if you wish to discard all the data, and start from a clean
 slate, without bringing the instance down.

examples/simple/README.md

@@ -15,11 +15,11 @@ set up the Dgraph cluster:
 version: "3.2"
 services:
   zero:
-    image: dgraph/dgraph:v1.1.0
+    image: dgraph/dgraph:latest
     restart: on-failure
     command: dgraph zero --my=zero:5080
   server:
-    image: dgraph/dgraph:v1.1.0
+    image: dgraph/dgraph:latest
     ports:
       - 8080:8080
       - 9080:9080

examples/simple/requirements.txt

@@ -1 +1 @@
-pydgraph>=1.0a1.0
+pydgraph>=2.0.2

examples/simple/simple.py

@@ -16,7 +16,7 @@ def create_client(client_stub):
 
 # Drop All - discard all data and start from a clean slate.
 def drop_all(client):
-    return client.alter(pydgraph.Operation(drop_all=True))
+    client.alter(pydgraph.Operation(drop_all=True))
 
 
 # Set schema.
@@ -38,7 +38,9 @@ def set_schema(client):
         dob
     }
     """
-    return client.alter(pydgraph.Operation(schema=schema))
+    client.alter(pydgraph.Operation(schema=schema))
+    pydgraph.util.wait_for_indexing(client, "name", ["exact"], False, False)
+    pydgraph.util.wait_for_indexing(client, "friend", [], False, True)
 
 
 # Create data using JSON.

examples/tls/tls_example.py

@@ -34,6 +34,7 @@ def main():
 '''
     op = pydgraph.Operation(schema=schema)
     client.alter(op)
+    pydgraph.util.wait_for_indexing(client, "name", ["exact"], False, False)
 
     # Mutate
     dgraph = {

pydgraph/util.py

@@ -14,7 +14,9 @@
 
 """Various utility functions."""
 
+import json
 import sys
+import time
 
 from pydgraph.meta import VERSION
 
@@ -34,3 +36,32 @@ def is_string(string):
 
 def is_jwt_expired(exception):
     return 'Token is expired' in str(exception)
+
+
+def wait_for_indexing(client, pred, tokenizers, count, reverse):
+    """Waits for indexes to be built."""
+    tokenizers.sort()
+    query = "schema(pred: [{}]){{tokenizer reverse count}}".format(pred)
+    while True:
+        resp = client.txn(read_only=True).query(query)
+        if has_indexes(resp, tokenizers, count, reverse):
+            break
+        time.sleep(0.1)
+
+def has_indexes(resp, tokenizers, count, reverse):
+    schema = json.loads(resp.json)
+    if len(schema["schema"]) != 1:
+        return False
+    index = schema["schema"][0]
+    if len(index.get("tokenizer", [])) != len(tokenizers):
+        return False
+    if index.get("count", False) != count or index.get("reverse", False) != reverse:
+        return False
+    # if no tokenizer is expected
+    if len(index.get("tokenizer", [])) == 0:
+        return True
+    index["tokenizer"].sort()
+    for i in range(len(tokenizers)):
+        if tokenizers[i] != index["tokenizer"][i]:
+            return False
+    return True

tests/helper.py

@@ -17,6 +17,7 @@
 __author__ = 'Garvit Pahal <[email protected]>'
 __maintainer__ = 'Martin Martinez Rivera <[email protected]>'
 
+import time
 import unittest
 
 import pydgraph
@@ -58,4 +59,11 @@ def setUp(self):
         """Sets up the client."""
 
         self.client = create_client(self.TEST_SERVER_ADDR)
-        self.client.login("groot", "password")
+        while True:
+            try:
+                self.client.login("groot", "password")
+                break
+            except Exception as e:
+                if not "user not found" in str(e):
+                    raise e
+            time.sleep(0.1)

tests/test_acct_upsert.py

@@ -52,6 +52,8 @@ def setUp(self):
             age:    int      @index(int)  @upsert .
             when:   int                   .
         """)
+        pydgraph.util.wait_for_indexing(self.client, "first", ["term"], False, False)
+        pydgraph.util.wait_for_indexing(self.client, "last", ["hash"], False, False)
 
     def test_account_upsert(self):
         """Run upserts concurrently."""
@@ -111,6 +113,7 @@ def assert_changes(self, firsts, accounts):
 def upsert_account(addr, account, success_ctr, retry_ctr):
     """Runs upsert operation."""
     client = helper.create_client(addr)
+    client.login("groot", "password")
     query = """{{
         acct(func:eq(first, "{first}")) @filter(eq(last, "{last}") AND eq(age, {age})) {{
             uid
@@ -167,6 +170,7 @@ def upsert_account(addr, account, success_ctr, retry_ctr):
 def upsert_account_upsert_block(addr, account, success_ctr, retry_ctr):
     """Runs upsert operation."""
     client = helper.create_client(addr)
+    client.login("groot", "password")
     query = """{{
         acct(func:eq(first, "{first}")) @filter(eq(last, "{last}") AND eq(age, {age})) {{
             u as uid

tests/test_acl.py

@@ -23,7 +23,7 @@
 import unittest
 
 from . import helper
-
+import pydgraph
 
 class TestACL(helper.ClientIntegrationTestCase):
     user_id = 'alice'
@@ -68,6 +68,8 @@ def change_permission(self, permission):
         bash_command = "dgraph acl -a " + self.server_addr + " mod -g " + self.group_id + \
                        " -p name -m " + str(permission) + " -x password"
         self.run_command(bash_command)
+        # wait for ACL cache to be refreshed.
+        time.sleep(6)
 
     def insert_sample_data(self):
         txn = self.client.txn()
@@ -126,11 +128,12 @@ def try_writing(self, expected):
     def try_altering(self, expected):
         try:
             helper.set_schema(self.alice_client, 'name: string @index(exact, term) .')
+            pydgraph.util.wait_for_indexing(self.alice_client, "name", ["exact", "term"], False, False)
             if not expected:
                 self.fail("Acl test failed: Alter successful without permission")
         except Exception as e:
             if expected:
-                self.fail("Acl test failed: Alter failed for altreble predicate.\n" + str(e))
+                self.fail("Acl test failed: Alter failed for alterable predicate.\n" + str(e))
 
 
 def suite():

tests/test_queries.py

@@ -35,6 +35,7 @@ def setUp(self):
 
         helper.drop_all(self.client)
         helper.set_schema(self.client, 'name: string @index(term) .')
+        pydgraph.util.wait_for_indexing(self.client, "name", ["term"], False, False)
 
     def test_mutation_and_query(self):
         """Runs mutation and verifies queries see the results."""

tests/test_txn.py

@@ -30,6 +30,7 @@ def setUp(self):
 
         helper.drop_all(self.client)
         helper.set_schema(self.client, 'name: string @index(fulltext) @upsert .')
+        pydgraph.util.wait_for_indexing(self.client, "name", ["fulltext"], False, False)
 
     def test_query_after_commit(self):
         txn = self.client.txn()
@@ -211,6 +212,7 @@ def test_read_from_new_client(self):
         txn.commit()
 
         client2 = helper.create_client(self.TEST_SERVER_ADDR)
+        client2.login("groot", "password")
         query = """{{
             me(func: uid("{uid:s}")) {{
                 name
@@ -258,6 +260,7 @@ def test_best_effort_txn(self):
 
         helper.drop_all(self.client)
         helper.set_schema(self.client, 'name: string @index(exact) .')
+        pydgraph.util.wait_for_indexing(self.client, "name", ["exact"], False, False)
 
         txn = self.client.txn()
         _ = txn.mutate(set_obj={'name': 'Manish'})
@@ -384,6 +387,7 @@ def test_read_index_key_same_txn(self):
 
         helper.drop_all(self.client)
         helper.set_schema(self.client, 'name: string @index(exact) .')
+        pydgraph.util.wait_for_indexing(self.client, "name", ["exact"], False, False)
 
         txn = self.client.txn()
         response = txn.mutate(set_obj={'name': 'Manish'})
@@ -407,6 +411,7 @@ def test_non_string_variable(self):
         in an Exception."""
         helper.drop_all(self.client)
         helper.set_schema(self.client, 'name: string @index(exact) .')
+        pydgraph.util.wait_for_indexing(self.client, "name", ["exact"], False, False)
 
         txn = self.client.txn()
         query = """

tests/test_type_system.py

@@ -20,8 +20,8 @@
 import logging
 import unittest
 
-from tests import helper
-
+from . import helper
+import pydgraph
 
 class TestTypeSystem(helper.ClientIntegrationTestCase):
     def setUp(self):
@@ -38,6 +38,7 @@ def setUp(self):
                 """
 
         helper.set_schema(self.client, schema)
+        pydgraph.util.wait_for_indexing(self.client, "name", ["term", "exact"], False, False)
 
     def test_type_deletion_failure(self):
         """It tries to delete all predicates of a node without having any type"""

tests/test_upsert_block.py

@@ -21,7 +21,7 @@
 import json
 
 from . import helper
-
+import pydgraph
 
 class TestUpsertBlock(helper.ClientIntegrationTestCase):
     """Tests for Upsert Block"""
@@ -30,6 +30,7 @@ def setUp(self):
         super(TestUpsertBlock, self).setUp()
         helper.drop_all(self.client)
         helper.set_schema(self.client, 'name: string @index(term) @upsert .')
+        pydgraph.util.wait_for_indexing(self.client, "name", ["term"], False, False)
 
     def test_upsert_block_one_mutation(self):
         txn = self.client.txn()