Merge remote-tracking branch 'origin/master' into symbol-icons-configurable

Author: haraldschilly

.github/workflows/make-and-test.yml

@@ -108,5 +108,4 @@ jobs:
 
       - run: cd src && npm install -g pnpm
       - run: cd src && pnpm run make
-      - run: source venv/bin/activate && cd src && pnpm run test || pnpm run test || pnpm run test
-      - run: cd src && pnpm run depcheck
+      - run: source venv/bin/activate && cd src && pnpm run test-github-ci

README.md

@@ -1,5 +1,7 @@
 # CoCalc
 
+[![Make all packages and run their tests](https://github.com/sagemathinc/cocalc/actions/workflows/make-and-test.yml/badge.svg)](https://github.com/sagemathinc/cocalc/actions/workflows/make-and-test.yml)
+
 #### <u>_**Co**_</u>_llaborative_ <u>_**Calc**_</u>_ulation_
 
 CoCalc is web-based software that enables collaboration in research, teaching, and scientific publishing. It includes [Jupyter Notebooks](https://cocalc.com/features/jupyter-notebook), [Sage Worksheets](https://cocalc.com/features/sage), a [LaTeX Editor](https://cocalc.com/features/latex-editor) and a [Linux Terminal](https://cocalc.com/features/terminal) to help people work together in real time from different locations. It also has a [Computational Whiteboard](https://cocalc.com/features/whiteboard) for expressing and sharing ideas and running code. It is available for free and [can be upgraded for internet access, better hosting quality, and other features](https://cocalc.com/store). It can also be used for [teaching courses](https://cocalc.com/features/teaching) with flexible [course license options](https://cocalc.com/pricing/courses). It is also possible to run CoCalc [on your own infrastructure](https://cocalc.com/pricing/onprem).

src/package.json

@@ -17,7 +17,8 @@
     "c": "LOGS=/tmp/ DEBUG='cocalc:*' ./scripts/c",
     "version-check": "pip3 install typing_extensions mypy  || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py",
     "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test",
-    "test": "unset DEBUG && pnpm run version-check && ./workspaces.py test",
+    "test": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test",
+    "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter --retries=1",
     "depcheck": "cd packages && pnpm run -r --parallel depcheck",
     "prettier-all": "cd packages/",
     "local-ci": "./scripts/ci.sh",

src/packages/backend/conat/test/cluster/add-link-timeout.test.ts

@@ -0,0 +1,20 @@
+import { before, after } from "@cocalc/backend/conat/test/setup";
+import { createClusterNode } from "./util";
+import getPort from "@cocalc/backend/get-port";
+
+beforeAll(before);
+
+describe("adding a node times out if we can't connect to it, rather than trying forever", () => {
+  it("tries to add a link to a node that doesn't exist", async () => {
+    const { server } = await createClusterNode({
+      clusterName: "cluster0",
+      id: "1",
+    });
+    const port = await getPort();
+    expect(async () => {
+      await server.join(`localhost:${port}`, { timeout: 500 });
+    }).rejects.toThrow("timeout");
+  });
+});
+
+afterAll(after);

src/packages/backend/conat/test/cluster/cluster-basics.test.ts

@@ -2,7 +2,6 @@
 
 pnpm test `pwd`/cluster-basics.test.ts
 
-
 */
 
 import {
@@ -26,6 +25,7 @@ import { sysApi } from "@cocalc/conat/core/sys";
 beforeAll(before);
 
 jest.setTimeout(20000);
+
 describe("create a cluster enabled socketio server and test that the streams update as they should", () => {
   let server, client;
   it("create a server with cluster support enabled", async () => {
@@ -562,4 +562,29 @@ describe("test automatic node discovery", () => {
   });
 });
 
+describe("test the clusterIpAddress option", () => {
+  it("create a server without explicit clusterIpAddress", async () => {
+    const { server } = await createClusterNode({
+      clusterName: "cluster0",
+      id: "0",
+    });
+
+    expect(server.options.clusterIpAddress).toBe(undefined);
+    expect(server.address()).toBe(`http://localhost:${server.options.port}`);
+    server.close();
+  });
+
+  it("create a server with explicit clusterIpAddress", async () => {
+    const { server } = await createClusterNode({
+      clusterName: "cluster0",
+      id: "0",
+      clusterIpAddress: "127.0.0.1",
+    });
+
+    expect(server.options.clusterIpAddress).toBe("127.0.0.1");
+    expect(server.address()).toBe(`http://127.0.0.1:${server.options.port}`);
+    server.close();
+  });
+});
+
 afterAll(after);

src/packages/backend/conat/test/cluster/cluster-sticky.test.ts

@@ -16,13 +16,15 @@ describe("create cluster of two nodes, and verify that *sticky* subs properly wo
     ({ server: server1, client: client1 } = await createClusterNode({
       clusterName: "cluster0",
       id: "1",
-      systemAccountPassword: "squeamish",
+      autoscanInterval: 100,
+      longAutoscanInterval: 1000,
     }));
     client1b = server1.client();
     ({ server: server2, client: client2 } = await createClusterNode({
       clusterName: "cluster0",
       id: "2",
-      systemAccountPassword: "ossifrage",
+      autoscanInterval: 100,
+      longAutoscanInterval: 1000,
     }));
     await server1.join(server2.address());
     await server2.join(server1.address());
@@ -60,11 +62,11 @@ describe("create cluster of two nodes, and verify that *sticky* subs properly wo
 
   it("send messages and note they all go to the same target -- next the hard case across the cluster", async () => {
     // NOTE: if we do this test without waiting for consistent state, it will definitely
-    // fail sometimes, since server2 literally doesn't know enough about the servers yet, 
+    // fail sometimes, since server2 literally doesn't know enough about the servers yet,
     // so has to make a different choice.  Services of course must account for the fact that
     // for the first moments of their existence, sticky routing can't work.
     await waitForConsistentState([server1, server2]);
-    
+
     await client2.waitForInterest(subject);
     for (let i = 0; i < count; i++) {
       await client2.publish(subject, "hi");
@@ -73,6 +75,45 @@ describe("create cluster of two nodes, and verify that *sticky* subs properly wo
     expect(recv1 + recv1b).toEqual(2 * count);
     expect(recv1 * recv1b).toEqual(0);
   });
+
+  let server3, client3;
+  it("add a third node", async () => {
+    const { client, server } = await createClusterNode({
+      clusterName: "cluster0",
+      autoscanInterval: 100,
+      longAutoscanInterval: 5000,
+      id: "3",
+    });
+    server3 = server;
+    client3 = client;
+    await server1.join(server.address());
+  });
+
+  it("waits for consistent state -- this verifies, e.g., that sticky state is equal", async () => {
+    await waitForConsistentState([server1, server2, server3]);
+  });
+
+  it("client connected to server3 also routes properly", async () => {
+    const total = recv1 + recv1b;
+    await client3.waitForInterest(subject);
+    for (let i = 0; i < count; i++) {
+      await client3.publish(subject, "hi");
+    }
+    await wait({ until: () => recv1 + recv1b >= total + count });
+    expect(recv1 + recv1b).toEqual(total + count);
+    expect(recv1 * recv1b).toEqual(0);
+  });
+
+  it("client1 and client2 still route properly", async () => {
+    const total = recv1 + recv1b;
+    for (let i = 0; i < count; i++) {
+      await client1.publish(subject, "hi");
+      await client2.publish(subject, "hi");
+    }
+    await wait({ until: () => recv1 + recv1b >= total + 2 * count });
+    expect(recv1 + recv1b).toEqual(total + 2 * count);
+    expect(recv1 * recv1b).toEqual(0);
+  });
 });
 
 afterAll(after);

src/packages/backend/conat/test/cluster/node-discovery.test.ts

@@ -10,7 +10,7 @@ import { isEqual } from "lodash";
 
 beforeAll(before);
 
-jest.setTimeout(10000);
+jest.setTimeout(20000);
 describe("test automatic node discovery (and forgetting)", () => {
   const nodes: { client; server }[] = [];
   const clusterName = "auto";

src/packages/backend/conat/test/cluster/pubsub.test.ts

@@ -9,6 +9,8 @@ import {
   addNodeToDefaultCluster,
   wait,
   delay,
+  waitForConsistentState,
+  defaultCluster,
 } from "@cocalc/backend/conat/test/setup";
 import { STICKY_QUEUE_GROUP } from "@cocalc/conat/core/client";
 
@@ -27,10 +29,13 @@ describe("the most basic pub/sub test with a 2-node cluster", () => {
     sub = await client0.subscribe("cocalc");
   });
 
-  it("publish -- message is initially dropped with no receiver because interest doesn't propogate instantly", async () => {
-    const { count } = await client1.publish("cocalc", "hi");
-    expect(count).toBe(0);
-  });
+  // commenting this out, since it is impossible to robustly test in all cases,
+  // since sometmes the subscription info can move very quickly and this publish
+  // could be slow!
+  //   it("publish -- message is initially dropped with no receiver because interest doesn't propogate instantly", async () => {
+  //     const { count } = await client1.publish("cocalc", "hi");
+  //     expect(count).toBe(0);
+  //   });
 
   it("publish after waiting for interest -- this works", async () => {
     await client1.waitForInterest("cocalc");
@@ -79,7 +84,7 @@ describe("same basic test, but in the other direction", () => {
   });
 });
 
-describe("with three nodes and two subscribers with different queue group on distinct nodes", () => {
+describe("three nodes and two subscribers with different queue group on distinct nodes", () => {
   let client0, client1, client2, server2;
   it("get the clients", async () => {
     client0 = server.client();
@@ -116,7 +121,7 @@ describe("with three nodes and two subscribers with different queue group on dis
     expect(count).toBe(1);
   });
 
-  it("make a sticky sub from all nodes and see that all messages go to the same receiver, no matter which node we send from", async () => {
+  it("make sticky subs from each node, and see that all messages go to the same receiver, no matter which node we send from", async () => {
     const sub0 = await client0.subscribe("sticky", {
       queue: STICKY_QUEUE_GROUP,
     });
@@ -150,20 +155,38 @@ describe("with three nodes and two subscribers with different queue group on dis
     await client2.waitForInterest("sticky");
 
     for (let i = 0; i < 20; i++) {
-      const { count } = await client2.publish("sticky");
+      const { count } = await client2.publish("sticky", null);
       expect(count).toBe(1);
     }
     for (let i = 0; i < 20; i++) {
-      const { count } = await client1.publish("sticky");
+      const { count } = await client1.publish("sticky", null);
       expect(count).toBe(1);
     }
     for (let i = 0; i < 20; i++) {
-      const { count } = await client0.publish("sticky");
+      const { count } = await client0.publish("sticky", null);
+      expect(count).toBe(1);
+    }
+    await wait({ until: () => n0 + n1 + n2 == 60 });
+    expect(n0 * n1).toBe(0);
+    expect(n0 * n2).toBe(0);
+    expect(n1 * n2).toBe(0);
+    expect(n0 + n1 + n2).toBe(60);
+
+    await waitForConsistentState(defaultCluster);
+
+    // add another node and test
+    const server3 = await addNodeToDefaultCluster();
+    const client3 = server3.client();
+    await waitForConsistentState(defaultCluster);
+    for (let i = 0; i < 20; i++) {
+      const { count } = await client3.publish("sticky", null);
       expect(count).toBe(1);
     }
+    await wait({ until: () => n0 + n1 + n2 == 80 });
     expect(n0 * n1).toBe(0);
     expect(n0 * n2).toBe(0);
     expect(n1 * n2).toBe(0);
+    expect(n0 + n1 + n2).toBe(80);
   });
 });
 

src/packages/backend/conat/test/core/services.test.ts

@@ -8,7 +8,7 @@ import {
   connect,
   delay,
 } from "@cocalc/backend/conat/test/setup";
-import { Client, type Message } from "@cocalc/conat/core/client";
+import type { Client, Message } from "@cocalc/conat/core/client";
 import { wait } from "@cocalc/backend/conat/test/util";
 
 beforeAll(before);

src/packages/backend/conat/test/persist/cluster.test.ts

@@ -138,13 +138,13 @@ describe("test using multiple persist servers in a cluster", () => {
       expect(t.getAll()).toEqual(["x"]);
       openStreams1.push(t);
     }
-    expect(openStreams0.length).toBeGreaterThan(1);
+    expect(openStreams0.length).toBeGreaterThan(0);
   });
 
   it("remove one persist server", async () => {
     persistServer1.close();
     // [ ] TODO: removing this delay leads to very consistent failures
-    // involving data loss, but things should just be slower, never broken, 
+    // involving data loss, but things should just be slower, never broken,
     // on automatic failover.
     await delay(3000);
   });