Merge remote-tracking branch 'origin/cocalc-api-20250927' into pr-20251208

Author: haraldschilly

.github/workflows/make-and-test.yml

@@ -106,7 +106,6 @@ jobs:
           pip install ipykernel
           python -m ipykernel install --prefix=./jupyter-local --name python3-local --display-name "Python 3 (Local)"
 
-
       - name: install pnpm
         uses: pnpm/action-setup@v4
         with:
@@ -128,6 +127,191 @@ jobs:
           name: "test-results-node-${{ matrix.node-version }}-pg-${{ matrix.pg-version }}"
           path: 'src/packages/*/junit.xml'
 
+      - name: Start CoCalc Hub
+        run: |
+          # Create conat password for hub internal authentication
+          mkdir -p src/data/secrets
+          echo "test-conat-password-$(date +%s)" > src/data/secrets/conat-password
+          chmod 600 src/data/secrets/conat-password
+
+          cd src/packages/hub
+          pnpm run hub-project-dev-nobuild > hub.log 2>&1 &
+          HUB_PID=$!
+          echo $HUB_PID > hub.pid
+          echo "Hub started with PID $HUB_PID"
+          # Check if process is still running after a moment
+          sleep 2
+          if ! kill -0 $HUB_PID 2>/dev/null; then
+            echo "Error: Hub process died immediately after starting"
+            echo "Hub log:"
+            cat hub.log
+            exit 1
+          fi
+        env:
+          PGDATABASE: smc
+          PGUSER: smc
+          PGHOST: localhost
+          COCALC_MODE: single-user
+          COCALC_TEST_MODE: yes
+          DEBUG: 'cocalc:*,-cocalc:silly:*,hub:*,project:*'
+
+      - name: Wait for hub readiness
+        run: |
+          MAX_ATTEMPTS=30
+          READY=false
+          for i in $(seq 1 $MAX_ATTEMPTS); do
+            if curl -sf --max-time 3 http://localhost:5000/healthcheck > /dev/null; then
+              echo "Hub is ready"
+              READY=true
+              break
+            fi
+            echo "Waiting for hub... ($i/$MAX_ATTEMPTS)"
+            sleep 3
+          done
+          if [ "$READY" = "false" ]; then
+            echo "Hub failed to become ready after $MAX_ATTEMPTS attempts"
+            echo "Hub log:"
+            cat src/packages/hub/hub.log || echo "No log file found"
+            exit 1
+          fi
+
+      - name: Create CI admin user and API key
+        run: |
+          cd src/packages/hub
+          OUTPUT=$(node dist/run/test-create-admin.js)
+          # Split output into account_id and API key (format: UUID;api-key)
+          ACCOUNT_ID=$(echo "$OUTPUT" | cut -d';' -f1)
+          API_KEY=$(echo "$OUTPUT" | cut -d';' -f2)
+
+          # Store in separate files
+          echo "$ACCOUNT_ID" > ../../account_id.txt
+          echo "$API_KEY" > ../../api_key.txt
+
+          # Validate account ID (UUID format)
+          if ! echo "$ACCOUNT_ID" | grep -qE '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'; then
+            echo "Error: Invalid account ID format: $ACCOUNT_ID"
+            exit 1
+          fi
+
+          # Validate API key was created
+          if [ ! -s ../../api_key.txt ]; then
+            echo "Error: API key file is empty or missing"
+            exit 1
+          fi
+          if ! echo "$API_KEY" | grep -qE '^sk-[A-Za-z0-9]+$'; then
+            echo "Error: Invalid API key format: $API_KEY"
+            exit 1
+          fi
+
+          echo "Account ID: $ACCOUNT_ID"
+          echo "API key created successfully"
+        env:
+          PGDATABASE: smc
+          PGUSER: smc
+          PGHOST: localhost
+
+      - name: Restart CoCalc Hub
+        run: |
+            HUB_PID=$(cat src/packages/hub/hub.pid)
+            echo "Stopping hub with PID $HUB_PID"
+            kill $HUB_PID || true
+            # Wait a bit for graceful shutdown
+            sleep 5
+            # Force kill if still running
+            kill -9 $HUB_PID 2>/dev/null || true
+
+            # Read the admin account ID created in the previous step
+            cd src/packages/hub
+            ADMIN_ACCOUNT_ID=$(cat ../../account_id.txt)
+            echo "Using admin account ID: $ADMIN_ACCOUNT_ID"
+
+            # Export env vars for the hub process
+            export COCALC_SETTING_JUPYTER_API_ENABLED=yes
+            export COCALC_SETTING_JUPYTER_ACCOUNT_ID=$ADMIN_ACCOUNT_ID
+            export COCALC_SETTING_JUPYTER_PROJECT_POOL_SIZE=1
+
+            # Start hub with new configuration
+            pnpm run hub-project-dev-nobuild > hub.log 2>&1 &
+            HUB_PID=$!
+            echo $HUB_PID > hub.pid
+            echo "Hub restarted with PID $HUB_PID"
+
+            # Check if process is still running after a moment
+            sleep 2
+            if ! kill -0 $HUB_PID 2>/dev/null; then
+              echo "Error: Hub process died immediately after restarting"
+              echo "Hub log:"
+              cat hub.log
+              exit 1
+            fi
+
+            # Wait for hub to be ready
+            MAX_ATTEMPTS=30
+            READY=false
+            for i in $(seq 1 $MAX_ATTEMPTS); do
+              if curl -sf --max-time 3 http://localhost:5000/healthcheck > /dev/null; then
+                echo "Hub is ready after restart"
+                READY=true
+                break
+              fi
+              echo "Waiting for hub to be ready... ($i/$MAX_ATTEMPTS)"
+              sleep 3
+            done
+            if [ "$READY" = "false" ]; then
+              echo "Hub failed to become ready after restart"
+              echo "Hub log:"
+              cat hub.log
+              exit 1
+            fi
+        env:
+          PGDATABASE: smc
+          PGUSER: smc
+          PGHOST: localhost
+          COCALC_MODE: single-user
+          COCALC_TEST_MODE: yes
+          DEBUG: 'cocalc:*,-cocalc:silly:*,hub:*,project:*'
+
+      - name: Install uv for cocalc-api tests
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh && echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Run cocalc-api tests
+        run: |
+          export COCALC_API_KEY=$(cat src/api_key.txt)
+          export COCALC_HOST=http://localhost:5000
+          export CI=true
+          cd src/python/cocalc-api && make ci
+        env:
+          PGDATABASE: smc
+          PGUSER: smc
+          PGHOST: localhost
+
+      - name: Stop CoCalc Hub
+        if: always()
+        run: |
+          if [ -f src/packages/hub/hub.pid ]; then
+            HUB_PID=$(cat src/packages/hub/hub.pid)
+            echo "Stopping hub with PID $HUB_PID"
+            kill $HUB_PID || true
+            # Wait a bit for graceful shutdown
+            sleep 5
+            # Force kill if still running
+            kill -9 $HUB_PID 2>/dev/null || true
+          fi
+
+      - name: Upload hub logs
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: "hub-logs-node-${{ matrix.node-version }}-pg-${{ matrix.pg-version }}"
+          path: 'src/packages/hub/hub.log'
+
+      - name: Upload cocalc-api test results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: "cocalc-api-test-results-node-${{ matrix.node-version }}-pg-${{ matrix.pg-version }}"
+          path: 'src/python/cocalc-api/test-results.xml'
+
   report:
     runs-on: ubuntu-latest
 

.gitignore

@@ -71,6 +71,12 @@ src/conf/tinc_hosts/
 # mocha: banket coverage report
 src/coverage
 src/*/coverage
+
+# Python coverage files
+.coverage
+.coverage.*
+htmlcov/
+**/htmlcov/
 # comes up when testing in that directory
 src/rethinkdb_data/
 src/dev/project/rethinkdb_data/

src/.claude/settings.json

@@ -35,6 +35,10 @@
       "Bash(pnpm exec tsc:*)",
       "Bash(pnpm i18n:*)",
       "Bash(pnpm i18n:*:*)",
+      "Bash(pnpm i18n:compile:*)",
+      "Bash(pnpm i18n:download:*)",
+      "Bash(pnpm i18n:extract:*)",
+      "Bash(pnpm i18n:upload:*)",
       "Bash(pnpm i18n:update:*)",
       "Bash(pnpm info:*)",
       "Bash(pnpm list:*)",
@@ -49,9 +53,12 @@
       "Bash(prettier -w:*)",
       "Bash(psql:*)",
       "Bash(python3:*)",
+      "Bash(uv:*)",
+      "Bash(timeout:*)",
       "Bash(uv sync:*)",
       "WebFetch",
       "WebSearch",
+      "mcp__cocalc__*",
       "mcp__cclsp__find_definition",
       "mcp__cclsp__find_references",
       "mcp__github__get_issue",

src/AGENTS.md

@@ -105,6 +105,22 @@ CoCalc is organized as a monorepo with key packages:
   5. **Authentication**: Each conat request includes account_id and is subject to permission checks at the hub level
   6. **Subjects**: Messages are routed using hierarchical subjects like `hub.account.{uuid}.{service}` or `project.{uuid}.{compute_server_id}.{service}`
 
+#### CoCalc Conat Hub API Architecture
+
+**API Method Registration Pattern:**
+
+- **Registry**: `packages/conat/hub/api/projects.ts` contains `export const projects = { methodName: authFirstRequireAccount }`
+- **Implementation**: `packages/server/conat/api/projects.ts` contains `export async function methodName() { ... }`
+- **Flow**: Python client `@api_method("projects.methodName")` → POST `/api/conat/hub` → `hubBridge()` → conat subject `hub.account.{account_id}.api` → registry lookup → implementation
+
+**Example - projects.createProject:**
+
+1. **Python**: `@api_method("projects.createProject")` decorator
+2. **HTTP**: `POST /api/conat/hub {"name": "projects.createProject", "args": [...]}`
+3. **Bridge**: `hubBridge()` routes to conat subject
+4. **Registry**: `packages/conat/hub/api/projects.ts: createProject: authFirstRequireAccount`
+5. **Implementation**: `packages/server/conat/api/projects.ts: export { createProject }` → `@cocalc/server/projects/create`
+
 ### Key Technologies
 
 - **TypeScript**: Primary language for all new code
@@ -158,7 +174,6 @@ CoCalc is organized as a monorepo with key packages:
 - Prefix git commits with the package and general area. e.g. 'frontend/latex: ...' if it concerns latex editor changes in the packages/frontend/... code.
 - When pushing a new branch to Github, track it upstream. e.g. `git push --set-upstream origin feature-foo` for branch "feature-foo".
 
-
 ## React-intl / Internationalization (i18n)
 
 CoCalc uses react-intl for internationalization with SimpleLocalize as the translation platform.
@@ -216,11 +231,42 @@ Same flow as above, but **before 3. i18n:upload**, delete the key. Only new keys
 - Ignore everything in `node_modules` or `dist` directories
 - Ignore all files not tracked by Git, unless they are newly created files
 
-# Important Instruction Reminders
+# CoCalc Python API Client Investigation
+
+## Overview
+
+The `python/cocalc-api/` directory contains a uv-based Python client library for the CoCalc API, published as the `cocalc-api` package on PyPI.
+
+It also contains a test framework (`python/cocalc-api/tests/README.md`) and an MCP client (`python/cocalc-api/src/cocalc_api/mcp/README.md`).
+For convenience, a `python/cocalc-api/Makefile` exists.
+
+## Client-Server Architecture Investigation
+
+### API Call Flow
+
+1. **cocalc-api Client** (Python) → HTTP POST requests
+2. **Next.js API Routes** (`/api/conat/{hub,project}`) → Bridge to conat messaging
+3. **ConatClient** (server-side) → NATS-like messaging protocol
+4. **Hub API Implementation** (`packages/conat/hub/api/`) → Actual business logic
+
+### Endpoints Discovered
+
+#### Hub API: `POST /api/conat/hub`
+
+- **Bridge**: `packages/next/pages/api/conat/hub.ts` → `hubBridge()` → conat subject `hub.account.{account_id}.api`
+- **Implementation**: `packages/conat/hub/api/projects.ts`
+- **Available Methods**: `createProject`, `start`, `stop`, `setQuotas`, `addCollaborator`, `removeCollaborator`, etc.
+
+#### Project API: `POST /api/conat/project`
+
+- **Bridge**: `packages/next/pages/api/conat/project.ts` → `projectBridge()` → conat project subjects
+- **Implementation**: `packages/conat/project/api/` (system.ping, system.exec, system.jupyterExecute)
+
+# important-instruction-reminders
 
-- Do what has been asked; nothing more, nothing less
-- NEVER create files unless they're absolutely necessary for achieving your goal
-- ALWAYS prefer editing an existing file to creating a new one
-- REFUSE to modify files when the git repository is on the `master` or `main` branch
-- NEVER proactively create documentation files (`*.md`) or README files. Only create documentation files if explicitly requested by the User
-- when modifying a file with a copyright banner at the top, make sure to fix/add the current year to indicate the copyright year
+- Do what has been asked; nothing more, nothing less.
+- NEVER create files unless they're absolutely necessary for achieving your goal.
+- ALWAYS prefer editing an existing file to creating a new one.
+- NEVER proactively create documentation files (\*.md) or README files. Only create documentation files if explicitly requested by the User.
+- ALWAYS ask questions if something is unclear. Only proceed to the implementation step if you have no questions left.
+- When modifying a file with a copyright banner at the top, make sure to fix/add the current year to indicate the copyright year.

src/cocalc.code-workspace

@@ -3,6 +3,9 @@
     {
       "name": "cocalc",
       "path": "."
+    },
+    {
+      "path": "../.github"
     }
   ],
   "settings": {

src/packages/conat/hub/api/projects.ts

@@ -1,6 +1,10 @@
 import { authFirstRequireAccount } from "./util";
 import { type CreateProjectOptions } from "@cocalc/util/db-schema/projects";
 import { type UserCopyOptions } from "@cocalc/util/db-schema/projects";
+import {
+  type ProjectState,
+  type ProjectStatus,
+} from "@cocalc/util/db-schema/projects";
 
 export const projects = {
   createProject: authFirstRequireAccount,
@@ -12,6 +16,10 @@ export const projects = {
   setQuotas: authFirstRequireAccount,
   start: authFirstRequireAccount,
   stop: authFirstRequireAccount,
+  deleteProject: authFirstRequireAccount,
+  touch: authFirstRequireAccount,
+  state: authFirstRequireAccount,
+  status: authFirstRequireAccount,
 };
 
 export type AddCollaborator =
@@ -103,4 +111,18 @@ export interface Projects {
 
   start: (opts: { account_id: string; project_id: string }) => Promise<void>;
   stop: (opts: { account_id: string; project_id: string }) => Promise<void>;
+  deleteProject: (opts: {
+    account_id: string;
+    project_id: string;
+  }) => Promise<void>;
+
+  touch: (opts: { account_id: string; project_id: string }) => Promise<void>;
+  state: (opts: {
+    account_id: string;
+    project_id: string;
+  }) => Promise<ProjectState>;
+  status: (opts: {
+    account_id: string;
+    project_id: string;
+  }) => Promise<ProjectStatus>;
 }

src/packages/conat/hub/api/system.ts

@@ -9,6 +9,7 @@ import { type UserSearchResult } from "@cocalc/util/db-schema/accounts";
 export const system = {
   getCustomize: noAuth,
   ping: noAuth,
+  test: authFirst,
   terminate: authFirst,
   userTracking: authFirst,
   logClientError: authFirst,
@@ -31,6 +32,8 @@ export interface System {
   getCustomize: (fields?: string[]) => Promise<Customize>;
   // ping server and get back the current time
   ping: () => { now: number };
+  // test API key and return scope information (account_id) and server time
+  test: () => Promise<{ account_id: string; server_time: number }>;
   // terminate a service:
   //   - only admin can do this.
   //   - useful for development