diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1a3eebd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.git +.github +.agents +.codex +**/node_modules +**/build +**/dist +backend/data +backend/data-* +*.db +*.db-wal +*.db-shm +SLICE* +SAVE_POINT.md +playwright-report +test-results diff --git a/.env.production.example b/.env.production.example new file mode 100644 index 0000000..1fe6b57 --- /dev/null +++ b/.env.production.example @@ -0,0 +1,14 @@ +# Copy this file to .env and replace both required keys before starting. +# Generate each independently with: openssl rand -hex 32 +CONDUCTOR_SECRET_KEY=replace-with-64-hex-characters +CONDUCTOR_SESSION_KEY=replace-with-an-independent-random-value-at-least-32-bytes + +CONDUCTOR_IMAGE=gitea.skeletonworks.online/vwiebe/conductor +CONDUCTOR_VERSION=v0.1.0 +CONDUCTOR_BIND_ADDRESS=0.0.0.0 +CONDUCTOR_PORT=8080 +CONDUCTOR_MEM_LIMIT=1g +CONDUCTOR_CPUS=1.5 + +# Optional comma-separated exact origins permitted by the proxy's internal-origin policy. +CONDUCTOR_PROXY_INTERNAL_ORIGINS= diff --git a/.gitea/workflows/release-image.yml b/.gitea/workflows/release-image.yml new file mode 100644 index 0000000..6242cc8 --- /dev/null +++ b/.gitea/workflows/release-image.yml @@ -0,0 +1,52 @@ +name: Release production image + +on: + push: + tags: + - 'v*.*.*' + workflow_dispatch: + +jobs: + production-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + - name: Log in to Gitea registry + uses: docker/login-action@v3 + with: + registry: gitea.skeletonworks.online + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + - name: Resolve release metadata + id: metadata + shell: bash + run: | + version="${GITHUB_REF_NAME:-manual}" + revision="$(git rev-parse HEAD)" + short_revision="$(git rev-parse --short=12 HEAD)" + created="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "revision=${revision}" >> "$GITHUB_OUTPUT" + echo "short_revision=${short_revision}" >> "$GITHUB_OUTPUT" + echo "created=${created}" >> "$GITHUB_OUTPUT" + - name: Build and publish immutable images + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile.production + platforms: linux/amd64,linux/arm64 + push: true + build-args: | + VERSION=${{ steps.metadata.outputs.version }} + REVISION=${{ steps.metadata.outputs.revision }} + CREATED=${{ steps.metadata.outputs.created }} + tags: | + gitea.skeletonworks.online/vwiebe/conductor:${{ steps.metadata.outputs.version }} + gitea.skeletonworks.online/vwiebe/conductor:git-${{ steps.metadata.outputs.short_revision }} + provenance: mode=max + sbom: true diff --git a/.gitignore b/.gitignore index 58390af..b0ecc17 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,9 @@ yarn-debug.log* yarn-error.log* *.log +# Production operator state and backups +backups/ + # OS .DS_Store Thumbs.db diff --git a/Dockerfile.production b/Dockerfile.production new file mode 100644 index 0000000..01fb973 --- /dev/null +++ b/Dockerfile.production @@ -0,0 +1,45 @@ +# syntax=docker/dockerfile:1.7 +FROM node:20-alpine AS frontend-build +WORKDIR /build/frontend +COPY frontend/package*.json ./ +RUN npm ci +COPY frontend/ ./ +RUN npm run build + +FROM node:20-alpine AS backend-build +RUN apk add --no-cache python3 make g++ +WORKDIR /build/backend +COPY backend/package*.json ./ +RUN npm ci +COPY backend/ ./ +RUN npm run build && npm prune --omit=dev + +FROM node:20-alpine AS runtime +ARG VERSION=development +ARG REVISION=unknown +ARG CREATED=unknown +LABEL org.opencontainers.image.title="Conductor" \ + org.opencontainers.image.description="Visual builder for REST-backed web applications" \ + org.opencontainers.image.version="${VERSION}" \ + org.opencontainers.image.revision="${REVISION}" \ + org.opencontainers.image.created="${CREATED}" \ + org.opencontainers.image.source="https://gitea.skeletonworks.online/vwiebe/conductor" +ENV NODE_ENV=production \ + PORT=8080 \ + CONDUCTOR_DATA_DIR=/data \ + CONDUCTOR_STATIC_DIR=/app/public \ + CONDUCTOR_VERSION=${VERSION} +RUN addgroup -S -g 10001 conductor && adduser -S -D -H -u 10001 -G conductor conductor \ + && mkdir -p /app /data /shared/schemas \ + && chown -R conductor:conductor /app /data +WORKDIR /app +COPY --from=backend-build --chown=conductor:conductor /build/backend/package*.json ./ +COPY --from=backend-build --chown=conductor:conductor /build/backend/node_modules ./node_modules +COPY --from=backend-build --chown=conductor:conductor /build/backend/dist ./dist +COPY --from=frontend-build --chown=conductor:conductor /build/frontend/build ./public +COPY --chown=conductor:conductor shared/schemas/conductor-project.schema.json /shared/schemas/conductor-project.schema.json +USER conductor +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ + CMD node -e "fetch('http://127.0.0.1:8080/api/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" +CMD ["node", "dist/index.js"] diff --git a/MVP_SCOPE.md b/MVP_SCOPE.md index b1b9209..4132569 100644 --- a/MVP_SCOPE.md +++ b/MVP_SCOPE.md @@ -83,7 +83,7 @@ Approved on 2026-08-07 for Slice 7a: ## Release Boundary -The MVP is complete only when all required component, configuration, authentication, security, validation, persistence, local-user/RBAC, publishing, first-run administration, multi-page/scoped-variable, Visual Editor command-ribbon polish, and deployment tasks in `ROADMAP.md` are complete; all six workflows above pass the Slice 6 release-validation process; and Slice 7a, Slice 7b, Slice 7c, and Slice 7d acceptance pass. +The MVP is complete only when all required component, configuration, authentication, security, validation, persistence, local-user/RBAC, publishing, first-run administration, multi-page/scoped-variable, Visual Editor command-ribbon polish, public production distribution, SkeletonWorks installation, and verified backup/restore tasks in `ROADMAP.md` are complete; all six workflows above pass the Slice 6 release-validation process; and Slice 7a, Slice 7b, Slice 7c, Slice 7d, and Slice 8 acceptance pass. Post-MVP scope includes AI assistance, OIDC/SSO beyond the local authentication architecture, OAuth 2.0 for REST actions, IBM Cloud IAM, mTLS, advanced orchestration, and the future capabilities listed in `ROADMAP.md`. diff --git a/README.md b/README.md index b93907e..a8ef409 100644 --- a/README.md +++ b/README.md @@ -67,14 +67,33 @@ The frontend development server proxies `/api/*` requests to `http://localhost:4 --- -## Docker +## Development Docker ```bash -docker-compose up --build +docker compose up --build # Frontend: http://localhost:3000 # Backend: http://localhost:4000 ``` +## Production installation + +Conductor's production distribution is one non-root container serving the compiled frontend, published application routes, and backend API on port 8080. SQLite data is stored in a persistent volume. The development stack above remains separate. + +```bash +curl -O https://gitea.skeletonworks.online/vwiebe/conductor/raw/tag/v0.1.0/compose.production.yml +curl -o .env https://gitea.skeletonworks.online/vwiebe/conductor/raw/tag/v0.1.0/.env.production.example + +# Replace both placeholder keys with independent values from: +openssl rand -hex 32 + +docker compose --env-file .env -f compose.production.yml pull +docker compose --env-file .env -f compose.production.yml up -d +``` + +Open `http://localhost:8080` and create the first administrator in the browser. There are no default credentials. + +For source builds, reverse proxies, upgrades, backups, restores, and SkeletonWorks installation, see [docs/INSTALL.md](docs/INSTALL.md) and [docs/OPERATIONS.md](docs/OPERATIONS.md). + --- ## Documentation @@ -89,6 +108,13 @@ docker-compose up --build | [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) | Technical architecture | | [docs/NICE-TO-HAVE.md](docs/NICE-TO-HAVE.md) | Future enhancements | | [docs/BUILD_AND_TEST_PLAN.md](docs/BUILD_AND_TEST_PLAN.md) | Incremental build and test plan | +| [docs/INSTALL.md](docs/INSTALL.md) | Public source/container and reverse-proxy installation | +| [docs/OPERATIONS.md](docs/OPERATIONS.md) | Production configuration, backup, restore, upgrade, and rollback | +| [docs/RELEASE.md](docs/RELEASE.md) | Image tagging, release construction, and publication | +| [docs/USER_GUIDE.md](docs/USER_GUIDE.md) | Admin/user authoring, runtime, and publishing workflows | +| [docs/DEVELOPER_GUIDE.md](docs/DEVELOPER_GUIDE.md) | Architecture, invariants, tests, and contribution workflow | +| [docs/TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md) | Production and application troubleshooting | +| [docs/KNOWN_LIMITATIONS.md](docs/KNOWN_LIMITATIONS.md) | Explicit v0.1.0 limitations and deferred capabilities | --- @@ -99,4 +125,4 @@ docker-compose up --build | Frontend | React 18, TypeScript 5 | | Backend | Node.js 20, Express 4, TypeScript 5 | | Database | SQLite (MVP) | -| Deployment | Docker Compose, NGINX/reverse proxy | +| Deployment | Single production container, Docker Compose, Caddy/NGINX reverse proxy | diff --git a/ROADMAP.md b/ROADMAP.md index de3fc6e..63d4186 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -313,19 +313,23 @@ Slice 8 release packaging now depends on completed Slice 7a authentication/RBAC/ ### Slices 6 and 8 — v0.1.0 packaging - [ ] Review examples, demo, open issues, deferred work, and release criteria. -- [ ] Complete fresh-clone validation and final Docker images. +- [ ] Build a single non-root production image serving the compiled frontend, SPA/published routes, and backend API on one port while retaining the two-container development stack. +- [ ] Publish production Compose/environment assets and anonymously downloadable source releases plus pinned multi-architecture container tags. +- [ ] Add application-owned consistent SQLite backup/restore with encryption keys, manifests, checksums, safety backup, and health verification. +- [ ] Add and validate SkeletonWorks `setup-conductor.sh`, `backup-conductor.sh`, and `restore-conductor.sh` behind Caddy. +- [ ] Complete fresh-clone, container-only, fresh-host, upgrade, rollback, backup, restore, restart, persistence, and health validation. - [ ] Prepare release notes and update version references. - [ ] Commit the release documentation and code. - [ ] Tag `v0.1.0` and push the release commit and tag. ## Post-MVP Backlog -- [ ] Multi-page application authoring and advanced page management. +- [ ] Nested/dynamic pages, route parameters, per-page authorization, and advanced page management. - [ ] OAuth 2.0, IBM Cloud IAM, mTLS, and custom authentication scripts. - [ ] OpenAPI import and generated forms/actions. - [ ] Reusable templates, themes, and component libraries. - [ ] Version history, Git integration, and team collaboration. -- [ ] Role-based access control and enterprise audit retention. +- [ ] Custom/per-application roles, groups, approval workflows, and enterprise audit retention. - [ ] Tabs, modals, date pickers, file uploads, charts, and progress indicators. - [ ] Table sorting, filtering, pagination, editing, CSV import/export, row actions, and multi-selection. - [ ] Advanced response transforms and expression languages. diff --git a/TESTING.md b/TESTING.md index 42d867e..f900143 100644 --- a/TESTING.md +++ b/TESTING.md @@ -707,3 +707,24 @@ After a manual workflow is completed: 4. Do not mark a broader workflow accepted when only one checkpoint passed. 5. File unresolved defects in `ROADMAP.md` and the owning slice with enough detail to reproduce them. +# Production packaging checks + +In addition to the development and acceptance suites below, Slice 8 validates the single production artifact: + +```bash +docker build -f Dockerfile.production -t conductor:local . + +# Must fail because production keys are absent. +docker run --rm conductor:local + +cp .env.production.example .env +# Replace both key placeholders, then: +docker compose --env-file .env \ + -f compose.production.yml -f compose.production.build.yml up -d --build +curl --fail http://127.0.0.1:8080/api/health + +scripts/production/backup-conductor.sh +scripts/production/restore-conductor.sh --list +``` + +The release gate includes a disposable destructive restore drill with a marker record, verification that the restored service runs as UID 10001 with a read-only root filesystem, and refresh checks for `/apps/:appSlug/:pageSlug`. diff --git a/backend/package.json b/backend/package.json index 84ff2b3..8670ca4 100644 --- a/backend/package.json +++ b/backend/package.json @@ -5,6 +5,7 @@ "scripts": { "dev": "ts-node-dev --respawn --transpile-only src/index.ts", "build": "tsc", + "backup-db": "node dist/scripts/backupDatabase.js", "bootstrap-admin": "npm run build && node dist/scripts/bootstrapAdmin.js", "test": "npm run build && node --test dist/**/*.test.js", "start": "node dist/index.js" diff --git a/backend/src/app.ts b/backend/src/app.ts index 7708bd0..1db535f 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -1,5 +1,7 @@ import express, { Application, Request, Response, NextFunction } from 'express'; import morgan from 'morgan'; +import fs from 'fs'; +import path from 'path'; import healthRouter from './routes/health'; import projectsRouter from './routes/projects'; @@ -19,6 +21,13 @@ const app: Application = express(); app.use(express.json({ limit: '2mb' })); app.use(morgan('dev')); +app.disable('x-powered-by'); +app.use((_req, res, next) => { + res.setHeader('X-Content-Type-Options', 'nosniff'); + res.setHeader('Referrer-Policy', 'same-origin'); + res.setHeader('X-Frame-Options', 'SAMEORIGIN'); + next(); +}); // ── Routes ──────────────────────────────────────────────────────────────────── @@ -33,6 +42,21 @@ app.use('/api/secrets', requireAdmin, requireMutationCsrf, secretsRouter); app.use('/api/executions', requireAdmin, requireMutationCsrf, executionsRouter); app.use('/api/proxy/execute', requireAdmin, requireMutationCsrf, proxyRouter); +// ── Production frontend ───────────────────────────────────────────────────── + +const staticDirectory = process.env.CONDUCTOR_STATIC_DIR + ? path.resolve(process.env.CONDUCTOR_STATIC_DIR) + : path.resolve(__dirname, '../public'); +const indexFile = path.join(staticDirectory, 'index.html'); +if (fs.existsSync(indexFile)) { + app.use(express.static(staticDirectory, { index: false, maxAge: '1h' })); + app.get('*', (req, res, next) => { + if (req.path.startsWith('/api/')) return next(); + res.setHeader('Cache-Control', 'no-cache'); + res.sendFile(indexFile); + }); +} + // ── Catch-all 404 ───────────────────────────────────────────────────────────── app.use((_req: Request, res: Response) => { diff --git a/backend/src/db/database.ts b/backend/src/db/database.ts index 182d781..4d81218 100644 --- a/backend/src/db/database.ts +++ b/backend/src/db/database.ts @@ -4,10 +4,10 @@ import fs from 'fs'; // Store the database file in backend/data/ by default. Tests and operators may // select an isolated directory without changing the application working tree. -const DATA_DIR = process.env.CONDUCTOR_DATA_DIR +export const DATA_DIR = process.env.CONDUCTOR_DATA_DIR ? path.resolve(process.env.CONDUCTOR_DATA_DIR) : path.resolve(__dirname, '../../data'); -const DB_PATH = path.join(DATA_DIR, 'conductor.db'); +export const DB_PATH = path.join(DATA_DIR, 'conductor.db'); // Ensure the data directory exists before opening the file fs.mkdirSync(DATA_DIR, { recursive: true }); diff --git a/backend/src/index.ts b/backend/src/index.ts index 5d93a84..63e8c69 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -1,13 +1,24 @@ import { initDatabase } from './db/init'; +import db from './db/database'; +import { validateProductionConfiguration } from './lib/productionConfig'; -const PORT = process.env.PORT ?? 4000; +const PORT = Number(process.env.PORT ?? 4000); +if (!Number.isInteger(PORT) || PORT < 1 || PORT > 65535) throw new Error('PORT must be an integer from 1 to 65535.'); +validateProductionConfiguration(); initDatabase(); // Import the app only after the schema exists. Project route modules prepare // their SQL statements during import and therefore require the projects table. void import('./app').then(({ default: app }) => { - app.listen(PORT, () => { + const server = app.listen(PORT, '0.0.0.0', () => { console.log(`Conductor backend listening on port ${PORT}`); }); + const shutdown = (signal: string) => { + console.log(`Received ${signal}; shutting down.`); + server.close(() => { db.close(); process.exit(0); }); + setTimeout(() => process.exit(1), 10_000).unref(); + }; + process.once('SIGTERM', () => shutdown('SIGTERM')); + process.once('SIGINT', () => shutdown('SIGINT')); }); diff --git a/backend/src/lib/productionConfig.test.ts b/backend/src/lib/productionConfig.test.ts new file mode 100644 index 0000000..9fefbf0 --- /dev/null +++ b/backend/src/lib/productionConfig.test.ts @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { validateProductionConfiguration } from './productionConfig'; + +test('development does not require production keys', () => { + assert.doesNotThrow(() => validateProductionConfiguration({ NODE_ENV: 'development' })); +}); + +test('production requires both correctly sized persistent keys', () => { + assert.throws(() => validateProductionConfiguration({ NODE_ENV: 'production' }), /CONDUCTOR_SECRET_KEY/); + assert.throws(() => validateProductionConfiguration({ NODE_ENV: 'production', CONDUCTOR_SECRET_KEY: '11'.repeat(32) }), /CONDUCTOR_SESSION_KEY/); + assert.doesNotThrow(() => validateProductionConfiguration({ + NODE_ENV: 'production', + CONDUCTOR_SECRET_KEY: Buffer.alloc(32, 7).toString('base64'), + CONDUCTOR_SESSION_KEY: 'session-key-material-that-is-at-least-32-bytes', + })); +}); diff --git a/backend/src/lib/productionConfig.ts b/backend/src/lib/productionConfig.ts new file mode 100644 index 0000000..1afd2cf --- /dev/null +++ b/backend/src/lib/productionConfig.ts @@ -0,0 +1,15 @@ +function decodeSecretKey(value: string): Buffer { + return /^[0-9a-fA-F]{64}$/.test(value) ? Buffer.from(value, 'hex') : Buffer.from(value, 'base64'); +} + +export function validateProductionConfiguration(environment: NodeJS.ProcessEnv = process.env): void { + if (environment.NODE_ENV !== 'production') return; + const secretKey = environment.CONDUCTOR_SECRET_KEY ?? ''; + if (!secretKey || decodeSecretKey(secretKey).length !== 32) { + throw new Error('CONDUCTOR_SECRET_KEY is required in production and must encode exactly 32 bytes as base64 or 64 hexadecimal characters.'); + } + const sessionKey = environment.CONDUCTOR_SESSION_KEY ?? ''; + if (Buffer.byteLength(sessionKey, 'utf8') < 32) { + throw new Error('CONDUCTOR_SESSION_KEY is required in production and must contain at least 32 bytes.'); + } +} diff --git a/backend/src/routes/health.ts b/backend/src/routes/health.ts index 5bd4173..1773f5b 100644 --- a/backend/src/routes/health.ts +++ b/backend/src/routes/health.ts @@ -6,6 +6,7 @@ router.get('/', (_req: Request, res: Response) => { res.json({ service: 'conductor-backend', status: 'ok', + version: process.env.CONDUCTOR_VERSION ?? 'development', timestamp: new Date().toISOString(), }); }); diff --git a/backend/src/scripts/backupDatabase.ts b/backend/src/scripts/backupDatabase.ts new file mode 100644 index 0000000..6a80c98 --- /dev/null +++ b/backend/src/scripts/backupDatabase.ts @@ -0,0 +1,29 @@ +import fs from 'fs'; +import path from 'path'; +import db, { DB_PATH } from '../db/database'; +import { initDatabase } from '../db/init'; + +async function main(): Promise { + const destinationArg = process.argv[2]; + if (!destinationArg) throw new Error('Usage: npm run backup-db -- '); + const destination = path.resolve(destinationArg); + if (destination === DB_PATH) throw new Error('Backup destination must differ from the live database.'); + fs.mkdirSync(path.dirname(destination), { recursive: true }); + if (fs.existsSync(destination)) throw new Error(`Backup destination already exists: ${destination}`); + const temporary = `${destination}.partial-${process.pid}`; + initDatabase(); + try { + await db.backup(temporary); + fs.renameSync(temporary, destination); + const stats = fs.statSync(destination); + console.log(JSON.stringify({ status: 'success', database: DB_PATH, backup: destination, sizeBytes: stats.size })); + } finally { + if (fs.existsSync(temporary)) fs.rmSync(temporary); + db.close(); + } +} + +void main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/compose.production.build.yml b/compose.production.build.yml new file mode 100644 index 0000000..0aea1fa --- /dev/null +++ b/compose.production.build.yml @@ -0,0 +1,10 @@ +services: + conductor: + image: conductor:local + build: + context: . + dockerfile: Dockerfile.production + args: + VERSION: local + REVISION: local + CREATED: local diff --git a/compose.production.yml b/compose.production.yml new file mode 100644 index 0000000..4e245af --- /dev/null +++ b/compose.production.yml @@ -0,0 +1,35 @@ +services: + conductor: + image: ${CONDUCTOR_IMAGE:-gitea.skeletonworks.online/vwiebe/conductor}:${CONDUCTOR_VERSION:-latest} + restart: unless-stopped + init: true + ports: + - "${CONDUCTOR_BIND_ADDRESS:-0.0.0.0}:${CONDUCTOR_PORT:-8080}:8080" + environment: + NODE_ENV: production + PORT: 8080 + CONDUCTOR_DATA_DIR: /data + CONDUCTOR_SECRET_KEY: ${CONDUCTOR_SECRET_KEY:?Set CONDUCTOR_SECRET_KEY in .env} + CONDUCTOR_SESSION_KEY: ${CONDUCTOR_SESSION_KEY:?Set CONDUCTOR_SESSION_KEY in .env} + CONDUCTOR_PROXY_INTERNAL_ORIGINS: ${CONDUCTOR_PROXY_INTERNAL_ORIGINS:-} + volumes: + - conductor_data:/data + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=64m + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + pids_limit: 512 + mem_limit: ${CONDUCTOR_MEM_LIMIT:-1g} + cpus: ${CONDUCTOR_CPUS:-1.5} + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8080/api/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + interval: 30s + timeout: 5s + start_period: 15s + retries: 3 + +volumes: + conductor_data: diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml new file mode 100644 index 0000000..e5af462 --- /dev/null +++ b/docker-compose.e2e.yml @@ -0,0 +1,11 @@ +services: + backend: + environment: + - CONDUCTOR_SECRET_KEY=1111111111111111111111111111111111111111111111111111111111111111 + - CONDUCTOR_SESSION_KEY=22222222222222222222222222222222 + volumes: !override + - conductor_e2e_data:/app/data + - ./shared:/shared:ro + +volumes: + conductor_e2e_data: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3f2719d..dfc0e0b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -288,12 +288,15 @@ The MVP deployment model should support: ```text Reverse Proxy - -> Frontend static assets - -> Backend API service + -> Single same-origin Conductor production service + -> Compiled frontend and published SPA routes + -> Backend API -> SQLite database file ``` -A simple Docker Compose deployment is recommended for local demos and early internal use. +Development retains separate React and backend processes. Production uses one non-root, read-only-root container with a dedicated writable `/data` volume, required persistent encryption/session keys, and a health check. Docker Compose is the supported self-hosted deployment contract; Caddy or NGINX terminates TLS and proxies all paths to the same internal port. + +Live SQLite backups use the application's online backup command. A recovery archive includes the database and its matching encryption key plus manifest/checksums; copying only the live WAL database is unsupported. Future deployment options may include: diff --git a/docs/DEVELOPER_GUIDE.md b/docs/DEVELOPER_GUIDE.md new file mode 100644 index 0000000..aae0d3a --- /dev/null +++ b/docs/DEVELOPER_GUIDE.md @@ -0,0 +1,47 @@ +# Developer Guide + +## Architecture + +The canonical `ProjectDocument` is shared by Visual Editor, JSON Editor, Preview, validation, persistence, and publication snapshots. React owns authoring and ephemeral runtime state; Express owns authentication, authorization, persistence, credential resolution, proxy policy, validation, and publication execution. SQLite is the v0.1.0 durable store. + +Development uses two containers/processes for fast React/backend iteration. Production uses `Dockerfile.production`: a multi-stage build copies compiled frontend assets and compiled backend output into one minimal runtime image. Express serves API and SPA fallbacks from the same origin. + +## Local commands + +```bash +cd frontend && npm ci && npm test -- --watchAll=false --runInBand && npm run build +cd ../backend && npm ci && npm test +cd .. && npm ci && npm run test:schema && npm run test:governance +``` + +Use the deterministic mock stack and Playwright commands in `docs/RELEASE_VALIDATION.md` for browser acceptance. + +## Database and migrations + +`backend/src/db/init.ts` applies idempotent startup schema creation/migrations. New migrations must preserve existing data and have explicit downgrade/rollback implications. Tests select isolated databases through `CONDUCTOR_DATA_DIR`. + +Never copy the live WAL database as a backup. `backend/src/scripts/backupDatabase.ts` uses `better-sqlite3`'s online backup facility. + +## Security invariants + +- Every admin mutation requires an authenticated admin and CSRF validation. +- Published execution resolves actions and credentials from server-owned snapshots. +- Proxy destinations are default-deny for internal/private networks unless an exact reviewed origin is allowed. +- Credential plaintext must not enter project JSON, browser responses, errors, or execution history. +- Production refuses missing/invalid encryption and session keys. +- New production files must work with a non-root UID/GID 10001 and read-only root filesystem. + +## Adding canonical behavior + +Update together: + +1. Shared JSON schema. +2. Frontend TypeScript model and authoring UI. +3. Backend semantic validation. +4. Preview and published runtime behavior. +5. Unit/integration/browser coverage. +6. Schema examples, user/operator docs, roadmap, and traceability. + +## Release artifacts + +Run `scripts/release-image.sh` for immutable OCI tags. `scripts/test-production.sh` performs the disposable same-origin/backup/restore gate. Release publication details are in `docs/RELEASE.md`. diff --git a/docs/INSTALL.md b/docs/INSTALL.md new file mode 100644 index 0000000..c2bfe50 --- /dev/null +++ b/docs/INSTALL.md @@ -0,0 +1,128 @@ +# Production Installation + +## Supported installation paths + +Conductor supports: + +1. A public container installation using `compose.production.yml` and a pinned release image. +2. A reproducible source build using `Dockerfile.production` and `compose.production.build.yml`. +3. SkeletonWorks deployment through `setup-conductor.sh` in the `skeletonworks-scripts` repository. + +The production distribution is one same-origin service. Express serves the compiled React application, `/api`, and refreshable published routes such as `/apps/inventory/details` on internal port 8080. The development-only frontend/backend containers are not production artifacts. + +## Requirements + +- Docker Engine with Compose v2 +- Approximately 1 GiB memory and 1.5 CPU available by default +- A persistent Docker volume or host directory for `/data` +- TLS reverse proxy for internet-facing deployments +- Two independently generated persistent keys + +## Public container installation + +Download `compose.production.yml` and `.env.production.example` from the same tagged release. Do not mix files from different releases. + +```bash +mkdir conductor && cd conductor +curl -O https://gitea.skeletonworks.online/vwiebe/conductor/raw/tag/v0.1.0/compose.production.yml +curl -o .env https://gitea.skeletonworks.online/vwiebe/conductor/raw/tag/v0.1.0/.env.production.example +``` + +Edit `.env`: + +```dotenv +CONDUCTOR_SECRET_KEY= +CONDUCTOR_SESSION_KEY= +CONDUCTOR_IMAGE=gitea.skeletonworks.online/vwiebe/conductor +CONDUCTOR_VERSION=v0.1.0 +CONDUCTOR_BIND_ADDRESS=127.0.0.1 +CONDUCTOR_PORT=8080 +``` + +`CONDUCTOR_SECRET_KEY` encrypts stored REST credentials. Losing or changing it makes those credentials unrecoverable. `CONDUCTOR_SESSION_KEY` protects sessions; changing it logs everyone out. Neither belongs in source control. + +Start and verify: + +```bash +docker compose --env-file .env -f compose.production.yml pull +docker compose --env-file .env -f compose.production.yml up -d +docker compose --env-file .env -f compose.production.yml ps +curl --fail http://127.0.0.1:8080/api/health +``` + +Open the configured URL. A new database presents the browser first-run administrator screen. Conductor never ships a default username or password. + +## Build from source + +Use a tagged source archive or checkout: + +```bash +git clone https://gitea.skeletonworks.online/vwiebe/conductor.git +cd conductor +git checkout v0.1.0 +cp .env.production.example .env +# Replace both key placeholders. +docker compose --env-file .env \ + -f compose.production.yml -f compose.production.build.yml \ + up -d --build +``` + +The multi-stage build compiles frontend and backend code, prunes development dependencies, includes the matching project schema, and runs as UID/GID 10001. + +## Caddy + +Bind Conductor to loopback and proxy it through Caddy: + +```caddyfile +conductor.example.com { + reverse_proxy 127.0.0.1:8080 + encode zstd gzip +} +``` + +Caddy provides TLS automatically when DNS and ports 80/443 are correctly configured. + +## NGINX + +```nginx +server { + listen 443 ssl http2; + server_name conductor.example.com; + + location / { + proxy_pass http://127.0.0.1:8080; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} +``` + +Configure certificates using the operator's normal NGINX/TLS process. + +## Internal REST destinations + +The proxy denies private/internal destinations by default. `CONDUCTOR_PROXY_INTERNAL_ORIGINS` is an optional comma-separated allowlist of exact origins. Add only reviewed origins, for example `https://api.internal.example.com:443`. It does not accept wildcard hosts. + +## SkeletonWorks + +On a prepared SkeletonWorks host: + +```bash +sudo bash /opt/skeletonworks/scripts/setup-conductor.sh \ + --domain example.com \ + --image gitea.skeletonworks.online/vwiebe/conductor:v0.1.0 \ + --restartcaddy +``` + +The script creates `/opt/skeletonworks/conductor/conductor.example.com`, preserves keys across reruns, joins `skeletonworks_net`, installs a Caddy block and backup cron entry, waits for health, and emits a JSON result. Prefer immutable version, Git-SHA, or digest references over `latest`. + +## Uninstalling + +Stopping/removing containers does not remove the named data volume: + +```bash +docker compose --env-file .env -f compose.production.yml down +``` + +Do not add `--volumes` unless a verified backup exists and permanent data deletion is intended. diff --git a/docs/KNOWN_LIMITATIONS.md b/docs/KNOWN_LIMITATIONS.md new file mode 100644 index 0000000..c149de4 --- /dev/null +++ b/docs/KNOWN_LIMITATIONS.md @@ -0,0 +1,13 @@ +# Known Limitations for v0.1.0 + +- SQLite supports one Conductor instance with a persistent local volume; active/active replicas and PostgreSQL are not supported. +- Local username/password authentication is the release provider. OIDC/SSO is planned for Slice 9. +- Roles are global `admin` and `user`; per-application assignments and custom roles are not implemented. +- Published visibility is application-wide (`public` or all authenticated users), not page-specific. +- Page navigation is flat. Nested routes, path parameters, per-page permissions, and cross-page component bindings are not supported. +- Runtime values are ephemeral and reset on full browser reload. +- REST proxy authentication supports Anonymous, Basic, Bearer, API-key header, and API-key query. OAuth 2.0, mTLS, and custom auth are deferred. +- Workflow orchestration, action graphs, loops, retries, and long-running jobs are out of scope. +- Binding transforms and advanced expression languages are deferred. +- The release image targets `amd64` and `arm64` only after both pass the native SQLite build/smoke gate. +- Backups are local archives by default; operators must arrange off-host copies and retention appropriate to their recovery objectives. diff --git a/docs/MVP_TRACEABILITY.md b/docs/MVP_TRACEABILITY.md index f4b1d4f..0e20995 100644 --- a/docs/MVP_TRACEABILITY.md +++ b/docs/MVP_TRACEABILITY.md @@ -10,7 +10,7 @@ This matrix maps every release-critical requirement area in `docs/REQUIREMENTS.m |---|---|---|---|---| | R1 | Browser-based application, React frontend, backend API, SQLite persistence, and Docker Compose deployment | Slices 1, 6, 8 | Production builds, Compose startup/health/restart/persistence, backup/restore in `docs/RELEASE_VALIDATION.md` and `TESTING.md` | Accepted | | R2 | Canonical JSON project definition shared by Visual Editor, JSON Editor, Preview, persistence, and validation | Slices 2, 5, 6 | Schema matrix, canonical round trips, cross-view synchronization, save/reload/restart tests | Accepted | -| R3 | Single-page visual canvas with the eleven schema-supported components, selection, movement, resizing, deletion, properties, and basic styling | Slices 1, 2 | Slice 1/2 component tests and accepted canvas/property/Preview manual workflows | Accepted | +| R3 | Per-page visual canvas with the eleven schema-supported components, selection, movement, resizing, deletion, properties, and basic styling | Slices 1, 2, 7c | Slice 1/2 component tests and accepted canvas/property/Preview/manual multi-page workflows | Accepted | | R4 | REST actions with supported methods, URL, headers, query/path parameters, body template, description, and fixed proxy-policy timeout | Slices 2, 4 | Action-editor tests, proxy integration/security suites, `docs/SCHEMA.md`, `docs/PROXY_SECURITY.md` | Accepted | | R5 | Button `onClick` and page `onLoad` execution; component/variable request templates; Table selection runtime binding | Slices 2, 6 | Frontend event/template/runtime tests and deterministic launcher/dashboard E2E | Accepted | | R6 | Top-level response/component bindings to supported component properties or typed runtime variables | Slices 2, 5, 6 | Binding diagnostics/runtime tests, compatibility validation, dependent-data/dashboard workflows | Accepted | @@ -27,6 +27,7 @@ This matrix maps every release-critical requirement area in `docs/REQUIREMENTS.m | R17 | A fresh installation creates its initial administrator through a secure browser first-run flow without requiring Docker commands | Slice 7b | Atomic setup/security integration tests, frontend setup/password tests, recovery verification, and accepted manual workflow | Accepted | | R18 | Authored and published applications support multiple deep-linked pages with page-local components and explicit global/page runtime-variable scope | Slice 7c | Schema/editor/runtime/publishing tests, multi-page browser E2E, security validation, and manual acceptance | Accepted | | R19 | The Visual Editor presents project, page, page-setting, and lifecycle commands in a professional, accessible, responsive command ribbon | Slice 7d | 28 frontend suites / 528 tests, production build, and accepted manual visual validation | Accepted | +| R20 | Public self-hosters and SkeletonWorks operators can reproducibly install, secure, upgrade, back up, restore, and roll back a production Conductor release | Slice 8 | Production image/Compose tests, anonymous release download, fresh-host deployment, SkeletonWorks script validation, and encrypted-data backup/restore drill | Planned | ## Approved acceptance workflows @@ -46,7 +47,7 @@ The following capabilities are not release requirements. Their presence in futur - Provider-neutral AI assistance, including chat, generation, explanation, documentation, and refactoring - IBM Bob, watsonx, or any provider-specific AI dependency - OIDC/enterprise SSO, external identity provisioning, and claim/group mapping (Slice 9) -- Multi-page authoring and advanced page management +- Nested/dynamic page routing, page parameters, and per-page authorization - Direct Dropdown-triggered REST execution and general action chaining/orchestration - Conditions, branches, loops, parallelism, retries, workflow graphs, and long-running task orchestration - OAuth 2.0, IBM Cloud IAM, mTLS, and custom authentication scripts @@ -55,7 +56,7 @@ The following capabilities are not release requirements. Their presence in futur - Binding transforms, expression languages, and configurable per-binding error policies - Canvas zoom/pan, undo/redo, advanced JSON-editor tooling, themes, and reusable component libraries - Tabs, modals, date pickers, file uploads, charts, progress indicators, and advanced Table capabilities -- RBAC, team collaboration, approval workflows, enterprise audit retention, and SIEM integration +- Custom/per-application roles, groups, team collaboration, approval workflows, enterprise audit retention, and SIEM integration - PostgreSQL/high-availability/centralized multi-user deployment ## Governance rule diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md new file mode 100644 index 0000000..527d827 --- /dev/null +++ b/docs/OPERATIONS.md @@ -0,0 +1,100 @@ +# Production Operations + +## Persistent state + +All durable application state is stored in `/data/conductor.db`: projects, users, password hashes, sessions, publications, encrypted credentials, and execution history. SQLite runs in WAL mode. + +The database alone is not a complete recovery set. A usable backup must include the exact `CONDUCTOR_SECRET_KEY` that encrypted stored credentials. Production backup archives therefore include the database, `.env`, Compose metadata, a manifest, and checksums. Treat archives as secrets and copy them off-host. + +## Backup + +From a source/release directory: + +```bash +scripts/production/backup-conductor.sh \ + --compose-file compose.production.yml \ + --env-file .env \ + --backup-dir ./backups \ + --retention-days 7 +``` + +The script invokes the application-owned SQLite online-backup command inside the running container. It does not copy a live WAL database. It then creates a `0600` archive with: + +- `database/conductor.db` +- `configuration/.env` +- `configuration/compose.production.yml` +- `manifest.json` +- `SHA256SUMS` + +It writes progress to stderr and a machine-readable result to stdout and `backup-result.json`. + +SkeletonWorks uses: + +```bash +sudo /opt/skeletonworks/scripts/backup-conductor.sh \ + --fqdn conductor.example.com --retention-days 7 +``` + +Its archive additionally contains `.secrets` and defaults to `/opt/skeletonworks/backups/conductor//`. + +## Restore + +Restore replaces live state. Verify that the chosen archive and the intended target match. + +```bash +scripts/production/restore-conductor.sh --list --backup-dir ./backups +scripts/production/restore-conductor.sh \ + --backup-file ./backups/conductor_.tar.gz \ + --compose-file compose.production.yml \ + --env-file .env +``` + +Interactive restore requires typing `RESTORE`. Automation must explicitly pass `--force`. By default, restore creates a current-state safety backup, validates paths/checksums/format/keys, stops Conductor, replaces the database, restores UID/GID 10001 and mode `0600`, restarts, and waits for health. + +SkeletonWorks: + +```bash +sudo /opt/skeletonworks/scripts/restore-conductor.sh --fqdn conductor.example.com --list +sudo /opt/skeletonworks/scripts/restore-conductor.sh \ + --fqdn conductor.example.com --backup-file +``` + +Never combine a database with a different encryption key. A checksum-valid but mismatched key cannot decrypt stored REST credentials. + +## Upgrade + +1. Read release notes and compatibility warnings. +2. Create and copy a verified backup off-host. +3. Change `CONDUCTOR_VERSION` or the pinned SkeletonWorks image. +4. Pull and recreate. +5. Wait for health and test login, project load, and a published deep link. + +```bash +scripts/production/backup-conductor.sh +docker compose --env-file .env -f compose.production.yml pull +docker compose --env-file .env -f compose.production.yml up -d +curl --fail http://127.0.0.1:8080/api/health +``` + +SkeletonWorks upgrades are idempotent reruns of `setup-conductor.sh --image `. + +## Rollback + +Application images and database schema must be treated as a pair. For a failed compatible deployment, return to the prior pinned image. If the release changed the database incompatibly, restore the pre-upgrade archive as well. + +1. Preserve logs and failed-state data. +2. Set the previous image tag/digest. +3. Restore the matching pre-upgrade backup. +4. Verify health and application smoke tests. + +## Health and logs + +- Liveness/ready endpoint: `GET /api/health` +- Container health: `docker compose ... ps` +- Logs: `docker compose ... logs --tail=200 conductor` + +The health response includes the packaged version when built with release metadata. + +## Recovery objectives + +The default scripts retain seven daily archives, but retention is not a substitute for off-host storage. Recovery point and recovery time objectives are operator decisions. At least one periodic clean-host restore drill should be scheduled and recorded. diff --git a/docs/RELEASE.md b/docs/RELEASE.md new file mode 100644 index 0000000..e1b2755 --- /dev/null +++ b/docs/RELEASE.md @@ -0,0 +1,45 @@ +# Release Construction and Publication + +## Image identity + +Each production image must correlate to one source revision and include OCI version, revision, creation-time, source, title, and description labels. + +Release tags: + +- `vX.Y.Z`: immutable release +- `git-<12-character-sha>`: immutable revision identity +- `latest`: optional convenience pointer; never recommended for controlled deployment + +## Build locally + +```bash +scripts/release-image.sh --version v0.1.0 --platforms linux/amd64 +``` + +## Publish multi-architecture images + +Authenticate to the registry using a scoped token, then: + +```bash +docker login gitea.skeletonworks.online +scripts/release-image.sh --version v0.1.0 --push +``` + +The default published platforms are `linux/amd64,linux/arm64`. Both must successfully compile and load the native `better-sqlite3` dependency. If a target cannot pass build and smoke tests, omit it and state that explicitly in release notes. + +## Release checklist + +1. Clean checkout of the intended commit. +2. Governance, schema, frontend, backend, and browser suites pass. +3. Production image builds without local dependencies. +4. Missing-key startup fails; configured startup becomes healthy. +5. Root UI, `/api/health`, and published deep-link refresh return correctly. +6. Fresh first-run administrator setup passes with no default credentials. +7. Restart preserves data. +8. Online backup, checksum verification, destructive restore, and marker recovery pass. +9. Upgrade and rollback drill passes with pinned images and matching database backup. +10. Public source archive and container image are anonymously downloadable. +11. Release notes list image digests, architectures, known limitations, and upgrade instructions. +12. Tag and push only after explicit release approval. + +Registry authentication is required only for publishing. Consumers must not require credentials for a public release; if the Gitea package cannot be pulled anonymously, publish the identical digest to a supported public registry and use it in public installation examples. diff --git a/docs/RELEASE_VALIDATION.md b/docs/RELEASE_VALIDATION.md index 6f2f7ad..87d5b80 100644 --- a/docs/RELEASE_VALIDATION.md +++ b/docs/RELEASE_VALIDATION.md @@ -43,10 +43,28 @@ npm ci npm run test:schema docker compose -f docker-compose.yml -f docker-compose.manual-test.yml up -d --build npm run test:e2e + +docker build -f Dockerfile.production -t conductor:slice8-test . +npm run test:production +shellcheck scripts/production/*.sh scripts/release-image.sh scripts/test-production.sh ``` The backend integration suite binds isolated localhost ports. In restricted command sandboxes it may require explicit permission for local listening sockets. +## Production distribution gate + +`npm run test:production` creates a uniquely named disposable Compose project and volume. It proves: + +- production startup refuses absent keys; +- the image runs as the non-root `conductor` user with a read-only root filesystem; +- health, root frontend, and a published deep-link refresh are served from one port; +- a live WAL-mode database is backed up using SQLite's online backup facility; +- the archive manifest and checksums validate; +- destructive restore resets ownership/mode, becomes healthy, and recovers a marker record; +- cleanup removes the disposable container, network, volume, environment, and archive. + +The release image workflow publishes immutable version and Git-SHA tags with OCI metadata, SBOM, and provenance for `linux/amd64` and `linux/arm64`. Before release approval, an unauthenticated client must pull the final digest. If Gitea denies anonymous package pulls, the same digest must be published to the documented public fallback registry. + ## Manual gate Manual acceptance begins only after all automated commands, Docker restart and persistence, backup/restore, repository credential scan, and cleanup checks pass. The manual checklist must cover all six workflows in `MVP_SCOPE.md`, including all authentication modes and failure recovery, and record any accepted limitations or defects. diff --git a/docs/REQUIREMENTS.md b/docs/REQUIREMENTS.md index f780f33..5bebb09 100644 --- a/docs/REQUIREMENTS.md +++ b/docs/REQUIREMENTS.md @@ -37,8 +37,7 @@ The system is not intended to be a full Appsmith replacement. The MVP will not include: * Full database integrations. -* Complex multi-user application publishing. -* Advanced permissions or RBAC. +* Per-application user assignments, custom roles, groups, or approval workflows beyond global admin/user roles. * Marketplace-style widget plugins. * Pixel-perfect design tooling. * Full workflow orchestration. @@ -535,7 +534,7 @@ The system should provide basic troubleshooting details: The MVP should include: -* Single-page project builder. +* Multi-page project builder with flat navigation and refreshable published deep links. * Basic canvas. * Core widgets. * REST action configuration. @@ -544,6 +543,9 @@ The MVP should include: * API response-to-component binding. * Preview mode. * Save/load project as JSON. +* Local authentication with global admin/user roles and browser first-run administration. +* Public or authenticated immutable application publishing. +* Single-image production distribution with verified backup and restore. The MVP does not include IBM Bob, watsonx, or any other AI dependency. @@ -553,8 +555,8 @@ The MVP does not include IBM Bob, watsonx, or any other AI dependency. Potential future enhancements: -* Multi-page applications. -* Role-based access control. +* Nested/dynamic pages and per-page authorization. +* Custom roles, groups, and per-application user assignments. * OAuth support. * IBM Cloud IAM integration. * Advanced or enterprise execution-history retention and integrations. @@ -565,7 +567,7 @@ Potential future enhancements: * Reusable templates. * Version history. * Approval workflows. -* Deployment to internal hosting. +* Managed public SaaS hosting. * Git integration. * Team collaboration. @@ -626,8 +628,7 @@ The application backend may include an embedded development server for local tes The reverse proxy should handle: * HTTPS termination -* Static frontend asset delivery -* Routing requests to the backend service +* Routing requests to the same-origin Conductor production service * Request size limits * Basic security headers * Optional access restrictions @@ -639,7 +640,7 @@ Browser ↓ NGINX / Apache / Caddy / IBM-approved reverse proxy ↓ -Frontend static assets + Backend API service +Single Conductor service (compiled frontend + Backend API) ↓ REST API endpoints / Concert / RIA / other systems ``` @@ -658,6 +659,7 @@ SQLite should store: * Canonical project JSON, which contains pages, components, layout, actions, bindings, variables, settings, and non-secret configuration * Encrypted secret records and metadata * Sanitized bounded execution history +* Local users, sessions, and immutable publication snapshots Secrets should not be stored directly in plain text in SQLite. @@ -680,9 +682,9 @@ If the project grows beyond MVP, the backend should be designed so SQLite can la A full RDBMS may be required if the system needs: -* Multiple concurrent users +* High-concurrency or horizontally scaled deployments * Team collaboration -* Role-based access control +* Custom/per-application authorization models * High availability * Centralized deployment * Enterprise backup/restore diff --git a/docs/SCHEMA.md b/docs/SCHEMA.md index dd796b1..7da6a82 100644 --- a/docs/SCHEMA.md +++ b/docs/SCHEMA.md @@ -25,7 +25,7 @@ The JSON schema: - Documents the exact shape of a valid project definition. - Enables offline validation during development. - Drives IDE autocomplete and inline error highlighting when `$schema` is set in a project file. -- Will be used by the backend's validation endpoint (Step 11) to reject malformed saves. +- Is used by the backend validation/persistence endpoints to reject malformed saves atomically. - Makes project definitions portable, diffable in Git, and importable/exportable. --- @@ -63,10 +63,10 @@ Consumers (backend, editor, preview runtime) must check the `MAJOR` version comp | `id` | `string` | ✅ | Stable unique identifier (UUID or URL-safe slug). Must not change. | | `name` | `string` | ✅ | Human-readable display name (1–200 characters). | | `description` | `string` | — | Optional free-text description. Defaults to `""`. | -| `pages` | `array` | ✅ | Ordered list of `Page` objects. May be empty. | +| `pages` | `array` | ✅ | Ordered non-empty list of `Page` objects. | | `actions` | `array` | ✅ | Project-level REST action definitions. May be empty. | | `bindings` | `array` | ✅ | Project-level binding definitions. May be empty. | -| `variables` | `object` | ✅ | Named global variable declarations. May be empty (`{}`). | +| `variables` | `object` | ✅ | Named global or page-scoped declarations. May be empty (`{}`). | | `settings` | `object` | ✅ | Project display and canvas settings. May be empty (`{}`). | --- @@ -81,6 +81,8 @@ Represents one view in the project. Required fields: `id`, `name`, `components`. { "id": "page_home", "name": "Home", + "slug": "home", + "showInNavigation": true, "description": "", "order": 0, "components": [], @@ -88,6 +90,8 @@ Represents one view in the project. Required fields: `id`, `name`, `components`. } ``` +Page slugs are unique URL-safe segments. Older documents may omit them; the runtime derives a deterministic slug. `showInNavigation: false` hides a page from automatic navigation but is not authorization. `project.settings.defaultPageId` selects the base-route page; otherwise the first ordered page is the compatibility default. Page events support `onLoad` (first visit in the loaded session) and `onEnter` (return visits). + ### `Component` A UI element placed on a page canvas. Required fields: `id`, `type`, `name`, `position`, `size`. @@ -110,7 +114,7 @@ Allowed `type` values: "events": [ { "event": "onClick", - "actionId": "action_trigger_workflow" + "navigateToPageId": "page_results" } ], "bindings": [] @@ -136,19 +140,20 @@ Allowed `authenticationType` values: `anonymous` · `bearerToken` · `basicAuth` "name": "Trigger Workflow", "description": "Calls the Concert RIA API to trigger a workflow run.", "method": "POST", - "url": "https://api.example.com/workflows/{{workflowId}}/run", + "url": "https://api.example.com/workflows/run", "headers": { "Content-Type": "application/json", "Accept": "application/json" }, "queryParameters": { - "environment": "{{environment}}" + "environment": "{{components.environment.value}}" }, "pathParameters": { - "workflowId": "{{workflowId}}" + "workflowId": "configured-static-id" }, - "bodyTemplate": "{\"params\": {{params}}}", - "authenticationType": "bearerToken" + "bodyTemplate": "{\"requestedBy\": \"{{variables.currentUser}}\"}", + "authenticationType": "bearerToken", + "secretReferenceId": "opaque-server-secret-id" } ``` @@ -173,11 +178,11 @@ New response mappings belong in top-level `project.bindings`, not inside the act | `name` | `string` | ✅ | Human-readable action name shown in the Actions panel. | | `description` | `string` | — | Optional description of what this action does. | | `method` | `string` | ✅ | HTTP method: `GET`, `POST`, `PUT`, `PATCH`, or `DELETE`. | -| `url` | `string` | ✅ | Target URL. Use `{{paramName}}` for path parameter placeholders. | -| `headers` | `object` | — | Static request headers. Values may use `{{variableName}}` syntax. | -| `queryParameters` | `object` | — | URL query string parameters. Values may use `{{variableName}}` syntax. | -| `pathParameters` | `object` | — | Path segment substitutions. Keys match `{{paramName}}` in the URL. | -| `bodyTemplate` | `string` | — | Request body template. Use `{{variableName}}` for runtime substitutions. | +| `url` | `string` | ✅ | Target URL; may use executed component/variable templates. | +| `headers` | `object` | — | Request headers; values may use executed templates. | +| `queryParameters` | `object` | — | Query parameters; values may use executed templates. | +| `pathParameters` | `object` | — | Static path substitutions; runtime templates are not supported here. | +| `bodyTemplate` | `string` | — | Request body string with component/variable templates. | | `authenticationType` | `string` | ✅ | Authentication strategy (see allowed values above). Credentials are never stored here. | | `responseMapping` | `array` | — | Deprecated legacy action-local mappings. New mappings use top-level `project.bindings`. | @@ -201,7 +206,7 @@ Declares data flow between a source and a target. Required fields: `id`, `source ### `Variable` -A named global variable. Required field: `type`. +A named global or page variable. Required field: `type`. Omitted `scope` means `global` for compatibility; `scope: "page"` requires a valid owning `pageId`. Allowed `type` values: `string` · `number` · `boolean` · `object` · `array` @@ -209,6 +214,8 @@ Allowed `type` values: `string` · `number` · `boolean` · `object` · `array` { "lastRunStatus": { "type": "string", + "scope": "page", + "pageId": "page_home", "defaultValue": "", "description": "Status returned by the most recent workflow trigger." } diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md new file mode 100644 index 0000000..02b977b --- /dev/null +++ b/docs/TROUBLESHOOTING.md @@ -0,0 +1,35 @@ +# Troubleshooting + +## Container exits immediately + +Inspect logs. Production intentionally refuses to start when `CONDUCTOR_SECRET_KEY` is not a valid 32-byte base64/hex value or `CONDUCTOR_SESSION_KEY` is shorter than 32 bytes. + +## Container is unhealthy + +```bash +docker compose --env-file .env -f compose.production.yml ps +docker compose --env-file .env -f compose.production.yml logs --tail=200 conductor +curl -v http://127.0.0.1:8080/api/health +``` + +Confirm `/data` is writable by UID/GID 10001. Restore tooling resets database ownership to `10001:10001` and mode `0600`. + +## Stored credential no longer works after restore + +The restored database and `CONDUCTOR_SECRET_KEY` likely do not match. Restore the complete archive, including its `.env`/`.secrets`. Credential encryption cannot be bypassed or recovered without the original key. + +## Published deep link returns proxy 404 + +The reverse proxy must send all non-API paths to Conductor rather than serving static files independently. Test the container directly at `/apps//` and use the Caddy/NGINX examples in `docs/INSTALL.md`. + +## REST request is blocked + +Read the safe error code and `docs/PROXY_SECURITY.md`. Private/internal destinations are denied unless their exact origin is configured in `CONDUCTOR_PROXY_INTERNAL_ORIGINS`. Do not add broad exceptions. + +## Browser shows old controls after upgrade + +Confirm the running image tag/digest, then perform a hard browser refresh. The production `index.html` is sent with `no-cache`; hashed assets may be cached safely. + +## Backup refuses to run + +The application container must be running and the environment file must exist because it contains recovery keys. Check Compose project selection and paths. SkeletonWorks commands require the exact `--fqdn` used during setup. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md new file mode 100644 index 0000000..c4a1515 --- /dev/null +++ b/docs/USER_GUIDE.md @@ -0,0 +1,46 @@ +# User Guide + +## Roles and entry points + +- **Admin** users create projects, manage credentials and users, inspect execution history, and publish applications. +- **User** users open restricted published applications and manage their own local password/profile. +- Public applications can be opened without signing in. + +A fresh installation creates its first administrator in the browser. Later accounts are managed from **Users**. + +## Building an application + +1. Create or load a project in **Visual Editor**. +2. Use **Pages** to add, duplicate, reorder, name, slug, hide, default, or delete pages. +3. Add components from the palette and configure the selected component in the inspector. +4. Define variables, REST actions, credentials, and response bindings in **Actions & Bindings**. +5. Configure Button action/navigation events and page `onLoad`/`onEnter` lifecycle actions. +6. Use **Preview** to exercise the application without changing canonical saved state. +7. Save or update the project. +8. Publish from **Publishing** as public or restricted. + +Public publications may use only anonymous actions. Restricted publications may use backend-held credentials. Publishing creates an immutable snapshot; edit/save and republish to release changes. + +## Variable and component scope + +Component names are local to a page. The same name may be reused on different pages. Global variables are available throughout the app; page variables belong to one page. Values persist while navigating in one loaded session and reset on full reload. + +## Lifecycle behavior + +- `onLoad`: first entry to that page during the loaded session. +- `onEnter`: subsequent entry when returning to the page. + +## JSON Editor + +The JSON Editor edits the same canonical document as the Visual Editor. Validate and apply before saving. Invalid JSON or semantic references do not replace the last valid in-memory document. + +## Credentials + +Credential values are submitted to encrypted server-side storage and are never displayed again. Projects contain opaque secret references, not values. Replacing a credential requires entering the new value twice where applicable. + +## Published URLs + +- `/apps/` opens the default page. +- `/apps//` is a refreshable page deep link. + +Hiding a page removes it from automatic navigation; it is not an authorization rule. diff --git a/docs/response-mapping-model.md b/docs/response-mapping-model.md index 950592d..58e9fd8 100644 --- a/docs/response-mapping-model.md +++ b/docs/response-mapping-model.md @@ -1,412 +1,81 @@ -# Response Mapping Model +# Canonical Response Mapping and Runtime Model -**Step 17.0 design document — established 2025** +## Ownership -This document records the canonical decisions about how action responses flow into -component state in the Conductor Preview runtime. It is the authoritative -reference for Steps 17.x implementation. +New response mappings are top-level `project.bindings`. The legacy `action.responseMapping` field remains schema-compatible but is not executed and must not be authored by new UI/examples. ---- - -## Table of Contents - -1. [Why `project.bindings` is canonical](#1-why-projectbindings-is-canonical) -2. [Why actions own runtime responses](#2-why-actions-own-runtime-responses) -3. [Why `actions..response` is a virtual namespace](#3-why-actionsidresponse-is-a-virtual-namespace) -4. [Source path grammar](#4-source-path-grammar) -5. [Target path grammar](#5-target-path-grammar) -6. [Trigger semantics](#6-trigger-semantics) -7. [Label runtime-value behaviour](#7-label-runtime-value-behaviour) -8. [Compatibility — existing `onClick` bindings](#8-compatibility--existing-onclick-bindings) -9. [Deprecation of `action.responseMapping`](#9-deprecation-of-actionresponsemapping) -10. [Step 17.1 scope](#10-step-171-scope) -11. [Deferred capabilities](#11-deferred-capabilities) -12. [Required diagnostics](#12-required-diagnostics) - ---- - -## 1. Why `project.bindings` is canonical - -Two mechanisms could in principle move action response data into components: - -| Mechanism | Location | Status | -|---|---|---| -| `action.responseMapping[]` | Inside each `Action` object | **Deprecated** | -| `project.bindings[]` | Top-level project array | **Canonical** | - -`project.bindings` was chosen as the sole canonical mechanism for the following -reasons: - -**Single locus of data-flow declarations.** All data movement in a Conductor -project — between components, from action responses, from variables — is -expressed in one flat array. Keeping response routing there avoids a split -where "inbound response routing" lives inside actions while all other data -movement lives at the project level. - -**Symmetry with component-sourced bindings.** A binding from a TextInput value -to a Label uses the same structure as a binding from an action response to a -JsonViewer. Same schema, same Inspector rendering, same future transform field. - -**Decoupled from action definition.** An action definition describes *how to -call* an endpoint (URL, method, auth). Where the response goes is a layout -concern, not an API concern. Separating them means a single action can feed -multiple targets without duplicating the action. - -**Better Inspector and diagnostics surface.** A flat array of bindings is easy -to enumerate, validate cross-references for, and display in a panel. -Distributing response mapping rules across individual action objects makes -completeness checks harder. - -**Incremental migration path.** Existing documents that carry -`action.responseMapping` values remain valid under the schema; they are simply -ignored at runtime. New documents use only `project.bindings`. - ---- - -## 2. Why actions own runtime responses - -Although `project.actions` is a design-time array of REST action definitions, -the Preview runtime must store ephemeral per-action state: the most recent -response, loading flag, and error. This state is: - -- **Not stored in canonical project JSON.** The project document is the - design-time source of truth. Mutating it with runtime values would corrupt - the save/load round-trip. -- **Keyed by action ID.** Even though actions are stored as an ordered array - in the project document, the runtime resolves them by `id` for O(1) lookup. -- **Discarded on navigation or refresh.** Preview state is ephemeral React - state (`useState`). - -The proposed runtime shape (to be implemented in Step 17.1) is: - -```ts -type ActionRuntimeState = { - response?: ProxyResponse; // the full ProxyResponse envelope - loading?: boolean; - error?: string; -}; - -type ActionRuntimeStateMap = Record; +```json +{ + "id": "binding_inventory", + "source": "actions.loadInventory.response.body.items", + "target": "components.inventoryTable.rows", + "trigger": "onSuccess" +} ``` -This is a parallel structure to the existing `componentState` map in -`usePreviewRuntime`. In Step 17.1 it will be introduced alongside it. +## Action execution -> **Step 17.0 note:** This runtime shape is defined here for design purposes. -> It is not implemented in Step 17.0. The current runtime continues to store -> the full `ProxyResponse` envelope directly in `componentState[id].value`. +Actions are project-level and reusable. A Button `onClick`, Dropdown `onChange` where supported, or page lifecycle event selects an `actionId`. A Button event may instead contain `navigateToPageId`; navigation is not represented as a REST action. ---- +Before execution, request fields interpolate: -## 3. Why `actions..response` is a virtual namespace +- `{{components..value}}` against components on the invoking page only. +- `{{variables.}}` against global variables or page variables owned by the invoking page. -In binding source and target expressions, paths of the form: +Credential-backed authentication is resolved by the backend through opaque `secretReferenceId`. Browser clients never supply authoritative URLs, authentication modes, actions, or secret values for published execution. -``` -actions..response -actions..response.body -actions..response.body. -``` +## Response source paths -refer to runtime action response data. This is a **virtual namespace** because: - -1. `project.actions` is an **array** in the canonical document, not an object - keyed by ID. At design time, `actions.action_httpbin` does not exist as a - JSON key path. - -2. The runtime resolves `` via `Array.find((a) => a.id === actionId)`, - not by property access. - -3. The path is only meaningful during Preview execution. It has no value in the - persisted project document. - -The path syntax is chosen for readability and symmetry with `components..*` -target paths. It is not JSONPath; it is a **Conductor runtime dot-path** with -a fixed `actions..response[.body[.]]` prefix. - ---- - -## 4. Source path grammar - -Step 17.1 will support the following source path forms for action-response -bindings: - -``` -actions..response -actions..response.body -actions..response.body. -actions..response.body.. -``` - -The v0.1.0 runtime additionally supports one bounded component source for the -read-only dashboard workflow: +Canonical action sources begin with: ```text -components..selectedRow +actions..response +actions..response.body +actions..response.body. ``` -It requires trigger `onChange` and may target a supported -`components..value` path. The selected row and resulting display -value are runtime-only and are discarded when Preview is reinitialized. +The current resolver supports ordinary dot-separated object traversal. JSONPath filters, wildcards, quoted dotted keys, and transform expressions are deferred. -### Resolution rules +## Targets -| Path | Resolved value | -|---|---| -| `actions..response` | The full `ProxyResponse` envelope `{ ok, status, statusText, headers, body, durationMs }` | -| `actions..response.body` | `ProxyResponse.body` — parsed JSON object or text string | -| `actions..response.body.` | `ProxyResponse.body.` where `body` is an object | -| `actions..response.body..` | Deep dot-path traversal on `body` | +Supported action-response targets are validated against component type/property compatibility: -If a field in the path does not exist on the response body, the resolved value -is `undefined`. Step 17.1 will surface a runtime warning in that case; it -will not throw. +- `components..value` +- `components..options` +- `components..rows` +- `variables.` -### Deferred source path forms +Component names are page-local. Response bindings execute in the invoking page context and may update components on that page, global variables, and page variables owned by that page. Cross-page component/page-variable writes are prohibited. -The following are explicitly deferred and must not be inferred from the above: +## Triggers -- `actions..response.headers.
` — reserved; not in Step 17.1 -- `actions..response.status` — reserved; not in Step 17.1 -- Wildcard `*` segments -- Filter expressions `[?(...)]` -- Array index syntax `[0]`, `[-1]` -- Keys containing literal `.` characters +- `onSuccess` is the canonical trigger for an action response and runs only when `ProxyResponse.ok` is true. +- Legacy action-response `onClick` is accepted for compatibility and should be migrated to `onSuccess`. +- Table `components..selectedRow` bindings use `onChange`. +- `onError`, action chaining, and arbitrary reactive component graphs are deferred. ---- +## Runtime state -## 5. Target path grammar +Configured defaults remain in canonical project JSON. Preview/published component, action, and variable values live in ephemeral runtime state and never mutate the canonical document. Global and page values survive in-session page navigation and reset on full reload. -Step 17.1 will support only: +Actions expose loading, success, upstream failure, network/policy failure, and mapping failure states. Mapped components receive loading/error state while requests run. When a successful action has no mapping, the triggering control surfaces the response for operator feedback. -``` -components..value -``` +## Lifecycle -### Semantic enforcement +- Page `onLoad` runs on the first entry to that page during one loaded runtime session. +- Page `onEnter` runs when returning to a previously visited page. +- Reopening/reloading the runtime begins a fresh session. -- `` must match `component.name` (not `component.id`) in the - project's page components, consistent with the existing binding model. -- The property segment **must be** `.value`. The Inspector must emit a warning - for targets with any other property segment (e.g. `.data`, `.message`, - `.label`). The runtime must not silently remap these to `.value`. +## Diagnostics -### Property segment rationale +Validation rejects or warns about: -`.value` is the single runtime-mutable property exposed by `ComponentRuntimeState` -for display purposes. Until per-property runtime targeting is designed: +- dangling action, binding, page, component, variable, and secret references; +- unsupported trigger/source/target combinations; +- duplicate page/component identities in their required scopes; +- malformed templates and unsupported component template properties; +- page-variable ownership violations and cross-page references; +- incompatible response target properties; +- deprecated `action.responseMapping` and inert `inputMap` compatibility fields. -- `components..value` is valid. -- `components..data`, `.message`, `.label`, `.text`, etc. are unsupported. - They must produce an Inspector diagnostic and a runtime warning. - -### Deferred target forms - -- `variables.` — variable runtime support is deferred -- `components..label` — Label runtime-value override is deferred to Step 17.1 - (see §7) -- Per-property routing for non-`.value` segments - ---- - -## 6. Trigger semantics - -### Canonical trigger for action-response bindings: `onSuccess` - -```json -{ "trigger": "onSuccess" } -``` - -`onSuccess` fires when the action completes and `ProxyResponse.ok === true`. - -This is the trigger all new action-response bindings should use. - -### Why not `onChange` (the schema default)? - -The schema's `binding.trigger` default is `"onChange"`. This default is -appropriate for component-to-component data flow (e.g. a TextInput value driving -a downstream Label). It is not appropriate for action-response bindings because: - -- Action responses are not "change" events on a component; they are completion - events on an async operation. -- `"onChange"` implies reactive/continuous behaviour; action execution is - discrete and user-triggered (or lifecycle-triggered). - -The `"onChange"` default is therefore **not the correct default for -action-response bindings**. The Inspector should surface a warning when -`"onChange"` is found on a binding whose source is `actions.*`. - -### Deferred triggers - -| Trigger | Status | -|---|---| -| `onSuccess` | **Step 17.1** | -| `onError` | Deferred | -| `onLoad` | Deferred | -| `onChange` | Component-change bindings only; deferred | -| Chained actions | Deferred | - ---- - -## 7. Label runtime-value behaviour - -**Current (Step 17.0):** Label renders `properties.label` only. It has no -`runtimeState` wiring. Runtime action responses cannot update a Label. - -**Planned (Step 17.1):** - -- `properties.label` — design-time / configured text. Displayed when no - runtime value is set. -- `componentState[id].value` — runtime display value. If set (not `undefined`), - it overrides `properties.label` in the Preview renderer. -- A binding of the form: - ```json - { - "source": "actions..response.body.", - "target": "components..value", - "trigger": "onSuccess" - } - ``` - writes the resolved field value (coerced to string for display) into - `componentState[labelId].value`. -- **Runtime mutations must not modify canonical project JSON.** The override - is ephemeral; it lives only in `useState` for the Preview session. - -This makes Label consistent with JsonViewer, which already reads -`runtimeState.value` in `PreviewComponent.tsx`. - ---- - -## 8. Compatibility — existing `onClick` bindings - -Steps 15 and 16 produced examples and docs using: - -```json -{ "trigger": "onClick" } -``` - -on action-response bindings. These bindings fire when the button's `onClick` -event causes the action to execute. - -**Step 17.0 treatment:** `"onClick"` bindings continue to work exactly as they -do today. The runtime (`usePreviewRuntime`) fires all bindings whose source -matches the action ID, regardless of their `trigger` value. - -**Recommended migration:** New bindings should use `"onSuccess"` rather than -`"onClick"`. The Inspector may surface an info-level diagnostic recommending -migration on bindings that use `"onClick"` as the trigger on an -`actions.*` source. This is not yet implemented. - -**Important:** `"onClick"` on an action-response binding is **legacy**, not -invalid. Documents using it will not fail schema validation and will not be -broken by Step 17.1. - ---- - -## 9. Deprecation of `action.responseMapping` - -`action.responseMapping` is **deprecated as of Step 17.0**. - -| Property | Status | -|---|---| -| Present in schema | Yes — retained for backward compatibility | -| `x-deprecated: true` annotation | Added in Step 17.0 | -| Schema `description` | Updated to say `project.bindings` is canonical | -| TypeScript `@deprecated` JSDoc | Added in Step 17.0 | -| TS type narrowed | Yes — changed from `unknown[]` to `ResponseMappingRule[]` | -| Executed by Preview runtime | **No** — never was, never will be | -| Executed by backend proxy | **No** — never was, never will be | -| Shown by Inspector | **No** | -| Removal timeline | Not scheduled; retained for backward compatibility only | - -**No new examples should use `action.responseMapping`.** Existing examples -(`valid-full.json`) that carry it are not updated; they remain valid because -the field is optional and the schema accepts (but ignores) its contents. - ---- - -## 10. Step 17.1 scope - -Step 17.1 will implement execution of `project.bindings` for -action-response-to-component data movement. Specifically: - -1. **`ActionRuntimeStateMap`** — introduce alongside the existing - `componentState` map in `usePreviewRuntime`. After each proxy call, - write the `ProxyResponse` into `actionState[actionId].response`. - -2. **Source path resolution** — implement dot-path traversal for - `actions..response`, `.response.body`, and `.response.body.`. - -3. **Binding execution loop** — after a successful proxy call, iterate - `project.bindings` whose `trigger` is `"onSuccess"` (or `"onClick"` for - legacy compatibility) and whose `source` parses to the completed action ID. - Resolve the source path against `actionState[actionId].response`. Write the - resolved value to `componentState[targetId].value`. - -4. **Label renderer update** — make `LabelRenderer` read `runtimeState?.value` - as an override over `properties.label`, consistent with `JsonViewerRenderer`. - -5. **Inspector diagnostics** — add warnings for: - - Unsupported target property segments (anything other than `.value`) - - `"onChange"` trigger on action-response bindings (recommend `"onSuccess"`) - - Bindings whose source action ID does not exist - - Bindings whose target component name does not exist - -6. **`valid-response-mapping-basic.json`** — the example created in Step 17.0 - becomes the primary Step 17.1 integration test document. - ---- - -## 11. Deferred capabilities - -The following are explicitly out of scope until separately designed and scheduled: - -| Capability | Reason deferred | -|---|---| -| `onError` trigger | Requires error-state routing design | -| `onLoad` trigger | Requires page lifecycle event system | -| Component-change data movement (`onChange`) | Requires reactive binding engine | -| Chained actions | Requires action dependency graph | -| Variable runtime support | Requires variable state store | -| Variable template interpolation in binding paths | Depends on variable runtime | -| Wildcard `*` in source paths | JSONPath-style engine not scoped | -| Filter expressions in source paths | Same | -| Array-index syntax `[0]` | Same | -| Keys with literal `.` in name | Requires path quoting syntax | -| `components..label` as writable target | Deferred to Step 17.1 label work | -| Any target property other than `.value` | Deferred to per-property routing design | -| Dropdown or Table runtime population | Separate step | -| `ComponentType` / `FullComponentType` cleanup | Separate step | -| Component-level `bindings[]` array | Schema field exists but is not wired | -| Duplicate component-name detection at save time | Future validation step | -| Transform expression execution | Deferred (`binding.transform` field retained) | -| Authentication credential injection | Separate step | -| `action.responseMapping` execution | Permanently deferred; field deprecated | - ---- - -## 12. Required diagnostics - -The following diagnostics must be implemented by Step 17.1. They are listed -here so the Inspector and runtime can be built consistently against this spec. - -### Inspector (static, design-time) - -| Condition | Severity | Message guidance | -|---|---|---| -| Binding `source` references action ID that does not exist in `project.actions` | Warning | "Source references action `` which does not exist." | -| Binding `target` references component name that does not exist on any page | Warning | "Target references component `` which does not exist." | -| Binding `target` uses property segment other than `.value` | Warning | "Target property `` is not supported. Use `.value`." | -| Two or more components share the same `name` on a page | Warning | "Duplicate component name ``. Binding resolution is ambiguous." | -| `trigger: "onChange"` on an `actions.*` source binding | Info | "Consider using `onSuccess` for action-response bindings." | -| `trigger: "onClick"` on an `actions.*` source binding | Info | "Legacy trigger. Consider migrating to `onSuccess`." | -| `action.responseMapping` is non-empty | Info | "Use `project.bindings` instead. `responseMapping` is deprecated and not executed." | - -### Runtime (execution-time, Preview only) - -| Condition | Behaviour | -|---|---| -| Source path resolves to `undefined` (field not on body) | Write `undefined` to target; emit console warning | -| Target component not found at runtime | Skip write; emit console warning | -| `ProxyResponse.ok === false` and trigger is `onSuccess` | Do not fire binding; optionally fire `onError` bindings (deferred) | -| Action ID in binding source does not match any executing action | Skip silently | +The backend performs schema and semantic validation before persistence/publication. The frontend shows the same actionable paths without replacing the last valid project state. diff --git a/package.json b/package.json index ebf8f9c..15ad8ca 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,9 @@ "scripts": { "test:e2e": "playwright test", "test:governance": "node scripts/check-mvp-governance.mjs", - "test:schema": "bash scripts/validate-schema.sh" + "test:docs": "node scripts/check-docs.mjs", + "test:schema": "bash scripts/validate-schema.sh", + "test:production": "bash scripts/test-production.sh --skip-build" }, "devDependencies": { "@playwright/test": "^1.54.2", diff --git a/scripts/check-docs.mjs b/scripts/check-docs.mjs new file mode 100644 index 0000000..cf2b4fa --- /dev/null +++ b/scripts/check-docs.mjs @@ -0,0 +1,31 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; + +const root = process.cwd(); +const markdown = []; +const walk = (directory) => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'node_modules') continue; + const absolute = path.join(directory, entry.name); + if (entry.isDirectory()) walk(absolute); + else if (entry.name.endsWith('.md')) markdown.push(absolute); + } +}; +walk(root); +const failures = []; +for (const file of markdown) { + const text = fs.readFileSync(file, 'utf8'); + for (const match of text.matchAll(/\[[^\]]*\]\(([^)]+)\)/g)) { + const raw = match[1].trim().replace(/^<|>$/g, ''); + const target = raw.split('#')[0]; + if (!target || /^[a-z][a-z0-9+.-]*:/i.test(target)) continue; + const resolved = path.resolve(path.dirname(file), decodeURIComponent(target)); + if (!fs.existsSync(resolved)) failures.push(`${path.relative(root, file)}: missing link target ${target}`); + } +} +if (failures.length) { + console.error(failures.join('\n')); + process.exit(1); +} +console.log(`Documentation links passed: ${markdown.length} Markdown files checked.`); diff --git a/scripts/check-mvp-governance.mjs b/scripts/check-mvp-governance.mjs index e80f9c9..75d369b 100644 --- a/scripts/check-mvp-governance.mjs +++ b/scripts/check-mvp-governance.mjs @@ -48,7 +48,7 @@ for (const file of linkedFiles) { } } -for (let id = 1; id <= 19; id += 1) { +for (let id = 1; id <= 20; id += 1) { if (!traceability.includes(`| R${id} |`)) failures.push(`docs/MVP_TRACEABILITY.md: missing R${id}`); } @@ -57,4 +57,4 @@ if (failures.length > 0) { process.exit(1); } -console.log('MVP governance check passed: scope, R1-R19 traceability, classifications, and links are consistent.'); +console.log('MVP governance check passed: scope, R1-R20 traceability, classifications, and links are consistent.'); diff --git a/scripts/production/backup-conductor.sh b/scripts/production/backup-conductor.sh new file mode 100755 index 0000000..66bebfa --- /dev/null +++ b/scripts/production/backup-conductor.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +COMPOSE_FILE="${ROOT_DIR}/compose.production.yml" +ENV_FILE="${ROOT_DIR}/.env" +BACKUP_DIR="${ROOT_DIR}/backups" +RETENTION_DAYS=7 +SERVICE="conductor" +TIMESTAMP="$(date -u +%Y-%m-%dT%H-%M-%SZ)" + +usage() { + printf '%s\n' "Usage: $0 [--compose-file PATH] [--env-file PATH] [--backup-dir PATH] [--retention-days N]" >&2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --compose-file) COMPOSE_FILE="$2"; shift 2 ;; + --env-file) ENV_FILE="$2"; shift 2 ;; + --backup-dir) BACKUP_DIR="$2"; shift 2 ;; + --retention-days) RETENTION_DAYS="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +[[ -f "$COMPOSE_FILE" ]] || { echo "Compose file not found: $COMPOSE_FILE" >&2; exit 1; } +[[ -f "$ENV_FILE" ]] || { echo "Environment file not found: $ENV_FILE (it contains required recovery keys)." >&2; exit 1; } +[[ "$RETENTION_DAYS" =~ ^[0-9]+$ ]] || { echo "--retention-days must be a non-negative integer." >&2; exit 1; } + +compose=(docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE") +container_id="$("${compose[@]}" ps -q "$SERVICE")" +[[ -n "$container_id" ]] || { echo "Conductor container is not created." >&2; exit 1; } +[[ "$(docker inspect -f '{{.State.Running}}' "$container_id")" == "true" ]] || { echo "Conductor container is not running." >&2; exit 1; } + +mkdir -p "$BACKUP_DIR" +chmod 700 "$BACKUP_DIR" +work_dir="$(mktemp -d)" +cleanup() { rm -rf -- "$work_dir"; } +trap cleanup EXIT +mkdir -p "$work_dir/database" "$work_dir/configuration" + +staged="/data/.conductor-backup-${TIMESTAMP}.db" +echo "[*] Creating consistent online SQLite backup..." >&2 +"${compose[@]}" exec -T "$SERVICE" node dist/scripts/backupDatabase.js "$staged" >&2 +docker cp "${container_id}:${staged}" "$work_dir/database/conductor.db" +"${compose[@]}" exec -T "$SERVICE" node -e 'const fs=require("fs");fs.unlinkSync(process.argv[1])' "$staged" + +cp "$ENV_FILE" "$work_dir/configuration/.env" +cp "$COMPOSE_FILE" "$work_dir/configuration/compose.production.yml" +image_ref="$(docker inspect -f '{{.Config.Image}}' "$container_id")" +image_id="$(docker inspect -f '{{.Image}}' "$container_id")" +cat > "$work_dir/manifest.json" < SHA256SUMS) + +archive="${BACKUP_DIR}/conductor_${TIMESTAMP}.tar.gz" +tar -czf "$archive" -C "$work_dir" database configuration manifest.json SHA256SUMS +chmod 600 "$archive" +sha256sum "$archive" > "${archive}.sha256" +chmod 600 "${archive}.sha256" + +removed=0 +while IFS= read -r -d '' old; do rm -f -- "$old" "${old}.sha256"; removed=$((removed + 1)); done < <(find "$BACKUP_DIR" -maxdepth 1 -name 'conductor_*.tar.gz' -mtime +"$RETENTION_DAYS" -print0) +size_bytes="$(stat -c '%s' "$archive")" +result="${BACKUP_DIR}/backup-result.json" +cat > "$result" <&2 +cat "$result" diff --git a/scripts/production/restore-conductor.sh b/scripts/production/restore-conductor.sh new file mode 100755 index 0000000..f3d0768 --- /dev/null +++ b/scripts/production/restore-conductor.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +COMPOSE_FILE="${ROOT_DIR}/compose.production.yml" +ENV_FILE="${ROOT_DIR}/.env" +BACKUP_DIR="${ROOT_DIR}/backups" +BACKUP_FILE="" +SERVICE="conductor" +LIST_ONLY=false +FORCE=false +SKIP_SAFETY_BACKUP=false + +usage() { + printf '%s\n' "Usage: $0 [--list] [--backup-file PATH] [--backup-dir PATH] [--compose-file PATH] [--env-file PATH] [--force]" >&2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --list) LIST_ONLY=true; shift ;; + --backup-file) BACKUP_FILE="$2"; shift 2 ;; + --backup-dir) BACKUP_DIR="$2"; shift 2 ;; + --compose-file) COMPOSE_FILE="$2"; shift 2 ;; + --env-file) ENV_FILE="$2"; shift 2 ;; + --force) FORCE=true; shift ;; + --skip-safety-backup) SKIP_SAFETY_BACKUP=true; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +mapfile -t available < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name 'conductor_*.tar.gz' -print 2>/dev/null | sort -r) +if $LIST_ONLY; then printf '%s\n' "${available[@]}"; exit 0; fi +if [[ -z "$BACKUP_FILE" ]]; then + [[ -t 0 ]] || { echo "--backup-file is required for non-interactive restore." >&2; exit 1; } + [[ ${#available[@]} -gt 0 ]] || { echo "No backups found in $BACKUP_DIR" >&2; exit 1; } + for i in "${!available[@]}"; do printf '%d) %s\n' "$((i + 1))" "${available[$i]}" >&2; done + read -r -p "Select backup number: " selection + if [[ ! "$selection" =~ ^[0-9]+$ ]] || (( selection < 1 || selection > ${#available[@]} )); then + echo "Invalid selection." >&2 + exit 1 + fi + BACKUP_FILE="${available[$((selection - 1))]}" +fi +BACKUP_FILE="$(realpath "$BACKUP_FILE")" +[[ -f "$BACKUP_FILE" ]] || { echo "Backup not found: $BACKUP_FILE" >&2; exit 1; } + +if ! $FORCE; then + [[ -t 0 ]] || { echo "Restore is destructive; pass --force for non-interactive use." >&2; exit 1; } + read -r -p "Replace the live Conductor database and recovery keys? Type RESTORE: " confirmation + [[ "$confirmation" == "RESTORE" ]] || { echo "Restore cancelled." >&2; exit 1; } +fi + +work_dir="$(mktemp -d)" +cleanup() { rm -rf -- "$work_dir"; } +trap cleanup EXIT +if tar -tzf "$BACKUP_FILE" | grep -Eq '(^/|(^|/)\.\.(/|$))'; then echo "Unsafe archive paths detected." >&2; exit 1; fi +tar -xzf "$BACKUP_FILE" -C "$work_dir" +[[ -f "$work_dir/manifest.json" && -f "$work_dir/SHA256SUMS" && -f "$work_dir/database/conductor.db" && -f "$work_dir/configuration/.env" ]] || { echo "Backup archive is incomplete." >&2; exit 1; } +(cd "$work_dir" && sha256sum -c SHA256SUMS >/dev/null) +grep -q '"format": "conductor-backup-v1"' "$work_dir/manifest.json" || { echo "Unsupported backup format." >&2; exit 1; } +grep -Eq '^CONDUCTOR_SECRET_KEY=.{32,}$' "$work_dir/configuration/.env" || { echo "Backup is missing its Conductor encryption key." >&2; exit 1; } +grep -Eq '^CONDUCTOR_SESSION_KEY=.{32,}$' "$work_dir/configuration/.env" || { echo "Backup is missing its Conductor session key." >&2; exit 1; } + +if ! $SKIP_SAFETY_BACKUP && [[ -f "$ENV_FILE" && -f "$COMPOSE_FILE" ]]; then + echo "[*] Creating pre-restore safety backup..." >&2 + "${ROOT_DIR}/scripts/production/backup-conductor.sh" --compose-file "$COMPOSE_FILE" --env-file "$ENV_FILE" --backup-dir "$BACKUP_DIR" >&2 +fi + +install -m 600 "$work_dir/configuration/.env" "$ENV_FILE" +[[ -f "$COMPOSE_FILE" ]] || install -m 600 "$work_dir/configuration/compose.production.yml" "$COMPOSE_FILE" +compose=(docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE") +container_id="$("${compose[@]}" ps -aq "$SERVICE")" +[[ -n "$container_id" ]] || { echo "Conductor container does not exist; run docker compose up once before restore." >&2; exit 1; } +image_ref="$(docker inspect -f '{{.Config.Image}}' "$container_id")" +echo "[*] Stopping Conductor and replacing database..." >&2 +"${compose[@]}" stop "$SERVICE" >&2 +docker cp "$work_dir/database/conductor.db" "${container_id}:/data/conductor.db.restore" +docker run --rm --user 0 --volumes-from "$container_id" --entrypoint node "$image_ref" -e ' +const fs=require("fs"); +for(const file of ["/data/conductor.db-wal","/data/conductor.db-shm"])if(fs.existsSync(file))fs.unlinkSync(file); +fs.renameSync("/data/conductor.db.restore","/data/conductor.db"); +fs.chownSync("/data/conductor.db",10001,10001); +fs.chmodSync("/data/conductor.db",0o600); +' >&2 +"${compose[@]}" up -d "$SERVICE" >&2 + +healthy=false +for _ in $(seq 1 30); do + status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$("${compose[@]}" ps -q "$SERVICE")")" + if [[ "$status" == "healthy" ]]; then healthy=true; break; fi + sleep 1 +done +$healthy || { echo "Conductor did not become healthy after restore." >&2; exit 1; } +result="${BACKUP_DIR}/restore-result.json" +cat > "$result" <&2 +cat "$result" diff --git a/scripts/release-image.sh b/scripts/release-image.sh new file mode 100755 index 0000000..8ad4261 --- /dev/null +++ b/scripts/release-image.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +IMAGE="${IMAGE:-gitea.skeletonworks.online/vwiebe/conductor}" +VERSION="${VERSION:-}" +PUSH=false +PLATFORMS="${PLATFORMS:-linux/amd64,linux/arm64}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --image) IMAGE="$2"; shift 2 ;; + --version) VERSION="$2"; shift 2 ;; + --platforms) PLATFORMS="$2"; shift 2 ;; + --push) PUSH=true; shift ;; + -h|--help) echo "Usage: $0 --version vX.Y.Z [--image REGISTRY/OWNER/NAME] [--platforms LIST] [--push]"; exit 0 ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac +done +[[ "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([.-][A-Za-z0-9.-]+)?$ ]] || { echo "--version must be a vX.Y.Z release tag." >&2; exit 1; } +revision="$(git rev-parse HEAD)" +short_revision="$(git rev-parse --short=12 HEAD)" +created="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +output=(--load) +$PUSH && output=(--push) +if ! $PUSH && [[ "$PLATFORMS" == *,* ]]; then + echo "Local --load supports one platform; use --push for a multi-platform release." >&2 + exit 1 +fi +docker buildx build --platform "$PLATFORMS" -f Dockerfile.production \ + --build-arg "VERSION=$VERSION" --build-arg "REVISION=$revision" --build-arg "CREATED=$created" \ + -t "${IMAGE}:${VERSION}" -t "${IMAGE}:git-${short_revision}" "${output[@]}" . +printf '{"image":"%s","version":"%s","revision":"%s","platforms":"%s","pushed":%s}\n' "$IMAGE" "$VERSION" "$revision" "$PLATFORMS" "$PUSH" diff --git a/scripts/test-production.sh b/scripts/test-production.sh new file mode 100755 index 0000000..c3a016a --- /dev/null +++ b/scripts/test-production.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +IMAGE="${CONDUCTOR_TEST_IMAGE:-conductor:slice8-test}" +PROJECT="conductor-release-test-${RANDOM}" +PORT="${CONDUCTOR_TEST_PORT:-3202}" +work="$(mktemp -d)" +env_file="${work}/.env" +backup_dir="${work}/backups" +compose=(docker compose -p "$PROJECT" --env-file "$env_file" -f "$ROOT/compose.production.yml") +cleanup() { "${compose[@]}" down -v --remove-orphans >/dev/null 2>&1 || true; rm -rf -- "$work"; } +trap cleanup EXIT + +if [[ "${1:-}" != "--skip-build" ]]; then docker build -f "$ROOT/Dockerfile.production" -t "$IMAGE" "$ROOT"; fi +if docker run --rm "$IMAGE" >/dev/null 2>&1; then echo "Production image started without required keys." >&2; exit 1; fi +cat > "$env_file" </dev/null +curl --fail --silent --show-error "http://127.0.0.1:${PORT}/" | grep -qi '' +curl --fail --silent --show-error "http://127.0.0.1:${PORT}/apps/test/details" | grep -qi '' +api_status="$(curl --silent --output /dev/null --write-out '%{http_code}' "http://127.0.0.1:${PORT}/api/not-a-route")" +[[ "$api_status" == 404 ]] || { echo "Unknown API routes must return 404, not the SPA." >&2; exit 1; } +"${compose[@]}" exec -T conductor node -e 'const D=require("better-sqlite3");const d=new D("/data/conductor.db");d.prepare("INSERT INTO projects(name,description,project_json) VALUES(?,?,?)").run("release-restore-marker","test","{}");d.close()' +COMPOSE_PROJECT_NAME="$PROJECT" "$ROOT/scripts/production/backup-conductor.sh" --compose-file "$ROOT/compose.production.yml" --env-file "$env_file" --backup-dir "$backup_dir" --retention-days 1 >/dev/null +archive="$(find "$backup_dir" -name 'conductor_*.tar.gz' -type f | head -n1)" +"${compose[@]}" exec -T conductor node -e 'const D=require("better-sqlite3");const d=new D("/data/conductor.db");d.prepare("DELETE FROM projects WHERE name=?").run("release-restore-marker");d.close()' +COMPOSE_PROJECT_NAME="$PROJECT" "$ROOT/scripts/production/restore-conductor.sh" --compose-file "$ROOT/compose.production.yml" --env-file "$env_file" --backup-dir "$backup_dir" --backup-file "$archive" --skip-safety-backup --force >/dev/null +count="$("${compose[@]}" exec -T conductor node -e 'const D=require("better-sqlite3");const d=new D("/data/conductor.db",{readonly:true});process.stdout.write(String(d.prepare("SELECT count(*) FROM projects WHERE name=?").pluck().get("release-restore-marker")));d.close()')" +[[ "$count" == 1 ]] || { echo "Restore marker was not recovered." >&2; exit 1; } +docker inspect "$("${compose[@]}" ps -q conductor)" --format '{{.Config.User}} {{.HostConfig.ReadonlyRootfs}}' | grep -q '^conductor true$' +echo '{"productionValidation":"passed"}' diff --git a/tests/e2e/release-workflows.spec.ts b/tests/e2e/release-workflows.spec.ts index f087dd7..5065e45 100644 --- a/tests/e2e/release-workflows.spec.ts +++ b/tests/e2e/release-workflows.spec.ts @@ -10,14 +10,34 @@ test('deterministic launcher, dependent data, dashboard, and save/reload workflo const document = structuredClone(demo); document.project.name = name; + await page.goto('/'); + const setup = page.getByRole('heading', { name: 'Set up Conductor' }); + if (await setup.isVisible().catch(() => false)) { + await page.getByLabel('Username').fill('e2e-admin'); + await page.getByLabel('Display name').fill('E2E Administrator'); + await page.getByLabel('Password', { exact: true }).fill('e2e-test-password-1234'); + await page.getByLabel('Confirm password').fill('e2e-test-password-1234'); + await page.getByRole('button', { name: 'Create administrator' }).click(); + } else if (await page.getByRole('heading', { name: 'Sign in to Conductor' }).isVisible().catch(() => false)) { + await page.getByLabel('Username').fill('e2e-admin'); + await page.getByLabel('Password').fill('e2e-test-password-1234'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + } + await expect(page.getByText('E2E Administrator')).toBeVisible(); + const cookies = await page.context().cookies(); + const cookieHeader = cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; '); + const csrf = cookies.find((cookie) => cookie.name === 'conductor_csrf')?.value; + expect(csrf).toBeTruthy(); + const authHeaders = { Cookie: cookieHeader, 'X-CSRF-Token': csrf! }; + const created = await request.post('http://127.0.0.1:4000/api/projects', { + headers: authHeaders, data: { name, description: document.project.description, project_json: JSON.stringify(document) }, }); expect(created.status()).toBe(201); const row = await created.json(); try { - await page.goto('/'); await page.getByRole('button', { name: 'Visual Editor', exact: true }).click(); await page.getByRole('button', { name: 'Load', exact: true }).click(); await page.getByRole('button', { name: new RegExp(name) }).click(); @@ -49,7 +69,7 @@ test('deterministic launcher, dependent data, dashboard, and save/reload workflo await page.getByRole('button', { name: 'Preview', exact: true }).click(); await expect(page.getByRole('button', { name: 'Launch workflow', exact: true })).toBeVisible(); } finally { - await request.delete(`http://127.0.0.1:4000/api/projects/${row.id}`); - await request.delete('http://127.0.0.1:4000/api/executions'); + await request.delete(`http://127.0.0.1:4000/api/projects/${row.id}`, { headers: authHeaders }); + await request.delete('http://127.0.0.1:4000/api/executions', { headers: authHeaders }); } });