vwiebe/conductor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add production packaging and operations

Browse Source
This commit is contained in:
Victor Wiebe 2026-08-07 16:48:40 -04:00
parent 24f04b9fc3
commit a4d64735c4
41 changed files with 1187 additions and 431 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.dockerignore Normal file
View File

@ -0,0 +1,16 @@
.git
.github
.agents
.codex
**/node_modules
**/build
**/dist
backend/data
backend/data-*
*.db
*.db-wal
*.db-shm
SLICE*
SAVE_POINT.md
playwright-report
test-results

14
.env.production.example Normal file
View File

@ -0,0 +1,14 @@
# Copy this file to .env and replace both required keys before starting.
# Generate each independently with: openssl rand -hex 32
CONDUCTOR_SECRET_KEY=replace-with-64-hex-characters
CONDUCTOR_SESSION_KEY=replace-with-an-independent-random-value-at-least-32-bytes
CONDUCTOR_IMAGE=gitea.skeletonworks.online/vwiebe/conductor
CONDUCTOR_VERSION=v0.1.0
CONDUCTOR_BIND_ADDRESS=0.0.0.0
CONDUCTOR_PORT=8080
CONDUCTOR_MEM_LIMIT=1g
CONDUCTOR_CPUS=1.5
# Optional comma-separated exact origins permitted by the proxy's internal-origin policy.
CONDUCTOR_PROXY_INTERNAL_ORIGINS=

52
.gitea/workflows/release-image.yml Normal file
View File

@ -0,0 +1,52 @@
name: Release production image
on:
push:
tags:
- 'v*.*.*'
workflow_dispatch:
jobs:
production-image:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v4
- uses: docker/setup-qemu-action@v3
- uses: docker/setup-buildx-action@v3
- name: Log in to Gitea registry
uses: docker/login-action@v3
with:
registry: gitea.skeletonworks.online
username: ${{ secrets.REGISTRY_USERNAME }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: Resolve release metadata
id: metadata
shell: bash
run: |
version="${GITHUB_REF_NAME:-manual}"
revision="$(git rev-parse HEAD)"
short_revision="$(git rev-parse --short=12 HEAD)"
created="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "version=${version}" >> "$GITHUB_OUTPUT"
echo "revision=${revision}" >> "$GITHUB_OUTPUT"
echo "short_revision=${short_revision}" >> "$GITHUB_OUTPUT"
echo "created=${created}" >> "$GITHUB_OUTPUT"
- name: Build and publish immutable images
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile.production
platforms: linux/amd64,linux/arm64
push: true
build-args: |
VERSION=${{ steps.metadata.outputs.version }}
REVISION=${{ steps.metadata.outputs.revision }}
CREATED=${{ steps.metadata.outputs.created }}
tags: |
gitea.skeletonworks.online/vwiebe/conductor:${{ steps.metadata.outputs.version }}
gitea.skeletonworks.online/vwiebe/conductor:git-${{ steps.metadata.outputs.short_revision }}
provenance: mode=max
sbom: true

3
.gitignore vendored
View File

@ -25,6 +25,9 @@ yarn-debug.log*
yarn-error.log*
*.log
# Production operator state and backups
backups/
# OS
.DS_Store
Thumbs.db

45
Dockerfile.production Normal file
View File

@ -0,0 +1,45 @@
# syntax=docker/dockerfile:1.7
FROM node:20-alpine AS frontend-build
WORKDIR /build/frontend
COPY frontend/package*.json ./
RUN npm ci
COPY frontend/ ./
RUN npm run build
FROM node:20-alpine AS backend-build
RUN apk add --no-cache python3 make g++
WORKDIR /build/backend
COPY backend/package*.json ./
RUN npm ci
COPY backend/ ./
RUN npm run build && npm prune --omit=dev
FROM node:20-alpine AS runtime
ARG VERSION=development
ARG REVISION=unknown
ARG CREATED=unknown
LABEL org.opencontainers.image.title="Conductor" \
org.opencontainers.image.description="Visual builder for REST-backed web applications" \
org.opencontainers.image.version="${VERSION}" \
org.opencontainers.image.revision="${REVISION}" \
org.opencontainers.image.created="${CREATED}" \
org.opencontainers.image.source="https://gitea.skeletonworks.online/vwiebe/conductor"
ENV NODE_ENV=production \
PORT=8080 \
CONDUCTOR_DATA_DIR=/data \
CONDUCTOR_STATIC_DIR=/app/public \
CONDUCTOR_VERSION=${VERSION}
RUN addgroup -S -g 10001 conductor && adduser -S -D -H -u 10001 -G conductor conductor \
&& mkdir -p /app /data /shared/schemas \
&& chown -R conductor:conductor /app /data
WORKDIR /app
COPY --from=backend-build --chown=conductor:conductor /build/backend/package*.json ./
COPY --from=backend-build --chown=conductor:conductor /build/backend/node_modules ./node_modules
COPY --from=backend-build --chown=conductor:conductor /build/backend/dist ./dist
COPY --from=frontend-build --chown=conductor:conductor /build/frontend/build ./public
COPY --chown=conductor:conductor shared/schemas/conductor-project.schema.json /shared/schemas/conductor-project.schema.json
USER conductor
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
CMD node -e "fetch('http://127.0.0.1:8080/api/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"
CMD ["node", "dist/index.js"]

2
MVP_SCOPE.md
View File

@ -83,7 +83,7 @@ Approved on 2026-08-07 for Slice 7a:
## Release Boundary
The MVP is complete only when all required component, configuration, authentication, security, validation, persistence, local-user/RBAC, publishing, first-run administration, multi-page/scoped-variable, Visual Editor command-ribbon polish, and deployment tasks in `ROADMAP.md` are complete; all six workflows above pass the Slice 6 release-validation process; and Slice 7a, Slice 7b, Slice 7c, and Slice 7d acceptance pass.
The MVP is complete only when all required component, configuration, authentication, security, validation, persistence, local-user/RBAC, publishing, first-run administration, multi-page/scoped-variable, Visual Editor command-ribbon polish, public production distribution, SkeletonWorks installation, and verified backup/restore tasks in `ROADMAP.md` are complete; all six workflows above pass the Slice 6 release-validation process; and Slice 7a, Slice 7b, Slice 7c, Slice 7d, and Slice 8 acceptance pass.
Post-MVP scope includes AI assistance, OIDC/SSO beyond the local authentication architecture, OAuth 2.0 for REST actions, IBM Cloud IAM, mTLS, advanced orchestration, and the future capabilities listed in `ROADMAP.md`.

32
README.md
View File

@ -67,14 +67,33 @@ The frontend development server proxies `/api/*` requests to `http://localhost:4
---
## Docker
## Development Docker
```bash
docker-compose up --build
docker compose up --build
# Frontend: http://localhost:3000
# Backend: http://localhost:4000
```
## Production installation
Conductor's production distribution is one non-root container serving the compiled frontend, published application routes, and backend API on port 8080. SQLite data is stored in a persistent volume. The development stack above remains separate.
```bash
curl -O https://gitea.skeletonworks.online/vwiebe/conductor/raw/tag/v0.1.0/compose.production.yml
curl -o .env https://gitea.skeletonworks.online/vwiebe/conductor/raw/tag/v0.1.0/.env.production.example
# Replace both placeholder keys with independent values from:
openssl rand -hex 32
docker compose --env-file .env -f compose.production.yml pull
docker compose --env-file .env -f compose.production.yml up -d
```
Open `http://localhost:8080` and create the first administrator in the browser. There are no default credentials.
For source builds, reverse proxies, upgrades, backups, restores, and SkeletonWorks installation, see [docs/INSTALL.md](docs/INSTALL.md) and [docs/OPERATIONS.md](docs/OPERATIONS.md).
---
## Documentation
@ -89,6 +108,13 @@ docker-compose up --build
| [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) | Technical architecture |
| [docs/NICE-TO-HAVE.md](docs/NICE-TO-HAVE.md) | Future enhancements |
| [docs/BUILD_AND_TEST_PLAN.md](docs/BUILD_AND_TEST_PLAN.md) | Incremental build and test plan |
| [docs/INSTALL.md](docs/INSTALL.md) | Public source/container and reverse-proxy installation |
| [docs/OPERATIONS.md](docs/OPERATIONS.md) | Production configuration, backup, restore, upgrade, and rollback |
| [docs/RELEASE.md](docs/RELEASE.md) | Image tagging, release construction, and publication |
| [docs/USER_GUIDE.md](docs/USER_GUIDE.md) | Admin/user authoring, runtime, and publishing workflows |
| [docs/DEVELOPER_GUIDE.md](docs/DEVELOPER_GUIDE.md) | Architecture, invariants, tests, and contribution workflow |
| [docs/TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md) | Production and application troubleshooting |
| [docs/KNOWN_LIMITATIONS.md](docs/KNOWN_LIMITATIONS.md) | Explicit v0.1.0 limitations and deferred capabilities |
---
@ -99,4 +125,4 @@ docker-compose up --build
| Frontend | React 18, TypeScript 5 |
| Backend | Node.js 20, Express 4, TypeScript 5 |
| Database | SQLite (MVP) |
| Deployment | Docker Compose, NGINX/reverse proxy |
| Deployment | Single production container, Docker Compose, Caddy/NGINX reverse proxy |

10
ROADMAP.md
View File

@ -313,19 +313,23 @@ Slice 8 release packaging now depends on completed Slice 7a authentication/RBAC/
### Slices 6 and 8 — v0.1.0 packaging
- [ ] Review examples, demo, open issues, deferred work, and release criteria.
- [ ] Complete fresh-clone validation and final Docker images.
- [ ] Build a single non-root production image serving the compiled frontend, SPA/published routes, and backend API on one port while retaining the two-container development stack.
- [ ] Publish production Compose/environment assets and anonymously downloadable source releases plus pinned multi-architecture container tags.
- [ ] Add application-owned consistent SQLite backup/restore with encryption keys, manifests, checksums, safety backup, and health verification.
- [ ] Add and validate SkeletonWorks `setup-conductor.sh`, `backup-conductor.sh`, and `restore-conductor.sh` behind Caddy.
- [ ] Complete fresh-clone, container-only, fresh-host, upgrade, rollback, backup, restore, restart, persistence, and health validation.
- [ ] Prepare release notes and update version references.
- [ ] Commit the release documentation and code.
- [ ] Tag `v0.1.0` and push the release commit and tag.
## Post-MVP Backlog
- [ ] Multi-page application authoring and advanced page management.
- [ ] Nested/dynamic pages, route parameters, per-page authorization, and advanced page management.
- [ ] OAuth 2.0, IBM Cloud IAM, mTLS, and custom authentication scripts.
- [ ] OpenAPI import and generated forms/actions.
- [ ] Reusable templates, themes, and component libraries.
- [ ] Version history, Git integration, and team collaboration.
- [ ] Role-based access control and enterprise audit retention.
- [ ] Custom/per-application roles, groups, approval workflows, and enterprise audit retention.
- [ ] Tabs, modals, date pickers, file uploads, charts, and progress indicators.
- [ ] Table sorting, filtering, pagination, editing, CSV import/export, row actions, and multi-selection.
- [ ] Advanced response transforms and expression languages.

21
TESTING.md
View File

@ -707,3 +707,24 @@ After a manual workflow is completed:
4. Do not mark a broader workflow accepted when only one checkpoint passed.
5. File unresolved defects in `ROADMAP.md` and the owning slice with enough detail to reproduce them.
# Production packaging checks
In addition to the development and acceptance suites below, Slice 8 validates the single production artifact:
```bash
docker build -f Dockerfile.production -t conductor:local .
# Must fail because production keys are absent.
docker run --rm conductor:local
cp .env.production.example .env
# Replace both key placeholders, then:
docker compose --env-file .env \
-f compose.production.yml -f compose.production.build.yml up -d --build
curl --fail http://127.0.0.1:8080/api/health
scripts/production/backup-conductor.sh
scripts/production/restore-conductor.sh --list
```
The release gate includes a disposable destructive restore drill with a marker record, verification that the restored service runs as UID 10001 with a read-only root filesystem, and refresh checks for `/apps/:appSlug/:pageSlug`.

1
backend/package.json
View File

@ -5,6 +5,7 @@
"scripts": {
"dev": "ts-node-dev --respawn --transpile-only src/index.ts",
"build": "tsc",
"backup-db": "node dist/scripts/backupDatabase.js",
"bootstrap-admin": "npm run build && node dist/scripts/bootstrapAdmin.js",
"test": "npm run build && node --test dist/**/*.test.js",
"start": "node dist/index.js"

24
backend/src/app.ts
View File

@ -1,5 +1,7 @@
import express, { Application, Request, Response, NextFunction } from 'express';
import morgan from 'morgan';
import fs from 'fs';
import path from 'path';
import healthRouter from './routes/health';
import projectsRouter from './routes/projects';
@ -19,6 +21,13 @@ const app: Application = express();
app.use(express.json({ limit: '2mb' }));
app.use(morgan('dev'));
app.disable('x-powered-by');
app.use((_req, res, next) => {
res.setHeader('X-Content-Type-Options', 'nosniff');
res.setHeader('Referrer-Policy', 'same-origin');
res.setHeader('X-Frame-Options', 'SAMEORIGIN');
next();
});
// ── Routes ────────────────────────────────────────────────────────────────────
@ -33,6 +42,21 @@ app.use('/api/secrets', requireAdmin, requireMutationCsrf, secretsRouter);
app.use('/api/executions', requireAdmin, requireMutationCsrf, executionsRouter);
app.use('/api/proxy/execute', requireAdmin, requireMutationCsrf, proxyRouter);
// ── Production frontend ─────────────────────────────────────────────────────
const staticDirectory = process.env.CONDUCTOR_STATIC_DIR
? path.resolve(process.env.CONDUCTOR_STATIC_DIR)
: path.resolve(__dirname, '../public');
const indexFile = path.join(staticDirectory, 'index.html');
if (fs.existsSync(indexFile)) {
app.use(express.static(staticDirectory, { index: false, maxAge: '1h' }));
app.get('*', (req, res, next) => {
if (req.path.startsWith('/api/')) return next();
res.setHeader('Cache-Control', 'no-cache');
res.sendFile(indexFile);
});
}
// ── Catch-all 404 ─────────────────────────────────────────────────────────────
app.use((_req: Request, res: Response) => {

4
backend/src/db/database.ts
View File

@ -4,10 +4,10 @@ import fs from 'fs';
// Store the database file in backend/data/ by default. Tests and operators may
// select an isolated directory without changing the application working tree.
const DATA_DIR = process.env.CONDUCTOR_DATA_DIR
export const DATA_DIR = process.env.CONDUCTOR_DATA_DIR
? path.resolve(process.env.CONDUCTOR_DATA_DIR)
: path.resolve(__dirname, '../../data');
const DB_PATH = path.join(DATA_DIR, 'conductor.db');
export const DB_PATH = path.join(DATA_DIR, 'conductor.db');
// Ensure the data directory exists before opening the file
fs.mkdirSync(DATA_DIR, { recursive: true });

15
backend/src/index.ts
View File

@ -1,13 +1,24 @@
import { initDatabase } from './db/init';
import db from './db/database';
import { validateProductionConfiguration } from './lib/productionConfig';
const PORT = process.env.PORT ?? 4000;
const PORT = Number(process.env.PORT ?? 4000);
if (!Number.isInteger(PORT) || PORT < 1 || PORT > 65535) throw new Error('PORT must be an integer from 1 to 65535.');
validateProductionConfiguration();
initDatabase();
// Import the app only after the schema exists. Project route modules prepare
// their SQL statements during import and therefore require the projects table.
void import('./app').then(({ default: app }) => {
app.listen(PORT, () => {
const server = app.listen(PORT, '0.0.0.0', () => {
console.log(`Conductor backend listening on port ${PORT}`);
});
const shutdown = (signal: string) => {
console.log(`Received ${signal}; shutting down.`);
server.close(() => { db.close(); process.exit(0); });
setTimeout(() => process.exit(1), 10_000).unref();
};
process.once('SIGTERM', () => shutdown('SIGTERM'));
process.once('SIGINT', () => shutdown('SIGINT'));
});

17
backend/src/lib/productionConfig.test.ts Normal file
View File

@ -0,0 +1,17 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { validateProductionConfiguration } from './productionConfig';
test('development does not require production keys', () => {
assert.doesNotThrow(() => validateProductionConfiguration({ NODE_ENV: 'development' }));
});
test('production requires both correctly sized persistent keys', () => {
assert.throws(() => validateProductionConfiguration({ NODE_ENV: 'production' }), /CONDUCTOR_SECRET_KEY/);
assert.throws(() => validateProductionConfiguration({ NODE_ENV: 'production', CONDUCTOR_SECRET_KEY: '11'.repeat(32) }), /CONDUCTOR_SESSION_KEY/);
assert.doesNotThrow(() => validateProductionConfiguration({
NODE_ENV: 'production',
CONDUCTOR_SECRET_KEY: Buffer.alloc(32, 7).toString('base64'),
CONDUCTOR_SESSION_KEY: 'session-key-material-that-is-at-least-32-bytes',
}));
});

15
backend/src/lib/productionConfig.ts Normal file
View File

@ -0,0 +1,15 @@
function decodeSecretKey(value: string): Buffer {
return /^[0-9a-fA-F]{64}$/.test(value) ? Buffer.from(value, 'hex') : Buffer.from(value, 'base64');
}
export function validateProductionConfiguration(environment: NodeJS.ProcessEnv = process.env): void {
if (environment.NODE_ENV !== 'production') return;
const secretKey = environment.CONDUCTOR_SECRET_KEY ?? '';
if (!secretKey || decodeSecretKey(secretKey).length !== 32) {
throw new Error('CONDUCTOR_SECRET_KEY is required in production and must encode exactly 32 bytes as base64 or 64 hexadecimal characters.');
}
const sessionKey = environment.CONDUCTOR_SESSION_KEY ?? '';
if (Buffer.byteLength(sessionKey, 'utf8') < 32) {
throw new Error('CONDUCTOR_SESSION_KEY is required in production and must contain at least 32 bytes.');
}
}

1
backend/src/routes/health.ts
View File

@ -6,6 +6,7 @@ router.get('/', (_req: Request, res: Response) => {
res.json({
service: 'conductor-backend',
status: 'ok',
version: process.env.CONDUCTOR_VERSION ?? 'development',
timestamp: new Date().toISOString(),
});
});

29
backend/src/scripts/backupDatabase.ts Normal file
View File

@ -0,0 +1,29 @@
import fs from 'fs';
import path from 'path';
import db, { DB_PATH } from '../db/database';
import { initDatabase } from '../db/init';
async function main(): Promise<void> {
const destinationArg = process.argv[2];
if (!destinationArg) throw new Error('Usage: npm run backup-db -- <destination.db>');
const destination = path.resolve(destinationArg);
if (destination === DB_PATH) throw new Error('Backup destination must differ from the live database.');
fs.mkdirSync(path.dirname(destination), { recursive: true });
if (fs.existsSync(destination)) throw new Error(`Backup destination already exists: ${destination}`);
const temporary = `${destination}.partial-${process.pid}`;
initDatabase();
try {
await db.backup(temporary);
fs.renameSync(temporary, destination);
const stats = fs.statSync(destination);
console.log(JSON.stringify({ status: 'success', database: DB_PATH, backup: destination, sizeBytes: stats.size }));
} finally {
if (fs.existsSync(temporary)) fs.rmSync(temporary);
db.close();
}
}
void main().catch((error: unknown) => {
console.error(error instanceof Error ? error.message : String(error));
process.exitCode = 1;
});

10
compose.production.build.yml Normal file
View File

@ -0,0 +1,10 @@
services:
conductor:
image: conductor:local
build:
context: .
dockerfile: Dockerfile.production
args:
VERSION: local
REVISION: local
CREATED: local

35
compose.production.yml Normal file
View File

@ -0,0 +1,35 @@
services:
conductor:
image: ${CONDUCTOR_IMAGE:-gitea.skeletonworks.online/vwiebe/conductor}:${CONDUCTOR_VERSION:-latest}
restart: unless-stopped
init: true
ports:
- "${CONDUCTOR_BIND_ADDRESS:-0.0.0.0}:${CONDUCTOR_PORT:-8080}:8080"
environment:
NODE_ENV: production
PORT: 8080
CONDUCTOR_DATA_DIR: /data
CONDUCTOR_SECRET_KEY: ${CONDUCTOR_SECRET_KEY:?Set CONDUCTOR_SECRET_KEY in .env}
CONDUCTOR_SESSION_KEY: ${CONDUCTOR_SESSION_KEY:?Set CONDUCTOR_SESSION_KEY in .env}
CONDUCTOR_PROXY_INTERNAL_ORIGINS: ${CONDUCTOR_PROXY_INTERNAL_ORIGINS:-}
volumes:
- conductor_data:/data
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=64m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 512
mem_limit: ${CONDUCTOR_MEM_LIMIT:-1g}
cpus: ${CONDUCTOR_CPUS:-1.5}
healthcheck:
test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8080/api/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"]
interval: 30s
timeout: 5s
start_period: 15s
retries: 3
volumes:
conductor_data:

11
docker-compose.e2e.yml Normal file
View File

@ -0,0 +1,11 @@
services:
backend:
environment:
- CONDUCTOR_SECRET_KEY=1111111111111111111111111111111111111111111111111111111111111111
- CONDUCTOR_SESSION_KEY=22222222222222222222222222222222
volumes: !override
- conductor_e2e_data:/app/data
- ./shared:/shared:ro
volumes:
conductor_e2e_data:

9
docs/ARCHITECTURE.md
View File

@ -288,12 +288,15 @@ The MVP deployment model should support:
```text
Reverse Proxy
-> Frontend static assets
-> Backend API service
-> Single same-origin Conductor production service
-> Compiled frontend and published SPA routes
-> Backend API
-> SQLite database file
```
A simple Docker Compose deployment is recommended for local demos and early internal use.
Development retains separate React and backend processes. Production uses one non-root, read-only-root container with a dedicated writable `/data` volume, required persistent encryption/session keys, and a health check. Docker Compose is the supported self-hosted deployment contract; Caddy or NGINX terminates TLS and proxies all paths to the same internal port.
Live SQLite backups use the application's online backup command. A recovery archive includes the database and its matching encryption key plus manifest/checksums; copying only the live WAL database is unsupported.
Future deployment options may include:

47
docs/DEVELOPER_GUIDE.md Normal file
View File

@ -0,0 +1,47 @@
# Developer Guide
## Architecture
The canonical `ProjectDocument` is shared by Visual Editor, JSON Editor, Preview, validation, persistence, and publication snapshots. React owns authoring and ephemeral runtime state; Express owns authentication, authorization, persistence, credential resolution, proxy policy, validation, and publication execution. SQLite is the v0.1.0 durable store.
Development uses two containers/processes for fast React/backend iteration. Production uses `Dockerfile.production`: a multi-stage build copies compiled frontend assets and compiled backend output into one minimal runtime image. Express serves API and SPA fallbacks from the same origin.
## Local commands
```bash
cd frontend && npm ci && npm test -- --watchAll=false --runInBand && npm run build
cd ../backend && npm ci && npm test
cd .. && npm ci && npm run test:schema && npm run test:governance
```
Use the deterministic mock stack and Playwright commands in `docs/RELEASE_VALIDATION.md` for browser acceptance.
## Database and migrations
`backend/src/db/init.ts` applies idempotent startup schema creation/migrations. New migrations must preserve existing data and have explicit downgrade/rollback implications. Tests select isolated databases through `CONDUCTOR_DATA_DIR`.
Never copy the live WAL database as a backup. `backend/src/scripts/backupDatabase.ts` uses `better-sqlite3`'s online backup facility.
## Security invariants
- Every admin mutation requires an authenticated admin and CSRF validation.
- Published execution resolves actions and credentials from server-owned snapshots.
- Proxy destinations are default-deny for internal/private networks unless an exact reviewed origin is allowed.
- Credential plaintext must not enter project JSON, browser responses, errors, or execution history.
- Production refuses missing/invalid encryption and session keys.
- New production files must work with a non-root UID/GID 10001 and read-only root filesystem.
## Adding canonical behavior
Update together:
1. Shared JSON schema.
2. Frontend TypeScript model and authoring UI.
3. Backend semantic validation.
4. Preview and published runtime behavior.
5. Unit/integration/browser coverage.
6. Schema examples, user/operator docs, roadmap, and traceability.
## Release artifacts
Run `scripts/release-image.sh` for immutable OCI tags. `scripts/test-production.sh` performs the disposable same-origin/backup/restore gate. Release publication details are in `docs/RELEASE.md`.

128
docs/INSTALL.md Normal file
View File

@ -0,0 +1,128 @@
# Production Installation
## Supported installation paths
Conductor supports:
1. A public container installation using `compose.production.yml` and a pinned release image.
2. A reproducible source build using `Dockerfile.production` and `compose.production.build.yml`.
3. SkeletonWorks deployment through `setup-conductor.sh` in the `skeletonworks-scripts` repository.
The production distribution is one same-origin service. Express serves the compiled React application, `/api`, and refreshable published routes such as `/apps/inventory/details` on internal port 8080. The development-only frontend/backend containers are not production artifacts.
## Requirements
- Docker Engine with Compose v2
- Approximately 1 GiB memory and 1.5 CPU available by default
- A persistent Docker volume or host directory for `/data`
- TLS reverse proxy for internet-facing deployments
- Two independently generated persistent keys
## Public container installation
Download `compose.production.yml` and `.env.production.example` from the same tagged release. Do not mix files from different releases.
```bash
mkdir conductor && cd conductor
curl -O https://gitea.skeletonworks.online/vwiebe/conductor/raw/tag/v0.1.0/compose.production.yml
curl -o .env https://gitea.skeletonworks.online/vwiebe/conductor/raw/tag/v0.1.0/.env.production.example
```
Edit `.env`:
```dotenv
CONDUCTOR_SECRET_KEY=<output of openssl rand -hex 32>
CONDUCTOR_SESSION_KEY=<different output of openssl rand -hex 32>
CONDUCTOR_IMAGE=gitea.skeletonworks.online/vwiebe/conductor
CONDUCTOR_VERSION=v0.1.0
CONDUCTOR_BIND_ADDRESS=127.0.0.1
CONDUCTOR_PORT=8080
```
`CONDUCTOR_SECRET_KEY` encrypts stored REST credentials. Losing or changing it makes those credentials unrecoverable. `CONDUCTOR_SESSION_KEY` protects sessions; changing it logs everyone out. Neither belongs in source control.
Start and verify:
```bash
docker compose --env-file .env -f compose.production.yml pull
docker compose --env-file .env -f compose.production.yml up -d
docker compose --env-file .env -f compose.production.yml ps
curl --fail http://127.0.0.1:8080/api/health
```
Open the configured URL. A new database presents the browser first-run administrator screen. Conductor never ships a default username or password.
## Build from source
Use a tagged source archive or checkout:
```bash
git clone https://gitea.skeletonworks.online/vwiebe/conductor.git
cd conductor
git checkout v0.1.0
cp .env.production.example .env
# Replace both key placeholders.
docker compose --env-file .env \
-f compose.production.yml -f compose.production.build.yml \
up -d --build
```
The multi-stage build compiles frontend and backend code, prunes development dependencies, includes the matching project schema, and runs as UID/GID 10001.
## Caddy
Bind Conductor to loopback and proxy it through Caddy:
```caddyfile
conductor.example.com {
reverse_proxy 127.0.0.1:8080
encode zstd gzip
}
```
Caddy provides TLS automatically when DNS and ports 80/443 are correctly configured.
## NGINX
```nginx
server {
listen 443 ssl http2;
server_name conductor.example.com;
location / {
proxy_pass http://127.0.0.1:8080;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
```
Configure certificates using the operator's normal NGINX/TLS process.
## Internal REST destinations
The proxy denies private/internal destinations by default. `CONDUCTOR_PROXY_INTERNAL_ORIGINS` is an optional comma-separated allowlist of exact origins. Add only reviewed origins, for example `https://api.internal.example.com:443`. It does not accept wildcard hosts.
## SkeletonWorks
On a prepared SkeletonWorks host:
```bash
sudo bash /opt/skeletonworks/scripts/setup-conductor.sh \
--domain example.com \
--image gitea.skeletonworks.online/vwiebe/conductor:v0.1.0 \
--restartcaddy
```
The script creates `/opt/skeletonworks/conductor/conductor.example.com`, preserves keys across reruns, joins `skeletonworks_net`, installs a Caddy block and backup cron entry, waits for health, and emits a JSON result. Prefer immutable version, Git-SHA, or digest references over `latest`.
## Uninstalling
Stopping/removing containers does not remove the named data volume:
```bash
docker compose --env-file .env -f compose.production.yml down
```
Do not add `--volumes` unless a verified backup exists and permanent data deletion is intended.

13
docs/KNOWN_LIMITATIONS.md Normal file
View File

@ -0,0 +1,13 @@
# Known Limitations for v0.1.0
- SQLite supports one Conductor instance with a persistent local volume; active/active replicas and PostgreSQL are not supported.
- Local username/password authentication is the release provider. OIDC/SSO is planned for Slice 9.
- Roles are global `admin` and `user`; per-application assignments and custom roles are not implemented.
- Published visibility is application-wide (`public` or all authenticated users), not page-specific.
- Page navigation is flat. Nested routes, path parameters, per-page permissions, and cross-page component bindings are not supported.
- Runtime values are ephemeral and reset on full browser reload.
- REST proxy authentication supports Anonymous, Basic, Bearer, API-key header, and API-key query. OAuth 2.0, mTLS, and custom auth are deferred.
- Workflow orchestration, action graphs, loops, retries, and long-running jobs are out of scope.
- Binding transforms and advanced expression languages are deferred.
- The release image targets `amd64` and `arm64` only after both pass the native SQLite build/smoke gate.
- Backups are local archives by default; operators must arrange off-host copies and retention appropriate to their recovery objectives.

7
docs/MVP_TRACEABILITY.md
View File

@ -10,7 +10,7 @@ This matrix maps every release-critical requirement area in `docs/REQUIREMENTS.m
|---|---|---|---|---|
| R1 | Browser-based application, React frontend, backend API, SQLite persistence, and Docker Compose deployment | Slices 1, 6, 8 | Production builds, Compose startup/health/restart/persistence, backup/restore in `docs/RELEASE_VALIDATION.md` and `TESTING.md` | Accepted |
| R2 | Canonical JSON project definition shared by Visual Editor, JSON Editor, Preview, persistence, and validation | Slices 2, 5, 6 | Schema matrix, canonical round trips, cross-view synchronization, save/reload/restart tests | Accepted |
| R3 | Single-page visual canvas with the eleven schema-supported components, selection, movement, resizing, deletion, properties, and basic styling | Slices 1, 2 | Slice 1/2 component tests and accepted canvas/property/Preview manual workflows | Accepted |
| R3 | Per-page visual canvas with the eleven schema-supported components, selection, movement, resizing, deletion, properties, and basic styling | Slices 1, 2, 7c | Slice 1/2 component tests and accepted canvas/property/Preview/manual multi-page workflows | Accepted |
| R4 | REST actions with supported methods, URL, headers, query/path parameters, body template, description, and fixed proxy-policy timeout | Slices 2, 4 | Action-editor tests, proxy integration/security suites, `docs/SCHEMA.md`, `docs/PROXY_SECURITY.md` | Accepted |
| R5 | Button `onClick` and page `onLoad` execution; component/variable request templates; Table selection runtime binding | Slices 2, 6 | Frontend event/template/runtime tests and deterministic launcher/dashboard E2E | Accepted |
| R6 | Top-level response/component bindings to supported component properties or typed runtime variables | Slices 2, 5, 6 | Binding diagnostics/runtime tests, compatibility validation, dependent-data/dashboard workflows | Accepted |
@ -27,6 +27,7 @@ This matrix maps every release-critical requirement area in `docs/REQUIREMENTS.m
| R17 | A fresh installation creates its initial administrator through a secure browser first-run flow without requiring Docker commands | Slice 7b | Atomic setup/security integration tests, frontend setup/password tests, recovery verification, and accepted manual workflow | Accepted |
| R18 | Authored and published applications support multiple deep-linked pages with page-local components and explicit global/page runtime-variable scope | Slice 7c | Schema/editor/runtime/publishing tests, multi-page browser E2E, security validation, and manual acceptance | Accepted |
| R19 | The Visual Editor presents project, page, page-setting, and lifecycle commands in a professional, accessible, responsive command ribbon | Slice 7d | 28 frontend suites / 528 tests, production build, and accepted manual visual validation | Accepted |
| R20 | Public self-hosters and SkeletonWorks operators can reproducibly install, secure, upgrade, back up, restore, and roll back a production Conductor release | Slice 8 | Production image/Compose tests, anonymous release download, fresh-host deployment, SkeletonWorks script validation, and encrypted-data backup/restore drill | Planned |
## Approved acceptance workflows
@ -46,7 +47,7 @@ The following capabilities are not release requirements. Their presence in futur
- Provider-neutral AI assistance, including chat, generation, explanation, documentation, and refactoring
- IBM Bob, watsonx, or any provider-specific AI dependency
- OIDC/enterprise SSO, external identity provisioning, and claim/group mapping (Slice 9)
- Multi-page authoring and advanced page management
- Nested/dynamic page routing, page parameters, and per-page authorization
- Direct Dropdown-triggered REST execution and general action chaining/orchestration
- Conditions, branches, loops, parallelism, retries, workflow graphs, and long-running task orchestration
- OAuth 2.0, IBM Cloud IAM, mTLS, and custom authentication scripts
@ -55,7 +56,7 @@ The following capabilities are not release requirements. Their presence in futur
- Binding transforms, expression languages, and configurable per-binding error policies
- Canvas zoom/pan, undo/redo, advanced JSON-editor tooling, themes, and reusable component libraries
- Tabs, modals, date pickers, file uploads, charts, progress indicators, and advanced Table capabilities
- RBAC, team collaboration, approval workflows, enterprise audit retention, and SIEM integration
- Custom/per-application roles, groups, team collaboration, approval workflows, enterprise audit retention, and SIEM integration
- PostgreSQL/high-availability/centralized multi-user deployment
## Governance rule

100
docs/OPERATIONS.md Normal file
View File

@ -0,0 +1,100 @@
# Production Operations
## Persistent state
All durable application state is stored in `/data/conductor.db`: projects, users, password hashes, sessions, publications, encrypted credentials, and execution history. SQLite runs in WAL mode.
The database alone is not a complete recovery set. A usable backup must include the exact `CONDUCTOR_SECRET_KEY` that encrypted stored credentials. Production backup archives therefore include the database, `.env`, Compose metadata, a manifest, and checksums. Treat archives as secrets and copy them off-host.
## Backup
From a source/release directory:
```bash
scripts/production/backup-conductor.sh \
--compose-file compose.production.yml \
--env-file .env \
--backup-dir ./backups \
--retention-days 7
```
The script invokes the application-owned SQLite online-backup command inside the running container. It does not copy a live WAL database. It then creates a `0600` archive with:
- `database/conductor.db`
- `configuration/.env`
- `configuration/compose.production.yml`
- `manifest.json`
- `SHA256SUMS`
It writes progress to stderr and a machine-readable result to stdout and `backup-result.json`.
SkeletonWorks uses:
```bash
sudo /opt/skeletonworks/scripts/backup-conductor.sh \
--fqdn conductor.example.com --retention-days 7
```
Its archive additionally contains `.secrets` and defaults to `/opt/skeletonworks/backups/conductor/<fqdn>/`.
## Restore
Restore replaces live state. Verify that the chosen archive and the intended target match.
```bash
scripts/production/restore-conductor.sh --list --backup-dir ./backups
scripts/production/restore-conductor.sh \
--backup-file ./backups/conductor_<timestamp>.tar.gz \
--compose-file compose.production.yml \
--env-file .env
```
Interactive restore requires typing `RESTORE`. Automation must explicitly pass `--force`. By default, restore creates a current-state safety backup, validates paths/checksums/format/keys, stops Conductor, replaces the database, restores UID/GID 10001 and mode `0600`, restarts, and waits for health.
SkeletonWorks:
```bash
sudo /opt/skeletonworks/scripts/restore-conductor.sh --fqdn conductor.example.com --list
sudo /opt/skeletonworks/scripts/restore-conductor.sh \
--fqdn conductor.example.com --backup-file <archive>
```
Never combine a database with a different encryption key. A checksum-valid but mismatched key cannot decrypt stored REST credentials.
## Upgrade
1. Read release notes and compatibility warnings.
2. Create and copy a verified backup off-host.
3. Change `CONDUCTOR_VERSION` or the pinned SkeletonWorks image.
4. Pull and recreate.
5. Wait for health and test login, project load, and a published deep link.
```bash
scripts/production/backup-conductor.sh
docker compose --env-file .env -f compose.production.yml pull
docker compose --env-file .env -f compose.production.yml up -d
curl --fail http://127.0.0.1:8080/api/health
```
SkeletonWorks upgrades are idempotent reruns of `setup-conductor.sh --image <new pinned ref>`.
## Rollback
Application images and database schema must be treated as a pair. For a failed compatible deployment, return to the prior pinned image. If the release changed the database incompatibly, restore the pre-upgrade archive as well.
1. Preserve logs and failed-state data.
2. Set the previous image tag/digest.
3. Restore the matching pre-upgrade backup.
4. Verify health and application smoke tests.
## Health and logs
- Liveness/ready endpoint: `GET /api/health`
- Container health: `docker compose ... ps`
- Logs: `docker compose ... logs --tail=200 conductor`
The health response includes the packaged version when built with release metadata.
## Recovery objectives
The default scripts retain seven daily archives, but retention is not a substitute for off-host storage. Recovery point and recovery time objectives are operator decisions. At least one periodic clean-host restore drill should be scheduled and recorded.

45
docs/RELEASE.md Normal file
View File

@ -0,0 +1,45 @@
# Release Construction and Publication
## Image identity
Each production image must correlate to one source revision and include OCI version, revision, creation-time, source, title, and description labels.
Release tags:
- `vX.Y.Z`: immutable release
- `git-<12-character-sha>`: immutable revision identity
- `latest`: optional convenience pointer; never recommended for controlled deployment
## Build locally
```bash
scripts/release-image.sh --version v0.1.0 --platforms linux/amd64
```
## Publish multi-architecture images
Authenticate to the registry using a scoped token, then:
```bash
docker login gitea.skeletonworks.online
scripts/release-image.sh --version v0.1.0 --push
```
The default published platforms are `linux/amd64,linux/arm64`. Both must successfully compile and load the native `better-sqlite3` dependency. If a target cannot pass build and smoke tests, omit it and state that explicitly in release notes.
## Release checklist
1. Clean checkout of the intended commit.
2. Governance, schema, frontend, backend, and browser suites pass.
3. Production image builds without local dependencies.
4. Missing-key startup fails; configured startup becomes healthy.
5. Root UI, `/api/health`, and published deep-link refresh return correctly.
6. Fresh first-run administrator setup passes with no default credentials.
7. Restart preserves data.
8. Online backup, checksum verification, destructive restore, and marker recovery pass.
9. Upgrade and rollback drill passes with pinned images and matching database backup.
10. Public source archive and container image are anonymously downloadable.
11. Release notes list image digests, architectures, known limitations, and upgrade instructions.
12. Tag and push only after explicit release approval.
Registry authentication is required only for publishing. Consumers must not require credentials for a public release; if the Gitea package cannot be pulled anonymously, publish the identical digest to a supported public registry and use it in public installation examples.

18
docs/RELEASE_VALIDATION.md
View File

@ -43,10 +43,28 @@ npm ci
npm run test:schema
docker compose -f docker-compose.yml -f docker-compose.manual-test.yml up -d --build
npm run test:e2e
docker build -f Dockerfile.production -t conductor:slice8-test .
npm run test:production
shellcheck scripts/production/*.sh scripts/release-image.sh scripts/test-production.sh
```
The backend integration suite binds isolated localhost ports. In restricted command sandboxes it may require explicit permission for local listening sockets.
## Production distribution gate
`npm run test:production` creates a uniquely named disposable Compose project and volume. It proves:
- production startup refuses absent keys;
- the image runs as the non-root `conductor` user with a read-only root filesystem;
- health, root frontend, and a published deep-link refresh are served from one port;
- a live WAL-mode database is backed up using SQLite's online backup facility;
- the archive manifest and checksums validate;
- destructive restore resets ownership/mode, becomes healthy, and recovers a marker record;
- cleanup removes the disposable container, network, volume, environment, and archive.
The release image workflow publishes immutable version and Git-SHA tags with OCI metadata, SBOM, and provenance for `linux/amd64` and `linux/arm64`. Before release approval, an unauthenticated client must pull the final digest. If Gitea denies anonymous package pulls, the same digest must be published to the documented public fallback registry.
## Manual gate
Manual acceptance begins only after all automated commands, Docker restart and persistence, backup/restore, repository credential scan, and cleanup checks pass. The manual checklist must cover all six workflows in `MVP_SCOPE.md`, including all authentication modes and failure recovery, and record any accepted limitations or defects.

24
docs/REQUIREMENTS.md
View File

@ -37,8 +37,7 @@ The system is not intended to be a full Appsmith replacement.
The MVP will not include:
* Full database integrations.
* Complex multi-user application publishing.
* Advanced permissions or RBAC.
* Per-application user assignments, custom roles, groups, or approval workflows beyond global admin/user roles.
* Marketplace-style widget plugins.
* Pixel-perfect design tooling.
* Full workflow orchestration.
@ -535,7 +534,7 @@ The system should provide basic troubleshooting details:
The MVP should include:
* Single-page project builder.
* Multi-page project builder with flat navigation and refreshable published deep links.
* Basic canvas.
* Core widgets.
* REST action configuration.
@ -544,6 +543,9 @@ The MVP should include:
* API response-to-component binding.
* Preview mode.
* Save/load project as JSON.
* Local authentication with global admin/user roles and browser first-run administration.
* Public or authenticated immutable application publishing.
* Single-image production distribution with verified backup and restore.
The MVP does not include IBM Bob, watsonx, or any other AI dependency.
@ -553,8 +555,8 @@ The MVP does not include IBM Bob, watsonx, or any other AI dependency.
Potential future enhancements:
* Multi-page applications.
* Role-based access control.
* Nested/dynamic pages and per-page authorization.
* Custom roles, groups, and per-application user assignments.
* OAuth support.
* IBM Cloud IAM integration.
* Advanced or enterprise execution-history retention and integrations.
@ -565,7 +567,7 @@ Potential future enhancements:
* Reusable templates.
* Version history.
* Approval workflows.
* Deployment to internal hosting.
* Managed public SaaS hosting.
* Git integration.
* Team collaboration.
@ -626,8 +628,7 @@ The application backend may include an embedded development server for local tes
The reverse proxy should handle:
* HTTPS termination
* Static frontend asset delivery
* Routing requests to the backend service
* Routing requests to the same-origin Conductor production service
* Request size limits
* Basic security headers
* Optional access restrictions
@ -639,7 +640,7 @@ Browser
↓
NGINX / Apache / Caddy / IBM-approved reverse proxy
↓
Frontend static assets + Backend API service
Single Conductor service (compiled frontend + Backend API)
↓
REST API endpoints / Concert / RIA / other systems
```
@ -658,6 +659,7 @@ SQLite should store:
* Canonical project JSON, which contains pages, components, layout, actions, bindings, variables, settings, and non-secret configuration
* Encrypted secret records and metadata
* Sanitized bounded execution history
* Local users, sessions, and immutable publication snapshots
Secrets should not be stored directly in plain text in SQLite.
@ -680,9 +682,9 @@ If the project grows beyond MVP, the backend should be designed so SQLite can la
A full RDBMS may be required if the system needs:
* Multiple concurrent users
* High-concurrency or horizontally scaled deployments
* Team collaboration
* Role-based access control
* Custom/per-application authorization models
* High availability
* Centralized deployment
* Enterprise backup/restore

37
docs/SCHEMA.md
View File

@ -25,7 +25,7 @@ The JSON schema:
- Documents the exact shape of a valid project definition.
- Enables offline validation during development.
- Drives IDE autocomplete and inline error highlighting when `$schema` is set in a project file.
- Will be used by the backend's validation endpoint (Step 11) to reject malformed saves.
- Is used by the backend validation/persistence endpoints to reject malformed saves atomically.
- Makes project definitions portable, diffable in Git, and importable/exportable.
---
@ -63,10 +63,10 @@ Consumers (backend, editor, preview runtime) must check the `MAJOR` version comp
| `id` | `string` | ✅ | Stable unique identifier (UUID or URL-safe slug). Must not change. |
| `name` | `string` | ✅ | Human-readable display name (1–200 characters). |
| `description` | `string` | — | Optional free-text description. Defaults to `""`. |
| `pages` | `array` | ✅ | Ordered list of `Page` objects. May be empty. |
| `pages` | `array` | ✅ | Ordered non-empty list of `Page` objects. |
| `actions` | `array` | ✅ | Project-level REST action definitions. May be empty. |
| `bindings` | `array` | ✅ | Project-level binding definitions. May be empty. |
| `variables` | `object` | ✅ | Named global variable declarations. May be empty (`{}`). |
| `variables` | `object` | ✅ | Named global or page-scoped declarations. May be empty (`{}`). |
| `settings` | `object` | ✅ | Project display and canvas settings. May be empty (`{}`). |
---
@ -81,6 +81,8 @@ Represents one view in the project. Required fields: `id`, `name`, `components`.
{
"id": "page_home",
"name": "Home",
"slug": "home",
"showInNavigation": true,
"description": "",
"order": 0,
"components": [],
@ -88,6 +90,8 @@ Represents one view in the project. Required fields: `id`, `name`, `components`.
}
```
Page slugs are unique URL-safe segments. Older documents may omit them; the runtime derives a deterministic slug. `showInNavigation: false` hides a page from automatic navigation but is not authorization. `project.settings.defaultPageId` selects the base-route page; otherwise the first ordered page is the compatibility default. Page events support `onLoad` (first visit in the loaded session) and `onEnter` (return visits).
### `Component`
A UI element placed on a page canvas. Required fields: `id`, `type`, `name`, `position`, `size`.
@ -110,7 +114,7 @@ Allowed `type` values:
"events": [
{
"event": "onClick",
"actionId": "action_trigger_workflow"
"navigateToPageId": "page_results"
}
],
"bindings": []
@ -136,19 +140,20 @@ Allowed `authenticationType` values: `anonymous` · `bearerToken` · `basicAuth`
"name": "Trigger Workflow",
"description": "Calls the Concert RIA API to trigger a workflow run.",
"method": "POST",
"url": "https://api.example.com/workflows/{{workflowId}}/run",
"url": "https://api.example.com/workflows/run",
"headers": {
"Content-Type": "application/json",
"Accept": "application/json"
},
"queryParameters": {
"environment": "{{environment}}"
"environment": "{{components.environment.value}}"
},
"pathParameters": {
"workflowId": "{{workflowId}}"
"workflowId": "configured-static-id"
},
"bodyTemplate": "{\"params\": {{params}}}",
"authenticationType": "bearerToken"
"bodyTemplate": "{\"requestedBy\": \"{{variables.currentUser}}\"}",
"authenticationType": "bearerToken",
"secretReferenceId": "opaque-server-secret-id"
}
```
@ -173,11 +178,11 @@ New response mappings belong in top-level `project.bindings`, not inside the act
| `name` | `string` | ✅ | Human-readable action name shown in the Actions panel. |
| `description` | `string` | — | Optional description of what this action does. |
| `method` | `string` | ✅ | HTTP method: `GET`, `POST`, `PUT`, `PATCH`, or `DELETE`. |
| `url` | `string` | ✅ | Target URL. Use `{{paramName}}` for path parameter placeholders. |
| `headers` | `object` | — | Static request headers. Values may use `{{variableName}}` syntax. |
| `queryParameters` | `object` | — | URL query string parameters. Values may use `{{variableName}}` syntax. |
| `pathParameters` | `object` | — | Path segment substitutions. Keys match `{{paramName}}` in the URL. |
| `bodyTemplate` | `string` | — | Request body template. Use `{{variableName}}` for runtime substitutions. |
| `url` | `string` | ✅ | Target URL; may use executed component/variable templates. |
| `headers` | `object` | — | Request headers; values may use executed templates. |
| `queryParameters` | `object` | — | Query parameters; values may use executed templates. |
| `pathParameters` | `object` | — | Static path substitutions; runtime templates are not supported here. |
| `bodyTemplate` | `string` | — | Request body string with component/variable templates. |
| `authenticationType` | `string` | ✅ | Authentication strategy (see allowed values above). Credentials are never stored here. |
| `responseMapping` | `array` | — | Deprecated legacy action-local mappings. New mappings use top-level `project.bindings`. |
@ -201,7 +206,7 @@ Declares data flow between a source and a target. Required fields: `id`, `source
### `Variable`
A named global variable. Required field: `type`.
A named global or page variable. Required field: `type`. Omitted `scope` means `global` for compatibility; `scope: "page"` requires a valid owning `pageId`.
Allowed `type` values: `string` · `number` · `boolean` · `object` · `array`
@ -209,6 +214,8 @@ Allowed `type` values: `string` · `number` · `boolean` · `object` · `array`
{
"lastRunStatus": {
"type": "string",
"scope": "page",
"pageId": "page_home",
"defaultValue": "",
"description": "Status returned by the most recent workflow trigger."
}

35
docs/TROUBLESHOOTING.md Normal file
View File

@ -0,0 +1,35 @@
# Troubleshooting
## Container exits immediately
Inspect logs. Production intentionally refuses to start when `CONDUCTOR_SECRET_KEY` is not a valid 32-byte base64/hex value or `CONDUCTOR_SESSION_KEY` is shorter than 32 bytes.
## Container is unhealthy
```bash
docker compose --env-file .env -f compose.production.yml ps
docker compose --env-file .env -f compose.production.yml logs --tail=200 conductor
curl -v http://127.0.0.1:8080/api/health
```
Confirm `/data` is writable by UID/GID 10001. Restore tooling resets database ownership to `10001:10001` and mode `0600`.
## Stored credential no longer works after restore
The restored database and `CONDUCTOR_SECRET_KEY` likely do not match. Restore the complete archive, including its `.env`/`.secrets`. Credential encryption cannot be bypassed or recovered without the original key.
## Published deep link returns proxy 404
The reverse proxy must send all non-API paths to Conductor rather than serving static files independently. Test the container directly at `/apps/<app>/<page>` and use the Caddy/NGINX examples in `docs/INSTALL.md`.
## REST request is blocked
Read the safe error code and `docs/PROXY_SECURITY.md`. Private/internal destinations are denied unless their exact origin is configured in `CONDUCTOR_PROXY_INTERNAL_ORIGINS`. Do not add broad exceptions.
## Browser shows old controls after upgrade
Confirm the running image tag/digest, then perform a hard browser refresh. The production `index.html` is sent with `no-cache`; hashed assets may be cached safely.
## Backup refuses to run
The application container must be running and the environment file must exist because it contains recovery keys. Check Compose project selection and paths. SkeletonWorks commands require the exact `--fqdn` used during setup.

46
docs/USER_GUIDE.md Normal file
View File

@ -0,0 +1,46 @@
# User Guide
## Roles and entry points
- **Admin** users create projects, manage credentials and users, inspect execution history, and publish applications.
- **User** users open restricted published applications and manage their own local password/profile.
- Public applications can be opened without signing in.
A fresh installation creates its first administrator in the browser. Later accounts are managed from **Users**.
## Building an application
1. Create or load a project in **Visual Editor**.
2. Use **Pages** to add, duplicate, reorder, name, slug, hide, default, or delete pages.
3. Add components from the palette and configure the selected component in the inspector.
4. Define variables, REST actions, credentials, and response bindings in **Actions & Bindings**.
5. Configure Button action/navigation events and page `onLoad`/`onEnter` lifecycle actions.
6. Use **Preview** to exercise the application without changing canonical saved state.
7. Save or update the project.
8. Publish from **Publishing** as public or restricted.
Public publications may use only anonymous actions. Restricted publications may use backend-held credentials. Publishing creates an immutable snapshot; edit/save and republish to release changes.
## Variable and component scope
Component names are local to a page. The same name may be reused on different pages. Global variables are available throughout the app; page variables belong to one page. Values persist while navigating in one loaded session and reset on full reload.
## Lifecycle behavior
- `onLoad`: first entry to that page during the loaded session.
- `onEnter`: subsequent entry when returning to the page.
## JSON Editor
The JSON Editor edits the same canonical document as the Visual Editor. Validate and apply before saving. Invalid JSON or semantic references do not replace the last valid in-memory document.
## Credentials
Credential values are submitted to encrypted server-side storage and are never displayed again. Projects contain opaque secret references, not values. Replacing a credential requires entering the new value twice where applicable.
## Published URLs
- `/apps/<app-slug>` opens the default page.
- `/apps/<app-slug>/<page-slug>` is a refreshable page deep link.
Hiding a page removes it from automatic navigation; it is not an authorization rule.

433
docs/response-mapping-model.md
View File

@ -1,412 +1,81 @@
# Response Mapping Model
# Canonical Response Mapping and Runtime Model
**Step 17.0 design document — established 2025**
## Ownership
This document records the canonical decisions about how action responses flow into
component state in the Conductor Preview runtime. It is the authoritative
reference for Steps 17.x implementation.
New response mappings are top-level `project.bindings`. The legacy `action.responseMapping` field remains schema-compatible but is not executed and must not be authored by new UI/examples.
---
## Table of Contents
1. [Why `project.bindings` is canonical](#1-why-projectbindings-is-canonical)
2. [Why actions own runtime responses](#2-why-actions-own-runtime-responses)
3. [Why `actions.<id>.response` is a virtual namespace](#3-why-actionsidresponse-is-a-virtual-namespace)
4. [Source path grammar](#4-source-path-grammar)
5. [Target path grammar](#5-target-path-grammar)
6. [Trigger semantics](#6-trigger-semantics)
7. [Label runtime-value behaviour](#7-label-runtime-value-behaviour)
8. [Compatibility — existing `onClick` bindings](#8-compatibility--existing-onclick-bindings)
9. [Deprecation of `action.responseMapping`](#9-deprecation-of-actionresponsemapping)
10. [Step 17.1 scope](#10-step-171-scope)
11. [Deferred capabilities](#11-deferred-capabilities)
12. [Required diagnostics](#12-required-diagnostics)
---
## 1. Why `project.bindings` is canonical
Two mechanisms could in principle move action response data into components:
| Mechanism | Location | Status |
|---|---|---|
| `action.responseMapping[]` | Inside each `Action` object | **Deprecated** |
| `project.bindings[]` | Top-level project array | **Canonical** |
`project.bindings` was chosen as the sole canonical mechanism for the following
reasons:
**Single locus of data-flow declarations.** All data movement in a Conductor
project — between components, from action responses, from variables — is
expressed in one flat array. Keeping response routing there avoids a split
where "inbound response routing" lives inside actions while all other data
movement lives at the project level.
**Symmetry with component-sourced bindings.** A binding from a TextInput value
to a Label uses the same structure as a binding from an action response to a
JsonViewer. Same schema, same Inspector rendering, same future transform field.
**Decoupled from action definition.** An action definition describes *how to
call* an endpoint (URL, method, auth). Where the response goes is a layout
concern, not an API concern. Separating them means a single action can feed
multiple targets without duplicating the action.
**Better Inspector and diagnostics surface.** A flat array of bindings is easy
to enumerate, validate cross-references for, and display in a panel.
Distributing response mapping rules across individual action objects makes
completeness checks harder.
**Incremental migration path.** Existing documents that carry
`action.responseMapping` values remain valid under the schema; they are simply
ignored at runtime. New documents use only `project.bindings`.
---
## 2. Why actions own runtime responses
Although `project.actions` is a design-time array of REST action definitions,
the Preview runtime must store ephemeral per-action state: the most recent
response, loading flag, and error. This state is:
- **Not stored in canonical project JSON.** The project document is the
design-time source of truth. Mutating it with runtime values would corrupt
the save/load round-trip.
- **Keyed by action ID.** Even though actions are stored as an ordered array
in the project document, the runtime resolves them by `id` for O(1) lookup.
- **Discarded on navigation or refresh.** Preview state is ephemeral React
state (`useState`).
The proposed runtime shape (to be implemented in Step 17.1) is:
```ts
type ActionRuntimeState = {
response?: ProxyResponse; // the full ProxyResponse envelope
loading?: boolean;
error?: string;
};
type ActionRuntimeStateMap = Record<string, ActionRuntimeState>;
```
This is a parallel structure to the existing `componentState` map in
`usePreviewRuntime`. In Step 17.1 it will be introduced alongside it.
> **Step 17.0 note:** This runtime shape is defined here for design purposes.
> It is not implemented in Step 17.0. The current runtime continues to store
> the full `ProxyResponse` envelope directly in `componentState[id].value`.
---
## 3. Why `actions.<id>.response` is a virtual namespace
In binding source and target expressions, paths of the form:
```
actions.<actionId>.response
actions.<actionId>.response.body
actions.<actionId>.response.body.<field>
```
refer to runtime action response data. This is a **virtual namespace** because:
1. `project.actions` is an **array** in the canonical document, not an object
keyed by ID. At design time, `actions.action_httpbin` does not exist as a
JSON key path.
2. The runtime resolves `<actionId>` via `Array.find((a) => a.id === actionId)`,
not by property access.
3. The path is only meaningful during Preview execution. It has no value in the
persisted project document.
The path syntax is chosen for readability and symmetry with `components.<name>.*`
target paths. It is not JSONPath; it is a **Conductor runtime dot-path** with
a fixed `actions.<id>.response[.body[.<fields>]]` prefix.
---
## 4. Source path grammar
Step 17.1 will support the following source path forms for action-response
bindings:
```
actions.<actionId>.response
actions.<actionId>.response.body
actions.<actionId>.response.body.<field>
actions.<actionId>.response.body.<nested>.<field>
```
The v0.1.0 runtime additionally supports one bounded component source for the
read-only dashboard workflow:
```text
components.<tableName>.selectedRow
```
It requires trigger `onChange` and may target a supported
`components.<displayName>.value` path. The selected row and resulting display
value are runtime-only and are discarded when Preview is reinitialized.
### Resolution rules
| Path | Resolved value |
|---|---|
| `actions.<id>.response` | The full `ProxyResponse` envelope `{ ok, status, statusText, headers, body, durationMs }` |
| `actions.<id>.response.body` | `ProxyResponse.body` — parsed JSON object or text string |
| `actions.<id>.response.body.<field>` | `ProxyResponse.body.<field>` where `body` is an object |
| `actions.<id>.response.body.<nested>.<field>` | Deep dot-path traversal on `body` |
If a field in the path does not exist on the response body, the resolved value
is `undefined`. Step 17.1 will surface a runtime warning in that case; it
will not throw.
### Deferred source path forms
The following are explicitly deferred and must not be inferred from the above:
- `actions.<id>.response.headers.<header>` — reserved; not in Step 17.1
- `actions.<id>.response.status` — reserved; not in Step 17.1
- Wildcard `*` segments
- Filter expressions `[?(...)]`
- Array index syntax `[0]`, `[-1]`
- Keys containing literal `.` characters
---
## 5. Target path grammar
Step 17.1 will support only:
```
components.<componentName>.value
```
### Semantic enforcement
- `<componentName>` must match `component.name` (not `component.id`) in the
project's page components, consistent with the existing binding model.
- The property segment **must be** `.value`. The Inspector must emit a warning
for targets with any other property segment (e.g. `.data`, `.message`,
`.label`). The runtime must not silently remap these to `.value`.
### Property segment rationale
`.value` is the single runtime-mutable property exposed by `ComponentRuntimeState`
for display purposes. Until per-property runtime targeting is designed:
- `components.<name>.value` is valid.
- `components.<name>.data`, `.message`, `.label`, `.text`, etc. are unsupported.
They must produce an Inspector diagnostic and a runtime warning.
### Deferred target forms
- `variables.<variableName>` — variable runtime support is deferred
- `components.<name>.label` — Label runtime-value override is deferred to Step 17.1
(see §7)
- Per-property routing for non-`.value` segments
---
## 6. Trigger semantics
### Canonical trigger for action-response bindings: `onSuccess`
```json
{ "trigger": "onSuccess" }
```
`onSuccess` fires when the action completes and `ProxyResponse.ok === true`.
This is the trigger all new action-response bindings should use.
### Why not `onChange` (the schema default)?
The schema's `binding.trigger` default is `"onChange"`. This default is
appropriate for component-to-component data flow (e.g. a TextInput value driving
a downstream Label). It is not appropriate for action-response bindings because:
- Action responses are not "change" events on a component; they are completion
events on an async operation.
- `"onChange"` implies reactive/continuous behaviour; action execution is
discrete and user-triggered (or lifecycle-triggered).
The `"onChange"` default is therefore **not the correct default for
action-response bindings**. The Inspector should surface a warning when
`"onChange"` is found on a binding whose source is `actions.*`.
### Deferred triggers
| Trigger | Status |
|---|---|
| `onSuccess` | **Step 17.1** |
| `onError` | Deferred |
| `onLoad` | Deferred |
| `onChange` | Component-change bindings only; deferred |
| Chained actions | Deferred |
---
## 7. Label runtime-value behaviour
**Current (Step 17.0):** Label renders `properties.label` only. It has no
`runtimeState` wiring. Runtime action responses cannot update a Label.
**Planned (Step 17.1):**
- `properties.label` — design-time / configured text. Displayed when no
runtime value is set.
- `componentState[id].value` — runtime display value. If set (not `undefined`),
it overrides `properties.label` in the Preview renderer.
- A binding of the form:
```json
{
"source": "actions.<id>.response.body.<field>",
"target": "components.<labelName>.value",
"id": "binding_inventory",
"source": "actions.loadInventory.response.body.items",
"target": "components.inventoryTable.rows",
"trigger": "onSuccess"
}
```
writes the resolved field value (coerced to string for display) into
`componentState[labelId].value`.
- **Runtime mutations must not modify canonical project JSON.** The override
is ephemeral; it lives only in `useState` for the Preview session.
This makes Label consistent with JsonViewer, which already reads
`runtimeState.value` in `PreviewComponent.tsx`.
## Action execution
---
Actions are project-level and reusable. A Button `onClick`, Dropdown `onChange` where supported, or page lifecycle event selects an `actionId`. A Button event may instead contain `navigateToPageId`; navigation is not represented as a REST action.
## 8. Compatibility — existing `onClick` bindings
Before execution, request fields interpolate:
Steps 15 and 16 produced examples and docs using:
- `{{components.<name>.value}}` against components on the invoking page only.
- `{{variables.<name>}}` against global variables or page variables owned by the invoking page.
```json
{ "trigger": "onClick" }
Credential-backed authentication is resolved by the backend through opaque `secretReferenceId`. Browser clients never supply authoritative URLs, authentication modes, actions, or secret values for published execution.
## Response source paths
Canonical action sources begin with:
```text
actions.<actionId>.response
actions.<actionId>.response.body
actions.<actionId>.response.body.<dot-separated-field>
```
on action-response bindings. These bindings fire when the button's `onClick`
event causes the action to execute.
The current resolver supports ordinary dot-separated object traversal. JSONPath filters, wildcards, quoted dotted keys, and transform expressions are deferred.
**Step 17.0 treatment:** `"onClick"` bindings continue to work exactly as they
do today. The runtime (`usePreviewRuntime`) fires all bindings whose source
matches the action ID, regardless of their `trigger` value.
## Targets
**Recommended migration:** New bindings should use `"onSuccess"` rather than
`"onClick"`. The Inspector may surface an info-level diagnostic recommending
migration on bindings that use `"onClick"` as the trigger on an
`actions.*` source. This is not yet implemented.
Supported action-response targets are validated against component type/property compatibility:
**Important:** `"onClick"` on an action-response binding is **legacy**, not
invalid. Documents using it will not fail schema validation and will not be
broken by Step 17.1.
- `components.<name>.value`
- `components.<dropdownName>.options`
- `components.<tableName>.rows`
- `variables.<name>`
---
Component names are page-local. Response bindings execute in the invoking page context and may update components on that page, global variables, and page variables owned by that page. Cross-page component/page-variable writes are prohibited.
## 9. Deprecation of `action.responseMapping`
## Triggers
`action.responseMapping` is **deprecated as of Step 17.0**.
- `onSuccess` is the canonical trigger for an action response and runs only when `ProxyResponse.ok` is true.
- Legacy action-response `onClick` is accepted for compatibility and should be migrated to `onSuccess`.
- Table `components.<name>.selectedRow` bindings use `onChange`.
- `onError`, action chaining, and arbitrary reactive component graphs are deferred.
| Property | Status |
|---|---|
| Present in schema | Yes — retained for backward compatibility |
| `x-deprecated: true` annotation | Added in Step 17.0 |
| Schema `description` | Updated to say `project.bindings` is canonical |
| TypeScript `@deprecated` JSDoc | Added in Step 17.0 |
| TS type narrowed | Yes — changed from `unknown[]` to `ResponseMappingRule[]` |
| Executed by Preview runtime | **No** — never was, never will be |
| Executed by backend proxy | **No** — never was, never will be |
| Shown by Inspector | **No** |
| Removal timeline | Not scheduled; retained for backward compatibility only |
## Runtime state
**No new examples should use `action.responseMapping`.** Existing examples
(`valid-full.json`) that carry it are not updated; they remain valid because
the field is optional and the schema accepts (but ignores) its contents.
Configured defaults remain in canonical project JSON. Preview/published component, action, and variable values live in ephemeral runtime state and never mutate the canonical document. Global and page values survive in-session page navigation and reset on full reload.
---
Actions expose loading, success, upstream failure, network/policy failure, and mapping failure states. Mapped components receive loading/error state while requests run. When a successful action has no mapping, the triggering control surfaces the response for operator feedback.
## 10. Step 17.1 scope
## Lifecycle
Step 17.1 will implement execution of `project.bindings` for
action-response-to-component data movement. Specifically:
- Page `onLoad` runs on the first entry to that page during one loaded runtime session.
- Page `onEnter` runs when returning to a previously visited page.
- Reopening/reloading the runtime begins a fresh session.
1. **`ActionRuntimeStateMap`** — introduce alongside the existing
`componentState` map in `usePreviewRuntime`. After each proxy call,
write the `ProxyResponse` into `actionState[actionId].response`.
## Diagnostics
2. **Source path resolution** — implement dot-path traversal for
`actions.<id>.response`, `.response.body`, and `.response.body.<field...>`.
Validation rejects or warns about:
3. **Binding execution loop** — after a successful proxy call, iterate
`project.bindings` whose `trigger` is `"onSuccess"` (or `"onClick"` for
legacy compatibility) and whose `source` parses to the completed action ID.
Resolve the source path against `actionState[actionId].response`. Write the
resolved value to `componentState[targetId].value`.
- dangling action, binding, page, component, variable, and secret references;
- unsupported trigger/source/target combinations;
- duplicate page/component identities in their required scopes;
- malformed templates and unsupported component template properties;
- page-variable ownership violations and cross-page references;
- incompatible response target properties;
- deprecated `action.responseMapping` and inert `inputMap` compatibility fields.
4. **Label renderer update** — make `LabelRenderer` read `runtimeState?.value`
as an override over `properties.label`, consistent with `JsonViewerRenderer`.
5. **Inspector diagnostics** — add warnings for:
- Unsupported target property segments (anything other than `.value`)
- `"onChange"` trigger on action-response bindings (recommend `"onSuccess"`)
- Bindings whose source action ID does not exist
- Bindings whose target component name does not exist
6. **`valid-response-mapping-basic.json`** — the example created in Step 17.0
becomes the primary Step 17.1 integration test document.
---
## 11. Deferred capabilities
The following are explicitly out of scope until separately designed and scheduled:
| Capability | Reason deferred |
|---|---|
| `onError` trigger | Requires error-state routing design |
| `onLoad` trigger | Requires page lifecycle event system |
| Component-change data movement (`onChange`) | Requires reactive binding engine |
| Chained actions | Requires action dependency graph |
| Variable runtime support | Requires variable state store |
| Variable template interpolation in binding paths | Depends on variable runtime |
| Wildcard `*` in source paths | JSONPath-style engine not scoped |
| Filter expressions in source paths | Same |
| Array-index syntax `[0]` | Same |
| Keys with literal `.` in name | Requires path quoting syntax |
| `components.<name>.label` as writable target | Deferred to Step 17.1 label work |
| Any target property other than `.value` | Deferred to per-property routing design |
| Dropdown or Table runtime population | Separate step |
| `ComponentType` / `FullComponentType` cleanup | Separate step |
| Component-level `bindings[]` array | Schema field exists but is not wired |
| Duplicate component-name detection at save time | Future validation step |
| Transform expression execution | Deferred (`binding.transform` field retained) |
| Authentication credential injection | Separate step |
| `action.responseMapping` execution | Permanently deferred; field deprecated |
---
## 12. Required diagnostics
The following diagnostics must be implemented by Step 17.1. They are listed
here so the Inspector and runtime can be built consistently against this spec.
### Inspector (static, design-time)
| Condition | Severity | Message guidance |
|---|---|---|
| Binding `source` references action ID that does not exist in `project.actions` | Warning | "Source references action `<id>` which does not exist." |
| Binding `target` references component name that does not exist on any page | Warning | "Target references component `<name>` which does not exist." |
| Binding `target` uses property segment other than `.value` | Warning | "Target property `<segment>` is not supported. Use `.value`." |
| Two or more components share the same `name` on a page | Warning | "Duplicate component name `<name>`. Binding resolution is ambiguous." |
| `trigger: "onChange"` on an `actions.*` source binding | Info | "Consider using `onSuccess` for action-response bindings." |
| `trigger: "onClick"` on an `actions.*` source binding | Info | "Legacy trigger. Consider migrating to `onSuccess`." |
| `action.responseMapping` is non-empty | Info | "Use `project.bindings` instead. `responseMapping` is deprecated and not executed." |
### Runtime (execution-time, Preview only)
| Condition | Behaviour |
|---|---|
| Source path resolves to `undefined` (field not on body) | Write `undefined` to target; emit console warning |
| Target component not found at runtime | Skip write; emit console warning |
| `ProxyResponse.ok === false` and trigger is `onSuccess` | Do not fire binding; optionally fire `onError` bindings (deferred) |
| Action ID in binding source does not match any executing action | Skip silently |
The backend performs schema and semantic validation before persistence/publication. The frontend shows the same actionable paths without replacing the last valid project state.

4
package.json
View File

@ -2,7 +2,9 @@
"scripts": {
"test:e2e": "playwright test",
"test:governance": "node scripts/check-mvp-governance.mjs",
"test:schema": "bash scripts/validate-schema.sh"
"test:docs": "node scripts/check-docs.mjs",
"test:schema": "bash scripts/validate-schema.sh",
"test:production": "bash scripts/test-production.sh --skip-build"
},
"devDependencies": {
"@playwright/test": "^1.54.2",

31
scripts/check-docs.mjs Normal file
View File

@ -0,0 +1,31 @@
import fs from 'node:fs';
import path from 'node:path';
import process from 'node:process';
const root = process.cwd();
const markdown = [];
const walk = (directory) => {
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
if (entry.name === '.git' || entry.name === 'node_modules') continue;
const absolute = path.join(directory, entry.name);
if (entry.isDirectory()) walk(absolute);
else if (entry.name.endsWith('.md')) markdown.push(absolute);
}
};
walk(root);
const failures = [];
for (const file of markdown) {
const text = fs.readFileSync(file, 'utf8');
for (const match of text.matchAll(/\[[^\]]*\]\(([^)]+)\)/g)) {
const raw = match[1].trim().replace(/^<|>$/g, '');
const target = raw.split('#')[0];
if (!target || /^[a-z][a-z0-9+.-]*:/i.test(target)) continue;
const resolved = path.resolve(path.dirname(file), decodeURIComponent(target));
if (!fs.existsSync(resolved)) failures.push(`${path.relative(root, file)}: missing link target ${target}`);
}
}
if (failures.length) {
console.error(failures.join('\n'));
process.exit(1);
}
console.log(`Documentation links passed: ${markdown.length} Markdown files checked.`);

4
scripts/check-mvp-governance.mjs
View File

@ -48,7 +48,7 @@ for (const file of linkedFiles) {
}
}
for (let id = 1; id <= 19; id += 1) {
for (let id = 1; id <= 20; id += 1) {
if (!traceability.includes(`| R${id} |`)) failures.push(`docs/MVP_TRACEABILITY.md: missing R${id}`);
}
@ -57,4 +57,4 @@ if (failures.length > 0) {
process.exit(1);
}
console.log('MVP governance check passed: scope, R1-R19 traceability, classifications, and links are consistent.');
console.log('MVP governance check passed: scope, R1-R20 traceability, classifications, and links are consistent.');

81
scripts/production/backup-conductor.sh Executable file
View File

@ -0,0 +1,81 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
COMPOSE_FILE="${ROOT_DIR}/compose.production.yml"
ENV_FILE="${ROOT_DIR}/.env"
BACKUP_DIR="${ROOT_DIR}/backups"
RETENTION_DAYS=7
SERVICE="conductor"
TIMESTAMP="$(date -u +%Y-%m-%dT%H-%M-%SZ)"
usage() {
printf '%s\n' "Usage: $0 [--compose-file PATH] [--env-file PATH] [--backup-dir PATH] [--retention-days N]" >&2
}
while [[ $# -gt 0 ]]; do
case "$1" in
--compose-file) COMPOSE_FILE="$2"; shift 2 ;;
--env-file) ENV_FILE="$2"; shift 2 ;;
--backup-dir) BACKUP_DIR="$2"; shift 2 ;;
--retention-days) RETENTION_DAYS="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "Unknown option: $1" >&2; usage; exit 1 ;;
esac
done
[[ -f "$COMPOSE_FILE" ]] || { echo "Compose file not found: $COMPOSE_FILE" >&2; exit 1; }
[[ -f "$ENV_FILE" ]] || { echo "Environment file not found: $ENV_FILE (it contains required recovery keys)." >&2; exit 1; }
[[ "$RETENTION_DAYS" =~ ^[0-9]+$ ]] || { echo "--retention-days must be a non-negative integer." >&2; exit 1; }
compose=(docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE")
container_id="$("${compose[@]}" ps -q "$SERVICE")"
[[ -n "$container_id" ]] || { echo "Conductor container is not created." >&2; exit 1; }
[[ "$(docker inspect -f '{{.State.Running}}' "$container_id")" == "true" ]] || { echo "Conductor container is not running." >&2; exit 1; }
mkdir -p "$BACKUP_DIR"
chmod 700 "$BACKUP_DIR"
work_dir="$(mktemp -d)"
cleanup() { rm -rf -- "$work_dir"; }
trap cleanup EXIT
mkdir -p "$work_dir/database" "$work_dir/configuration"
staged="/data/.conductor-backup-${TIMESTAMP}.db"
echo "[*] Creating consistent online SQLite backup..." >&2
"${compose[@]}" exec -T "$SERVICE" node dist/scripts/backupDatabase.js "$staged" >&2
docker cp "${container_id}:${staged}" "$work_dir/database/conductor.db"
"${compose[@]}" exec -T "$SERVICE" node -e 'const fs=require("fs");fs.unlinkSync(process.argv[1])' "$staged"
cp "$ENV_FILE" "$work_dir/configuration/.env"
cp "$COMPOSE_FILE" "$work_dir/configuration/compose.production.yml"
image_ref="$(docker inspect -f '{{.Config.Image}}' "$container_id")"
image_id="$(docker inspect -f '{{.Image}}' "$container_id")"
cat > "$work_dir/manifest.json" <<EOF
{
"format": "conductor-backup-v1",
"createdAt": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
"image": "${image_ref}",
"imageId": "${image_id}",
"database": "database/conductor.db",
"environment": "configuration/.env"
}
EOF
(cd "$work_dir" && sha256sum database/conductor.db configuration/.env configuration/compose.production.yml manifest.json > SHA256SUMS)
archive="${BACKUP_DIR}/conductor_${TIMESTAMP}.tar.gz"
tar -czf "$archive" -C "$work_dir" database configuration manifest.json SHA256SUMS
chmod 600 "$archive"
sha256sum "$archive" > "${archive}.sha256"
chmod 600 "${archive}.sha256"
removed=0
while IFS= read -r -d '' old; do rm -f -- "$old" "${old}.sha256"; removed=$((removed + 1)); done < <(find "$BACKUP_DIR" -maxdepth 1 -name 'conductor_*.tar.gz' -mtime +"$RETENTION_DAYS" -print0)
size_bytes="$(stat -c '%s' "$archive")"
result="${BACKUP_DIR}/backup-result.json"
cat > "$result" <<EOF
{"script":"backup-conductor","status":"success","timestamp":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","backup":{"file":"${archive}","sizeBytes":${size_bytes},"checksumFile":"${archive}.sha256"},"rotation":{"retentionDays":${RETENTION_DAYS},"filesRemoved":${removed}},"coverage":{"database":true,"encryptionKeys":true,"composeMetadata":true}}
EOF
chmod 600 "$result"
echo "[*] Backup complete: $archive" >&2
cat "$result"

102
scripts/production/restore-conductor.sh Executable file
View File

@ -0,0 +1,102 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
COMPOSE_FILE="${ROOT_DIR}/compose.production.yml"
ENV_FILE="${ROOT_DIR}/.env"
BACKUP_DIR="${ROOT_DIR}/backups"
BACKUP_FILE=""
SERVICE="conductor"
LIST_ONLY=false
FORCE=false
SKIP_SAFETY_BACKUP=false
usage() {
printf '%s\n' "Usage: $0 [--list] [--backup-file PATH] [--backup-dir PATH] [--compose-file PATH] [--env-file PATH] [--force]" >&2
}
while [[ $# -gt 0 ]]; do
case "$1" in
--list) LIST_ONLY=true; shift ;;
--backup-file) BACKUP_FILE="$2"; shift 2 ;;
--backup-dir) BACKUP_DIR="$2"; shift 2 ;;
--compose-file) COMPOSE_FILE="$2"; shift 2 ;;
--env-file) ENV_FILE="$2"; shift 2 ;;
--force) FORCE=true; shift ;;
--skip-safety-backup) SKIP_SAFETY_BACKUP=true; shift ;;
-h|--help) usage; exit 0 ;;
*) echo "Unknown option: $1" >&2; usage; exit 1 ;;
esac
done
mapfile -t available < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name 'conductor_*.tar.gz' -print 2>/dev/null | sort -r)
if $LIST_ONLY; then printf '%s\n' "${available[@]}"; exit 0; fi
if [[ -z "$BACKUP_FILE" ]]; then
[[ -t 0 ]] || { echo "--backup-file is required for non-interactive restore." >&2; exit 1; }
[[ ${#available[@]} -gt 0 ]] || { echo "No backups found in $BACKUP_DIR" >&2; exit 1; }
for i in "${!available[@]}"; do printf '%d) %s\n' "$((i + 1))" "${available[$i]}" >&2; done
read -r -p "Select backup number: " selection
if [[ ! "$selection" =~ ^[0-9]+$ ]] || (( selection < 1 || selection > ${#available[@]} )); then
echo "Invalid selection." >&2
exit 1
fi
BACKUP_FILE="${available[$((selection - 1))]}"
fi
BACKUP_FILE="$(realpath "$BACKUP_FILE")"
[[ -f "$BACKUP_FILE" ]] || { echo "Backup not found: $BACKUP_FILE" >&2; exit 1; }
if ! $FORCE; then
[[ -t 0 ]] || { echo "Restore is destructive; pass --force for non-interactive use." >&2; exit 1; }
read -r -p "Replace the live Conductor database and recovery keys? Type RESTORE: " confirmation
[[ "$confirmation" == "RESTORE" ]] || { echo "Restore cancelled." >&2; exit 1; }
fi
work_dir="$(mktemp -d)"
cleanup() { rm -rf -- "$work_dir"; }
trap cleanup EXIT
if tar -tzf "$BACKUP_FILE" | grep -Eq '(^/|(^|/)\.\.(/|$))'; then echo "Unsafe archive paths detected." >&2; exit 1; fi
tar -xzf "$BACKUP_FILE" -C "$work_dir"
[[ -f "$work_dir/manifest.json" && -f "$work_dir/SHA256SUMS" && -f "$work_dir/database/conductor.db" && -f "$work_dir/configuration/.env" ]] || { echo "Backup archive is incomplete." >&2; exit 1; }
(cd "$work_dir" && sha256sum -c SHA256SUMS >/dev/null)
grep -q '"format": "conductor-backup-v1"' "$work_dir/manifest.json" || { echo "Unsupported backup format." >&2; exit 1; }
grep -Eq '^CONDUCTOR_SECRET_KEY=.{32,}$' "$work_dir/configuration/.env" || { echo "Backup is missing its Conductor encryption key." >&2; exit 1; }
grep -Eq '^CONDUCTOR_SESSION_KEY=.{32,}$' "$work_dir/configuration/.env" || { echo "Backup is missing its Conductor session key." >&2; exit 1; }
if ! $SKIP_SAFETY_BACKUP && [[ -f "$ENV_FILE" && -f "$COMPOSE_FILE" ]]; then
echo "[*] Creating pre-restore safety backup..." >&2
"${ROOT_DIR}/scripts/production/backup-conductor.sh" --compose-file "$COMPOSE_FILE" --env-file "$ENV_FILE" --backup-dir "$BACKUP_DIR" >&2
fi
install -m 600 "$work_dir/configuration/.env" "$ENV_FILE"
[[ -f "$COMPOSE_FILE" ]] || install -m 600 "$work_dir/configuration/compose.production.yml" "$COMPOSE_FILE"
compose=(docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE")
container_id="$("${compose[@]}" ps -aq "$SERVICE")"
[[ -n "$container_id" ]] || { echo "Conductor container does not exist; run docker compose up once before restore." >&2; exit 1; }
image_ref="$(docker inspect -f '{{.Config.Image}}' "$container_id")"
echo "[*] Stopping Conductor and replacing database..." >&2
"${compose[@]}" stop "$SERVICE" >&2
docker cp "$work_dir/database/conductor.db" "${container_id}:/data/conductor.db.restore"
docker run --rm --user 0 --volumes-from "$container_id" --entrypoint node "$image_ref" -e '
const fs=require("fs");
for(const file of ["/data/conductor.db-wal","/data/conductor.db-shm"])if(fs.existsSync(file))fs.unlinkSync(file);
fs.renameSync("/data/conductor.db.restore","/data/conductor.db");
fs.chownSync("/data/conductor.db",10001,10001);
fs.chmodSync("/data/conductor.db",0o600);
' >&2
"${compose[@]}" up -d "$SERVICE" >&2
healthy=false
for _ in $(seq 1 30); do
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$("${compose[@]}" ps -q "$SERVICE")")"
if [[ "$status" == "healthy" ]]; then healthy=true; break; fi
sleep 1
done
$healthy || { echo "Conductor did not become healthy after restore." >&2; exit 1; }
result="${BACKUP_DIR}/restore-result.json"
cat > "$result" <<EOF
{"script":"restore-conductor","status":"success","timestamp":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","backupFile":"${BACKUP_FILE}","health":"healthy"}
EOF
chmod 600 "$result"
echo "[*] Restore complete and healthy." >&2
cat "$result"

32
scripts/release-image.sh Executable file
View File

@ -0,0 +1,32 @@
#!/usr/bin/env bash
set -euo pipefail
IMAGE="${IMAGE:-gitea.skeletonworks.online/vwiebe/conductor}"
VERSION="${VERSION:-}"
PUSH=false
PLATFORMS="${PLATFORMS:-linux/amd64,linux/arm64}"
while [[ $# -gt 0 ]]; do
case "$1" in
--image) IMAGE="$2"; shift 2 ;;
--version) VERSION="$2"; shift 2 ;;
--platforms) PLATFORMS="$2"; shift 2 ;;
--push) PUSH=true; shift ;;
-h|--help) echo "Usage: $0 --version vX.Y.Z [--image REGISTRY/OWNER/NAME] [--platforms LIST] [--push]"; exit 0 ;;
*) echo "Unknown option: $1" >&2; exit 1 ;;
esac
done
[[ "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([.-][A-Za-z0-9.-]+)?$ ]] || { echo "--version must be a vX.Y.Z release tag." >&2; exit 1; }
revision="$(git rev-parse HEAD)"
short_revision="$(git rev-parse --short=12 HEAD)"
created="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
output=(--load)
$PUSH && output=(--push)
if ! $PUSH && [[ "$PLATFORMS" == *,* ]]; then
echo "Local --load supports one platform; use --push for a multi-platform release." >&2
exit 1
fi
docker buildx build --platform "$PLATFORMS" -f Dockerfile.production \
--build-arg "VERSION=$VERSION" --build-arg "REVISION=$revision" --build-arg "CREATED=$created" \
-t "${IMAGE}:${VERSION}" -t "${IMAGE}:git-${short_revision}" "${output[@]}" .
printf '{"image":"%s","version":"%s","revision":"%s","platforms":"%s","pushed":%s}\n' "$IMAGE" "$VERSION" "$revision" "$PLATFORMS" "$PUSH"

39
scripts/test-production.sh Executable file
View File

@ -0,0 +1,39 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
IMAGE="${CONDUCTOR_TEST_IMAGE:-conductor:slice8-test}"
PROJECT="conductor-release-test-${RANDOM}"
PORT="${CONDUCTOR_TEST_PORT:-3202}"
work="$(mktemp -d)"
env_file="${work}/.env"
backup_dir="${work}/backups"
compose=(docker compose -p "$PROJECT" --env-file "$env_file" -f "$ROOT/compose.production.yml")
cleanup() { "${compose[@]}" down -v --remove-orphans >/dev/null 2>&1 || true; rm -rf -- "$work"; }
trap cleanup EXIT
if [[ "${1:-}" != "--skip-build" ]]; then docker build -f "$ROOT/Dockerfile.production" -t "$IMAGE" "$ROOT"; fi
if docker run --rm "$IMAGE" >/dev/null 2>&1; then echo "Production image started without required keys." >&2; exit 1; fi
cat > "$env_file" <<EOF
CONDUCTOR_SECRET_KEY=1111111111111111111111111111111111111111111111111111111111111111
CONDUCTOR_SESSION_KEY=22222222222222222222222222222222
CONDUCTOR_IMAGE=${IMAGE%:*}
CONDUCTOR_VERSION=${IMAGE##*:}
CONDUCTOR_BIND_ADDRESS=127.0.0.1
CONDUCTOR_PORT=${PORT}
EOF
"${compose[@]}" up -d --no-build
curl --fail --silent --show-error --retry 20 --retry-delay 1 --retry-all-errors "http://127.0.0.1:${PORT}/api/health" >/dev/null
curl --fail --silent --show-error "http://127.0.0.1:${PORT}/" | grep -qi '<!doctype html>'
curl --fail --silent --show-error "http://127.0.0.1:${PORT}/apps/test/details" | grep -qi '<!doctype html>'
api_status="$(curl --silent --output /dev/null --write-out '%{http_code}' "http://127.0.0.1:${PORT}/api/not-a-route")"
[[ "$api_status" == 404 ]] || { echo "Unknown API routes must return 404, not the SPA." >&2; exit 1; }
"${compose[@]}" exec -T conductor node -e 'const D=require("better-sqlite3");const d=new D("/data/conductor.db");d.prepare("INSERT INTO projects(name,description,project_json) VALUES(?,?,?)").run("release-restore-marker","test","{}");d.close()'
COMPOSE_PROJECT_NAME="$PROJECT" "$ROOT/scripts/production/backup-conductor.sh" --compose-file "$ROOT/compose.production.yml" --env-file "$env_file" --backup-dir "$backup_dir" --retention-days 1 >/dev/null
archive="$(find "$backup_dir" -name 'conductor_*.tar.gz' -type f | head -n1)"
"${compose[@]}" exec -T conductor node -e 'const D=require("better-sqlite3");const d=new D("/data/conductor.db");d.prepare("DELETE FROM projects WHERE name=?").run("release-restore-marker");d.close()'
COMPOSE_PROJECT_NAME="$PROJECT" "$ROOT/scripts/production/restore-conductor.sh" --compose-file "$ROOT/compose.production.yml" --env-file "$env_file" --backup-dir "$backup_dir" --backup-file "$archive" --skip-safety-backup --force >/dev/null
count="$("${compose[@]}" exec -T conductor node -e 'const D=require("better-sqlite3");const d=new D("/data/conductor.db",{readonly:true});process.stdout.write(String(d.prepare("SELECT count(*) FROM projects WHERE name=?").pluck().get("release-restore-marker")));d.close()')"
[[ "$count" == 1 ]] || { echo "Restore marker was not recovered." >&2; exit 1; }
docker inspect "$("${compose[@]}" ps -q conductor)" --format '{{.Config.User}} {{.HostConfig.ReadonlyRootfs}}' | grep -q '^conductor true$'
echo '{"productionValidation":"passed"}'

26
tests/e2e/release-workflows.spec.ts
View File

@ -10,14 +10,34 @@ test('deterministic launcher, dependent data, dashboard, and save/reload workflo
const document = structuredClone(demo);
document.project.name = name;
await page.goto('/');
const setup = page.getByRole('heading', { name: 'Set up Conductor' });
if (await setup.isVisible().catch(() => false)) {
await page.getByLabel('Username').fill('e2e-admin');
await page.getByLabel('Display name').fill('E2E Administrator');
await page.getByLabel('Password', { exact: true }).fill('e2e-test-password-1234');
await page.getByLabel('Confirm password').fill('e2e-test-password-1234');
await page.getByRole('button', { name: 'Create administrator' }).click();
} else if (await page.getByRole('heading', { name: 'Sign in to Conductor' }).isVisible().catch(() => false)) {
await page.getByLabel('Username').fill('e2e-admin');
await page.getByLabel('Password').fill('e2e-test-password-1234');
await page.getByRole('button', { name: 'Sign in', exact: true }).click();
}
await expect(page.getByText('E2E Administrator')).toBeVisible();
const cookies = await page.context().cookies();
const cookieHeader = cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ');
const csrf = cookies.find((cookie) => cookie.name === 'conductor_csrf')?.value;
expect(csrf).toBeTruthy();
const authHeaders = { Cookie: cookieHeader, 'X-CSRF-Token': csrf! };
const created = await request.post('http://127.0.0.1:4000/api/projects', {
headers: authHeaders,
data: { name, description: document.project.description, project_json: JSON.stringify(document) },
});
expect(created.status()).toBe(201);
const row = await created.json();
try {
await page.goto('/');
await page.getByRole('button', { name: 'Visual Editor', exact: true }).click();
await page.getByRole('button', { name: 'Load', exact: true }).click();
await page.getByRole('button', { name: new RegExp(name) }).click();
@ -49,7 +69,7 @@ test('deterministic launcher, dependent data, dashboard, and save/reload workflo
await page.getByRole('button', { name: 'Preview', exact: true }).click();
await expect(page.getByRole('button', { name: 'Launch workflow', exact: true })).toBeVisible();
} finally {
await request.delete(`http://127.0.0.1:4000/api/projects/${row.id}`);
await request.delete('http://127.0.0.1:4000/api/executions');
await request.delete(`http://127.0.0.1:4000/api/projects/${row.id}`, { headers: authHeaders });
await request.delete('http://127.0.0.1:4000/api/executions', { headers: authHeaders });
}
});