200 lines
19 KiB
Python
200 lines
19 KiB
Python
"""End-to-end upgrade acceptance in disposable Ubuntu with its own Docker daemon.
|
|
No customer host, Docker socket or data directory is mounted. Host Node is absent.
|
|
Build conductor:slice13-legacy from the archived 0.4.0 kit as described in TESTING.md.
|
|
"""
|
|
import pathlib,subprocess,secrets,json,time,os,hashlib,sys
|
|
ROOT=pathlib.Path(__file__).resolve().parent.parent
|
|
ARTIFACTS=ROOT/'test-results/slice13';ARTIFACTS.mkdir(parents=True,exist_ok=True)
|
|
name='conductor-slice13-'+secrets.token_hex(5)
|
|
fqdn='conductor.customer.test';stack='/opt/skeletonworks/conductor/'+fqdn
|
|
inner='conductor-conductor_customer_test';compose=stack+'/docker-compose.yml'
|
|
kit='/opt/conductor-test/conductor-upgrade';update=kit+'/update-conductor.sh'
|
|
target='gitea.skeletonworks.online/vwiebe/conductor:v1.1.0'
|
|
def command(args,input=None,check=True):
|
|
r=subprocess.run(args,input=input,text=True,capture_output=True)
|
|
if check and r.returncode:raise RuntimeError('Command failed: '+str(args[:6])+'\n'+r.stderr[-5000:])
|
|
return r
|
|
def host(args,input=None,check=True):return command(['docker','exec','-i',name]+args,input,check)
|
|
def write_files(files):
|
|
host(['python3','-c','import sys,json,pathlib\nfor name,content in json.load(sys.stdin).items():\n p=pathlib.Path(name);p.parent.mkdir(parents=True,exist_ok=True);p.write_text(content)\n'],json.dumps(files))
|
|
def milestone(text):print(text,flush=True)
|
|
def invoke(args=[],check=True,prefix=[]):
|
|
r=host(prefix+['bash',update,'--fqdn',fqdn]+args,check=check)
|
|
if check:return json.loads(r.stdout)
|
|
assert r.returncode!=0,'Expected update refusal'
|
|
return r
|
|
def metadata():return json.loads(host(['docker','inspect',inner]).stdout)[0]
|
|
def files():
|
|
return json.loads(host(['python3','-c','import pathlib,hashlib,json\np=pathlib.Path("'+stack+'")\nprint(json.dumps({str(f.relative_to(p)):hashlib.sha256(f.read_bytes()).hexdigest() for f in p.iterdir() if f.is_file() and f.name!=".maintenance.lock"}))']).stdout)
|
|
def check_healthy():
|
|
for _ in range(60):
|
|
r=host(['docker','exec',inner,'node','-e','fetch("http://127.0.0.1:8080/api/health").then(r=>r.json()).then(x=>console.log(JSON.stringify(x)))'],check=False)
|
|
if r.returncode==0:return json.loads(r.stdout)
|
|
time.sleep(.5)
|
|
raise RuntimeError('Fixture unhealthy.')
|
|
def browser(phase):
|
|
r=command(['node',str(ROOT/'scripts/test-update-conductor.mjs'),name,port,phase],check=False)
|
|
(ARTIFACTS/(phase+'-browser.log')).write_text(r.stdout+'\n'+r.stderr)
|
|
if r.returncode:raise RuntimeError(r.stderr[-6000:])
|
|
milestone(r.stdout.strip())
|
|
def node(code):return host(['docker','exec',inner,'node','-e',code]).stdout
|
|
def copy_scripts():
|
|
for script in ['backup-conductor.sh','restore-conductor.sh']:
|
|
host(['cp',kit+'/'+script,'/opt/skeletonworks/scripts/'+script])
|
|
host(['chmod','755','/opt/skeletonworks/scripts/'+script])
|
|
try:
|
|
command(['docker','run','-d','--privileged','--cgroupns=private','--name',name,'--publish','127.0.0.1::443','conductor:slice12-test-host'])
|
|
for _ in range(80):
|
|
if host(['docker','info'],check=False).returncode==0:break
|
|
time.sleep(.25)
|
|
else:raise RuntimeError('Nested Docker did not start.')
|
|
assert host(['bash','-c','command -v node'],check=False).returncode!=0
|
|
host(['mkdir','-p','/opt/conductor-test','/opt/skeletonworks/scripts',stack])
|
|
command(['docker','cp',str(ROOT/'release/conductor-upgrade-1.1.0.tar.gz'),name+':/opt/conductor-test/kit.tar.gz'])
|
|
host(['tar','-xzf','/opt/conductor-test/kit.tar.gz','-C','/opt/conductor-test'])
|
|
copy_scripts()
|
|
# Transfer already built public/legacy images through a pipe into the isolated engine.
|
|
producer=subprocess.Popen(['docker','image','save',target,'conductor:slice13-legacy','gitea.skeletonworks.online/vwiebe/conductor:v1.0.0'],stdout=subprocess.PIPE)
|
|
consumer=subprocess.run(['docker','exec','-i',name,'docker','load'],stdin=producer.stdout,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
|
|
producer.stdout.close();assert producer.wait()==0 and consumer.returncode==0
|
|
host(['docker','network','create','skeletonworks_net'])
|
|
env='NODE_ENV=production\nPORT=8080\nCONDUCTOR_DATA_DIR=/data\nCONDUCTOR_SECRET_KEY='+('31'*32)+'\nCONDUCTOR_SESSION_KEY='+('42'*32)+'\n'
|
|
initial={'name':'conductor-customer-test','services':{'conductor':{'image':'conductor:slice13-legacy','container_name':inner,'restart':'unless-stopped','read_only':True,'tmpfs':['/tmp:rw,noexec,nosuid,size=64m'],'env_file':['.env'],'environment':{'SOCIAL_API_ORIGIN':'http://nodered-fixture:1880/social-scheduler','SOCIAL_INTERNAL_TOKEN':'slice13-internal-private-fixture-token','SOCIAL_WORKSPACE_ID':'customer'},'volumes':['conductor_data:/data'],'networks':['skeletonworks_net']}},'volumes':{'conductor_data':{'name':inner+'-data'}},'networks':{'skeletonworks_net':{'external':True}}}
|
|
result={'script':'setup-conductor','status':'success','container':inner,'url':'https://'+fqdn,'domain':fqdn,'files':{'stack':stack,'compose':compose,'env':stack+'/.env','secrets':stack+'/.secrets','result':stack+'/setup-result.json'}}
|
|
fixture=r'''
|
|
const http=require('http');
|
|
const snapshot={accounts:[{network:'bluesky',id:'fixture-account',handle:'customer.bsky.social',displayName:'Customer Bluesky',did:'did:plc:fixture',status:'connected',revision:7}],posts:[{id:'fixture-post',network:'bluesky',accountId:'fixture-account',accountHandle:'customer.bsky.social',text:'Scheduled before the Conductor upgrade.',media:null,mode:'scheduled',localDateTime:'2030-12-01T12:00',timeZone:'UTC',disambiguation:'reject',scheduledAt:'2030-12-01T12:00:00Z',status:'queued',revision:4,message:'',canEdit:true,canCancel:true,canRetry:false,publishedUrl:null,uri:null,cid:null,attempts:[]}],publishingEnabled:true,pendingCount:1,attentionCount:0};
|
|
let writes=0;
|
|
http.createServer((req,res)=>{
|
|
if(req.url==='/fixture-status'){res.end(JSON.stringify({writes,snapshot}));return;}
|
|
if(req.method!=='GET'){writes++;res.writeHead(405);res.end();return;}
|
|
if(req.headers.authorization!=='Bearer slice13-internal-private-fixture-token'||req.headers['x-social-workspace']!=='customer'){res.writeHead(401);res.end('{}');return;}
|
|
const value=req.url==='/social-scheduler/session'?{displayName:'Customer Administrator',role:req.headers['x-conductor-role'],workspaces:[{id:'customer',name:'Customer workspace'}],preview:false,publishingEnabled:true}:snapshot;
|
|
res.writeHead(200,{'content-type':'application/json'});res.end(JSON.stringify(value));
|
|
}).listen(1880,'0.0.0.0');
|
|
'''
|
|
caddy='{\n local_certs\n auto_https disable_redirects\n}\n'+fqdn+' {\n reverse_proxy '+inner+':8080\n}\nunrelated.customer.test {\n respond "Neighbor preserved"\n}\n'
|
|
caddycompose={'services':{'caddy':{'image':'caddy:2.10.2','ports':['443:443'],'volumes':['./Caddyfile:/etc/caddy/Caddyfile','caddy_data:/data','caddy_config:/config'],'networks':['skeletonworks_net']}},'volumes':{'caddy_data':{},'caddy_config':{}},'networks':{'skeletonworks_net':{'external':True}}}
|
|
write_files({compose:json.dumps(initial),stack+'/.env':env,stack+'/.secrets':'CONDUCTOR_SECRET_KEY='+('31'*32)+'\nCONDUCTOR_SESSION_KEY='+('42'*32)+'\n',stack+'/setup-result.json':json.dumps(result),'/opt/conductor-test/api.cjs':fixture,'/opt/skeletonworks/caddy/Caddyfile':caddy,'/opt/skeletonworks/caddy/docker-compose.yml':json.dumps(caddycompose)})
|
|
host(['chmod','700',stack]);host(['chmod','600',compose,stack+'/.env',stack+'/.secrets',stack+'/setup-result.json'])
|
|
host(['chown','-R','admin:admin',stack])
|
|
host(['docker','run','-d','--name','nodered-fixture','--network','skeletonworks_net','--entrypoint','node','--mount','type=bind,src=/opt/conductor-test/api.cjs,dst=/fixture.cjs,readonly',target,'/fixture.cjs'])
|
|
host(['docker','compose','-f',compose,'up','-d'])
|
|
host(['docker','compose','-f','/opt/skeletonworks/caddy/docker-compose.yml','up','-d'])
|
|
check_healthy()
|
|
node('const d=require("./dist/db/database").default;d.prepare("INSERT INTO included_apps(id,manifest_json) VALUES(?,?)").run("social-scheduler",JSON.stringify({schemaVersion:1,id:"social-scheduler",name:"Social Scheduler",description:"Schedule posts.",version:"0.4.0",launchPath:"/social-scheduler",configurationPath:"/social-scheduler?view=accounts",audience:"all-users"}));d.close();')
|
|
port=json.loads(command(['docker','inspect',name]).stdout)[0]['NetworkSettings']['Ports']['443/tcp'][0]['HostPort']
|
|
browser('seed')
|
|
node('const d=require("./dist/db/database").default,e=require("./dist/lib/secretCrypto").encryptSecret({token:"customer-encrypted-secret"});d.prepare("INSERT INTO secrets(id,name,authentication_type,encrypted_value,iv,auth_tag) VALUES(?,?,?,?,?,?)").run("preserved-secret","Saved customer API","bearer",e.encryptedValue,e.iv,e.authTag);d.close();')
|
|
original_files=files();old_image=metadata()['Image'];old_volume=[x['Name'] for x in metadata()['Mounts'] if x['Destination']=='/data'][0]
|
|
backend=json.loads(host(['docker','inspect','nodered-fixture']).stdout)[0]
|
|
dry=invoke(['--dry-run']);assert dry['legacyAppsToMigrate']==['social-scheduler']
|
|
assert files()==original_files and metadata()['Image']==old_image
|
|
nonroot=host(['runuser','-u','admin','--','bash',update,'--fqdn',fqdn,'--dry-run'],check=False)
|
|
assert nonroot.returncode!=0 and 'sudo' in nonroot.stderr
|
|
# Corrupt migration assets must be rejected before service/data mutation.
|
|
asset=kit+'/conductor-app-packages/social-scheduler/app.css'
|
|
host(['cp',asset,asset+'.original']);host(['bash','-c','echo corrupted >> "$1"','fixture',asset])
|
|
assert 'checksum' in invoke(['--dry-run'],False).stderr
|
|
host(['mv',asset+'.original',asset])
|
|
assert files()==original_files and metadata()['Image']==old_image
|
|
host(['docker','tag','conductor:slice13-legacy','conductor-incompatible:v1.1.0'])
|
|
invoke(['--image','conductor-incompatible:v1.1.0'],False)
|
|
assert files()==original_files and metadata()['Image']==old_image
|
|
milestone('PASS: protected read-only dry-run; incompatible image/corrupt package refusal preserves live service.')
|
|
# A test-only Docker shim kills the updater before candidate creation, after its verified snapshot.
|
|
shim=r'''#!/usr/bin/python3
|
|
import os,sys,signal,pathlib
|
|
args=sys.argv[1:];flag=pathlib.Path('/opt/conductor-test/interrupt')
|
|
if flag.exists() and args[:2]==['run','-d'] and '--name' in args and args[args.index('--name')+1].startswith('conductor-upgrade-'):
|
|
flag.unlink();os.kill(os.getppid(),signal.SIGKILL);sys.exit(70)
|
|
os.execv('/usr/local/bin/docker',['docker']+args)
|
|
'''
|
|
write_files({'/opt/conductor-test/bin/docker':shim,'/opt/conductor-test/interrupt':'yes'})
|
|
host(['chmod','755','/opt/conductor-test/bin/docker'])
|
|
interrupted=invoke(['--image',target],False,['env','PATH=/opt/conductor-test/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'])
|
|
pending=json.loads(host(['cat',stack+'/.upgrade-state.json']).stdout)
|
|
assert pending['phase']=='backed-up',interrupted.stderr
|
|
assert invoke(['--dry-run'])['status']=='recovery-required'
|
|
r=invoke(['--image',target]);assert r['status']=='recovered'
|
|
assert metadata()['Image']==old_image
|
|
for file,value in original_files.items():assert files()[file]==value
|
|
milestone('PASS: actual SIGKILL interruption recovers the untouched original image, configuration and data.')
|
|
# A target with valid capability metadata but a broken server proves health-failure recovery.
|
|
write_files({'/opt/conductor-test/failure/Dockerfile':'FROM '+target+'\nCMD ["node","-e","process.exit(42)"]\n'})
|
|
host(['docker','build','-t','conductor-failure:v1.1.0','/opt/conductor-test/failure'])
|
|
failed=invoke(['--image','conductor-failure:v1.1.0'],False)
|
|
assert 'previous Conductor service is running again' in failed.stderr,failed.stderr
|
|
assert metadata()['Image']==old_image
|
|
for file,value in original_files.items():assert files()[file]==value
|
|
milestone('PASS: failed candidate health automatically restores the previous working service.')
|
|
upgraded=invoke(['--image',target]);assert upgraded['status']=='upgraded' and upgraded['migratedApps']==['social-scheduler']
|
|
success_backup=upgraded['backup']
|
|
assert metadata()['Image']!=old_image
|
|
assert files()['.env']==original_files['.env'] and files()['.secrets']==original_files['.secrets']
|
|
apps=json.loads(node('const a=require("./dist/db/installedApps");console.log(JSON.stringify(a.allInstalled().map(x=>({id:x.manifest.application.id,digest:x.digest}))));require("./dist/db/database").default.close();'))
|
|
assert len(apps)==1 and apps[0]['id']=='social-scheduler'
|
|
browser('upgraded')
|
|
before=files();assert invoke(['--image',target])['status']=='already-current';assert files()==before
|
|
# Generic independent app installed alongside Social must survive another image identity.
|
|
command(['docker','cp',str(ROOT/'examples/installed-app'),name+':/opt/conductor-test/example-package'])
|
|
host(['docker','exec',inner,'mkdir','-p','/tmp/example-package'])
|
|
host(['bash','-c','tar -C /opt/conductor-test/example-package -cf - . | docker exec -i "$1" tar -xf - -C /tmp/example-package','fixture',inner])
|
|
host(['docker','exec','-i',inner,'node','dist/scripts/installedApps.js','install','/tmp/example-package'],'')
|
|
write_files({'/opt/conductor-test/compatible/Dockerfile':'FROM '+target+'\nLABEL works.skeleton.upgrade-fixture="second-compatible-image"\n'})
|
|
host(['docker','build','-t','conductor-compatible:v1.1.0','/opt/conductor-test/compatible'])
|
|
assert invoke(['--image','conductor-compatible:v1.1.0'])['migratedApps']==[]
|
|
assert len(json.loads(node('console.log(JSON.stringify(require("./dist/db/installedApps").allInstalled().map(x=>x.manifest.application.id)));require("./dist/db/database").default.close();')))==2
|
|
milestone('PASS: legacy migration and repeat preserve passwords/keys, project/publication, API connection, queue view and a second independent app.')
|
|
# Matched wrappers still work with the shared lock, including restore's nested safety backup.
|
|
backup=json.loads(host(['runuser','-u','admin','--','/opt/skeletonworks/scripts/backup-conductor.sh','--fqdn',fqdn]).stdout)
|
|
restored=host(['bash','/opt/skeletonworks/scripts/restore-conductor.sh','--fqdn',fqdn,'--backup-file',backup['backup']['file'],'--force'])
|
|
check_healthy()
|
|
milestone('PASS: routine backup and restore, including its safety backup, share the upgrade maintenance lock.')
|
|
# Damaged upgrade snapshots must not stop the live service.
|
|
host(['cp',success_backup+'/data.tar',success_backup+'/data.original'])
|
|
host(['bash','-c','echo corrupt >> "$1"','fixture',success_backup+'/data.tar'])
|
|
current_image=metadata()['Image']
|
|
assert 'checksum' in invoke(['--rollback',success_backup,'--dry-run'],False).stderr
|
|
assert metadata()['Image']==current_image and metadata()['State']['Running']
|
|
host(['mv',success_backup+'/data.original',success_backup+'/data.tar'])
|
|
# Remove the unused legacy container image/volume to prove recovery uses the saved archive.
|
|
host(['docker','image','rm','-f','conductor:slice13-legacy','conductor-incompatible:v1.1.0'])
|
|
host(['docker','volume','rm',old_volume])
|
|
host(['docker','compose','-f',compose,'stop','conductor'])
|
|
rollback=invoke(['--rollback',success_backup,'--accept-data-rewind'])
|
|
assert rollback['status']=='restored' and metadata()['Image']==old_image
|
|
assert invoke(['--rollback',success_backup,'--accept-data-rewind'])['status']=='already-restored'
|
|
browser('legacy')
|
|
after_backend=json.loads(host(['docker','inspect','nodered-fixture']).stdout)[0]
|
|
assert after_backend['Id']==backend['Id'] and after_backend['State']['StartedAt']==backend['State']['StartedAt']
|
|
observed=json.loads(host(['docker','exec','nodered-fixture','node','-e','fetch("http://127.0.0.1:1880/fixture-status").then(r=>r.json()).then(x=>console.log(JSON.stringify(x)))']).stdout)
|
|
assert observed['writes']==0 and observed['snapshot']['posts'][0]['revision']==4
|
|
assert host(['cat','/opt/skeletonworks/caddy/Caddyfile']).stdout==caddy
|
|
assert host(['bash','-c','command -v node'],check=False).returncode!=0
|
|
# A separate standard 1.0.0 installation must upgrade without falsely adding Social Scheduler.
|
|
base_fqdn='conductor.standalone.test';base_stack='/opt/skeletonworks/conductor/'+base_fqdn
|
|
base_compose=base_stack+'/docker-compose.yml';base_inner='conductor-standalone'
|
|
base_config=json.loads(json.dumps(initial))
|
|
base_config['name']='conductor-standalone'
|
|
base_service=base_config['services']['conductor']
|
|
base_service['image']='gitea.skeletonworks.online/vwiebe/conductor:v1.0.0'
|
|
base_service['container_name']=base_inner;base_service['environment']={}
|
|
base_config['volumes']['conductor_data']['name']='conductor-standalone-data'
|
|
base_record={'script':'setup-conductor','status':'success','container':base_inner,'url':'https://'+base_fqdn,'domain':base_fqdn,'files':{'stack':base_stack,'compose':base_compose,'env':base_stack+'/.env','secrets':base_stack+'/.secrets','result':base_stack+'/setup-result.json'}}
|
|
write_files({base_compose:json.dumps(base_config),base_stack+'/.env':env,base_stack+'/.secrets':env,base_stack+'/setup-result.json':json.dumps(base_record)})
|
|
host(['docker','compose','-f',base_compose,'up','-d'])
|
|
for _ in range(50):
|
|
ready=host(['docker','exec',base_inner,'node','-e','fetch("http://127.0.0.1:8080/api/health").then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))'],check=False)
|
|
if ready.returncode==0:break
|
|
time.sleep(.5)
|
|
standalone=json.loads(host(['bash',update,'--fqdn',base_fqdn,'--image',target]).stdout)
|
|
assert standalone['status']=='upgraded' and standalone['migratedApps']==[]
|
|
assert json.loads(host(['docker','exec',base_inner,'node','dist/scripts/installedApps.js','list']).stdout)==[]
|
|
assert json.loads(host(['bash',update,'--fqdn',base_fqdn,'--image',target]).stdout)['status']=='already-current'
|
|
milestone('PASS: published standalone v1.0.0 upgrades to v1.1.0 and remains empty of uninstalled apps.')
|
|
(ARTIFACTS/'result.json').write_text(json.dumps({'passed':True,'environment':'Isolated Ubuntu 24.04 Docker daemon; no host Node.js','legacySource':'Original Social Scheduler 0.4.0 release kit, rebuilt without source changes','checks':['read-only protected dry-run','source/target compatibility failure','corrupt app refusal','actual SIGKILL and recovery','candidate health failure recovery','legacy frontend migration','preserved database records/encrypted secrets','same account/password','project/publication preserved','authenticated Social Scheduler connection and queued post display','title/favicon','repeat no-op','independent second app survives core replacement','routine backup/restore locking','damaged rollback archive refusal','saved image/data restore after original image/volume removal','rollback repeat no-op','backend container and state untouched','Caddy unchanged','published standalone v1.0.0 upgrade without seeded apps'],'liveVmChanged':False,'socialBackend':'Read-only API fixture, no real social connection/post'},indent=2)+'\n')
|
|
milestone('PASS: SLICE13 acceptance complete, including archived-image/data recovery and untouched backend/Caddy.')
|
|
finally:
|
|
command(['docker','rm','-f','-v',name],check=False)
|