[ Oct 23 15:45:44 Enabled. ]
[ Oct 23 15:45:44 Rereading configuration. ]
[ Oct 23 15:45:46 Rereading configuration. ]
[ Oct 23 15:45:49 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: f85cf0dc3631e2f0c13384787efb0efc1b642162\nCommit timestamp: 2026-10-10T00:38:51.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:45:49.793577508Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428}
{"msg":"dataset: \"oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:45:49.793802966Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:45:49.793822236Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:45:49.793830887Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:45:49.812554691Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:45:49.876983715Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.661002885Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:59674"}
{"msg":"region 97fb94b9-45c9-44e7-9391-35274d213d96 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.691870559Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.699037623Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.702955292Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"uri":"/crucible/0/regions","method":"POST","req_id":"d812a3bf-aef2-4528-8c55-ec79624da919","remote_addr":"[fd00:1122:3344:101::c]:59674","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":20363,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible/regions/97fb94b9-45c9-44e7-9391-35274d213d96","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.704459926Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible/regions/97fb94b9-45c9-44e7-9391-35274d213d96","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.704470406Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"97fb94b9-45c9-44e7-9391-35274d213d96\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/97fb94b9-45c9-44e7-9391-35274d213d96\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.76663103Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"region":"97fb94b9-45c9-44e7-9391-35274d213d96","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"97fb94b9-45c9-44e7-9391-35274d213d96\" \"--data\" \"/data/regions/97fb94b9-45c9-44e7-9391-35274d213d96\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.766722563Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"region":"97fb94b9-45c9-44e7-9391-35274d213d96","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.901440279Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"region":"97fb94b9-45c9-44e7-9391-35274d213d96","component":"worker"}
{"msg":"region 97fb94b9-45c9-44e7-9391-35274d213d96 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.903703299Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.904163946Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-97fb94b9-45c9-44e7-9391-35274d213d96","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.90456614Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-97fb94b9-45c9-44e7-9391-35274d213d96","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.910841133Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.921309854Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-97fb94b9-45c9-44e7-9391-35274d213d96","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.923028155Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/97fb94b9-45c9-44e7-9391-35274d213d96","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.923846944Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.923907496Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.923945287Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.923984029Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.927057387Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:17.930085985Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:46:18.063105791Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"uri":"/crucible/0/regions","method":"POST","req_id":"ee391333-3c54-4687-96d0-794b831ad218","remote_addr":"[fd00:1122:3344:101::c]:59674","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":93,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.028332043Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:49168"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.028420826Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"uri":"/crucible/0/regions/97fb94b9-45c9-44e7-9391-35274d213d96","method":"GET","req_id":"d47eec20-82a7-4c3d-9a88-ffabcf9849e2","remote_addr":"[fd00:1122:3344:101::c]:49168","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":44,"response_code":200}
{"msg":"path is \"/data/regions/97fb94b9-45c9-44e7-9391-35274d213d96\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.034561553Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"datafile"}
{"msg":"dataset is oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible/regions/97fb94b9-45c9-44e7-9391-35274d213d96","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.040417901Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"datafile"}
{"msg":"snapshot is oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible/regions/97fb94b9-45c9-44e7-9391-35274d213d96@4673ecac-0ac4-491e-abcb-d3f2e51923e3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.045775031Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible/regions/97fb94b9-45c9-44e7-9391-35274d213d96\", \"4673ecac-0ac4-491e-abcb-d3f2e51923e3\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.045785411Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.056377967Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"uri":"/crucible/0/regions/97fb94b9-45c9-44e7-9391-35274d213d96/snapshots/4673ecac-0ac4-491e-abcb-d3f2e51923e3","method":"GET","req_id":"d586b87e-da49-46db-b7a4-3ce4064b6605","remote_addr":"[fd00:1122:3344:101::c]:49168","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":24791,"response_code":200}
{"msg":"path is \"/data/regions/97fb94b9-45c9-44e7-9391-35274d213d96\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.059339572Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"datafile"}
{"msg":"dataset is oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible/regions/97fb94b9-45c9-44e7-9391-35274d213d96","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.0651995Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"datafile"}
{"msg":"snapshot is oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible/regions/97fb94b9-45c9-44e7-9391-35274d213d96@4673ecac-0ac4-491e-abcb-d3f2e51923e3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.071460582Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible/regions/97fb94b9-45c9-44e7-9391-35274d213d96\", \"4673ecac-0ac4-491e-abcb-d3f2e51923e3\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.071474232Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 97fb94b9-45c9-44e7-9391-35274d213d96-4673ecac-0ac4-491e-abcb-d3f2e51923e3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.078059556Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"datafile"}
{"msg":"applying SMF actions for region 97fb94b9-45c9-44e7-9391-35274d213d96 running snapshot 4673ecac-0ac4-491e-abcb-d3f2e51923e3 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.078732799Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.07874616Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"uri":"/crucible/0/regions/97fb94b9-45c9-44e7-9391-35274d213d96/snapshots/4673ecac-0ac4-491e-abcb-d3f2e51923e3/run","method":"POST","req_id":"5fc06f03-d2f8-4bb4-bfe7-714a025581b5","remote_addr":"[fd00:1122:3344:101::c]:49168","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19426,"response_code":200}
{"msg":"creating missing snapshot instance snapshot-97fb94b9-45c9-44e7-9391-35274d213d96-4673ecac-0ac4-491e-abcb-d3f2e51923e3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.092063802Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-97fb94b9-45c9-44e7-9391-35274d213d96-4673ecac-0ac4-491e-abcb-d3f2e51923e3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.095247905Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-97fb94b9-45c9-44e7-9391-35274d213d96-4673ecac-0ac4-491e-abcb-d3f2e51923e3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.106265556Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/97fb94b9-45c9-44e7-9391-35274d213d96/.zfs/snapshot/4673ecac-0ac4-491e-abcb-d3f2e51923e3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.106588457Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.106631719Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.10667671Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.106713382Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.106761843Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.109235611Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.112949763Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"worker"}
{"msg":"region 97fb94b9-45c9-44e7-9391-35274d213d96 running snapshot 4673ecac-0ac4-491e-abcb-d3f2e51923e3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.112964313Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"datafile"}
{"msg":"path is \"/data/regions/97fb94b9-45c9-44e7-9391-35274d213d96\"","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.24345029Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"datafile"}
{"msg":"dataset is oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible/regions/97fb94b9-45c9-44e7-9391-35274d213d96","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.249483424Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"datafile"}
{"msg":"snapshot is oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible/regions/97fb94b9-45c9-44e7-9391-35274d213d96@4673ecac-0ac4-491e-abcb-d3f2e51923e3","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.254398848Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_47f9ec7d-7d34-49a4-bfaf-371cc3c96e4a/crucible/regions/97fb94b9-45c9-44e7-9391-35274d213d96\", \"4673ecac-0ac4-491e-abcb-d3f2e51923e3\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.254419188Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-10-23T15:57:29.258707671Z","hostname":"oxz_crucible_b46489f1-bb76-4375-97b7-087659d3a9f1","pid":24428,"uri":"/crucible/0/regions/97fb94b9-45c9-44e7-9391-35274d213d96/snapshots/4673ecac-0ac4-491e-abcb-d3f2e51923e3/run","method":"POST","req_id":"09d94dd1-08d2-4be8-a1ea-70b2f8295ea7","remote_addr":"[fd00:1122:3344:101::c]:49168","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":15309,"response_code":200}