[ Jul 22 17:45:25 Enabled. ]
[ Jul 22 17:45:25 Rereading configuration. ]
[ Jul 22 17:45:27 Rereading configuration. ]
[ Jul 22 17:45:33 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 2bfe090eb5318ec8c467157018db2429d4df535b\nCommit timestamp: 2026-06-25T19:33:01.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:45:33.930327984Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692}
{"msg":"dataset: \"oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:45:33.930530929Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:45:33.930600321Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:45:33.930606671Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:45:33.944935428Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:45:34.070242283Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:12.959570717Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:58925"}
{"msg":"region a794f8d9-448b-4773-bc24-9c493e45fd21 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:12.990849057Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:12.995590878Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:12.998243646Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"uri":"/crucible/0/regions","method":"POST","req_id":"d75c9089-0fef-480e-9559-ffb2f2b23dff","remote_addr":"[fd00:1122:3344:101::b]:58925","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":16102,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.00544942Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.005475431Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a794f8d9-448b-4773-bc24-9c493e45fd21\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.096732035Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"region":"a794f8d9-448b-4773-bc24-9c493e45fd21","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a794f8d9-448b-4773-bc24-9c493e45fd21\" \"--data\" \"/data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.098765557Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"region":"a794f8d9-448b-4773-bc24-9c493e45fd21","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.259802456Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"region":"a794f8d9-448b-4773-bc24-9c493e45fd21","component":"worker"}
{"msg":"region a794f8d9-448b-4773-bc24-9c493e45fd21 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.263554222Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.263819819Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.266325653Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.273125377Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.284659872Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.288030708Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.288800438Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.291499317Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.291566729Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.29159292Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.300751104Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.305364272Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:46:13.36586235Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"uri":"/crucible/0/regions","method":"POST","req_id":"4b61ff08-eeed-4d83-ae92-a8d76335756b","remote_addr":"[fd00:1122:3344:101::b]:58925","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":81,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.022583042Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:52593"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.023170977Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"uri":"/crucible/0/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","method":"GET","req_id":"8fe5fd11-142b-4126-a6fa-d3ca2a3bbfec","remote_addr":"[fd00:1122:3344:101::b]:52593","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":54,"response_code":200}
{"msg":"path is \"/data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.026565094Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"dataset is oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.033963133Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"snapshot is oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/a794f8d9-448b-4773-bc24-9c493e45fd21@c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.040276264Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/a794f8d9-448b-4773-bc24-9c493e45fd21\", \"c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.040294735Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.053263857Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"uri":"/crucible/0/regions/a794f8d9-448b-4773-bc24-9c493e45fd21/snapshots/c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e","method":"GET","req_id":"3aebe003-93cf-4626-95c1-d5045a0f93e7","remote_addr":"[fd00:1122:3344:101::b]:52593","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":26716,"response_code":200}
{"msg":"path is \"/data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.062684417Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"dataset is oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.06867234Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"snapshot is oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/a794f8d9-448b-4773-bc24-9c493e45fd21@c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.0757073Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/a794f8d9-448b-4773-bc24-9c493e45fd21\", \"c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.075737931Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot a794f8d9-448b-4773-bc24-9c493e45fd21-c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.085272665Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"applying SMF actions for region a794f8d9-448b-4773-bc24-9c493e45fd21 running snapshot c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.088290562Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.088305212Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"uri":"/crucible/0/regions/a794f8d9-448b-4773-bc24-9c493e45fd21/snapshots/c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e/run","method":"POST","req_id":"88441676-3b84-473a-87ee-4175dab8b267","remote_addr":"[fd00:1122:3344:101::b]:52593","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":25669,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.103949492Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.104057395Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.104166038Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.104174638Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.10423218Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.104297321Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.104431955Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.104504667Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.104559458Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.104587699Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.112916802Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-a794f8d9-448b-4773-bc24-9c493e45fd21-c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.115777935Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-a794f8d9-448b-4773-bc24-9c493e45fd21-c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.118765222Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-a794f8d9-448b-4773-bc24-9c493e45fd21-c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.131700502Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21/.zfs/snapshot/c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.13201205Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.132072372Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.132111823Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.132150544Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.132203255Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.135692635Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.1402087Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"region a794f8d9-448b-4773-bc24-9c493e45fd21 running snapshot c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.140234361Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"path is \"/data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.346023955Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"dataset is oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.352969393Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"snapshot is oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/a794f8d9-448b-4773-bc24-9c493e45fd21@c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.360128296Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/a794f8d9-448b-4773-bc24-9c493e45fd21\", \"c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.360160827Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:24.366718385Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"uri":"/crucible/0/regions/a794f8d9-448b-4773-bc24-9c493e45fd21/snapshots/c0a0d6bb-ed64-421d-81a8-f5a00b3ec07e/run","method":"POST","req_id":"44afe196-e645-4aa7-acf4-37210ca24bfc","remote_addr":"[fd00:1122:3344:101::b]:52593","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":20749,"response_code":200}
{"msg":"region 7b57620c-82e9-4d69-9152-2dfeaa4cf817 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.527330119Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.527521724Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"uri":"/crucible/0/regions","method":"POST","req_id":"3beddd1a-b884-4cef-9a93-cedd66d6e80b","remote_addr":"[fd00:1122:3344:101::b]:52593","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":251,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.527546155Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/7b57620c-82e9-4d69-9152-2dfeaa4cf817","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.533572319Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/7b57620c-82e9-4d69-9152-2dfeaa4cf817","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.533602619Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"7b57620c-82e9-4d69-9152-2dfeaa4cf817\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7b57620c-82e9-4d69-9152-2dfeaa4cf817\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.588381371Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"region":"7b57620c-82e9-4d69-9152-2dfeaa4cf817","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7b57620c-82e9-4d69-9152-2dfeaa4cf817\" \"--data\" \"/data/regions/7b57620c-82e9-4d69-9152-2dfeaa4cf817\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.588421322Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"region":"7b57620c-82e9-4d69-9152-2dfeaa4cf817","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.663738689Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"region":"7b57620c-82e9-4d69-9152-2dfeaa4cf817","component":"worker"}
{"msg":"region 7b57620c-82e9-4d69-9152-2dfeaa4cf817 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.66377368Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.663890593Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-7b57620c-82e9-4d69-9152-2dfeaa4cf817","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.664277813Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7b57620c-82e9-4d69-9152-2dfeaa4cf817","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.676882376Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.686225285Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7b57620c-82e9-4d69-9152-2dfeaa4cf817","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.689736594Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7b57620c-82e9-4d69-9152-2dfeaa4cf817","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.68995815Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.689994911Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.690038442Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.690074283Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.693554042Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.698491429Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.698594471Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.698685603Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.698696134Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.698739405Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.698792786Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.703004764Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.703055265Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.703101296Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.703145168Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.707742565Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.709540371Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:29.820995353Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"uri":"/crucible/0/regions","method":"POST","req_id":"96f5369f-ca8d-4762-a769-d1a42058306a","remote_addr":"[fd00:1122:3344:101::b]:52593","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.883608458Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"uri":"/crucible/0/regions/7b57620c-82e9-4d69-9152-2dfeaa4cf817","method":"GET","req_id":"13c5a1c3-cf91-4688-9399-2ee9ee380054","remote_addr":"[fd00:1122:3344:101::b]:52593","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":69,"response_code":200}
{"msg":"path is \"/data/regions/7b57620c-82e9-4d69-9152-2dfeaa4cf817\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.886278696Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"dataset is oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/7b57620c-82e9-4d69-9152-2dfeaa4cf817","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.898192201Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"region 7b57620c-82e9-4d69-9152-2dfeaa4cf817 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.905203501Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.905398716Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.908223408Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"uri":"/crucible/0/regions/7b57620c-82e9-4d69-9152-2dfeaa4cf817","method":"DELETE","req_id":"5ea6b73f-4967-4314-beed-4cfc406a3179","remote_addr":"[fd00:1122:3344:101::b]:52593","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21974,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-7b57620c-82e9-4d69-9152-2dfeaa4cf817 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.908918796Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.911866361Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"uri":"/crucible/0/regions/7b57620c-82e9-4d69-9152-2dfeaa4cf817","method":"GET","req_id":"3844f9db-70ab-432c-ad16-ab5e0c5a53be","remote_addr":"[fd00:1122:3344:101::b]:52593","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":29,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.913666797Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.913740359Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.913815731Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.913829211Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.913860152Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.913902713Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a794f8d9-448b-4773-bc24-9c493e45fd21","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.914211741Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.914275003Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.914318504Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.914368285Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.918544202Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.920112102Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d15d1651-8f0a-4765-8957-6a6d16d4c890/crucible/regions/7b57620c-82e9-4d69-9152-2dfeaa4cf817\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.927943733Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"region":"7b57620c-82e9-4d69-9152-2dfeaa4cf817","component":"worker"}
{"msg":"region 7b57620c-82e9-4d69-9152-2dfeaa4cf817 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:55.980280641Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T17:57:56.000960481Z","hostname":"oxz_crucible_c7ac0ae7-4030-4b28-a1da-10cf518e3f4e","pid":28692,"uri":"/crucible/0/regions/7b57620c-82e9-4d69-9152-2dfeaa4cf817","method":"GET","req_id":"87355caa-a0d3-4353-b49d-bbf49556c1eb","remote_addr":"[fd00:1122:3344:101::b]:52593","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":61,"response_code":200}