[ May  8 03:00:05 Enabled. ]
[ May  8 03:00:05 Rereading configuration. ]
[ May  8 03:00:07 Rereading configuration. ]
[ May  8 03:00:09 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_67f5652e-ea35-441e-91ae-f730371a2544/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:09.845233242Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491}
{"msg":"dataset: \"oxp_67f5652e-ea35-441e-91ae-f730371a2544/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:09.84614854Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:09.84616193Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:09.846170061Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:09.862387714Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:00:09.930508663Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.146851607Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:42621"}
{"msg":"region 4c0dbf32-9fea-4fdc-81c6-fed475037c77 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.193935668Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.1953225Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.195354541Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions","method":"POST","req_id":"6d4aeff5-e166-40db-8db8-c4488fd946da","remote_addr":"[fd00:1122:3344:101::b]:42621","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":17286,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_67f5652e-ea35-441e-91ae-f730371a2544/crucible/regions/4c0dbf32-9fea-4fdc-81c6-fed475037c77","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.204134518Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_67f5652e-ea35-441e-91ae-f730371a2544/crucible/regions/4c0dbf32-9fea-4fdc-81c6-fed475037c77","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.204152038Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4c0dbf32-9fea-4fdc-81c6-fed475037c77\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4c0dbf32-9fea-4fdc-81c6-fed475037c77\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.24831983Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"region":"4c0dbf32-9fea-4fdc-81c6-fed475037c77","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4c0dbf32-9fea-4fdc-81c6-fed475037c77\" \"--data\" \"/data/regions/4c0dbf32-9fea-4fdc-81c6-fed475037c77\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.250725403Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"region":"4c0dbf32-9fea-4fdc-81c6-fed475037c77","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.383673682Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"region":"4c0dbf32-9fea-4fdc-81c6-fed475037c77","component":"worker"}
{"msg":"region 4c0dbf32-9fea-4fdc-81c6-fed475037c77 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.383718714Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.383791446Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4c0dbf32-9fea-4fdc-81c6-fed475037c77","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.384156427Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4c0dbf32-9fea-4fdc-81c6-fed475037c77","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.389119198Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.39940253Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4c0dbf32-9fea-4fdc-81c6-fed475037c77","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.40071085Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4c0dbf32-9fea-4fdc-81c6-fed475037c77","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.401405101Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.401471163Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.401527575Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.401568856Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.403754792Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.406217587Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.423838973Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions","method":"POST","req_id":"dea6133a-953d-43e6-9cf7-0279afefcce7","remote_addr":"[fd00:1122:3344:101::b]:42621","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.859968703Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions/4c0dbf32-9fea-4fdc-81c6-fed475037c77","method":"GET","req_id":"a8ca76f4-a8c8-4d03-b3cd-4b85d66a38bb","remote_addr":"[fd00:1122:3344:101::b]:42621","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":90,"response_code":200}
{"msg":"path is \"/data/regions/4c0dbf32-9fea-4fdc-81c6-fed475037c77\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.863475269Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"dataset is oxp_67f5652e-ea35-441e-91ae-f730371a2544/crucible/regions/4c0dbf32-9fea-4fdc-81c6-fed475037c77","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.870357458Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"region 4c0dbf32-9fea-4fdc-81c6-fed475037c77 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.877351551Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.877492525Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions/4c0dbf32-9fea-4fdc-81c6-fed475037c77","method":"DELETE","req_id":"fa89766d-5a0c-458c-9267-e72a29d242d6","remote_addr":"[fd00:1122:3344:101::b]:42621","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":14776,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.877533096Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-4c0dbf32-9fea-4fdc-81c6-fed475037c77 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.878589718Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.880851937Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.882090375Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions/4c0dbf32-9fea-4fdc-81c6-fed475037c77","method":"GET","req_id":"0719e230-6776-42ae-a4a8-3f46d5ba9852","remote_addr":"[fd00:1122:3344:101::b]:42621","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"deleting zfs dataset \"oxp_67f5652e-ea35-441e-91ae-f730371a2544/crucible/regions/4c0dbf32-9fea-4fdc-81c6-fed475037c77\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.889133198Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"region":"4c0dbf32-9fea-4fdc-81c6-fed475037c77","component":"worker"}
{"msg":"region 4c0dbf32-9fea-4fdc-81c6-fed475037c77 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.937165758Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:01:15.939452487Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions/4c0dbf32-9fea-4fdc-81c6-fed475037c77","method":"GET","req_id":"369deac0-9044-4e21-9fab-8922c24534a8","remote_addr":"[fd00:1122:3344:101::b]:42621","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":41,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.654327595Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:61847"}
{"msg":"region 09560d06-24d9-4a06-9a18-3c023d6d3d55 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.655215052Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.655234362Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.655240153Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions","method":"POST","req_id":"afb275bb-6dbc-4657-a2e3-bd0aad416442","remote_addr":"[fd00:1122:3344:101::b]:61847","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":832,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_67f5652e-ea35-441e-91ae-f730371a2544/crucible/regions/09560d06-24d9-4a06-9a18-3c023d6d3d55","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.66405503Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_67f5652e-ea35-441e-91ae-f730371a2544/crucible/regions/09560d06-24d9-4a06-9a18-3c023d6d3d55","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.664095372Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"09560d06-24d9-4a06-9a18-3c023d6d3d55\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/09560d06-24d9-4a06-9a18-3c023d6d3d55\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.700617381Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"region":"09560d06-24d9-4a06-9a18-3c023d6d3d55","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"09560d06-24d9-4a06-9a18-3c023d6d3d55\" \"--data\" \"/data/regions/09560d06-24d9-4a06-9a18-3c023d6d3d55\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.700659773Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"region":"09560d06-24d9-4a06-9a18-3c023d6d3d55","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.750691553Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"region":"09560d06-24d9-4a06-9a18-3c023d6d3d55","component":"worker"}
{"msg":"region 09560d06-24d9-4a06-9a18-3c023d6d3d55 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.750731504Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.750810766Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-09560d06-24d9-4a06-9a18-3c023d6d3d55","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.751619891Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-09560d06-24d9-4a06-9a18-3c023d6d3d55","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.754939612Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.769828964Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-09560d06-24d9-4a06-9a18-3c023d6d3d55","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.771195185Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/09560d06-24d9-4a06-9a18-3c023d6d3d55","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.771624269Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.77166838Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.771730822Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.771766453Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.77429956Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.776578059Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:15.976332228Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions","method":"POST","req_id":"fc56e5b1-aa19-490e-a535-a2aca60bcad8","remote_addr":"[fd00:1122:3344:101::b]:61847","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"region 2c4aea39-73a7-4656-bfd8-48b083cda377 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.367700579Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.367863864Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions","method":"POST","req_id":"492ed9e5-4d2d-4f69-89e3-8f1a74711265","remote_addr":"[fd00:1122:3344:101::b]:61847","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":208,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.367892325Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_67f5652e-ea35-441e-91ae-f730371a2544/crucible/regions/2c4aea39-73a7-4656-bfd8-48b083cda377","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.374769114Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_67f5652e-ea35-441e-91ae-f730371a2544/crucible/regions/2c4aea39-73a7-4656-bfd8-48b083cda377","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.374809215Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2c4aea39-73a7-4656-bfd8-48b083cda377\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2c4aea39-73a7-4656-bfd8-48b083cda377\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.416036758Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"region":"2c4aea39-73a7-4656-bfd8-48b083cda377","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2c4aea39-73a7-4656-bfd8-48b083cda377\" \"--data\" \"/data/regions/2c4aea39-73a7-4656-bfd8-48b083cda377\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.41612776Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"region":"2c4aea39-73a7-4656-bfd8-48b083cda377","component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.490291424Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:46791"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.491058557Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions/2c4aea39-73a7-4656-bfd8-48b083cda377","method":"GET","req_id":"550f1bd2-05d2-458e-8ad0-29a027fc8eb0","remote_addr":"[fd00:1122:3344:101::b]:46791","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":69,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.569438678Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions","method":"POST","req_id":"00fbfce1-b8dd-41b9-976e-cf5e1840e6e4","remote_addr":"[fd00:1122:3344:101::b]:61847","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":81,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.596509081Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"region":"2c4aea39-73a7-4656-bfd8-48b083cda377","component":"worker"}
{"msg":"region 2c4aea39-73a7-4656-bfd8-48b083cda377 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.596571162Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.596754098Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.599082519Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44086"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.59911419Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions/2c4aea39-73a7-4656-bfd8-48b083cda377","method":"GET","req_id":"8aded91f-845a-4f22-b8ff-915e4ce0cb3c","remote_addr":"[fd00:1122:3344:101::a]:44086","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":79,"response_code":200}
{"msg":"creating missing downstairs instance downstairs-2c4aea39-73a7-4656-bfd8-48b083cda377","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.609706441Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2c4aea39-73a7-4656-bfd8-48b083cda377","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.623462119Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.628967937Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2c4aea39-73a7-4656-bfd8-48b083cda377","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.631104432Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2c4aea39-73a7-4656-bfd8-48b083cda377","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.631544305Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.631617197Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.631679039Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.631735851Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.63663875Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:52559"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.636782954Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions/2c4aea39-73a7-4656-bfd8-48b083cda377","method":"GET","req_id":"78b5c744-4d02-464c-bc2e-e0817ed7946a","remote_addr":"[fd00:1122:3344:101::c]:52559","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":63,"response_code":200}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.641548009Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:22.646249862Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:23.093368175Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions","method":"POST","req_id":"5ef39fc9-0282-45a8-bf07-38804c7127f6","remote_addr":"[fd00:1122:3344:101::b]:61847","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":76,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.708290575Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions/2c4aea39-73a7-4656-bfd8-48b083cda377","method":"GET","req_id":"a8138812-5754-4fc0-8d32-18e7eba65f39","remote_addr":"[fd00:1122:3344:101::b]:61847","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":81,"response_code":200}
{"msg":"path is \"/data/regions/2c4aea39-73a7-4656-bfd8-48b083cda377\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.711613376Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"dataset is oxp_67f5652e-ea35-441e-91ae-f730371a2544/crucible/regions/2c4aea39-73a7-4656-bfd8-48b083cda377","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.721643911Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"region 2c4aea39-73a7-4656-bfd8-48b083cda377 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.729536611Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.729721746Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions/2c4aea39-73a7-4656-bfd8-48b083cda377","method":"DELETE","req_id":"161c4262-cfc8-4dea-b3b1-22e541740107","remote_addr":"[fd00:1122:3344:101::b]:61847","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":18154,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.729751887Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-2c4aea39-73a7-4656-bfd8-48b083cda377 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.731512831Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.733601844Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions/2c4aea39-73a7-4656-bfd8-48b083cda377","method":"GET","req_id":"74fc74ed-05f6-4662-b5d5-2fd6bccc8d5a","remote_addr":"[fd00:1122:3344:101::b]:61847","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.735228624Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_67f5652e-ea35-441e-91ae-f730371a2544/crucible/regions/2c4aea39-73a7-4656-bfd8-48b083cda377\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.743477244Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"region":"2c4aea39-73a7-4656-bfd8-48b083cda377","component":"worker"}
{"msg":"region 2c4aea39-73a7-4656-bfd8-48b083cda377 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.789330287Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-08T03:12:47.830074215Z","hostname":"oxz_crucible_27139c60-8738-41f4-b456-74affbf57aea","pid":25491,"uri":"/crucible/0/regions/2c4aea39-73a7-4656-bfd8-48b083cda377","method":"GET","req_id":"7c2d8315-4558-4c2a-bf61-3232da9db737","remote_addr":"[fd00:1122:3344:101::b]:61847","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":54,"response_code":200}