[ Aug 29 21:37:28 Enabled. ]
[ Aug 29 21:37:28 Rereading configuration. ]
[ Aug 29 21:37:31 Rereading configuration. ]
[ Aug 29 21:37:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_3883d706-5670-490a-b3cd-441531713e44/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ad8a31742adc45e925e63443a5b43c8e30604022\nCommit timestamp: 2026-07-28T19:41:59.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:37:38.999912597Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439}
{"msg":"dataset: \"oxp_3883d706-5670-490a-b3cd-441531713e44/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:37:39.00006667Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:37:39.000084241Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:37:39.00008995Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:37:39.014446711Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:37:39.332176127Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:36.956468261Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:32990"}
{"msg":"region c1c10ed0-0544-4c96-a07b-81f6b2732d0c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:36.986305143Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:36.990337288Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:36.994317673Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"uri":"/crucible/0/regions","method":"POST","req_id":"676da40f-f06b-4210-a645-3505563c839e","remote_addr":"[fd00:1122:3344:101::b]:32990","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":14710,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_3883d706-5670-490a-b3cd-441531713e44/crucible/regions/c1c10ed0-0544-4c96-a07b-81f6b2732d0c","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.00002371Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_3883d706-5670-490a-b3cd-441531713e44/crucible/regions/c1c10ed0-0544-4c96-a07b-81f6b2732d0c","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.000044381Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c1c10ed0-0544-4c96-a07b-81f6b2732d0c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c1c10ed0-0544-4c96-a07b-81f6b2732d0c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.061803833Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"region":"c1c10ed0-0544-4c96-a07b-81f6b2732d0c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c1c10ed0-0544-4c96-a07b-81f6b2732d0c\" \"--data\" \"/data/regions/c1c10ed0-0544-4c96-a07b-81f6b2732d0c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.063857951Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"region":"c1c10ed0-0544-4c96-a07b-81f6b2732d0c","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.179569898Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"region":"c1c10ed0-0544-4c96-a07b-81f6b2732d0c","component":"worker"}
{"msg":"region c1c10ed0-0544-4c96-a07b-81f6b2732d0c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.183471441Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.183629964Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c1c10ed0-0544-4c96-a07b-81f6b2732d0c","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.185883797Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c1c10ed0-0544-4c96-a07b-81f6b2732d0c","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.198258759Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.211437727Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c1c10ed0-0544-4c96-a07b-81f6b2732d0c","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.214111138Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c1c10ed0-0544-4c96-a07b-81f6b2732d0c","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.214805031Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.214866832Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.214910733Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.214946324Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.21847814Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.222846782Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:37.350774588Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"uri":"/crucible/0/regions","method":"POST","req_id":"52c5d34a-70ec-4b3b-82b2-c390c9a5d32e","remote_addr":"[fd00:1122:3344:101::b]:32990","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":83,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:38.182025424Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"uri":"/crucible/0/regions/c1c10ed0-0544-4c96-a07b-81f6b2732d0c","method":"GET","req_id":"cd48cd62-7284-4e99-ae17-0cf61580e0f9","remote_addr":"[fd00:1122:3344:101::b]:32990","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":80,"response_code":200}
{"msg":"path is \"/data/regions/c1c10ed0-0544-4c96-a07b-81f6b2732d0c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:38.185349687Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"datafile"}
{"msg":"dataset is oxp_3883d706-5670-490a-b3cd-441531713e44/crucible/regions/c1c10ed0-0544-4c96-a07b-81f6b2732d0c","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:38.194179463Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"datafile"}
{"msg":"region c1c10ed0-0544-4c96-a07b-81f6b2732d0c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:38.201285526Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:38.201399019Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:38.201409999Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"uri":"/crucible/0/regions/c1c10ed0-0544-4c96-a07b-81f6b2732d0c","method":"DELETE","req_id":"13589789-73bf-4477-b973-2b3cda959da9","remote_addr":"[fd00:1122:3344:101::b]:32990","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":16104,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-c1c10ed0-0544-4c96-a07b-81f6b2732d0c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:38.202364317Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:38.204598149Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"uri":"/crucible/0/regions/c1c10ed0-0544-4c96-a07b-81f6b2732d0c","method":"GET","req_id":"b10456d9-abe5-477a-ab9f-903b7c48be3a","remote_addr":"[fd00:1122:3344:101::b]:32990","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":57,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:38.20678072Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_3883d706-5670-490a-b3cd-441531713e44/crucible/regions/c1c10ed0-0544-4c96-a07b-81f6b2732d0c\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:38.215236299Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"region":"c1c10ed0-0544-4c96-a07b-81f6b2732d0c","component":"worker"}
{"msg":"region c1c10ed0-0544-4c96-a07b-81f6b2732d0c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:38.278068981Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:38.326286228Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"uri":"/crucible/0/regions/c1c10ed0-0544-4c96-a07b-81f6b2732d0c","method":"GET","req_id":"916830e8-da00-4345-a5eb-b72a6cf3ca66","remote_addr":"[fd00:1122:3344:101::b]:32990","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":163,"response_code":200}
{"msg":"region b7f9337b-ca83-4825-8f19-423590adbb21 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.295481419Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.295616702Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"uri":"/crucible/0/regions","method":"POST","req_id":"3d78ef29-a82e-4906-94a0-9ccff60f7c7f","remote_addr":"[fd00:1122:3344:101::b]:32990","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":182,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.295639212Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_3883d706-5670-490a-b3cd-441531713e44/crucible/regions/b7f9337b-ca83-4825-8f19-423590adbb21","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.301054894Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_3883d706-5670-490a-b3cd-441531713e44/crucible/regions/b7f9337b-ca83-4825-8f19-423590adbb21","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.301085555Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b7f9337b-ca83-4825-8f19-423590adbb21\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b7f9337b-ca83-4825-8f19-423590adbb21\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.355528939Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"region":"b7f9337b-ca83-4825-8f19-423590adbb21","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b7f9337b-ca83-4825-8f19-423590adbb21\" \"--data\" \"/data/regions/b7f9337b-ca83-4825-8f19-423590adbb21\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.355564669Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"region":"b7f9337b-ca83-4825-8f19-423590adbb21","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.434520794Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"region":"b7f9337b-ca83-4825-8f19-423590adbb21","component":"worker"}
{"msg":"region b7f9337b-ca83-4825-8f19-423590adbb21 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.434603786Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.43478978Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b7f9337b-ca83-4825-8f19-423590adbb21","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.436075714Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.43908497Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"uri":"/crucible/0/regions","method":"POST","req_id":"3686b225-4e07-4772-9a0f-bd647aad2070","remote_addr":"[fd00:1122:3344:101::b]:32990","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b7f9337b-ca83-4825-8f19-423590adbb21","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.43961853Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.452672086Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b7f9337b-ca83-4825-8f19-423590adbb21","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.45767484Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b7f9337b-ca83-4825-8f19-423590adbb21","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.458009536Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.458077338Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.458109048Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.458143219Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.461671525Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T21:38:40.466923704Z","hostname":"oxz_crucible_367c1be7-82c4-473c-b9f8-3d652c0042a0","pid":24439,"component":"worker"}