[ Aug 27 01:40:14 Enabled. ]
[ Aug 27 01:40:14 Rereading configuration. ]
[ Aug 27 01:40:21 Rereading configuration. ]
[ Aug 27 01:40:29 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ea971bec-da00-45a9-a92b-a92a4c74a66f/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ad8a31742adc45e925e63443a5b43c8e30604022\nCommit timestamp: 2026-07-28T19:41:59.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:40:29.636058118Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284}
{"msg":"dataset: \"oxp_ea971bec-da00-45a9-a92b-a92a4c74a66f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:40:29.63624924Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:40:29.63626262Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:40:29.63627013Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:40:29.652721061Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:40:30.012127986Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.704165419Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:50740"}
{"msg":"region 838d7089-75d9-4586-8108-06c0a5e6ece8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.737705987Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.741948373Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.746198059Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"uri":"/crucible/0/regions","method":"POST","req_id":"da2b4018-6614-49c5-a26a-1966036325bd","remote_addr":"[fd00:1122:3344:101::c]:50740","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":16652,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_ea971bec-da00-45a9-a92b-a92a4c74a66f/crucible/regions/838d7089-75d9-4586-8108-06c0a5e6ece8","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.750530357Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_ea971bec-da00-45a9-a92b-a92a4c74a66f/crucible/regions/838d7089-75d9-4586-8108-06c0a5e6ece8","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.750545907Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"838d7089-75d9-4586-8108-06c0a5e6ece8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/838d7089-75d9-4586-8108-06c0a5e6ece8\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.801235732Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"region":"838d7089-75d9-4586-8108-06c0a5e6ece8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"838d7089-75d9-4586-8108-06c0a5e6ece8\" \"--data\" \"/data/regions/838d7089-75d9-4586-8108-06c0a5e6ece8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.803528612Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"region":"838d7089-75d9-4586-8108-06c0a5e6ece8","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.972171589Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"region":"838d7089-75d9-4586-8108-06c0a5e6ece8","component":"worker"}
{"msg":"region 838d7089-75d9-4586-8108-06c0a5e6ece8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.976109953Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.976242654Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-838d7089-75d9-4586-8108-06c0a5e6ece8","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.978426863Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-838d7089-75d9-4586-8108-06c0a5e6ece8","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:16.990249954Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.003369147Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-838d7089-75d9-4586-8108-06c0a5e6ece8","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.006178031Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/838d7089-75d9-4586-8108-06c0a5e6ece8","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.006953358Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.007015969Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.007058719Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.007095829Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.011578857Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.012290994Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"uri":"/crucible/0/regions","method":"POST","req_id":"8d5977a6-a1ed-4c07-9df5-2a013eb88084","remote_addr":"[fd00:1122:3344:101::c]:50740","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":71,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.017476158Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.908872449Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"uri":"/crucible/0/regions/838d7089-75d9-4586-8108-06c0a5e6ece8","method":"GET","req_id":"13fbd480-6c45-476f-8eae-5537af03100f","remote_addr":"[fd00:1122:3344:101::c]:50740","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"path is \"/data/regions/838d7089-75d9-4586-8108-06c0a5e6ece8\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.912131447Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"datafile"}
{"msg":"dataset is oxp_ea971bec-da00-45a9-a92b-a92a4c74a66f/crucible/regions/838d7089-75d9-4586-8108-06c0a5e6ece8","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.919919984Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"datafile"}
{"msg":"region 838d7089-75d9-4586-8108-06c0a5e6ece8 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.926828483Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.926956344Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"uri":"/crucible/0/regions/838d7089-75d9-4586-8108-06c0a5e6ece8","method":"DELETE","req_id":"0f3d0a59-5db0-4cb9-bd96-b1afae86349f","remote_addr":"[fd00:1122:3344:101::c]:50740","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":14860,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.926974014Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-838d7089-75d9-4586-8108-06c0a5e6ece8 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.928096244Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.929612577Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"uri":"/crucible/0/regions/838d7089-75d9-4586-8108-06c0a5e6ece8","method":"GET","req_id":"ee45d96a-4fdc-465c-b30d-5b32882989c6","remote_addr":"[fd00:1122:3344:101::c]:50740","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":42,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.933068557Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ea971bec-da00-45a9-a92b-a92a4c74a66f/crucible/regions/838d7089-75d9-4586-8108-06c0a5e6ece8\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:17.941898032Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"region":"838d7089-75d9-4586-8108-06c0a5e6ece8","component":"worker"}
{"msg":"region 838d7089-75d9-4586-8108-06c0a5e6ece8 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:18.00339618Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:18.076042074Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"uri":"/crucible/0/regions/838d7089-75d9-4586-8108-06c0a5e6ece8","method":"GET","req_id":"71da73da-aa1a-40d0-9088-f8ae934d46b3","remote_addr":"[fd00:1122:3344:101::c]:50740","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"region d981c9e9-b282-43f9-ae48-5c0c28d4b50f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.582126312Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.582280833Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"uri":"/crucible/0/regions","method":"POST","req_id":"562120d6-0805-4302-8a9b-efa8943252ca","remote_addr":"[fd00:1122:3344:101::c]:50740","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":192,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.582333174Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ea971bec-da00-45a9-a92b-a92a4c74a66f/crucible/regions/d981c9e9-b282-43f9-ae48-5c0c28d4b50f","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.588816399Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ea971bec-da00-45a9-a92b-a92a4c74a66f/crucible/regions/d981c9e9-b282-43f9-ae48-5c0c28d4b50f","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.588842639Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d981c9e9-b282-43f9-ae48-5c0c28d4b50f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d981c9e9-b282-43f9-ae48-5c0c28d4b50f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.641035937Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"region":"d981c9e9-b282-43f9-ae48-5c0c28d4b50f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d981c9e9-b282-43f9-ae48-5c0c28d4b50f\" \"--data\" \"/data/regions/d981c9e9-b282-43f9-ae48-5c0c28d4b50f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.641084128Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"region":"d981c9e9-b282-43f9-ae48-5c0c28d4b50f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.713429329Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"region":"d981c9e9-b282-43f9-ae48-5c0c28d4b50f","component":"worker"}
{"msg":"region d981c9e9-b282-43f9-ae48-5c0c28d4b50f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.713465229Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.71355569Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d981c9e9-b282-43f9-ae48-5c0c28d4b50f","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.714342696Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d981c9e9-b282-43f9-ae48-5c0c28d4b50f","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.717864117Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.725488132Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d981c9e9-b282-43f9-ae48-5c0c28d4b50f","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.728773311Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d981c9e9-b282-43f9-ae48-5c0c28d4b50f","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.729089403Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.729121554Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.729165444Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.729211374Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.732408182Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.737534376Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-27T01:41:20.8429554Z","hostname":"oxz_crucible_1358c012-f0e3-4496-9887-7d8b3f10c855","pid":26284,"uri":"/crucible/0/regions","method":"POST","req_id":"2f234e49-a2b8-4abe-958c-1b2b65deee0c","remote_addr":"[fd00:1122:3344:101::c]:50740","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":68,"response_code":200}