[ Mar 18 18:17:19 Enabled. ]
[ Mar 18 18:17:19 Rereading configuration. ]
[ Mar 18 18:17:21 Rereading configuration. ]
[ Mar 18 18:17:27 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_2cff8c67-719c-42dd-8a3a-14f2726b082f/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:27.409241225Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544}
{"msg":"dataset: \"oxp_2cff8c67-719c-42dd-8a3a-14f2726b082f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:27.409394329Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:27.409403999Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:27.40940933Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:27.423299602Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:17:27.522541415Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:08.940056168Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42449"}
{"msg":"region 848c3e3d-54f1-465f-a1e1-ccf794b58c61 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:08.98951392Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:08.992521833Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:08.992536773Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"uri":"/crucible/0/regions","method":"POST","req_id":"74934949-f014-49ee-abc0-6529eb4d5465","remote_addr":"[fd00:1122:3344:101::a]:42449","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":20859,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_2cff8c67-719c-42dd-8a3a-14f2726b082f/crucible/regions/848c3e3d-54f1-465f-a1e1-ccf794b58c61","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.003579347Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_2cff8c67-719c-42dd-8a3a-14f2726b082f/crucible/regions/848c3e3d-54f1-465f-a1e1-ccf794b58c61","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.003592857Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"848c3e3d-54f1-465f-a1e1-ccf794b58c61\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/848c3e3d-54f1-465f-a1e1-ccf794b58c61\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.062567522Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"region":"848c3e3d-54f1-465f-a1e1-ccf794b58c61","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"848c3e3d-54f1-465f-a1e1-ccf794b58c61\" \"--data\" \"/data/regions/848c3e3d-54f1-465f-a1e1-ccf794b58c61\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.06505409Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"region":"848c3e3d-54f1-465f-a1e1-ccf794b58c61","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.135386467Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"uri":"/crucible/0/regions","method":"POST","req_id":"c65cf34f-4ea4-4554-a604-f61d585a6127","remote_addr":"[fd00:1122:3344:101::a]:42449","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.428121748Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"region":"848c3e3d-54f1-465f-a1e1-ccf794b58c61","component":"worker"}
{"msg":"region 848c3e3d-54f1-465f-a1e1-ccf794b58c61 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.428166899Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.428248732Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-848c3e3d-54f1-465f-a1e1-ccf794b58c61","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.428583421Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-848c3e3d-54f1-465f-a1e1-ccf794b58c61","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.440286533Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.465608921Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-848c3e3d-54f1-465f-a1e1-ccf794b58c61","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.468926592Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/848c3e3d-54f1-465f-a1e1-ccf794b58c61","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.469682373Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.469742815Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.469781185Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.469817046Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.473991052Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.478338611Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:09.590675565Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"uri":"/crucible/0/regions","method":"POST","req_id":"d9751748-7584-42a4-9f91-cdea72006c7f","remote_addr":"[fd00:1122:3344:101::a]:42449","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:34.418444165Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"uri":"/crucible/0/regions/848c3e3d-54f1-465f-a1e1-ccf794b58c61","method":"GET","req_id":"81ee4784-9a8c-401a-8b7d-df4f564e4dfc","remote_addr":"[fd00:1122:3344:101::a]:42449","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":93,"response_code":200}
{"msg":"path is \"/data/regions/848c3e3d-54f1-465f-a1e1-ccf794b58c61\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:34.422030634Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"datafile"}
{"msg":"dataset is oxp_2cff8c67-719c-42dd-8a3a-14f2726b082f/crucible/regions/848c3e3d-54f1-465f-a1e1-ccf794b58c61","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:34.431554856Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"datafile"}
{"msg":"region 848c3e3d-54f1-465f-a1e1-ccf794b58c61 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:34.437917051Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:34.438090426Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:34.438104326Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"uri":"/crucible/0/regions/848c3e3d-54f1-465f-a1e1-ccf794b58c61","method":"DELETE","req_id":"dadeb3b8-140f-4a3c-8f72-915fe92736ef","remote_addr":"[fd00:1122:3344:101::a]:42449","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":16077,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-848c3e3d-54f1-465f-a1e1-ccf794b58c61 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:34.439444393Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:34.441563492Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"uri":"/crucible/0/regions/848c3e3d-54f1-465f-a1e1-ccf794b58c61","method":"GET","req_id":"a58b4c90-ad1b-49f3-a4ea-69a01b5e7e33","remote_addr":"[fd00:1122:3344:101::a]:42449","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":33,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:34.447464057Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_2cff8c67-719c-42dd-8a3a-14f2726b082f/crucible/regions/848c3e3d-54f1-465f-a1e1-ccf794b58c61\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:34.454645642Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"region":"848c3e3d-54f1-465f-a1e1-ccf794b58c61","component":"worker"}
{"msg":"region 848c3e3d-54f1-465f-a1e1-ccf794b58c61 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:34.504241537Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T18:28:34.581390681Z","hostname":"oxz_crucible_7b45784a-3001-412a-92ac-d1068909ce42","pid":24544,"uri":"/crucible/0/regions/848c3e3d-54f1-465f-a1e1-ccf794b58c61","method":"GET","req_id":"fbbfb035-5421-4668-bc5c-8de6a8110e8f","remote_addr":"[fd00:1122:3344:101::a]:42449","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":51,"response_code":200}